- Add safety checks to Zig build - Add TUI with job management and narrative views - Add WebSocket support and export services - Add smart configuration defaults - Update API routes with security headers - Update SECURITY.md with comprehensive policy - Add Makefile security scanning targets
189 lines
5.1 KiB
Go
189 lines
5.1 KiB
Go
// Package services provides TUI service implementations
|
|
package services
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/jfraeys/fetch_ml/cmd/tui/internal/config"
|
|
"github.com/jfraeys/fetch_ml/cmd/tui/internal/model"
|
|
"github.com/jfraeys/fetch_ml/internal/domain"
|
|
"github.com/jfraeys/fetch_ml/internal/experiment"
|
|
"github.com/jfraeys/fetch_ml/internal/network"
|
|
"github.com/jfraeys/fetch_ml/internal/queue"
|
|
)
|
|
|
|
// Task is an alias for domain.Task for TUI compatibility
|
|
type Task = domain.Task
|
|
|
|
// TaskQueue provides TUI-specific task operations by embedding queue.TaskQueue
|
|
// and extending it with experiment management capabilities.
|
|
type TaskQueue struct {
|
|
*queue.TaskQueue // Embed to inherit all queue methods directly
|
|
expManager *experiment.Manager
|
|
ctx context.Context
|
|
config *config.Config
|
|
}
|
|
|
|
// NewTaskQueue creates a new task queue service
|
|
func NewTaskQueue(cfg *config.Config) (*TaskQueue, error) {
|
|
// Create internal queue config
|
|
queueCfg := queue.Config{
|
|
RedisAddr: cfg.RedisAddr,
|
|
RedisPassword: cfg.RedisPassword,
|
|
RedisDB: cfg.RedisDB,
|
|
}
|
|
|
|
internalQueue, err := queue.NewTaskQueue(queueCfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create task queue: %w", err)
|
|
}
|
|
|
|
// Initialize experiment manager with proper path
|
|
// BasePath already includes the mode-based experiments path (e.g., ./data/dev/experiments)
|
|
expDir := cfg.BasePath
|
|
os.MkdirAll(expDir, 0755)
|
|
expManager := experiment.NewManager(expDir)
|
|
|
|
return &TaskQueue{
|
|
TaskQueue: internalQueue,
|
|
expManager: expManager,
|
|
ctx: context.Background(),
|
|
config: cfg,
|
|
}, nil
|
|
}
|
|
|
|
// EnqueueTask adds a new task to the queue (TUI-specific: creates task with proper defaults)
|
|
func (tq *TaskQueue) EnqueueTask(jobName, args string, priority int64) (*Task, error) {
|
|
// Create internal task
|
|
internalTask := &queue.Task{
|
|
JobName: jobName,
|
|
Args: args,
|
|
Priority: priority,
|
|
}
|
|
|
|
// Use embedded queue to enqueue
|
|
err := tq.TaskQueue.AddTask(internalTask)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Return domain.Task directly (no conversion needed)
|
|
return internalTask, nil
|
|
}
|
|
|
|
// GetQueuedTasks retrieves all queued tasks (TUI-specific alias for GetAllTasks)
|
|
func (tq *TaskQueue) GetQueuedTasks() ([]*Task, error) {
|
|
return tq.TaskQueue.GetAllTasks()
|
|
}
|
|
|
|
// GetJobStatus gets the status of a job by name (TUI-specific convenience method)
|
|
func (tq *TaskQueue) GetJobStatus(jobName string) (map[string]string, error) {
|
|
task, err := tq.TaskQueue.GetTaskByName(jobName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if task == nil {
|
|
return map[string]string{"status": "not_found"}, nil
|
|
}
|
|
|
|
return map[string]string{
|
|
"status": task.Status,
|
|
"task_id": task.ID,
|
|
}, nil
|
|
}
|
|
|
|
// GetMetrics retrieves metrics for a job (TUI-specific: currently returns empty)
|
|
func (tq *TaskQueue) GetMetrics(_ string) (map[string]string, error) {
|
|
// This method doesn't exist in internal queue, return empty for now
|
|
return map[string]string{}, nil
|
|
}
|
|
|
|
// ListDatasets retrieves available datasets from the filesystem
|
|
func (tq *TaskQueue) ListDatasets() ([]model.DatasetInfo, error) {
|
|
var datasets []model.DatasetInfo
|
|
|
|
// Scan the active data directory for datasets
|
|
dataDir := tq.config.BasePath
|
|
if dataDir == "" {
|
|
return datasets, nil
|
|
}
|
|
|
|
entries, err := os.ReadDir(dataDir)
|
|
if err != nil {
|
|
// Directory might not exist yet, return empty
|
|
return datasets, nil
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
datasets = append(datasets, model.DatasetInfo{
|
|
Name: entry.Name(),
|
|
SizeBytes: info.Size(),
|
|
Location: filepath.Join(dataDir, entry.Name()),
|
|
LastAccess: time.Now(),
|
|
})
|
|
}
|
|
}
|
|
|
|
return datasets, nil
|
|
}
|
|
|
|
// ListExperiments retrieves experiment list
|
|
func (tq *TaskQueue) ListExperiments() ([]string, error) {
|
|
return tq.expManager.ListExperiments()
|
|
}
|
|
|
|
// GetExperimentDetails retrieves formatted experiment details
|
|
func (tq *TaskQueue) GetExperimentDetails(commitID string) (string, error) {
|
|
meta, err := tq.expManager.ReadMetadata(commitID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
metrics, err := tq.expManager.GetMetrics(commitID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
output := fmt.Sprintf("Experiment: %s\n", meta.JobName)
|
|
output += fmt.Sprintf("Commit ID: %s\n", meta.CommitID)
|
|
output += fmt.Sprintf("User: %s\n", meta.User)
|
|
output += fmt.Sprintf("Timestamp: %d\n\n", meta.Timestamp)
|
|
output += "Metrics:\n"
|
|
|
|
if len(metrics) == 0 {
|
|
output += " No metrics logged.\n"
|
|
} else {
|
|
for _, m := range metrics {
|
|
output += fmt.Sprintf(" %s: %.4f (Step: %d)\n", m.Name, m.Value, m.Step)
|
|
}
|
|
}
|
|
|
|
return output, nil
|
|
}
|
|
|
|
// Close closes the task queue
|
|
func (tq *TaskQueue) Close() error {
|
|
return tq.TaskQueue.Close()
|
|
}
|
|
|
|
// MLServer is an alias for network.MLServer for backward compatibility
|
|
type MLServer = network.MLServer
|
|
|
|
// NewMLServer creates a new ML server connection
|
|
func NewMLServer(cfg *config.Config) (*MLServer, error) {
|
|
// Local mode: skip SSH entirely
|
|
if cfg.Host == "" {
|
|
return network.NewMLServer("", "", "", 0, "")
|
|
}
|
|
|
|
return network.NewMLServer(cfg.Host, cfg.User, cfg.SSHKey, cfg.Port, cfg.KnownHosts)
|
|
}
|