5f53104fcd
- Update E2E tests for consolidated docker-compose.test.yml - Remove references to obsolete logs-debug.yml - Enhance test fixtures and utilities - Improve integration test coverage for KMS, queue, scheduler - Update unit tests for config constants and worker execution - Modernize cleanup-status.sh with new Makefile targets
90 lines
2.5 KiB
Go
90 lines
2.5 KiB
Go
package worker_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jfraeys/fetch_ml/internal/experiment"
|
|
"github.com/jfraeys/fetch_ml/internal/manifest"
|
|
"github.com/jfraeys/fetch_ml/internal/queue"
|
|
"github.com/jfraeys/fetch_ml/internal/worker"
|
|
tests "github.com/jfraeys/fetch_ml/tests/fixtures"
|
|
)
|
|
|
|
func TestRunManifest_WrittenForLocalModeRun(t *testing.T) {
|
|
base := t.TempDir()
|
|
cfg := &worker.Config{
|
|
BasePath: base,
|
|
LocalMode: true,
|
|
Entrypoint: "train.py",
|
|
PodmanImage: "python:3.11",
|
|
WorkerID: "worker-test",
|
|
}
|
|
w := tests.NewTestWorker(cfg)
|
|
|
|
commitID := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // 40 hex
|
|
expMgr := experiment.NewManager(base)
|
|
if err := expMgr.CreateExperiment(commitID); err != nil {
|
|
t.Fatalf("CreateExperiment: %v", err)
|
|
}
|
|
filesPath := expMgr.GetFilesPath(commitID)
|
|
if err := os.WriteFile(filepath.Join(filesPath, "train.py"), []byte("print('ok')\n"), 0600); err != nil {
|
|
t.Fatalf("write train.py: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(filesPath, "requirements.txt"), []byte("numpy==1.0.0\n"), 0600); err != nil {
|
|
t.Fatalf("write requirements.txt: %v", err)
|
|
}
|
|
man, err := expMgr.GenerateManifest(commitID)
|
|
if err != nil {
|
|
t.Fatalf("GenerateManifest: %v", err)
|
|
}
|
|
if err := expMgr.WriteManifest(man); err != nil {
|
|
t.Fatalf("WriteManifest: %v", err)
|
|
}
|
|
|
|
task := &queue.Task{
|
|
ID: "task-1234",
|
|
JobName: "job-1",
|
|
CreatedAt: time.Now().UTC(),
|
|
Metadata: map[string]string{
|
|
"commit_id": commitID,
|
|
"experiment_manifest_overall_sha": man.OverallSHA,
|
|
"deps_manifest_name": "requirements.txt",
|
|
"deps_manifest_sha256": "deadbeef",
|
|
},
|
|
}
|
|
|
|
if err := w.RunJob(context.Background(), task, ""); err != nil {
|
|
t.Fatalf("RunJob: %v", err)
|
|
}
|
|
|
|
finishedDir := filepath.Join(base, "finished", task.JobName)
|
|
loaded, err := manifest.LoadFromDir(finishedDir)
|
|
if err != nil {
|
|
t.Fatalf("LoadFromDir: %v", err)
|
|
}
|
|
if loaded.RunID == "" {
|
|
t.Fatalf("expected run_id")
|
|
}
|
|
if loaded.TaskID != task.ID {
|
|
t.Fatalf("task_id mismatch: got %q want %q", loaded.TaskID, task.ID)
|
|
}
|
|
if loaded.JobName != task.JobName {
|
|
t.Fatalf("job_name mismatch: got %q want %q", loaded.JobName, task.JobName)
|
|
}
|
|
if loaded.CommitID != commitID {
|
|
t.Fatalf("commit_id mismatch: got %q want %q", loaded.CommitID, commitID)
|
|
}
|
|
if loaded.DepsManifestName == "" {
|
|
t.Fatalf("expected deps_manifest_name")
|
|
}
|
|
if loaded.Command == "" {
|
|
t.Fatalf("expected command")
|
|
}
|
|
if loaded.ExitCode == nil {
|
|
t.Fatalf("expected exit_code")
|
|
}
|
|
}
|