fetch_ml/internal/queue/queue_spec_test.go
Jeremie Fraeys 8271277dc3
feat: implement research-grade maintainability phases 2, 5, 8, 10
Phase 2: Deterministic Manifests
- Add manifest.Validator with required field checking
- Support Validate() and ValidateStrict() modes
- Integrate validation into worker executor before execution
- Block execution if manifest missing commit_id or deps_manifest_sha256

Phase 5: Pinned Dependencies
- Add hermetic.dockerfile template with pinned system deps
- Frozen package versions: libblas3, libcudnn8, etc.
- Support for deps_manifest.json and requirements.txt with hashes
- Image tagging strategy: deps-<first-8-of-sha256>

Phase 8: Tests as Specifications
- Add queue_spec_test.go with executable scheduler specs
- Document priority ordering (higher first)
- Document FIFO tiebreaker for same priority
- Test cases for negative/zero priorities

Phase 10: Local Dev Parity
- Create root-level docker-compose.dev.yml
- Simplified from deployments/ for quick local dev
- Redis + API server + Worker with hot reload volumes
- Debug ports: 9101 (API), 6379 (Redis)
2026-02-18 15:34:28 -05:00

188 lines
4.8 KiB
Go

package queue
import (
"testing"
"time"
)
// TestTaskPrioritizationSpec documents the scheduler's priority and FIFO behavior.
// These tests serve as executable specifications for the queue system.
func TestTaskPrioritizationSpec(t *testing.T) {
tests := []struct {
name string
tasks []Task
expected []string // IDs in expected execution order
}{
{
name: "higher priority runs first",
tasks: []Task{
{ID: "low", Priority: 1, CreatedAt: time.Unix(100, 0)},
{ID: "high", Priority: 10, CreatedAt: time.Unix(100, 0)},
},
expected: []string{"high", "low"},
},
{
name: "FIFO for same priority",
tasks: []Task{
{ID: "first", Priority: 5, CreatedAt: time.Unix(100, 0)},
{ID: "second", Priority: 5, CreatedAt: time.Unix(200, 0)},
},
expected: []string{"first", "second"},
},
{
name: "mixed priorities and creation times",
tasks: []Task{
{ID: "medium-early", Priority: 5, CreatedAt: time.Unix(100, 0)},
{ID: "high-late", Priority: 10, CreatedAt: time.Unix(300, 0)},
{ID: "low-early", Priority: 1, CreatedAt: time.Unix(50, 0)},
},
expected: []string{"high-late", "medium-early", "low-early"},
},
{
name: "negative priority is lowest",
tasks: []Task{
{ID: "negative", Priority: -1, CreatedAt: time.Unix(100, 0)},
{ID: "positive", Priority: 1, CreatedAt: time.Unix(100, 0)},
},
expected: []string{"positive", "negative"},
},
{
name: "zero priority is default",
tasks: []Task{
{ID: "zero", Priority: 0, CreatedAt: time.Unix(100, 0)},
{ID: "positive", Priority: 1, CreatedAt: time.Unix(100, 0)},
},
expected: []string{"positive", "zero"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create a queue and add tasks
tmpDir := t.TempDir()
q, err := NewFilesystemQueue(tmpDir)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
defer q.Close()
// Add all tasks
for _, task := range tt.tasks {
task := task // capture range variable
if err := q.AddTask(&task); err != nil {
t.Fatalf("failed to add task %s: %v", task.ID, err)
}
}
// Get tasks in order and verify
var actual []string
for i := 0; i < len(tt.tasks); i++ {
task, err := q.GetNextTask()
if err != nil {
t.Fatalf("failed to get task at position %d: %v", i, err)
}
if task == nil {
t.Fatalf("expected task at position %d, got nil", i)
}
actual = append(actual, task.ID)
}
// Verify order
if len(actual) != len(tt.expected) {
t.Errorf("expected %d tasks, got %d", len(tt.expected), len(actual))
}
for i, expectedID := range tt.expected {
if i >= len(actual) {
break
}
if actual[i] != expectedID {
t.Errorf("position %d: expected %s, got %s", i, expectedID, actual[i])
}
}
})
}
}
// TestQueueSpec_ClaimAndComplete documents the claim-complete lifecycle
func TestQueueSpec_ClaimAndComplete(t *testing.T) {
tmpDir := t.TempDir()
q, err := NewFilesystemQueue(tmpDir)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
defer q.Close()
// Add a task
task := &Task{
ID: "task-1",
JobName: "test-job",
Priority: 5,
CreatedAt: time.Now(),
}
if err := q.AddTask(task); err != nil {
t.Fatalf("failed to add task: %v", err)
}
// Get the task
claimed, err := q.GetNextTask()
if err != nil {
t.Fatalf("failed to get task: %v", err)
}
if claimed == nil {
t.Fatal("expected to claim a task, got nil")
}
if claimed.ID != task.ID {
t.Errorf("expected task %s, got %s", task.ID, claimed.ID)
}
// Verify task is no longer in queue
tasks, err := q.GetAllTasks()
if err != nil {
t.Fatalf("failed to get tasks: %v", err)
}
for _, tsk := range tasks {
if tsk.ID == task.ID {
t.Error("claimed task should not be in queue")
}
}
}
// TestQueueSpec_TaskPriorityOrdering documents numeric priority ordering
func TestQueueSpec_TaskPriorityOrdering(t *testing.T) {
tmpDir := t.TempDir()
q, err := NewFilesystemQueue(tmpDir)
if err != nil {
t.Fatalf("failed to create queue: %v", err)
}
defer q.Close()
// Add tasks with various priorities
priorities := []int64{100, 50, 200, 1, 75}
for i, p := range priorities {
task := &Task{
ID: "task-" + string(rune('a'+i)),
JobName: "job-" + string(rune('a'+i)),
Priority: p,
CreatedAt: time.Now(),
}
if err := q.AddTask(task); err != nil {
t.Fatalf("failed to add task: %v", err)
}
}
// Expected order: 200, 100, 75, 50, 1 (descending)
expected := []string{"task-c", "task-a", "task-e", "task-b", "task-d"}
for i, expID := range expected {
task, err := q.GetNextTask()
if err != nil {
t.Fatalf("position %d: failed to get task: %v", i, err)
}
if task == nil {
t.Fatalf("position %d: expected task %s, got nil", i, expID)
}
if task.ID != expID {
t.Errorf("position %d: expected %s, got %s", i, expID, task.ID)
}
}
}