- Move queue_spec_test.go from internal/queue/ to tests/unit/queue/ - Update imports to use github.com/jfraeys/fetch_ml/internal/queue - Remove duplicate docker-compose.dev.yml from root (exists in deployments/) - Fix spec tests: add required Status field, JobName field - Fix loop variable capture in priority ordering test - Fix missing closing brace between test functions - Fix existing queue_test.go: change 50ms to 1s for Redis min duration All tests pass: go test ./tests/unit/queue/...
204 lines
5.8 KiB
Go
204 lines
5.8 KiB
Go
package queue
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jfraeys/fetch_ml/internal/queue"
|
|
)
|
|
|
|
// TestTaskPrioritizationSpec documents the scheduler's priority and FIFO behavior.
|
|
// These tests serve as executable specifications for the queue system.
|
|
func TestTaskPrioritizationSpec(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
tasks []queue.Task
|
|
expected []string // IDs in expected execution order
|
|
}{
|
|
{
|
|
name: "higher priority runs first",
|
|
tasks: []queue.Task{
|
|
{ID: "low", JobName: "low-job", Status: "queued", Priority: 1, CreatedAt: time.Unix(100, 0)},
|
|
{ID: "high", JobName: "high-job", Status: "queued", Priority: 10, CreatedAt: time.Unix(100, 0)},
|
|
},
|
|
expected: []string{"high", "low"},
|
|
},
|
|
{
|
|
name: "FIFO for same priority",
|
|
tasks: []queue.Task{
|
|
{ID: "first", JobName: "first-job", Status: "queued", Priority: 5, CreatedAt: time.Unix(100, 0)},
|
|
{ID: "second", JobName: "second-job", Status: "queued", Priority: 5, CreatedAt: time.Unix(200, 0)},
|
|
},
|
|
expected: []string{"first", "second"},
|
|
},
|
|
{
|
|
name: "mixed priorities and creation times",
|
|
tasks: []queue.Task{
|
|
{ID: "medium-early", JobName: "me-job", Status: "queued", Priority: 5, CreatedAt: time.Unix(100, 0)},
|
|
{ID: "high-late", JobName: "hl-job", Status: "queued", Priority: 10, CreatedAt: time.Unix(300, 0)},
|
|
{ID: "low-early", JobName: "le-job", Status: "queued", Priority: 1, CreatedAt: time.Unix(50, 0)},
|
|
},
|
|
expected: []string{"high-late", "medium-early", "low-early"},
|
|
},
|
|
{
|
|
name: "negative priority is lowest",
|
|
tasks: []queue.Task{
|
|
{ID: "negative", JobName: "neg-job", Status: "queued", Priority: -1, CreatedAt: time.Unix(100, 0)},
|
|
{ID: "positive", JobName: "pos-job", Status: "queued", Priority: 1, CreatedAt: time.Unix(100, 0)},
|
|
},
|
|
expected: []string{"positive", "negative"},
|
|
},
|
|
{
|
|
name: "zero priority is default",
|
|
tasks: []queue.Task{
|
|
{ID: "zero", JobName: "zero-job", Status: "queued", Priority: 0, CreatedAt: time.Unix(100, 0)},
|
|
{ID: "positive", JobName: "pos-job", Status: "queued", Priority: 1, CreatedAt: time.Unix(100, 0)},
|
|
},
|
|
expected: []string{"positive", "zero"},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// Create a queue and add tasks
|
|
tmpDir := t.TempDir()
|
|
q, err := queue.NewFilesystemQueue(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("failed to create queue: %v", err)
|
|
}
|
|
defer q.Close()
|
|
|
|
// Add all tasks
|
|
for _, task := range tt.tasks {
|
|
task := task // capture range variable
|
|
if err := q.AddTask(&task); err != nil {
|
|
t.Fatalf("failed to add task %s: %v", task.ID, err)
|
|
}
|
|
}
|
|
|
|
// Get tasks in order and verify
|
|
var actual []string
|
|
for i := 0; i < len(tt.tasks); i++ {
|
|
task, err := q.GetNextTask()
|
|
if err != nil {
|
|
t.Fatalf("failed to get task at position %d: %v", i, err)
|
|
}
|
|
if task == nil {
|
|
t.Fatalf("expected task at position %d, got nil", i)
|
|
}
|
|
actual = append(actual, task.ID)
|
|
}
|
|
|
|
// Verify order
|
|
if len(actual) != len(tt.expected) {
|
|
t.Errorf("expected %d tasks, got %d", len(tt.expected), len(actual))
|
|
}
|
|
for i, expectedID := range tt.expected {
|
|
if i >= len(actual) {
|
|
break
|
|
}
|
|
if actual[i] != expectedID {
|
|
t.Errorf("position %d: expected %s, got %s", i, expectedID, actual[i])
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestQueueSpec_ClaimAndComplete documents the claim-complete lifecycle
|
|
func TestQueueSpec_ClaimAndComplete(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
q, err := queue.NewFilesystemQueue(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("failed to create queue: %v", err)
|
|
}
|
|
defer q.Close()
|
|
|
|
// Add a task
|
|
task := &queue.Task{
|
|
ID: "task-1",
|
|
JobName: "test-job",
|
|
Status: "queued",
|
|
Priority: 5,
|
|
CreatedAt: time.Now(),
|
|
}
|
|
if err := q.AddTask(task); err != nil {
|
|
t.Fatalf("failed to add task: %v", err)
|
|
}
|
|
|
|
// Get the task (moves it from pending to running)
|
|
claimed, err := q.GetNextTask()
|
|
if err != nil {
|
|
t.Fatalf("failed to get task: %v", err)
|
|
}
|
|
if claimed == nil {
|
|
t.Fatal("expected to get a task, got nil")
|
|
}
|
|
if claimed.ID != task.ID {
|
|
t.Errorf("expected task %s, got %s", task.ID, claimed.ID)
|
|
}
|
|
|
|
// Verify task is no longer in pending (it's now in running)
|
|
pendingDir := filepath.Join(tmpDir, "pending", "entries")
|
|
entries, err := os.ReadDir(pendingDir)
|
|
if err != nil {
|
|
t.Fatalf("failed to read pending dir: %v", err)
|
|
}
|
|
for _, e := range entries {
|
|
if strings.Contains(e.Name(), task.ID) {
|
|
t.Error("task should not be in pending after GetNextTask")
|
|
}
|
|
}
|
|
|
|
// Verify task is in running
|
|
runningPath := filepath.Join(tmpDir, "running", task.ID+".json")
|
|
if _, err := os.Stat(runningPath); os.IsNotExist(err) {
|
|
t.Error("task should be in running directory after GetNextTask")
|
|
}
|
|
}
|
|
|
|
// TestQueueSpec_TaskPriorityOrdering documents numeric priority ordering
|
|
func TestQueueSpec_TaskPriorityOrdering(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
q, err := queue.NewFilesystemQueue(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("failed to create queue: %v", err)
|
|
}
|
|
defer q.Close()
|
|
|
|
// Add tasks with various priorities
|
|
priorities := []int64{100, 50, 200, 1, 75}
|
|
i := 0
|
|
for _, p := range priorities {
|
|
task := &queue.Task{
|
|
ID: "task-" + string(rune('a'+i)),
|
|
JobName: "job-" + string(rune('a'+i)),
|
|
Status: "queued",
|
|
Priority: p,
|
|
CreatedAt: time.Now(),
|
|
}
|
|
if err := q.AddTask(task); err != nil {
|
|
t.Fatalf("failed to add task: %v", err)
|
|
}
|
|
i++
|
|
}
|
|
|
|
// Expected order: 200, 100, 75, 50, 1 (descending)
|
|
expected := []string{"task-c", "task-a", "task-e", "task-b", "task-d"}
|
|
|
|
for i, expID := range expected {
|
|
task, err := q.GetNextTask()
|
|
if err != nil {
|
|
t.Fatalf("position %d: failed to get task: %v", i, err)
|
|
}
|
|
if task == nil {
|
|
t.Fatalf("position %d: expected task %s, got nil", i, expID)
|
|
}
|
|
if task.ID != expID {
|
|
t.Errorf("position %d: expected %s, got %s", i, expID, task.ID)
|
|
}
|
|
}
|
|
}
|