fetch_ml/tests/unit/scheduler/priority_queue_test.go
Jeremie Fraeys 43e6446587
feat(scheduler): implement multi-tenant job scheduler with gang scheduling
Add new scheduler component for distributed ML workload orchestration:
- Hub-based coordination for multi-worker clusters
- Pacing controller for rate limiting job submissions
- Priority queue with preemption support
- Port allocator for dynamic service discovery
- Protocol handlers for worker-scheduler communication
- Service manager with OS-specific implementations
- Connection management and state persistence
- Template system for service deployment

Includes comprehensive test suite:
- Unit tests for all core components
- Integration tests for distributed scenarios
- Benchmark tests for performance validation
- Mock fixtures for isolated testing

Refs: scheduler-architecture.md
2026-02-26 12:03:23 -05:00

214 lines
4.9 KiB
Go

package scheduler_test
import (
"fmt"
"testing"
"time"
"github.com/jfraeys/fetch_ml/internal/scheduler"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPriorityQueue_BasicOperations(t *testing.T) {
q := scheduler.NewPriorityQueue(0.1)
task1 := &scheduler.Task{
ID: "task-1",
Priority: 10,
SubmittedAt: time.Now(),
Spec: scheduler.JobSpec{ID: "task-1"},
}
task2 := &scheduler.Task{
ID: "task-2",
Priority: 5,
SubmittedAt: time.Now(),
Spec: scheduler.JobSpec{ID: "task-2"},
}
// Add tasks
q.Add(task1)
q.Add(task2)
require.Equal(t, 2, q.Len())
// Should return highest priority first (task1 with priority 10)
first := q.Take()
require.NotNil(t, first)
assert.Equal(t, "task-1", first.ID)
// Second task
second := q.Take()
require.NotNil(t, second)
assert.Equal(t, "task-2", second.ID)
// Queue empty
third := q.Take()
assert.Nil(t, third)
}
func TestPriorityQueue_EffectivePriority_WithAging(t *testing.T) {
now := time.Now()
// Task with lower priority but older submission
oldTask := &scheduler.Task{
ID: "old-task",
Priority: 5,
SubmittedAt: now.Add(-10 * time.Minute), // 10 min old
}
// Task with higher priority but recent submission
newTask := &scheduler.Task{
ID: "new-task",
Priority: 10,
SubmittedAt: now, // Just submitted
}
// Calculate effective priorities
oldEffective := oldTask.EffectivePriority(0.1, now)
newEffective := newTask.EffectivePriority(0.1, now)
// Old task should have higher effective priority due to aging
// 5 + (10 min * 0.1) = 6.0
// 10 + (0 min * 0.1) = 10.0
assert.Less(t, oldEffective, newEffective)
assert.InDelta(t, 6.0, oldEffective, 0.1)
assert.InDelta(t, 10.0, newEffective, 0.1)
}
func TestPriorityQueue_FIFOOnTie(t *testing.T) {
now := time.Now()
q := scheduler.NewPriorityQueue(0.1)
// Two tasks with same priority, submitted at different times
task1 := &scheduler.Task{
ID: "task-1",
Priority: 10,
SubmittedAt: now.Add(-5 * time.Minute),
Spec: scheduler.JobSpec{ID: "task-1"},
}
task2 := &scheduler.Task{
ID: "task-2",
Priority: 10,
SubmittedAt: now.Add(-1 * time.Minute),
Spec: scheduler.JobSpec{ID: "task-2"},
}
// Add in reverse order
q.Add(task2)
q.Add(task1)
// Should return older task first (FIFO on tie)
first := q.Take()
require.NotNil(t, first)
assert.Equal(t, "task-1", first.ID)
second := q.Take()
require.NotNil(t, second)
assert.Equal(t, "task-2", second.ID)
}
func TestPriorityQueue_Remove(t *testing.T) {
q := scheduler.NewPriorityQueue(0.1)
task1 := &scheduler.Task{ID: "task-1", Priority: 10, Spec: scheduler.JobSpec{ID: "task-1"}}
task2 := &scheduler.Task{ID: "task-2", Priority: 5, Spec: scheduler.JobSpec{ID: "task-2"}}
task3 := &scheduler.Task{ID: "task-3", Priority: 1, Spec: scheduler.JobSpec{ID: "task-3"}}
q.Add(task1)
q.Add(task2)
q.Add(task3)
// Remove middle task
removed := q.Remove("task-2")
assert.True(t, removed)
assert.Equal(t, 2, q.Len())
// Try to remove non-existent
removed = q.Remove("non-existent")
assert.False(t, removed)
// Verify remaining order
first := q.Take()
assert.Equal(t, "task-1", first.ID)
second := q.Take()
assert.Equal(t, "task-3", second.ID)
}
func TestPriorityQueue_Get(t *testing.T) {
q := scheduler.NewPriorityQueue(0.1)
task1 := &scheduler.Task{ID: "task-1", Priority: 10, Spec: scheduler.JobSpec{ID: "task-1"}}
q.Add(task1)
// Get existing task
found := q.Get("task-1")
assert.NotNil(t, found)
assert.Equal(t, "task-1", found.ID)
// Get non-existent
notFound := q.Get("non-existent")
assert.Nil(t, notFound)
}
func TestPriorityQueue_Items(t *testing.T) {
q := scheduler.NewPriorityQueue(0.1)
tasks := []*scheduler.Task{
{ID: "task-1", Priority: 10, Spec: scheduler.JobSpec{ID: "task-1"}},
{ID: "task-2", Priority: 5, Spec: scheduler.JobSpec{ID: "task-2"}},
{ID: "task-3", Priority: 1, Spec: scheduler.JobSpec{ID: "task-3"}},
}
for _, task := range tasks {
q.Add(task)
}
items := q.Items()
require.Len(t, items, 3)
// Items should be in priority order (highest first)
assert.Equal(t, "task-1", items[0].ID)
assert.Equal(t, "task-2", items[1].ID)
assert.Equal(t, "task-3", items[2].ID)
}
func TestPriorityQueue_ConcurrentAccess(t *testing.T) {
q := scheduler.NewPriorityQueue(0.1)
done := make(chan bool, 3)
// Concurrent adds
go func() {
for i := 0; i < 100; i++ {
q.Add(&scheduler.Task{ID: fmt.Sprintf("task-%d", i), Priority: i})
}
done <- true
}()
// Concurrent takes
go func() {
for i := 0; i < 50; i++ {
q.Take()
}
done <- true
}()
// Concurrent peeks
go func() {
for i := 0; i < 100; i++ {
q.Peek()
}
done <- true
}()
// Wait for all goroutines
for i := 0; i < 3; i++ {
<-done
}
// Queue should be in consistent state
assert.GreaterOrEqual(t, q.Len(), 0)
assert.LessOrEqual(t, q.Len(), 100)
}