Stress Tests: - TestStress_WorkerConnectBurst: 30 workers, p99 latency validation - TestStress_JobSubmissionBurst: 1K job submissions - TestStress_WorkerChurn: 50 connect/disconnect cycles, memory leak detection - TestStress_ConcurrentScheduling: 10 workers x 20 jobs contention Long-Running Tests: - TestLongRunning_MemoryLeak: heap growth monitoring - TestLongRunning_OrphanRecovery: worker death/requeue stability - TestLongRunning_WebSocketStability: 20 worker connection stability Infrastructure: - Add testreport package with JSON output, flaky test tracking - Add TestTimer for timing/budget enforcement - Add WaitForEvent, WaitForTaskStatus helpers - Fix worker IDs to use valid bench-worker token patterns
237 lines
5.1 KiB
Go
237 lines
5.1 KiB
Go
// Package fixtures provides test fixtures and helpers
|
|
package tests
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jfraeys/fetch_ml/internal/scheduler"
|
|
)
|
|
|
|
// WaitForEvent waits for a specific event type from the scheduler's state events
|
|
// with a timeout. It polls the state events until the event is found or timeout.
|
|
// Returns the matching event and true if found, nil and false if timeout.
|
|
func WaitForEvent(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
eventType scheduler.StateEventType,
|
|
timeout time.Duration,
|
|
) (*scheduler.StateEvent, bool) {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
events, err := hub.GetStateEvents()
|
|
if err != nil {
|
|
t.Logf("WaitForEvent: error getting state events: %v", err)
|
|
time.Sleep(50 * time.Millisecond)
|
|
continue
|
|
}
|
|
|
|
for _, event := range events {
|
|
if event.Type == eventType {
|
|
return &event, true
|
|
}
|
|
}
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
return nil, false
|
|
}
|
|
|
|
// WaitForEventWithFilter waits for a specific event type that matches a filter function
|
|
func WaitForEventWithFilter(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
eventType scheduler.StateEventType,
|
|
filter func(scheduler.StateEvent) bool,
|
|
timeout time.Duration,
|
|
) (*scheduler.StateEvent, bool) {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
events, err := hub.GetStateEvents()
|
|
if err != nil {
|
|
t.Logf("WaitForEventWithFilter: error getting state events: %v", err)
|
|
time.Sleep(50 * time.Millisecond)
|
|
continue
|
|
}
|
|
|
|
for _, event := range events {
|
|
if event.Type == eventType && filter(event) {
|
|
return &event, true
|
|
}
|
|
}
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
return nil, false
|
|
}
|
|
|
|
// WaitForTaskStatus waits for a task to reach a specific status
|
|
func WaitForTaskStatus(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
taskID string,
|
|
status string,
|
|
timeout time.Duration,
|
|
) bool {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
task := hub.GetTask(taskID)
|
|
if task != nil && task.Status == status {
|
|
return true
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// WaitForMetric waits for a metric to satisfy a condition
|
|
func WaitForMetric(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
metricKey string,
|
|
condition func(interface{}) bool,
|
|
timeout time.Duration,
|
|
) bool {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
metrics := hub.GetMetricsPayload()
|
|
if value, ok := metrics[metricKey]; ok {
|
|
if condition(value) {
|
|
return true
|
|
}
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// PollWithTimeout repeatedly calls a function until it returns true or timeout
|
|
func PollWithTimeout(
|
|
t *testing.T,
|
|
name string,
|
|
fn func() bool,
|
|
timeout time.Duration,
|
|
interval time.Duration,
|
|
) bool {
|
|
t.Helper()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Logf("PollWithTimeout %s: timeout after %v", name, timeout)
|
|
return false
|
|
case <-ticker.C:
|
|
if fn() {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// AssertEventReceived asserts that an event of the specified type was received
|
|
func AssertEventReceived(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
eventType scheduler.StateEventType,
|
|
timeout time.Duration,
|
|
) *scheduler.StateEvent {
|
|
t.Helper()
|
|
|
|
event, found := WaitForEvent(t, hub, eventType, timeout)
|
|
if !found {
|
|
t.Fatalf("Expected event type %v within %v, but was not received", eventType, timeout)
|
|
}
|
|
return event
|
|
}
|
|
|
|
// AssertTaskStatus asserts that a task reaches the expected status
|
|
func AssertTaskStatus(
|
|
t *testing.T,
|
|
hub *scheduler.SchedulerHub,
|
|
taskID string,
|
|
expectedStatus string,
|
|
timeout time.Duration,
|
|
) {
|
|
t.Helper()
|
|
|
|
if !WaitForTaskStatus(t, hub, taskID, expectedStatus, timeout) {
|
|
task := hub.GetTask(taskID)
|
|
if task == nil {
|
|
t.Fatalf("Task %s not found (expected status: %s)", taskID, expectedStatus)
|
|
}
|
|
t.Fatalf("Task %s has status %s, expected %s (timeout: %v)",
|
|
taskID, task.Status, expectedStatus, timeout)
|
|
}
|
|
}
|
|
|
|
// WaitForCondition waits for a condition to be true with a timeout
|
|
// Returns true if condition was met, false if timeout
|
|
func WaitForCondition(
|
|
t *testing.T,
|
|
name string,
|
|
condition func() bool,
|
|
timeout time.Duration,
|
|
) bool {
|
|
t.Helper()
|
|
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
if condition() {
|
|
return true
|
|
}
|
|
time.Sleep(50 * time.Millisecond)
|
|
}
|
|
|
|
t.Logf("WaitForCondition %s: timeout after %v", name, timeout)
|
|
return false
|
|
}
|
|
|
|
// RetryWithBackoff retries an operation with exponential backoff
|
|
func RetryWithBackoff(
|
|
t *testing.T,
|
|
name string,
|
|
maxRetries int,
|
|
baseDelay time.Duration,
|
|
fn func() error,
|
|
) error {
|
|
t.Helper()
|
|
|
|
var err error
|
|
delay := baseDelay
|
|
|
|
for i := 0; i < maxRetries; i++ {
|
|
err = fn()
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
t.Logf("RetryWithBackoff %s: attempt %d/%d failed: %v", name, i+1, maxRetries, err)
|
|
|
|
if i < maxRetries-1 {
|
|
time.Sleep(delay)
|
|
delay *= 2 // exponential backoff
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("%s failed after %d attempts: %w", name, maxRetries, err)
|
|
}
|