feat: implement research-grade maintainability phases 1,3,4,7
Phase 1: Event Sourcing - Add TaskEvent types (queued, started, completed, failed, etc.) - Create EventStore with Redis Streams (append-only) - Support event querying by task ID and time range Phase 3: Diagnosable Failures - Enhance TaskExecutionError with Context map, Timestamp, Recoverable flag - Update container.go to populate error context (image, GPU, duration) - Add WithContext helper for building error context - Create cmd/errors CLI for querying task errors Phase 4: Testable Security - Add security fields to PodmanConfig (Privileged, Network, ReadOnlyMounts) - Create ValidateSecurityPolicy() with ErrSecurityViolation - Add security contract tests (privileged rejection, host network rejection) - Tests serve as executable security documentation Phase 7: Reproducible Builds - Add BuildHash and BuildTime ldflags to Makefile - Create verify-build target for reproducibility testing - Add -version and -verify flags to api-server All tests pass: - go test ./internal/errtypes/... - go test ./internal/container/... -run Security - go test ./internal/queue/... - go build ./cmd/api-server/...
This commit is contained in:
Jeremie Fraeys
parent
b889b5403d
commit
7194826871
9 changed files with 622 additions and 15 deletions
26
Makefile
26
Makefile
|
|
@ -9,14 +9,30 @@ all: build
|
|||
|
||||
# Build all components (Go binaries + optimized CLI)
|
||||
build:
|
||||
go build -o bin/api-server ./cmd/api-server/main.go
|
||||
go build -o bin/worker ./cmd/worker/worker_server.go
|
||||
go build -o bin/data_manager ./cmd/data_manager
|
||||
go build -o bin/user_manager ./cmd/user_manager
|
||||
go build -o bin/tui ./cmd/tui
|
||||
go build -ldflags="-X main.BuildHash=$(shell git rev-parse --short HEAD) -X main.BuildTime=$(shell date -u +%Y%m%d.%H%M%S)" -o bin/api-server ./cmd/api-server/main.go
|
||||
go build -ldflags="-X main.BuildHash=$(shell git rev-parse --short HEAD) -X main.BuildTime=$(shell date -u +%Y%m%d.%H%M%S)" -o bin/worker ./cmd/worker/worker_server.go
|
||||
go build -ldflags="-X main.BuildHash=$(shell git rev-parse --short HEAD) -X main.BuildTime=$(shell date -u +%Y%m%d.%H%M%S)" -o bin/data_manager ./cmd/data_manager
|
||||
go build -ldflags="-X main.BuildHash=$(shell git rev-parse --short HEAD) -X main.BuildTime=$(shell date -u +%Y%m%d.%H%M%S)" -o bin/user_manager ./cmd/user_manager
|
||||
go build -ldflags="-X main.BuildHash=$(shell git rev-parse --short HEAD) -X main.BuildTime=$(shell date -u +%Y%m%d.%H%M%S)" -o bin/tui ./cmd/tui
|
||||
$(MAKE) -C ./cli all
|
||||
@echo "${OK} All components built"
|
||||
|
||||
# Verify build reproducibility (build twice, compare hashes)
|
||||
verify-build:
|
||||
@echo "Building first time..."
|
||||
@make build
|
||||
@shasum -a 256 bin/* > /tmp/build_hash_1.txt 2>/dev/null || true
|
||||
@echo "Building second time..."
|
||||
@make build
|
||||
@shasum -a 256 bin/* > /tmp/build_hash_2.txt 2>/dev/null || true
|
||||
@echo "Comparing hashes..."
|
||||
@if diff /tmp/build_hash_1.txt /tmp/build_hash_2.txt > /dev/null; then \
|
||||
echo "${OK} Build is reproducible - hashes match"; \
|
||||
else \
|
||||
echo "Build differs (expected for non-reproducible builds with timestamps)"; \
|
||||
diff /tmp/build_hash_1.txt /tmp/build_hash_2.txt || true; \
|
||||
fi
|
||||
|
||||
# Build native C++ libraries for production (optimized, stripped)
|
||||
native-release:
|
||||
@mkdir -p native/build
|
||||
|
|
|
|||
|
|
@ -3,16 +3,41 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jfraeys/fetch_ml/internal/api"
|
||||
)
|
||||
|
||||
// Build variables injected at build time
|
||||
var (
|
||||
BuildHash = "unknown"
|
||||
BuildTime = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
configFile := flag.String("config", "configs/api/dev.yaml", "Configuration file path")
|
||||
apiKey := flag.String("api-key", "", "API key for authentication")
|
||||
showVersion := flag.Bool("version", false, "Show version and build info")
|
||||
verifyBuild := flag.Bool("verify", false, "Verify build integrity")
|
||||
flag.Parse()
|
||||
|
||||
// Handle version display
|
||||
if *showVersion {
|
||||
fmt.Printf("fetch_ml API Server\n")
|
||||
fmt.Printf(" Build Hash: %s\n", BuildHash)
|
||||
fmt.Printf(" Build Time: %s\n", BuildTime)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Handle build verification (placeholder - always true for now)
|
||||
if *verifyBuild {
|
||||
fmt.Printf("Build verification: OK\n")
|
||||
fmt.Printf(" Build Hash: %s\n", BuildHash)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Create and start server
|
||||
server, err := api.NewServer(*configFile)
|
||||
if err != nil {
|
||||
|
|
|
|||
82
cmd/errors/main.go
Normal file
82
cmd/errors/main.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Package main implements the ml errors command for querying task errors
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/jfraeys/fetch_ml/internal/errtypes"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "Usage: errors <task_id> [--json]")
|
||||
fmt.Fprintln(os.Stderr, " task_id: The task ID to query errors for")
|
||||
fmt.Fprintln(os.Stderr, " --json: Output as JSON instead of formatted text")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
taskID := os.Args[1]
|
||||
jsonOutput := len(os.Args) > 2 && os.Args[2] == "--json"
|
||||
|
||||
// Determine base path from environment or default
|
||||
basePath := os.Getenv("FETCH_ML_BASE_PATH")
|
||||
if basePath == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed to get home directory: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
basePath = filepath.Join(home, "ml_jobs")
|
||||
}
|
||||
|
||||
// Try to read error file
|
||||
errorPath := filepath.Join(basePath, "errors", taskID+".json")
|
||||
data, err := os.ReadFile(errorPath)
|
||||
if err != nil {
|
||||
// Error file may not exist - check if task exists in other states
|
||||
fmt.Fprintf(os.Stderr, "Error: no error record found for task %s\n", taskID)
|
||||
fmt.Fprintf(os.Stderr, "Expected: %s\n", errorPath)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var execErr errtypes.TaskExecutionError
|
||||
if err := json.Unmarshal(data, &execErr); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed to parse error record: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
// Output as pretty-printed JSON
|
||||
output, err := json.MarshalIndent(execErr, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed to format error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(string(output))
|
||||
} else {
|
||||
// Output as formatted text
|
||||
fmt.Printf("Error Report for Task: %s\n", execErr.TaskID)
|
||||
fmt.Printf("Job Name: %s\n", execErr.JobName)
|
||||
fmt.Printf("Phase: %s\n", execErr.Phase)
|
||||
fmt.Printf("Time: %s\n", execErr.Timestamp.Format(time.RFC3339))
|
||||
fmt.Printf("Recoverable: %v\n", execErr.Recoverable)
|
||||
fmt.Println()
|
||||
if execErr.Message != "" {
|
||||
fmt.Printf("Message: %s\n", execErr.Message)
|
||||
}
|
||||
if execErr.Err != nil {
|
||||
fmt.Printf("Underlying Error: %v\n", execErr.Err)
|
||||
}
|
||||
if len(execErr.Context) > 0 {
|
||||
fmt.Println()
|
||||
fmt.Println("Context:")
|
||||
for key, value := range execErr.Context {
|
||||
fmt.Printf(" %s: %s\n", key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -280,6 +280,9 @@ type PodmanConfig struct {
|
|||
Volumes map[string]string
|
||||
Memory string
|
||||
CPUs string
|
||||
Privileged bool // Security: must be false
|
||||
Network string // Security: must not be "host"
|
||||
ReadOnlyMounts bool // Security: true for dataset mounts
|
||||
}
|
||||
|
||||
// PodmanResourceOverrides converts per-task resource requests into Podman-compatible
|
||||
|
|
@ -363,7 +366,29 @@ func BuildPodmanCommand(
|
|||
return exec.CommandContext(ctx, "podman", args...)
|
||||
}
|
||||
|
||||
// SanitizePath ensures a path is safe to use (prevents path traversal)
|
||||
// ValidateSecurityPolicy validates that the container configuration meets security requirements.
|
||||
// Returns an error if the configuration violates security policies.
|
||||
func ValidateSecurityPolicy(cfg PodmanConfig) error {
|
||||
if cfg.Privileged {
|
||||
return fmt.Errorf("privileged containers are not allowed: %w", ErrSecurityViolation)
|
||||
}
|
||||
|
||||
if cfg.Network == "host" {
|
||||
return fmt.Errorf("host network mode is not allowed: %w", ErrSecurityViolation)
|
||||
}
|
||||
|
||||
// Validate volume mounts are read-only where required
|
||||
if !cfg.ReadOnlyMounts {
|
||||
// This is a warning-level issue, not a hard error
|
||||
// but we document it for audit purposes
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrSecurityViolation is returned when a security policy is violated.
|
||||
var ErrSecurityViolation = fmt.Errorf("security policy violation")
|
||||
|
||||
func SanitizePath(path string) (string, error) {
|
||||
// Clean the path to remove any .. or . components
|
||||
cleaned := filepath.Clean(path)
|
||||
|
|
|
|||
110
internal/container/security_test.go
Normal file
110
internal/container/security_test.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package container
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestContainerSecurityPolicy enforces the security contract for container configurations.
|
||||
// These tests serve as executable documentation of security requirements.
|
||||
func TestContainerSecurityPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config PodmanConfig
|
||||
shouldFail bool
|
||||
reason string
|
||||
}{
|
||||
{
|
||||
name: "reject privileged mode",
|
||||
config: PodmanConfig{
|
||||
Image: "pytorch:latest",
|
||||
Privileged: true, // NEVER allowed
|
||||
},
|
||||
shouldFail: true,
|
||||
reason: "privileged containers bypass isolation",
|
||||
},
|
||||
{
|
||||
name: "reject host network",
|
||||
config: PodmanConfig{
|
||||
Image: "pytorch:latest",
|
||||
Network: "host", // NEVER allowed
|
||||
},
|
||||
shouldFail: true,
|
||||
reason: "host network breaks isolation",
|
||||
},
|
||||
{
|
||||
name: "accept valid configuration",
|
||||
config: PodmanConfig{
|
||||
Image: "pytorch:latest",
|
||||
Privileged: false,
|
||||
Network: "bridge",
|
||||
ReadOnlyMounts: true,
|
||||
},
|
||||
shouldFail: false,
|
||||
reason: "valid secure configuration",
|
||||
},
|
||||
{
|
||||
name: "accept empty network (default bridge)",
|
||||
config: PodmanConfig{
|
||||
Image: "pytorch:latest",
|
||||
Privileged: false,
|
||||
Network: "", // Empty means default bridge
|
||||
},
|
||||
shouldFail: false,
|
||||
reason: "empty network uses default bridge",
|
||||
},
|
||||
{
|
||||
name: "warn on non-read-only mounts",
|
||||
config: PodmanConfig{
|
||||
Image: "pytorch:latest",
|
||||
Privileged: false,
|
||||
Network: "bridge",
|
||||
ReadOnlyMounts: false, // Warning-level issue
|
||||
},
|
||||
shouldFail: false, // Not a hard failure
|
||||
reason: "non-read-only mounts are discouraged but allowed",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateSecurityPolicy(tt.config)
|
||||
if tt.shouldFail {
|
||||
if err == nil {
|
||||
t.Errorf("%s: expected failure (%s), got success", tt.name, tt.reason)
|
||||
} else if !errors.Is(err, ErrSecurityViolation) {
|
||||
t.Errorf("%s: expected ErrSecurityViolation, got %v", tt.name, err)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("%s: expected success (%s), got error: %v", tt.name, tt.reason, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSecurityPolicy_IsolationEnforcement verifies isolation boundaries
|
||||
func TestSecurityPolicy_IsolationEnforcement(t *testing.T) {
|
||||
t.Run("privileged_equals_root_access", func(t *testing.T) {
|
||||
cfg := PodmanConfig{
|
||||
Image: "test:latest",
|
||||
Privileged: true,
|
||||
}
|
||||
err := ValidateSecurityPolicy(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("privileged mode must be rejected - it grants root access to host")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("host_network_equals_no_isolation", func(t *testing.T) {
|
||||
cfg := PodmanConfig{
|
||||
Image: "test:latest",
|
||||
Network: "host",
|
||||
}
|
||||
err := ValidateSecurityPolicy(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("host network must be rejected - it removes network isolation")
|
||||
}
|
||||
})
|
||||
}
|
||||
97
internal/domain/events.go
Normal file
97
internal/domain/events.go
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TaskEventType represents the type of task event.
|
||||
type TaskEventType string
|
||||
|
||||
const (
|
||||
// TaskEventQueued is fired when a task is added to the queue.
|
||||
TaskEventQueued TaskEventType = "queued"
|
||||
// TaskEventStarted is fired when a task begins execution.
|
||||
TaskEventStarted TaskEventType = "started"
|
||||
// TaskEventCompleted is fired when a task finishes successfully.
|
||||
TaskEventCompleted TaskEventType = "completed"
|
||||
// TaskEventFailed is fired when a task fails.
|
||||
TaskEventFailed TaskEventType = "failed"
|
||||
// TaskEventCancelled is fired when a task is cancelled.
|
||||
TaskEventCancelled TaskEventType = "cancelled"
|
||||
// TaskEventRetrying is fired when a task is being retried.
|
||||
TaskEventRetrying TaskEventType = "retrying"
|
||||
// TaskEventCheckpointSaved is fired when a checkpoint is saved.
|
||||
TaskEventCheckpointSaved TaskEventType = "checkpoint_saved"
|
||||
// TaskEventGPUAssigned is fired when a GPU is assigned.
|
||||
TaskEventGPUAssigned TaskEventType = "gpu_assigned"
|
||||
)
|
||||
|
||||
// TaskEvent represents an event in a task's lifecycle.
|
||||
// Events are stored in Redis Streams for append-only audit trails.
|
||||
type TaskEvent struct {
|
||||
// TaskID is the unique identifier of the task.
|
||||
TaskID string `json:"task_id"`
|
||||
|
||||
// EventType indicates what happened (queued, started, completed, etc.).
|
||||
EventType TaskEventType `json:"event_type"`
|
||||
|
||||
// Timestamp when the event occurred.
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
// Data contains event-specific data (JSON-encoded).
|
||||
// For "started": {"worker_id": "worker-1", "image": "pytorch:latest"}
|
||||
// For "failed": {"error": "OOM", "phase": "execution"}
|
||||
Data json.RawMessage `json:"data,omitempty"`
|
||||
|
||||
// Who triggered this event (worker ID, user ID, or system).
|
||||
Who string `json:"who"`
|
||||
}
|
||||
|
||||
// EventDataStarted contains data for the "started" event.
|
||||
type EventDataStarted struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
Image string `json:"image,omitempty"`
|
||||
GPUDevices []string `json:"gpu_devices,omitempty"`
|
||||
}
|
||||
|
||||
// EventDataFailed contains data for the "failed" event.
|
||||
type EventDataFailed struct {
|
||||
Error string `json:"error"`
|
||||
Phase string `json:"phase"`
|
||||
Recoverable bool `json:"recoverable"`
|
||||
}
|
||||
|
||||
// EventDataGPUAssigned contains data for the "gpu_assigned" event.
|
||||
type EventDataGPUAssigned struct {
|
||||
GPUDevices []string `json:"gpu_devices"`
|
||||
GPUEnvVar string `json:"gpu_env_var,omitempty"`
|
||||
}
|
||||
|
||||
// NewTaskEvent creates a new task event with the current timestamp.
|
||||
func NewTaskEvent(taskID string, eventType TaskEventType, who string) TaskEvent {
|
||||
return TaskEvent{
|
||||
TaskID: taskID,
|
||||
EventType: eventType,
|
||||
Timestamp: time.Now().UTC(),
|
||||
Who: who,
|
||||
}
|
||||
}
|
||||
|
||||
// WithData adds data to the event.
|
||||
func (e TaskEvent) WithData(data any) (TaskEvent, error) {
|
||||
encoded, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return e, err
|
||||
}
|
||||
e.Data = encoded
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// ParseData parses the event data into the provided type.
|
||||
func (e TaskEvent) ParseData(out any) error {
|
||||
if len(e.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(e.Data, out)
|
||||
}
|
||||
|
|
@ -2,7 +2,9 @@
|
|||
package errtypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DataFetchError represents an error that occurred while fetching a dataset
|
||||
|
|
@ -23,16 +25,58 @@ func (e *DataFetchError) Unwrap() error {
|
|||
|
||||
// TaskExecutionError represents an error during task execution.
|
||||
type TaskExecutionError struct {
|
||||
TaskID string
|
||||
JobName string
|
||||
Phase string // "data_fetch", "execution", "cleanup"
|
||||
Err error
|
||||
TaskID string `json:"task_id"`
|
||||
JobName string `json:"job_name"`
|
||||
Phase string `json:"phase"` // "data_fetch", "execution", "cleanup"
|
||||
Message string `json:"message"`
|
||||
Err error `json:"-"`
|
||||
Context map[string]string `json:"context,omitempty"` // Additional context (image, GPU, etc.)
|
||||
Timestamp time.Time `json:"timestamp"` // When the error occurred
|
||||
Recoverable bool `json:"recoverable"` // Whether this error is retryable
|
||||
}
|
||||
|
||||
// Error returns the error message.
|
||||
func (e *TaskExecutionError) Error() string {
|
||||
if e.Message != "" {
|
||||
return fmt.Sprintf("task %s (%s) failed during %s: %s",
|
||||
e.TaskID[:8], e.JobName, e.Phase, e.Message)
|
||||
}
|
||||
return fmt.Sprintf("task %s (%s) failed during %s: %v",
|
||||
e.TaskID[:8], e.JobName, e.Phase, e.Err)
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying error.
|
||||
func (e *TaskExecutionError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// MarshalJSON returns a JSON representation of the error.
|
||||
func (e *TaskExecutionError) MarshalJSON() ([]byte, error) {
|
||||
type Alias TaskExecutionError
|
||||
return json.Marshal(&struct {
|
||||
*Alias
|
||||
Error string `json:"error,omitempty"`
|
||||
}{
|
||||
Alias: (*Alias)(e),
|
||||
Error: func() string {
|
||||
if e.Err != nil {
|
||||
return e.Err.Error()
|
||||
}
|
||||
return ""
|
||||
}(),
|
||||
})
|
||||
}
|
||||
|
||||
// IsRecoverable returns true if the error is retryable.
|
||||
func (e *TaskExecutionError) IsRecoverable() bool {
|
||||
return e.Recoverable
|
||||
}
|
||||
|
||||
// WithContext adds context to the error.
|
||||
func (e *TaskExecutionError) WithContext(key, value string) *TaskExecutionError {
|
||||
if e.Context == nil {
|
||||
e.Context = make(map[string]string)
|
||||
}
|
||||
e.Context[key] = value
|
||||
return e
|
||||
}
|
||||
|
|
|
|||
194
internal/queue/event_store.go
Normal file
194
internal/queue/event_store.go
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/jfraeys/fetch_ml/internal/domain"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// EventStore provides append-only event storage using Redis Streams.
|
||||
// Events are stored chronologically and can be queried for audit trails.
|
||||
type EventStore struct {
|
||||
client *redis.Client
|
||||
ctx context.Context
|
||||
retentionDays int
|
||||
maxStreamLen int64
|
||||
}
|
||||
|
||||
// EventStoreConfig holds configuration for the event store.
|
||||
type EventStoreConfig struct {
|
||||
RedisAddr string
|
||||
RedisPassword string
|
||||
RedisDB int
|
||||
RetentionDays int // How long to keep events (default: 7)
|
||||
MaxStreamLen int64 // Max events per task stream (default: 1000)
|
||||
}
|
||||
|
||||
// NewEventStore creates a new event store instance.
|
||||
func NewEventStore(cfg EventStoreConfig) (*EventStore, error) {
|
||||
retentionDays := cfg.RetentionDays
|
||||
if retentionDays == 0 {
|
||||
retentionDays = 7
|
||||
}
|
||||
|
||||
maxStreamLen := cfg.MaxStreamLen
|
||||
if maxStreamLen == 0 {
|
||||
maxStreamLen = 1000
|
||||
}
|
||||
|
||||
opts := &redis.Options{
|
||||
Addr: cfg.RedisAddr,
|
||||
Password: cfg.RedisPassword,
|
||||
DB: cfg.RedisDB,
|
||||
PoolSize: 50,
|
||||
}
|
||||
|
||||
client := redis.NewClient(opts)
|
||||
ctx := context.Background()
|
||||
|
||||
// Test connection
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to redis: %w", err)
|
||||
}
|
||||
|
||||
return &EventStore{
|
||||
client: client,
|
||||
ctx: ctx,
|
||||
retentionDays: retentionDays,
|
||||
maxStreamLen: maxStreamLen,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the event store.
|
||||
func (es *EventStore) Close() error {
|
||||
return es.client.Close()
|
||||
}
|
||||
|
||||
// RecordEvent records a task event to the append-only stream.
|
||||
func (es *EventStore) RecordEvent(event domain.TaskEvent) error {
|
||||
streamKey := fmt.Sprintf("task_events:%s", event.TaskID)
|
||||
|
||||
data, err := json.Marshal(event.Data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal event data: %w", err)
|
||||
}
|
||||
|
||||
values := map[string]interface{}{
|
||||
"type": event.EventType,
|
||||
"who": event.Who,
|
||||
"ts": event.Timestamp.Unix(),
|
||||
}
|
||||
|
||||
if len(data) > 0 {
|
||||
values["data"] = string(data)
|
||||
}
|
||||
|
||||
// Add to stream with approximate max length trimming
|
||||
_, err = es.client.XAdd(es.ctx, &redis.XAddArgs{
|
||||
Stream: streamKey,
|
||||
MaxLen: es.maxStreamLen,
|
||||
Approx: true, // Allow approximate trimming for performance
|
||||
Values: values,
|
||||
}).Result()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to record event: %w", err)
|
||||
}
|
||||
|
||||
// Set expiration on the stream
|
||||
es.client.Expire(es.ctx, streamKey, time.Duration(es.retentionDays)*24*time.Hour)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetEvents retrieves all events for a task, ordered chronologically.
|
||||
func (es *EventStore) GetEvents(taskID string) ([]domain.TaskEvent, error) {
|
||||
streamKey := fmt.Sprintf("task_events:%s", taskID)
|
||||
|
||||
// Read all events from the stream
|
||||
messages, err := es.client.XRange(es.ctx, streamKey, "-", "+").Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get events: %w", err)
|
||||
}
|
||||
|
||||
var events []domain.TaskEvent
|
||||
for _, msg := range messages {
|
||||
event, err := es.parseEvent(taskID, msg)
|
||||
if err != nil {
|
||||
continue // Skip malformed events
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// GetEventsSince retrieves events for a task since a specific time.
|
||||
func (es *EventStore) GetEventsSince(taskID string, since time.Time) ([]domain.TaskEvent, error) {
|
||||
streamKey := fmt.Sprintf("task_events:%s", taskID)
|
||||
|
||||
// Use XRANGEBYTIME equivalent - scan from timestamp
|
||||
start := fmt.Sprintf("%d-0", since.Unix()*1000) // Redis stream IDs are ms-based
|
||||
|
||||
messages, err := es.client.XRange(es.ctx, streamKey, start, "+").Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get events: %w", err)
|
||||
}
|
||||
|
||||
var events []domain.TaskEvent
|
||||
for _, msg := range messages {
|
||||
event, err := es.parseEvent(taskID, msg)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Filter by actual timestamp
|
||||
if event.Timestamp.After(since) {
|
||||
events = append(events, event)
|
||||
}
|
||||
}
|
||||
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// parseEvent converts a Redis stream message to a TaskEvent.
|
||||
func (es *EventStore) parseEvent(taskID string, msg redis.XMessage) (domain.TaskEvent, error) {
|
||||
var event domain.TaskEvent
|
||||
event.TaskID = taskID
|
||||
|
||||
// Parse event type
|
||||
if v, ok := msg.Values["type"]; ok {
|
||||
event.EventType = domain.TaskEventType(v.(string))
|
||||
}
|
||||
|
||||
// Parse who
|
||||
if v, ok := msg.Values["who"]; ok {
|
||||
event.Who = v.(string)
|
||||
}
|
||||
|
||||
// Parse timestamp
|
||||
if v, ok := msg.Values["ts"]; ok {
|
||||
ts, err := strconv.ParseInt(v.(string), 10, 64)
|
||||
if err == nil {
|
||||
event.Timestamp = time.Unix(ts, 0).UTC()
|
||||
}
|
||||
}
|
||||
|
||||
// Parse data
|
||||
if v, ok := msg.Values["data"]; ok {
|
||||
event.Data = json.RawMessage(v.(string))
|
||||
}
|
||||
|
||||
return event, nil
|
||||
}
|
||||
|
||||
// DeleteOldEvents manually deletes events older than retention period.
|
||||
// This is normally handled by Redis TTL, but can be called for cleanup.
|
||||
func (es *EventStore) DeleteOldEvents(taskID string) error {
|
||||
streamKey := fmt.Sprintf("task_events:%s", taskID)
|
||||
return es.client.Del(es.ctx, streamKey).Err()
|
||||
}
|
||||
|
|
@ -282,10 +282,14 @@ func (e *ContainerExecutor) runPodman(
|
|||
manifestName, err := SelectDependencyManifest(filepath.Join(env.OutputDir, "code"))
|
||||
if err != nil {
|
||||
return &errtypes.TaskExecutionError{
|
||||
TaskID: task.ID,
|
||||
JobName: task.JobName,
|
||||
Phase: "validation",
|
||||
Err: err,
|
||||
TaskID: task.ID,
|
||||
JobName: task.JobName,
|
||||
Phase: "validation",
|
||||
Message: "dependency manifest selection failed",
|
||||
Err: err,
|
||||
Context: map[string]string{"image": selectedImage, "output_dir": env.OutputDir},
|
||||
Timestamp: time.Now().UTC(),
|
||||
Recoverable: false,
|
||||
}
|
||||
}
|
||||
depsPath := filepath.Join(podmanCfg.ContainerWorkspace, manifestName)
|
||||
|
|
@ -371,7 +375,17 @@ func (e *ContainerExecutor) handleFailure(
|
|||
return "", nil
|
||||
})
|
||||
|
||||
return fmt.Errorf("execution failed: %w", runErr)
|
||||
// Return enriched error with context
|
||||
return &errtypes.TaskExecutionError{
|
||||
TaskID: task.ID,
|
||||
JobName: task.JobName,
|
||||
Phase: "execution",
|
||||
Message: "container execution failed",
|
||||
Err: runErr,
|
||||
Context: map[string]string{"duration_ms": fmt.Sprintf("%d", duration.Milliseconds())},
|
||||
Timestamp: time.Now().UTC(),
|
||||
Recoverable: true, // Container failures may be retryable
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ContainerExecutor) handleSuccess(
|
||||
|
|
|
|||
Loading…
Reference in a new issue