package tests import ( "context" "crypto/tls" "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "time" tests "github.com/jfraeys/fetch_ml/tests/fixtures" ) // TestCLIAndAPIE2E tests the complete CLI and API integration end-to-end func TestCLIAndAPIE2E(t *testing.T) { t.Parallel() cliPath := "../../cli/zig-out/bin/ml" if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } manageScript := "../../tools/manage.sh" if _, err := os.Stat(manageScript); os.IsNotExist(err) { t.Skip("manage.sh not found") } ms := tests.NewManageScript(manageScript) defer ms.StopAndCleanup() ctx := context.Background() testDir := t.TempDir() cliConfigDir := filepath.Join(testDir, "cli_config") t.Run("ServiceManagementE2E", func(t *testing.T) { runServiceManagementPhase(t, ms) }) t.Run("CLIConfigurationE2E", func(t *testing.T) { runCLIConfigurationPhase(t, cliPath, cliConfigDir) }) t.Run("APIHealthCheckE2E", func(t *testing.T) { runAPIHealthPhase(t) }) t.Run("RedisIntegrationE2E", func(t *testing.T) { runRedisIntegrationPhase(ctx, t) }) t.Run("MLExperimentWorkflowE2E", func(t *testing.T) { runMLExperimentPhase(t, cliPath, cliConfigDir, testDir) }) t.Run("HealthCheckScenariosE2E", func(t *testing.T) { runHealthCheckScenariosPhase(t, ms) }) } func runServiceManagementPhase(t *testing.T, ms *tests.ManageScript) { output, err := ms.Status() switch { case err != nil: t.Fatalf("Failed to get status: %v", err) default: t.Logf("Initial status: %s", output) } if err := ms.Start(); err != nil { t.Skipf("Failed to start services: %v", err) } time.Sleep(2 * time.Second) healthOutput, err := ms.Health() switch { case err != nil: t.Logf("Health check failed (services may not be fully started)") case !strings.Contains(healthOutput, "API is healthy") && !strings.Contains(healthOutput, "Port 9101 is open"): t.Errorf("Unexpected health check output: %s", healthOutput) default: t.Log("Health check passed") } t.Cleanup(func() { _ = ms.Stop() }) } func runCLIConfigurationPhase(t *testing.T, cliPath, cliConfigDir string) { if err := os.MkdirAll(cliConfigDir, 0750); err != nil { t.Fatalf("Failed to create CLI config dir: %v", err) } initCmd := exec.CommandContext(context.Background(), cliPath, "init") initCmd.Dir = cliConfigDir output, err := initCmd.CombinedOutput() t.Logf("CLI init output: %s", string(output)) if err != nil { t.Logf("CLI init failed (may be due to server connection): %v", err) } minimalConfig := `{ "server_url": "wss://localhost:9101/ws", "api_key": "password", "working_dir": "` + cliConfigDir + `" }` configPath := filepath.Join(cliConfigDir, "config.json") if err := os.WriteFile(configPath, []byte(minimalConfig), 0600); err != nil { t.Fatalf("Failed to create minimal config: %v", err) } statusCmd := exec.CommandContext(context.Background(), cliPath, "status") statusCmd.Dir = cliConfigDir statusOutput, err := statusCmd.CombinedOutput() t.Logf("CLI status output: %s", string(statusOutput)) if err != nil { t.Logf("CLI status failed (may be due to server): %v", err) } if strings.Contains(string(statusOutput), "Getting status for user") { t.Errorf("Expected clean output without debug messages, got: %s", string(statusOutput)) } } func runAPIHealthPhase(t *testing.T) { client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec }, } req, err := http.NewRequestWithContext(context.Background(), "GET", "https://localhost:9101/health", nil) if err != nil { t.Skipf("Failed to create request: %v", err) } req.Header.Set("X-API-Key", "password") req.Header.Set("X-Forwarded-For", "127.0.0.1") resp, err := client.Do(req) if err != nil { t.Skipf("API not available: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { t.Errorf("Expected status 200, got %d", resp.StatusCode) } } func runRedisIntegrationPhase(ctx context.Context, t *testing.T) { redisHelper, err := tests.NewRedisHelper("localhost:6379", 13) if err != nil { t.Skipf("Redis not available, skipping Redis integration test: %v", err) } defer func() { _ = redisHelper.Close() }() if err := redisHelper.GetClient().Ping(ctx).Err(); err != nil { t.Errorf("Redis ping failed: %v", err) } key := "test_key" value := "test_value" if err := redisHelper.GetClient().Set(ctx, key, value, 0).Err(); err != nil { t.Errorf("Redis set failed: %v", err) } result, err := redisHelper.GetClient().Get(ctx, key).Result() if err != nil { t.Errorf("Redis get failed: %v", err) } if result != value { t.Errorf("Expected %s, got %s", value, result) } redisHelper.GetClient().Del(ctx, key) } func runMLExperimentPhase(t *testing.T, cliPath, cliConfigDir, testDir string) { expDir := filepath.Join(testDir, "experiments", "test_experiment") if err := os.MkdirAll(expDir, 0750); err != nil { t.Fatalf("Failed to create experiment dir: %v", err) } trainScript := filepath.Join(expDir, "train.py") trainCode := `#!/usr/bin/env python3 import json import sys import time from pathlib import Path # Simple training script print("Starting training...") time.sleep(2) # Simulate training # Create results results = { "accuracy": 0.85, "loss": 0.15, "epochs": 10, "status": "completed" } # Save results with open("results.json", "w") as f: json.dump(results, f) print("Training completed successfully!") print(f"Results: {results}") sys.exit(0) ` if err := os.WriteFile(trainScript, []byte(trainCode), 0600); err != nil { t.Fatalf("Failed to create train.py: %v", err) } reqFile := filepath.Join(expDir, "requirements.txt") reqContent := `numpy==1.21.0 scikit-learn==1.0.0 ` if err := os.WriteFile(reqFile, []byte(reqContent), 0600); err != nil { t.Fatalf("Failed to create requirements.txt: %v", err) } readmeFile := filepath.Join(expDir, "README.md") readmeContent := `# Test ML Experiment A simple machine learning experiment for testing purposes. ## Usage ` + "```bash" + ` python train.py ` + "```" if err := os.WriteFile(readmeFile, []byte(readmeContent), 0600); err != nil { t.Fatalf("Failed to create README.md: %v", err) } t.Logf("Created ML experiment in: %s", expDir) syncCmd := exec.CommandContext(context.Background(), cliPath, "sync", expDir) //nolint:gosec syncCmd.Dir = cliConfigDir syncOutput, err := syncCmd.CombinedOutput() t.Logf("CLI sync output: %s", string(syncOutput)) if err != nil { t.Logf("CLI sync failed (may be expected): %v", err) } if strings.Contains(string(syncOutput), "Calculating commit ID") { t.Errorf("Expected clean sync output without debug messages, got: %s", string(syncOutput)) } cancelCmd := exec.CommandContext(context.Background(), cliPath, "cancel", "test_job") cancelCmd.Dir = cliConfigDir cancelOutput, err := cancelCmd.CombinedOutput() t.Logf("CLI cancel output: %s", string(cancelOutput)) if err != nil { t.Logf("CLI cancel failed (may be expected): %v", err) } if strings.Contains(string(cancelOutput), "Cancelling job") { t.Errorf("Expected clean cancel output without debug messages, got: %s", string(cancelOutput)) } } func runHealthCheckScenariosPhase(t *testing.T, ms *tests.ManageScript) { initialOutput, _ := ms.Health() if err := ms.Stop(); err != nil { t.Logf("Failed to stop services: %v", err) } time.Sleep(2 * time.Second) output, err := ms.Health() if err == nil && strings.Contains(output, "API is healthy") { t.Log("Services are still running after stop command (may be persistent)") t.Skip("Services persist after stop command, skipping stopped state test") } go func() { _ = ms.Start() }() if !waitForHealthDuringStartup(t, ms) { t.Log("Health check did not pass during startup (expected if services not fully started)") } t.Cleanup(func() { if strings.Contains(initialOutput, "API is healthy") { t.Log("Services were originally running, keeping them running") return } _ = ms.Stop() t.Log("Services were originally stopped, stopping them again") }) } func waitForHealthDuringStartup(t *testing.T, ms *tests.ManageScript) bool { for range 5 { time.Sleep(1 * time.Second) output, err := ms.Health() if err == nil && strings.Contains(output, "API is healthy") { t.Log("Health check passed during startup") return true } } return false } // TestCLICommandsE2E tests CLI command workflows end-to-end func TestCLICommandsE2E(t *testing.T) { t.Parallel() // Enable parallel execution // Ensure Redis is available cleanup := tests.EnsureRedis(t) defer cleanup() cliPath := "../../cli/zig-out/bin/ml" if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } testDir := t.TempDir() // Test 1: CLI Help and Commands t.Run("CLIHelpCommands", func(t *testing.T) { helpCmd := exec.CommandContext(context.Background(), cliPath, "--help") output, err := helpCmd.CombinedOutput() if err != nil { t.Logf("CLI help failed (CLI may not be built): %v", err) t.Skip("CLI not available - run 'make build' first") } outputStr := string(output) expectedCommands := []string{ "init", "sync", "queue", "status", "monitor", "cancel", "prune", "watch", } for _, cmd := range expectedCommands { if !strings.Contains(outputStr, cmd) { t.Errorf("Missing command in help: %s", cmd) } } }) // Test 2: CLI Error Handling t.Run("CLIErrorHandling", func(t *testing.T) { // Test invalid command invalidCmd := exec.CommandContext(context.Background(), cliPath, "invalid_command") output, err := invalidCmd.CombinedOutput() if err == nil { t.Error("Expected CLI to fail with invalid command") } if !strings.Contains(string(output), "Invalid command arguments") && !strings.Contains(string(output), "Unknown command") { t.Errorf("Expected command error, got: %s", string(output)) } // Test without config noConfigCmd := exec.CommandContext(context.Background(), cliPath, "status") noConfigCmd.Dir = testDir output, err = noConfigCmd.CombinedOutput() if err != nil { if strings.Contains(err.Error(), "no such file") { t.Skip("CLI binary not available") } // Expected to fail without config if !strings.Contains(string(output), "Config file not found") { t.Errorf("Expected config error, got: %s", string(output)) } } }) // Test 3: CLI Performance t.Run("CLIPerformance", func(t *testing.T) { commands := []string{"--help", "status", "queue", "list"} for _, cmd := range commands { start := time.Now() //nolint:gosec // G204: Subprocess launched with potential tainted input - this is a test testCmd := exec.CommandContext(context.Background(), cliPath, strings.Fields(cmd)...) output, err := testCmd.CombinedOutput() duration := time.Since(start) t.Logf("Command '%s' took %v", cmd, duration) if duration > 5*time.Second { t.Errorf("Command '%s' took too long: %v", cmd, duration) } t.Logf("Command '%s' output length: %d", cmd, len(string(output))) if err != nil { t.Logf("Command '%s' failed: %v", cmd, err) } } }) }