package tests import ( "context" "crypto/tls" "net/http" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" "time" tests "github.com/jfraeys/fetch_ml/tests/fixtures" ) func e2eRepoRoot(t *testing.T) string { t.Helper() _, filename, _, ok := runtime.Caller(0) if !ok { t.Fatalf("failed to resolve caller path") } return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..")) } func e2eCLIPath(t *testing.T) string { t.Helper() return filepath.Join(e2eRepoRoot(t), "cli", "zig-out", "bin", "ml") } // TestCLIAndAPIE2E tests the complete CLI and API integration end-to-end func TestCLIAndAPIE2E(t *testing.T) { t.Parallel() cliPath := e2eCLIPath(t) if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } manageScript := "../../tools/manage.sh" if _, err := os.Stat(manageScript); os.IsNotExist(err) { t.Skip("manage.sh not found") } ms := tests.NewManageScript(manageScript) defer ms.StopAndCleanup() ctx := context.Background() testDir := t.TempDir() cliConfigDir := filepath.Join(testDir, "cli_config") t.Run("ServiceManagementE2E", func(t *testing.T) { runServiceManagementPhase(t, ms) }) t.Run("CLIConfigurationE2E", func(t *testing.T) { runCLIConfigurationPhase(t, cliPath, cliConfigDir) }) t.Run("APIHealthCheckE2E", func(t *testing.T) { runAPIHealthPhase(t) }) t.Run("RedisIntegrationE2E", func(t *testing.T) { runRedisIntegrationPhase(ctx, t) }) t.Run("MLExperimentWorkflowE2E", func(t *testing.T) { runMLExperimentPhase(t, cliPath, cliConfigDir, testDir) }) t.Run("HealthCheckScenariosE2E", func(t *testing.T) { runHealthCheckScenariosPhase(t, ms) }) } func runServiceManagementPhase(t *testing.T, ms *tests.ManageScript) { output, err := ms.Status() switch { case err != nil: t.Fatalf("Failed to get status: %v", err) default: t.Logf("Initial status: %s", output) } if err := ms.Start(); err != nil { t.Skipf("Failed to start services: %v", err) } _ = waitForHealthDuringStartup(t, ms) healthOutput, err := ms.Health() switch { case err != nil: t.Logf("Health check failed (services may not be fully started)") case !strings.Contains(healthOutput, "API is healthy") && !strings.Contains(healthOutput, "Port 9101 is open"): t.Errorf("Unexpected health check output: %s", healthOutput) default: t.Log("Health check passed") } t.Cleanup(func() { _ = ms.Stop() }) } func runCLIConfigurationPhase(t *testing.T, cliPath, cliConfigDir string) { if err := os.MkdirAll(cliConfigDir, 0750); err != nil { t.Fatalf("Failed to create CLI config dir: %v", err) } initCmd := exec.CommandContext(context.Background(), cliPath, "init") initCmd.Dir = cliConfigDir output, err := initCmd.CombinedOutput() t.Logf("CLI init output: %s", string(output)) if err != nil { t.Logf("CLI init failed (may be due to server connection): %v", err) } minimalConfig := `{ "server_url": "wss://localhost:9101/ws", "api_key": "password", "working_dir": "` + cliConfigDir + `" }` configPath := filepath.Join(cliConfigDir, "config.json") if err := os.WriteFile(configPath, []byte(minimalConfig), 0600); err != nil { t.Fatalf("Failed to create minimal config: %v", err) } statusCmd := exec.CommandContext(context.Background(), cliPath, "status") statusCmd.Dir = cliConfigDir statusOutput, err := statusCmd.CombinedOutput() t.Logf("CLI status output: %s", string(statusOutput)) if err != nil { t.Logf("CLI status failed (may be due to server): %v", err) } if strings.Contains(string(statusOutput), "Getting status for user") { t.Errorf("Expected clean output without debug messages, got: %s", string(statusOutput)) } } func runAPIHealthPhase(t *testing.T) { client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec }, } req, err := http.NewRequestWithContext(context.Background(), "GET", "https://localhost:9101/health", nil) if err != nil { t.Skipf("Failed to create request: %v", err) } req.Header.Set("X-API-Key", "password") req.Header.Set("X-Forwarded-For", "127.0.0.1") resp, err := client.Do(req) if err != nil { t.Skipf("API not available: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { t.Errorf("Expected status 200, got %d", resp.StatusCode) } } func runRedisIntegrationPhase(ctx context.Context, t *testing.T) { redisHelper, err := tests.NewRedisHelper("localhost:6379", 13) if err != nil { t.Skipf("Redis not available, skipping Redis integration test: %v", err) } defer func() { _ = redisHelper.Close() }() if err := redisHelper.GetClient().Ping(ctx).Err(); err != nil { t.Errorf("Redis ping failed: %v", err) } key := "test_key" value := "test_value" if err := redisHelper.GetClient().Set(ctx, key, value, 0).Err(); err != nil { t.Errorf("Redis set failed: %v", err) } result, err := redisHelper.GetClient().Get(ctx, key).Result() if err != nil { t.Errorf("Redis get failed: %v", err) } if result != value { t.Errorf("Expected %s, got %s", value, result) } redisHelper.GetClient().Del(ctx, key) } func runMLExperimentPhase(t *testing.T, cliPath, cliConfigDir, testDir string) { expDir := filepath.Join(testDir, "experiments", "test_experiment") if err := os.MkdirAll(expDir, 0750); err != nil { t.Fatalf("Failed to create experiment dir: %v", err) } trainScript := filepath.Join(expDir, "train.py") trainCode := `#!/usr/bin/env python3 import json import sys import time from pathlib import Path # Simple training script print("Starting training...") time.sleep(2) # Simulate training # Create results results = { "accuracy": 0.85, "loss": 0.15, "epochs": 10, "status": "completed" } # Save results with open("results.json", "w") as f: json.dump(results, f) print("Training completed successfully!") print(f"Results: {results}") sys.exit(0) ` if err := os.WriteFile(trainScript, []byte(trainCode), 0600); err != nil { t.Fatalf("Failed to create train.py: %v", err) } reqFile := filepath.Join(expDir, "requirements.txt") reqContent := `numpy==1.21.0 scikit-learn==1.0.0 ` if err := os.WriteFile(reqFile, []byte(reqContent), 0600); err != nil { t.Fatalf("Failed to create requirements.txt: %v", err) } readmeFile := filepath.Join(expDir, "README.md") readmeContent := `# Test ML Experiment A simple machine learning experiment for testing purposes. ## Usage ` + "```bash" + ` python train.py ` + "```" if err := os.WriteFile(readmeFile, []byte(readmeContent), 0600); err != nil { t.Fatalf("Failed to create README.md: %v", err) } t.Logf("Created ML experiment in: %s", expDir) syncCmd := exec.CommandContext(context.Background(), cliPath, "sync", expDir) //nolint:gosec syncCmd.Dir = cliConfigDir syncOutput, err := syncCmd.CombinedOutput() t.Logf("CLI sync output: %s", string(syncOutput)) if err != nil { t.Logf("CLI sync failed (may be expected): %v", err) } if strings.Contains(string(syncOutput), "Calculating commit ID") { t.Errorf("Expected clean sync output without debug messages, got: %s", string(syncOutput)) } cancelCmd := exec.CommandContext(context.Background(), cliPath, "cancel", "test_job") cancelCmd.Dir = cliConfigDir cancelOutput, err := cancelCmd.CombinedOutput() t.Logf("CLI cancel output: %s", string(cancelOutput)) if err != nil { t.Logf("CLI cancel failed (may be expected): %v", err) } if strings.Contains(string(cancelOutput), "Cancelling job") { t.Errorf("Expected clean cancel output without debug messages, got: %s", string(cancelOutput)) } } func runHealthCheckScenariosPhase(t *testing.T, ms *tests.ManageScript) { initialOutput, _ := ms.Health() if err := ms.Stop(); err != nil { t.Logf("Failed to stop services: %v", err) } for range 10 { output, err := ms.Health() if err != nil || !strings.Contains(output, "API is healthy") { break } time.Sleep(200 * time.Millisecond) } output, err := ms.Health() if err == nil && strings.Contains(output, "API is healthy") { t.Log("Services are still running after stop command (may be persistent)") t.Skip("Services persist after stop command, skipping stopped state test") } go func() { _ = ms.Start() }() if !waitForHealthDuringStartup(t, ms) { t.Log("Health check did not pass during startup (expected if services not fully started)") } t.Cleanup(func() { if strings.Contains(initialOutput, "API is healthy") { t.Log("Services were originally running, keeping them running") return } _ = ms.Stop() t.Log("Services were originally stopped, stopping them again") }) } func waitForHealthDuringStartup(t *testing.T, ms *tests.ManageScript) bool { for range 5 { time.Sleep(1 * time.Second) output, err := ms.Health() if err == nil && strings.Contains(output, "API is healthy") { t.Log("Health check passed during startup") return true } } return false } // TestCLICommandsE2E tests CLI command workflows end-to-end func TestCLICommandsE2E(t *testing.T) { t.Parallel() // Enable parallel execution // Ensure Redis is available cleanup := tests.EnsureRedis(t) defer cleanup() cliPath := e2eCLIPath(t) if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } testDir := t.TempDir() // Test 1: CLI Help and Commands t.Run("CLIHelpCommands", func(t *testing.T) { helpCmd := exec.CommandContext(context.Background(), cliPath, "--help") output, err := helpCmd.CombinedOutput() if err != nil { t.Logf("CLI help failed (CLI may not be built): %v", err) t.Skip("CLI not available - run 'make build' first") } outputStr := string(output) expectedCommands := []string{ "init", "sync", "queue", "status", "monitor", "cancel", "prune", "watch", } for _, cmd := range expectedCommands { if !strings.Contains(outputStr, cmd) { t.Errorf("Missing command in help: %s", cmd) } } }) // Test 2: CLI Error Handling t.Run("CLIErrorHandling", func(t *testing.T) { // Skip if CLI binary doesn't exist if _, statErr := os.Stat(cliPath); os.IsNotExist(statErr) { t.Skip("CLI binary not found at: " + cliPath) } // Test invalid command invalidCmd := exec.CommandContext(context.Background(), cliPath, "invalid_command") output, err := invalidCmd.CombinedOutput() outputStr := string(output) // Check for binary execution issues if err != nil { if strings.Contains(err.Error(), "no such file") || strings.Contains(err.Error(), "not found") { t.Skip("CLI binary not executable: " + err.Error()) } // CLI exited with error - this is expected for invalid commands t.Logf("CLI exited with error (expected): %v", err) } // Validate error output contains expected error message, or skip if CLI doesn't validate commands hasErrorMsg := strings.Contains(outputStr, "Invalid") || strings.Contains(outputStr, "Unknown") || strings.Contains(outputStr, "Error") || strings.Contains(outputStr, "invalid") || strings.Contains(outputStr, "not found") || strings.Contains(outputStr, "usage") || strings.Contains(outputStr, "help") if !hasErrorMsg { if len(outputStr) == 0 { t.Skip("CLI produced no output - may be incompatible binary or accepts all commands") } else { t.Logf("CLI output (no recognizable error message): %s", outputStr) // Don't fail - CLI might accept unknown commands or have different error format t.Skip("CLI error format differs from expected - may need test update") } } else { t.Logf("CLI correctly rejected invalid command: %s", outputStr) } // Test without config noConfigCmd := exec.CommandContext(context.Background(), cliPath, "status") noConfigCmd.Dir = testDir noConfigCmd.Env = append(os.Environ(), "HOME="+testDir, "XDG_CONFIG_HOME="+testDir, ) output, err = noConfigCmd.CombinedOutput() if err != nil { if strings.Contains(err.Error(), "no such file") { t.Skip("CLI binary not available") } // Expected to fail without config if !strings.Contains(string(output), "Config file not found") { t.Errorf("Expected config error, got: %s", string(output)) } } }) // Test 3: CLI Performance t.Run("CLIPerformance", func(t *testing.T) { commands := []string{"--help", "status", "queue", "list"} for _, cmd := range commands { start := time.Now() //nolint:gosec // G204: Subprocess launched with potential tainted input - this is a test testCmd := exec.CommandContext(context.Background(), cliPath, strings.Fields(cmd)...) output, err := testCmd.CombinedOutput() duration := time.Since(start) t.Logf("Command '%s' took %v", cmd, duration) if duration > 5*time.Second { t.Errorf("Command '%s' took too long: %v", cmd, duration) } t.Logf("Command '%s' output length: %d", cmd, len(string(output))) if err != nil { t.Logf("Command '%s' failed: %v", cmd, err) } } }) }