package tests import ( "context" "crypto/tls" "net/http" "os" "os/exec" "path/filepath" "strings" "testing" "time" tests "github.com/jfraeys/fetch_ml/tests/fixtures" ) // TestCLIAndAPIE2E tests the complete CLI and API integration end-to-end func TestCLIAndAPIE2E(t *testing.T) { t.Parallel() // Enable parallel execution // Skip if CLI not built cliPath := "../../cli/zig-out/bin/ml" if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } // Skip if manage.sh not available manageScript := "../../tools/manage.sh" if _, err := os.Stat(manageScript); os.IsNotExist(err) { t.Skip("manage.sh not found") } // Use fixtures for manage script operations ms := tests.NewManageScript(manageScript) defer ms.StopAndCleanup() // Ensure cleanup ctx := context.Background() testDir := t.TempDir() // Create CLI config directory for use across tests cliConfigDir := filepath.Join(testDir, "cli_config") // Phase 1: Service Management E2E t.Run("ServiceManagementE2E", func(t *testing.T) { // Test initial status output, err := ms.Status() if err != nil { t.Fatalf("Failed to get status: %v", err) } t.Logf("Initial status: %s", output) // Start services if err := ms.Start(); err != nil { t.Skipf("Failed to start services: %v", err) } // Give services time to start time.Sleep(2 * time.Second) // Reduced from 3 seconds // Verify with health check healthOutput, err := ms.Health() if err != nil { t.Logf("Health check failed (services may not be fully started)") } else { if !strings.Contains(healthOutput, "API is healthy") && !strings.Contains(healthOutput, "Port 9101 is open") { t.Errorf("Unexpected health check output: %s", healthOutput) } t.Log("Health check passed") } // Cleanup defer ms.Stop() }) // Phase 2: CLI Configuration E2E t.Run("CLIConfigurationE2E", func(t *testing.T) { // Create CLI config directory if it doesn't exist if err := os.MkdirAll(cliConfigDir, 0755); err != nil { t.Fatalf("Failed to create CLI config dir: %v", err) } // Test CLI init initCmd := exec.Command(cliPath, "init") initCmd.Dir = cliConfigDir output, err := initCmd.CombinedOutput() t.Logf("CLI init output: %s", string(output)) if err != nil { t.Logf("CLI init failed (may be due to server connection): %v", err) } // Create minimal config for testing minimalConfig := `{ "server_url": "wss://localhost:9101/ws", "api_key": "password", "working_dir": "` + cliConfigDir + `" }` configPath := filepath.Join(cliConfigDir, "config.json") if err := os.WriteFile(configPath, []byte(minimalConfig), 0644); err != nil { t.Fatalf("Failed to create minimal config: %v", err) } // Test CLI status with config statusCmd := exec.Command(cliPath, "status") statusCmd.Dir = cliConfigDir statusOutput, err := statusCmd.CombinedOutput() t.Logf("CLI status output: %s", string(statusOutput)) if err != nil { t.Logf("CLI status failed (may be due to server): %v", err) } // Verify the output doesn't contain debug messages outputStr := string(statusOutput) if strings.Contains(outputStr, "Getting status for user") { t.Errorf("Expected clean output without debug messages, got: %s", outputStr) } }) // Phase 3: API Health Check E2E t.Run("APIHealthCheckE2E", func(t *testing.T) { client := &http.Client{ Timeout: 5 * time.Second, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } req, err := http.NewRequest("GET", "https://localhost:9101/health", nil) if err != nil { t.Skipf("Failed to create request: %v", err) } req.Header.Set("X-API-Key", "password") req.Header.Set("X-Forwarded-For", "127.0.0.1") resp, err := client.Do(req) if err != nil { t.Skipf("API not available: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("Expected status 200, got %d", resp.StatusCode) } }) // Phase 4: Redis Integration E2E t.Run("RedisIntegrationE2E", func(t *testing.T) { // Use fixtures for Redis operations redisHelper, err := tests.NewRedisHelper("localhost:6379", 13) if err != nil { t.Skipf("Redis not available, skipping Redis integration test: %v", err) } defer redisHelper.Close() // Test Redis connection if err := redisHelper.GetClient().Ping(ctx).Err(); err != nil { t.Errorf("Redis ping failed: %v", err) } // Test basic operations key := "test_key" value := "test_value" if err := redisHelper.GetClient().Set(ctx, key, value, 0).Err(); err != nil { t.Errorf("Redis set failed: %v", err) } result, err := redisHelper.GetClient().Get(ctx, key).Result() if err != nil { t.Errorf("Redis get failed: %v", err) } if result != value { t.Errorf("Expected %s, got %s", value, result) } // Cleanup test data redisHelper.GetClient().Del(ctx, key) }) // Phase 5: ML Experiment Workflow E2E t.Run("MLExperimentWorkflowE2E", func(t *testing.T) { // Create experiment directory expDir := filepath.Join(testDir, "experiments", "test_experiment") if err := os.MkdirAll(expDir, 0755); err != nil { t.Fatalf("Failed to create experiment dir: %v", err) } // Create simple ML script trainScript := filepath.Join(expDir, "train.py") trainCode := `#!/usr/bin/env python3 import json import sys import time from pathlib import Path # Simple training script print("Starting training...") time.sleep(2) # Simulate training # Create results results = { "accuracy": 0.85, "loss": 0.15, "epochs": 10, "status": "completed" } # Save results with open("results.json", "w") as f: json.dump(results, f) print("Training completed successfully!") print(f"Results: {results}") sys.exit(0) ` if err := os.WriteFile(trainScript, []byte(trainCode), 0755); err != nil { t.Fatalf("Failed to create train.py: %v", err) } // Create requirements.txt reqFile := filepath.Join(expDir, "requirements.txt") reqContent := `numpy==1.21.0 scikit-learn==1.0.0 ` if err := os.WriteFile(reqFile, []byte(reqContent), 0644); err != nil { t.Fatalf("Failed to create requirements.txt: %v", err) } // Create README.md readmeFile := filepath.Join(expDir, "README.md") readmeContent := `# Test ML Experiment A simple machine learning experiment for testing purposes. ## Usage ` + "```bash" + ` python train.py ` + "```" if err := os.WriteFile(readmeFile, []byte(readmeContent), 0644); err != nil { t.Fatalf("Failed to create README.md: %v", err) } t.Logf("Created ML experiment in: %s", expDir) // Test CLI sync (if available) syncCmd := exec.Command(cliPath, "sync", expDir) syncCmd.Dir = cliConfigDir syncOutput, err := syncCmd.CombinedOutput() t.Logf("CLI sync output: %s", string(syncOutput)) if err != nil { t.Logf("CLI sync failed (may be expected): %v", err) } // Verify the output doesn't contain debug messages syncOutputStr := string(syncOutput) if strings.Contains(syncOutputStr, "Calculating commit ID") { t.Errorf("Expected clean sync output without debug messages, got: %s", syncOutputStr) } // Test CLI cancel command cancelCmd := exec.Command(cliPath, "cancel", "test_job") cancelCmd.Dir = cliConfigDir cancelOutput, err := cancelCmd.CombinedOutput() t.Logf("CLI cancel output: %s", string(cancelOutput)) if err != nil { t.Logf("CLI cancel failed (may be expected): %v", err) } // Verify the output doesn't contain debug messages cancelOutputStr := string(cancelOutput) if strings.Contains(cancelOutputStr, "Cancelling job") { t.Errorf("Expected clean cancel output without debug messages, got: %s", cancelOutputStr) } }) // Phase 6: Health Check Scenarios E2E t.Run("HealthCheckScenariosE2E", func(t *testing.T) { // Check initial state first initialOutput, _ := ms.Health() // Try to stop services to test stopped state if err := ms.Stop(); err != nil { t.Logf("Failed to stop services: %v", err) } time.Sleep(2 * time.Second) // Give more time for shutdown output, err := ms.Health() // If services are still running, that's okay - they might be persistent if err == nil { if strings.Contains(output, "API is healthy") { t.Log("Services are still running after stop command (may be persistent)") // Skip the stopped state test since services won't stop t.Skip("Services persist after stop command, skipping stopped state test") } } // Test health check during service startup go func() { ms.Start() }() // Check health multiple times during startup healthPassed := false for i := 0; i < 5; i++ { time.Sleep(1 * time.Second) output, err := ms.Health() if err == nil && strings.Contains(output, "API is healthy") { t.Log("Health check passed during startup") healthPassed = true break } } if !healthPassed { t.Log("Health check did not pass during startup (expected if services not fully started)") } // Cleanup: Restore original state t.Cleanup(func() { // If services were originally running, keep them running // If they were originally stopped, stop them again if strings.Contains(initialOutput, "API is healthy") { t.Log("Services were originally running, keeping them running") } else { ms.Stop() t.Log("Services were originally stopped, stopping them again") } }) }) } // TestCLICommandsE2E tests CLI command workflows end-to-end func TestCLICommandsE2E(t *testing.T) { t.Parallel() // Enable parallel execution // Ensure Redis is available cleanup := tests.EnsureRedis(t) defer cleanup() cliPath := "../../cli/zig-out/bin/ml" if _, err := os.Stat(cliPath); os.IsNotExist(err) { t.Skip("CLI not built - run 'make build' first") } testDir := t.TempDir() // Test 1: CLI Help and Commands t.Run("CLIHelpCommands", func(t *testing.T) { helpCmd := exec.Command(cliPath, "--help") output, err := helpCmd.CombinedOutput() if err != nil { t.Logf("CLI help failed (CLI may not be built): %v", err) t.Skip("CLI not available - run 'make build' first") } outputStr := string(output) expectedCommands := []string{ "init", "sync", "queue", "status", "monitor", "cancel", "prune", "watch", } for _, cmd := range expectedCommands { if !strings.Contains(outputStr, cmd) { t.Errorf("Missing command in help: %s", cmd) } } }) // Test 2: CLI Error Handling t.Run("CLIErrorHandling", func(t *testing.T) { // Test invalid command invalidCmd := exec.Command(cliPath, "invalid_command") output, err := invalidCmd.CombinedOutput() if err == nil { t.Error("Expected CLI to fail with invalid command") } if !strings.Contains(string(output), "Invalid command arguments") && !strings.Contains(string(output), "Unknown command") { t.Errorf("Expected command error, got: %s", string(output)) } // Test without config noConfigCmd := exec.Command(cliPath, "status") noConfigCmd.Dir = testDir output, err = noConfigCmd.CombinedOutput() if err != nil { if strings.Contains(string(err.Error()), "no such file") { t.Skip("CLI binary not available") } // Expected to fail without config if !strings.Contains(string(output), "Config file not found") { t.Errorf("Expected config error, got: %s", string(output)) } } }) // Test 3: CLI Performance t.Run("CLIPerformance", func(t *testing.T) { commands := []string{"--help", "status", "queue", "list"} for _, cmd := range commands { start := time.Now() testCmd := exec.Command(cliPath, strings.Fields(cmd)...) output, err := testCmd.CombinedOutput() duration := time.Since(start) t.Logf("Command '%s' took %v", cmd, duration) if duration > 5*time.Second { t.Errorf("Command '%s' took too long: %v", cmd, duration) } t.Logf("Command '%s' output length: %d", cmd, len(string(output))) if err != nil { t.Logf("Command '%s' failed: %v", cmd, err) } } }) }