From e08feae6ab14bafba1e6a4f6164642b53ab309f9 Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Wed, 4 Mar 2026 13:22:26 -0500 Subject: [PATCH] chore: migrate scripts from docker-compose v1 to v2 - Update all scripts to use 'docker compose' instead of 'docker-compose' - Fix compose file paths after consolidation (test.yml, prod.yml) - Update cleanup.sh to handle --profile debug and --profile smoke - Update test fixtures to reference consolidated compose files --- scripts/deploy/deploy.sh | 241 ++++++++++++++++++++++ scripts/deploy/rollback.sh | 110 ++++++++++ scripts/dev/smoke-test.sh | 2 +- scripts/release/cleanup.sh | 6 +- scripts/testing/test-native-with-redis.sh | 10 +- scripts/testing/test-prod.sh | 12 +- scripts/testing/tui-ssh-test.sh | 12 +- tests/e2e/logs_debug_e2e_test.go | 2 +- tests/fixtures/ssh_server.go | 2 +- 9 files changed, 374 insertions(+), 23 deletions(-) create mode 100755 scripts/deploy/deploy.sh create mode 100755 scripts/deploy/rollback.sh diff --git a/scripts/deploy/deploy.sh b/scripts/deploy/deploy.sh new file mode 100755 index 0000000..69b81f4 --- /dev/null +++ b/scripts/deploy/deploy.sh @@ -0,0 +1,241 @@ +#!/bin/bash +# Quick deployment script for fetch_ml + +set -e + + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) + export FETCHML_REPO_ROOT="${FETCHML_REPO_ROOT:-${REPO_ROOT}}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored output +print_status() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to show usage +show_usage() { + echo "Usage: $0 [ENVIRONMENT] [ACTION]" + echo "" + echo "Environments:" + echo " dev Development environment" + echo " staging Staging environment (pre-production)" + echo " secure Secure homelab environment" + echo " prod Production environment" + echo "" + echo "Actions:" + echo " up Start services" + echo " down Stop services" + echo " restart Restart services" + echo " logs Show logs" + echo " status Show status" + echo " rollback Rollback to previous deployment (image only)" + echo " health-check Check service health and compliance mode" + echo " check-audit-sink Verify audit sink reachability" + echo "" + echo "Examples:" + echo " $0 dev up # Start development environment" + echo " $0 staging up # Start staging environment" + echo " $0 prod down # Stop production environment" + echo " $0 staging rollback # Rollback staging deployment" + echo " $0 prod health-check # Check production health" + echo " $0 prod check-audit-sink # Verify audit sink before deploy" +} + +# Function to check if docker-compose file exists +check_compose_file() { + local env=$1 + local compose_file="" + + case $env in + "dev") + compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.dev.yml" + ;; + "staging") + compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.staging.yml" + ;; + "secure") + compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.homelab-secure.yml" + ;; + "prod") + compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.prod.yml" + ;; + *) + print_error "Unknown environment: $env" + show_usage + exit 1 + ;; + esac + + if [ ! -f "$compose_file" ]; then + print_error "Docker Compose file not found: $compose_file" + exit 1 + fi + + echo "$compose_file" +} + +# Function to check if .env file exists +check_env_file() { + local env=$1 + + if [ ! -f "${FETCHML_REPO_ROOT}/.env" ]; then + print_warning ".env file not found. Creating from example..." + if [ "$env" = "dev" ]; then + cp "${FETCHML_REPO_ROOT}/deployments/env.dev.example" "${FETCHML_REPO_ROOT}/.env" + elif [ "$env" = "prod" ]; then + cp "${FETCHML_REPO_ROOT}/deployments/env.prod.example" "${FETCHML_REPO_ROOT}/.env" + else + cp "${FETCHML_REPO_ROOT}/deployments/env.dev.example" "${FETCHML_REPO_ROOT}/.env" + fi + print_warning "Please edit .env file with your configuration" + fi +} + +# Main script +main() { + if [ $# -ne 2 ]; then + show_usage + exit 1 + fi + + local environment=$1 + local action=$2 + + print_status "Environment: $environment" + print_status "Action: $action" + + # Check compose file + compose_file=$(check_compose_file "$environment") + print_status "Using: $compose_file" + + # Check .env file + check_env_file "$environment" + + # Execute action + case $action in + "up") + print_status "Starting $environment environment..." + docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" up -d + print_success "$environment environment started successfully!" + + # Show service URLs + echo "" + print_status "Service URLs:" + echo " API Server: http://localhost:9101" + if [ "$environment" = "dev" ]; then + echo " Grafana: http://localhost:3000 (admin/admin123)" + echo " Prometheus: http://localhost:9090" + fi + ;; + "down") + print_status "Stopping $environment environment..." + docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" down + print_success "$environment environment stopped successfully!" + ;; + "restart") + print_status "Restarting $environment environment..." + docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" restart + print_success "$environment environment restarted successfully!" + ;; + "logs") + print_status "Showing logs for $environment environment..." + docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" logs -f + ;; + "status") + print_status "Status of $environment environment:" + docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" ps + ;; + "rollback") + print_warning "Rolling back $environment environment..." + print_warning "⚠ This rolls back the image only - queue state and audit log are NOT rolled back" + + if [ "$environment" = "prod" ]; then + print_warning "⚠ CRITICAL: Production rollback" + print_warning "⚠ Queue state is NOT rolled back" + print_warning "⚠ Audit log chain is NOT rolled back (must never break chain)" + read -p "CONFIRM PRODUCTION ROLLBACK? [yes/N] " confirm + if [ "$confirm" != "yes" ]; then + print_error "Rollback cancelled" + exit 1 + fi + fi + + # Get previous deployment info + LOG_FILE="${FETCHML_REPO_ROOT}/deployments/.${environment}-audit.log" + if [ -f "$LOG_FILE" ]; then + PREVIOUS_SHA=$(tail -2 "$LOG_FILE" | head -1 | grep -o 'sha-[a-f0-9]*' || echo "") + if [ -n "$PREVIOUS_SHA" ]; then + print_status "Rolling back to: $PREVIOUS_SHA" + fi + fi + + docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" down + docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" up -d + + # Write rollback entry to audit log + echo "$(date -Iseconds) | rollback | $environment | actor=$(whoami)" >> "$LOG_FILE" 2>/dev/null || true + + print_success "$environment rollback complete!" + print_status "Verify health with: $0 $environment health-check" + ;; + "health-check"|"health") + print_status "Health check for $environment environment..." + + # Determine port based on environment + case $environment in + dev) PORT=9101 ;; + staging) PORT=9102 ;; + prod) PORT=9101 ;; + *) PORT=9101 ;; + esac + + # Check API health + if curl -fsS "http://localhost:${PORT}/health" > /dev/null 2>&1; then + print_success "API is healthy (port $PORT)" + + # Check compliance_mode + COMPLIANCE_MODE=$(curl -fsS "http://localhost:${PORT}/health" 2>/dev/null | grep -o '"compliance_mode":"[^"]*"' | cut -d'"' -f4 || echo "unknown") + print_status "Compliance mode: $COMPLIANCE_MODE" + else + print_error "API health check failed (port $PORT)" + exit 1 + fi + ;; + "check-audit-sink") + print_status "Checking audit sink for $environment..." + + if [ -f "${FETCHML_REPO_ROOT}/scripts/check-audit-sink.sh" ]; then + "${FETCHML_REPO_ROOT}/scripts/check-audit-sink.sh" --env "$environment" + else + print_warning "Audit sink check script not found" + fi + ;; + *) + print_error "Unknown action: $action" + show_usage + exit 1 + ;; + esac +} + +# Run main function +main "$@" diff --git a/scripts/deploy/rollback.sh b/scripts/deploy/rollback.sh new file mode 100755 index 0000000..9bf674b --- /dev/null +++ b/scripts/deploy/rollback.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# Rollback procedures for fetch_ml deployments + +set -e + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +DEPLOYMENTS_DIR="${REPO_ROOT}/deployments" + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' + +print_success() { echo -e "${GREEN}[OK]${NC} $1"; } +print_error() { echo -e "${RED}[ERROR]${NC} $1"; } +print_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +print_info() { echo -e "[INFO] $1"; } + +show_usage() { + echo "Usage: $0 [ENVIRONMENT]" + echo "" + echo "Environments:" + echo " staging Rollback staging environment" + echo " prod Rollback production environment" + echo "" + echo "Examples:" + echo " $0 staging # Rollback staging" + echo " $0 prod # Rollback production (requires confirmation)" +} + +rollback_environment() { + local env=$1 + local compose_file + + case $env in + staging) + compose_file="${DEPLOYMENTS_DIR}/docker compose.staging.yml" + ;; + prod) + compose_file="${DEPLOYMENTS_DIR}/docker compose.prod.yml" + ;; + *) + print_error "Unknown environment: $env" + show_usage + exit 1 + ;; + esac + + if [ ! -f "$compose_file" ]; then + print_error "Compose file not found: $compose_file" + exit 1 + fi + + print_warn "Rolling back ${env} environment..." + print_warn "⚠ This rolls back the image only - queue state and audit log are NOT rolled back" + + if [ "$env" = "prod" ]; then + print_warn "⚠ CRITICAL: Production rollback" + print_warn "⚠ Queue state is NOT rolled back" + print_warn "⚠ Audit log chain is NOT rolled back (must never break chain)" + print_warn "⚠ New artifacts remain in storage" + read -p "CONFIRM PRODUCTION ROLLBACK? [yes/N] " confirm + if [ "$confirm" != "yes" ]; then + print_error "Rollback cancelled" + exit 1 + fi + else + read -p "Continue with rollback? [y/N] " confirm + if [ "$confirm" != "y" ]; then + print_error "Rollback cancelled" + exit 1 + fi + fi + + # Get previous deployment info from audit log + local log_file="${DEPLOYMENTS_DIR}/.${env}-audit.log" + local previous_sha="" + + if [ -f "$log_file" ]; then + previous_sha=$(tail -2 "$log_file" | head -1 | grep -o 'sha-[a-f0-9]*' || echo "") + if [ -n "$previous_sha" ]; then + print_info "Previous deployment: $previous_sha" + fi + fi + + # Perform rollback + print_info "Stopping ${env} environment..." + docker compose -f "$compose_file" down + + print_info "Starting ${env} environment..." + docker compose -f "$compose_file" up -d + + # Log the rollback + echo "$(date -Iseconds) | rollback | ${env} | actor=$(whoami)" >> "$log_file" 2>/dev/null || true + + print_success "${env} rollback complete!" + print_info "Verify health with: ./scripts/deploy/health-check.sh ${env}" +} + +main() { + if [ $# -ne 1 ]; then + show_usage + exit 1 + fi + + rollback_environment "$1" +} + +main "$@" diff --git a/scripts/dev/smoke-test.sh b/scripts/dev/smoke-test.sh index 1f5f4bf..63cb01c 100755 --- a/scripts/dev/smoke-test.sh +++ b/scripts/dev/smoke-test.sh @@ -37,7 +37,7 @@ cp -r "$repo_root/configs/"* "$DATA_DIR/configs/" # Build arguments BUILD_PROGRESS="${BUILD_PROGRESS:-auto}" # Set to 'plain' for full logs -compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose") +compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose") # Determine build flags build_flags="" diff --git a/scripts/release/cleanup.sh b/scripts/release/cleanup.sh index d0f20eb..646c806 100755 --- a/scripts/release/cleanup.sh +++ b/scripts/release/cleanup.sh @@ -35,12 +35,12 @@ print_error() { cleanup_docker() { print_header "Docker Compose Cleanup" - local compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose") + local compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose") # Stop all project-related containers $compose_cmd -f deployments/docker-compose.dev.yml down --volumes --remove-orphans 2>/dev/null || true - $compose_cmd -f deployments/docker-compose.local.yml down --volumes --remove-orphans 2>/dev/null || true - $compose_cmd -f tests/e2e/docker-compose.logs-debug.yml down --volumes --remove-orphans 2>/dev/null || true + $compose_cmd -f deployments/docker-compose.test.yml --profile debug down --volumes --remove-orphans 2>/dev/null || true + $compose_cmd -f deployments/docker-compose.prod.yml --profile smoke down --volumes --remove-orphans 2>/dev/null || true # Remove project-specific images (keep base images) docker images --filter "reference=fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true diff --git a/scripts/testing/test-native-with-redis.sh b/scripts/testing/test-native-with-redis.sh index a9c5e36..63ad8ff 100755 --- a/scripts/testing/test-native-with-redis.sh +++ b/scripts/testing/test-native-with-redis.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Test script for native libraries with Redis via docker-compose +# Test script for native libraries with Redis via docker compose # Usage: ./scripts/test-native-with-redis.sh set -e @@ -13,12 +13,12 @@ if [ ! -f "native/build/libqueue_index.so" ] && [ ! -f "native/build/libqueue_in make native-build fi -compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose") +compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose") -# Start Redis via docker-compose +# Start Redis via docker compose echo "Starting Redis..." cd deployments -$compose_cmd -f docker-compose.dev.yml up -d redis +$compose_cmd -f docker compose.dev.yml up -d redis cd .. # Wait for Redis to be ready @@ -49,7 +49,7 @@ fi echo "" echo "Stopping Redis..." cd deployments -$compose_cmd -f docker-compose.dev.yml stop redis +$compose_cmd -f docker compose.dev.yml stop redis cd .. exit $TEST_EXIT diff --git a/scripts/testing/test-prod.sh b/scripts/testing/test-prod.sh index 53d8d84..c27063a 100755 --- a/scripts/testing/test-prod.sh +++ b/scripts/testing/test-prod.sh @@ -5,11 +5,11 @@ set -e echo "Starting Full Production Test Environment with Podman and SQLite..." -compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose") +compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose") # Clean up any existing containers echo "Cleaning up existing containers..." -$compose_cmd -f deployments/docker-compose.prod.yml down -v +$compose_cmd -f deployments/docker compose.prod.yml down -v # Create necessary directories echo "Creating directories..." @@ -17,7 +17,7 @@ mkdir -p data logs # Build and start services echo "Building and starting services..." -$compose_cmd -f deployments/docker-compose.prod.yml up --build -d +$compose_cmd -f deployments/docker compose.prod.yml up --build -d # Wait for services to be healthy echo "Waiting for services to be healthy..." @@ -25,7 +25,7 @@ sleep 15 # Check service health echo "Checking service health..." -$compose_cmd -f deployments/docker-compose.prod.yml ps +$compose_cmd -f deployments/docker compose.prod.yml ps # Test API server echo "Testing API server..." @@ -61,7 +61,7 @@ echo " ./cli/zig-out/bin/ml queue prod-test-job" echo " ./cli/zig-out/bin/ml status" echo "" echo "To view logs:" -echo " $compose_cmd -f deployments/docker-compose.prod.yml logs -f worker" +echo " $compose_cmd -f deployments/docker compose.prod.yml logs -f worker" echo "" echo "To stop:" -echo " $compose_cmd -f deployments/docker-compose.prod.yml down" +echo " $compose_cmd -f deployments/docker compose.prod.yml down" diff --git a/scripts/testing/tui-ssh-test.sh b/scripts/testing/tui-ssh-test.sh index 9eb1ff0..8193349 100755 --- a/scripts/testing/tui-ssh-test.sh +++ b/scripts/testing/tui-ssh-test.sh @@ -24,25 +24,25 @@ if [[ ! -f "$SSH_KEY" ]]; then exit 1 fi -# Check if docker-compose services are running -compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose") +# Check if docker compose services are running +compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose") echo "=== Checking Docker Compose Services ===" cd "$REPO_ROOT" -if $compose_cmd -f deployments/docker-compose.prod.smoke.yml ps | grep -q "ml-smoke-caddy"; then +if $compose_cmd -f deployments/docker compose.prod.smoke.yml ps | grep -q "ml-smoke-caddy"; then echo "Caddy container running" else echo "✗ Caddy container not running" - echo "Start services: $compose_cmd -f deployments/docker-compose.prod.smoke.yml up -d" + echo "Start services: $compose_cmd -f deployments/docker compose.prod.smoke.yml up -d" exit 1 fi -if $compose_cmd -f deployments/docker-compose.prod.smoke.yml ps | grep -q "ml-ssh-test"; then +if $compose_cmd -f deployments/docker compose.prod.smoke.yml ps | grep -q "ml-ssh-test"; then echo "SSH test container running" else echo "✗ SSH test container not running" - echo "Start services: $compose_cmd -f deployments/docker-compose.prod.smoke.yml up -d" + echo "Start services: $compose_cmd -f deployments/docker compose.prod.smoke.yml up -d" exit 1 fi diff --git a/tests/e2e/logs_debug_e2e_test.go b/tests/e2e/logs_debug_e2e_test.go index e0c49cc..0f0a01b 100644 --- a/tests/e2e/logs_debug_e2e_test.go +++ b/tests/e2e/logs_debug_e2e_test.go @@ -18,7 +18,7 @@ import ( ) const ( - logsDebugComposeFile = "docker-compose.logs-debug.yml" + logsDebugComposeFile = "../../deployments/docker-compose.test.yml" apiPort = "9102" // Use docker-compose port apiHost = "localhost" ) diff --git a/tests/fixtures/ssh_server.go b/tests/fixtures/ssh_server.go index 59dc73b..2a4a9e8 100644 --- a/tests/fixtures/ssh_server.go +++ b/tests/fixtures/ssh_server.go @@ -67,7 +67,7 @@ func NewSSHTestServer(t *testing.T) *SSHTestServer { // Verify container is running if err := server.waitForContainer(); err != nil { - t.Skipf("SSH test container not available: %v. Run: docker-compose -f deployments/docker-compose.prod.smoke.yml up -d ssh-test-server", err) + t.Skipf("SSH test container not available: %v. Run: docker compose -f deployments/docker-compose.prod.yml --profile smoke up -d ssh-test-server", err) } // Verify SSH connectivity