chore: migrate scripts from docker-compose v1 to v2
- Update all scripts to use 'docker compose' instead of 'docker-compose' - Fix compose file paths after consolidation (test.yml, prod.yml) - Update cleanup.sh to handle --profile debug and --profile smoke - Update test fixtures to reference consolidated compose files
This commit is contained in:
Jeremie Fraeys
parent
537faa6825
commit
e08feae6ab
9 changed files with 374 additions and 23 deletions
241
scripts/deploy/deploy.sh
Executable file
241
scripts/deploy/deploy.sh
Executable file
|
|
@ -0,0 +1,241 @@
|
|||
#!/bin/bash
|
||||
# Quick deployment script for fetch_ml
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd)
|
||||
export FETCHML_REPO_ROOT="${FETCHML_REPO_ROOT:-${REPO_ROOT}}"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Function to print colored output
|
||||
print_status() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Function to show usage
|
||||
show_usage() {
|
||||
echo "Usage: $0 [ENVIRONMENT] [ACTION]"
|
||||
echo ""
|
||||
echo "Environments:"
|
||||
echo " dev Development environment"
|
||||
echo " staging Staging environment (pre-production)"
|
||||
echo " secure Secure homelab environment"
|
||||
echo " prod Production environment"
|
||||
echo ""
|
||||
echo "Actions:"
|
||||
echo " up Start services"
|
||||
echo " down Stop services"
|
||||
echo " restart Restart services"
|
||||
echo " logs Show logs"
|
||||
echo " status Show status"
|
||||
echo " rollback Rollback to previous deployment (image only)"
|
||||
echo " health-check Check service health and compliance mode"
|
||||
echo " check-audit-sink Verify audit sink reachability"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 dev up # Start development environment"
|
||||
echo " $0 staging up # Start staging environment"
|
||||
echo " $0 prod down # Stop production environment"
|
||||
echo " $0 staging rollback # Rollback staging deployment"
|
||||
echo " $0 prod health-check # Check production health"
|
||||
echo " $0 prod check-audit-sink # Verify audit sink before deploy"
|
||||
}
|
||||
|
||||
# Function to check if docker-compose file exists
|
||||
check_compose_file() {
|
||||
local env=$1
|
||||
local compose_file=""
|
||||
|
||||
case $env in
|
||||
"dev")
|
||||
compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.dev.yml"
|
||||
;;
|
||||
"staging")
|
||||
compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.staging.yml"
|
||||
;;
|
||||
"secure")
|
||||
compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.homelab-secure.yml"
|
||||
;;
|
||||
"prod")
|
||||
compose_file="${FETCHML_REPO_ROOT}/deployments/docker-compose.prod.yml"
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown environment: $env"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ ! -f "$compose_file" ]; then
|
||||
print_error "Docker Compose file not found: $compose_file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$compose_file"
|
||||
}
|
||||
|
||||
# Function to check if .env file exists
|
||||
check_env_file() {
|
||||
local env=$1
|
||||
|
||||
if [ ! -f "${FETCHML_REPO_ROOT}/.env" ]; then
|
||||
print_warning ".env file not found. Creating from example..."
|
||||
if [ "$env" = "dev" ]; then
|
||||
cp "${FETCHML_REPO_ROOT}/deployments/env.dev.example" "${FETCHML_REPO_ROOT}/.env"
|
||||
elif [ "$env" = "prod" ]; then
|
||||
cp "${FETCHML_REPO_ROOT}/deployments/env.prod.example" "${FETCHML_REPO_ROOT}/.env"
|
||||
else
|
||||
cp "${FETCHML_REPO_ROOT}/deployments/env.dev.example" "${FETCHML_REPO_ROOT}/.env"
|
||||
fi
|
||||
print_warning "Please edit .env file with your configuration"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main script
|
||||
main() {
|
||||
if [ $# -ne 2 ]; then
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local environment=$1
|
||||
local action=$2
|
||||
|
||||
print_status "Environment: $environment"
|
||||
print_status "Action: $action"
|
||||
|
||||
# Check compose file
|
||||
compose_file=$(check_compose_file "$environment")
|
||||
print_status "Using: $compose_file"
|
||||
|
||||
# Check .env file
|
||||
check_env_file "$environment"
|
||||
|
||||
# Execute action
|
||||
case $action in
|
||||
"up")
|
||||
print_status "Starting $environment environment..."
|
||||
docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" up -d
|
||||
print_success "$environment environment started successfully!"
|
||||
|
||||
# Show service URLs
|
||||
echo ""
|
||||
print_status "Service URLs:"
|
||||
echo " API Server: http://localhost:9101"
|
||||
if [ "$environment" = "dev" ]; then
|
||||
echo " Grafana: http://localhost:3000 (admin/admin123)"
|
||||
echo " Prometheus: http://localhost:9090"
|
||||
fi
|
||||
;;
|
||||
"down")
|
||||
print_status "Stopping $environment environment..."
|
||||
docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" down
|
||||
print_success "$environment environment stopped successfully!"
|
||||
;;
|
||||
"restart")
|
||||
print_status "Restarting $environment environment..."
|
||||
docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" restart
|
||||
print_success "$environment environment restarted successfully!"
|
||||
;;
|
||||
"logs")
|
||||
print_status "Showing logs for $environment environment..."
|
||||
docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" logs -f
|
||||
;;
|
||||
"status")
|
||||
print_status "Status of $environment environment:"
|
||||
docker-compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" ps
|
||||
;;
|
||||
"rollback")
|
||||
print_warning "Rolling back $environment environment..."
|
||||
print_warning "⚠ This rolls back the image only - queue state and audit log are NOT rolled back"
|
||||
|
||||
if [ "$environment" = "prod" ]; then
|
||||
print_warning "⚠ CRITICAL: Production rollback"
|
||||
print_warning "⚠ Queue state is NOT rolled back"
|
||||
print_warning "⚠ Audit log chain is NOT rolled back (must never break chain)"
|
||||
read -p "CONFIRM PRODUCTION ROLLBACK? [yes/N] " confirm
|
||||
if [ "$confirm" != "yes" ]; then
|
||||
print_error "Rollback cancelled"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get previous deployment info
|
||||
LOG_FILE="${FETCHML_REPO_ROOT}/deployments/.${environment}-audit.log"
|
||||
if [ -f "$LOG_FILE" ]; then
|
||||
PREVIOUS_SHA=$(tail -2 "$LOG_FILE" | head -1 | grep -o 'sha-[a-f0-9]*' || echo "")
|
||||
if [ -n "$PREVIOUS_SHA" ]; then
|
||||
print_status "Rolling back to: $PREVIOUS_SHA"
|
||||
fi
|
||||
fi
|
||||
|
||||
docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" down
|
||||
docker compose --project-directory "${FETCHML_REPO_ROOT}" -f "$compose_file" up -d
|
||||
|
||||
# Write rollback entry to audit log
|
||||
echo "$(date -Iseconds) | rollback | $environment | actor=$(whoami)" >> "$LOG_FILE" 2>/dev/null || true
|
||||
|
||||
print_success "$environment rollback complete!"
|
||||
print_status "Verify health with: $0 $environment health-check"
|
||||
;;
|
||||
"health-check"|"health")
|
||||
print_status "Health check for $environment environment..."
|
||||
|
||||
# Determine port based on environment
|
||||
case $environment in
|
||||
dev) PORT=9101 ;;
|
||||
staging) PORT=9102 ;;
|
||||
prod) PORT=9101 ;;
|
||||
*) PORT=9101 ;;
|
||||
esac
|
||||
|
||||
# Check API health
|
||||
if curl -fsS "http://localhost:${PORT}/health" > /dev/null 2>&1; then
|
||||
print_success "API is healthy (port $PORT)"
|
||||
|
||||
# Check compliance_mode
|
||||
COMPLIANCE_MODE=$(curl -fsS "http://localhost:${PORT}/health" 2>/dev/null | grep -o '"compliance_mode":"[^"]*"' | cut -d'"' -f4 || echo "unknown")
|
||||
print_status "Compliance mode: $COMPLIANCE_MODE"
|
||||
else
|
||||
print_error "API health check failed (port $PORT)"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
"check-audit-sink")
|
||||
print_status "Checking audit sink for $environment..."
|
||||
|
||||
if [ -f "${FETCHML_REPO_ROOT}/scripts/check-audit-sink.sh" ]; then
|
||||
"${FETCHML_REPO_ROOT}/scripts/check-audit-sink.sh" --env "$environment"
|
||||
else
|
||||
print_warning "Audit sink check script not found"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown action: $action"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
110
scripts/deploy/rollback.sh
Executable file
110
scripts/deploy/rollback.sh
Executable file
|
|
@ -0,0 +1,110 @@
|
|||
#!/bin/bash
|
||||
# Rollback procedures for fetch_ml deployments
|
||||
|
||||
set -e
|
||||
|
||||
REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
DEPLOYMENTS_DIR="${REPO_ROOT}/deployments"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
print_success() { echo -e "${GREEN}[OK]${NC} $1"; }
|
||||
print_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
print_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
print_info() { echo -e "[INFO] $1"; }
|
||||
|
||||
show_usage() {
|
||||
echo "Usage: $0 [ENVIRONMENT]"
|
||||
echo ""
|
||||
echo "Environments:"
|
||||
echo " staging Rollback staging environment"
|
||||
echo " prod Rollback production environment"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 staging # Rollback staging"
|
||||
echo " $0 prod # Rollback production (requires confirmation)"
|
||||
}
|
||||
|
||||
rollback_environment() {
|
||||
local env=$1
|
||||
local compose_file
|
||||
|
||||
case $env in
|
||||
staging)
|
||||
compose_file="${DEPLOYMENTS_DIR}/docker compose.staging.yml"
|
||||
;;
|
||||
prod)
|
||||
compose_file="${DEPLOYMENTS_DIR}/docker compose.prod.yml"
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown environment: $env"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ ! -f "$compose_file" ]; then
|
||||
print_error "Compose file not found: $compose_file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_warn "Rolling back ${env} environment..."
|
||||
print_warn "⚠ This rolls back the image only - queue state and audit log are NOT rolled back"
|
||||
|
||||
if [ "$env" = "prod" ]; then
|
||||
print_warn "⚠ CRITICAL: Production rollback"
|
||||
print_warn "⚠ Queue state is NOT rolled back"
|
||||
print_warn "⚠ Audit log chain is NOT rolled back (must never break chain)"
|
||||
print_warn "⚠ New artifacts remain in storage"
|
||||
read -p "CONFIRM PRODUCTION ROLLBACK? [yes/N] " confirm
|
||||
if [ "$confirm" != "yes" ]; then
|
||||
print_error "Rollback cancelled"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
read -p "Continue with rollback? [y/N] " confirm
|
||||
if [ "$confirm" != "y" ]; then
|
||||
print_error "Rollback cancelled"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get previous deployment info from audit log
|
||||
local log_file="${DEPLOYMENTS_DIR}/.${env}-audit.log"
|
||||
local previous_sha=""
|
||||
|
||||
if [ -f "$log_file" ]; then
|
||||
previous_sha=$(tail -2 "$log_file" | head -1 | grep -o 'sha-[a-f0-9]*' || echo "")
|
||||
if [ -n "$previous_sha" ]; then
|
||||
print_info "Previous deployment: $previous_sha"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Perform rollback
|
||||
print_info "Stopping ${env} environment..."
|
||||
docker compose -f "$compose_file" down
|
||||
|
||||
print_info "Starting ${env} environment..."
|
||||
docker compose -f "$compose_file" up -d
|
||||
|
||||
# Log the rollback
|
||||
echo "$(date -Iseconds) | rollback | ${env} | actor=$(whoami)" >> "$log_file" 2>/dev/null || true
|
||||
|
||||
print_success "${env} rollback complete!"
|
||||
print_info "Verify health with: ./scripts/deploy/health-check.sh ${env}"
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ $# -ne 1 ]; then
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rollback_environment "$1"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
@ -37,7 +37,7 @@ cp -r "$repo_root/configs/"* "$DATA_DIR/configs/"
|
|||
# Build arguments
|
||||
BUILD_PROGRESS="${BUILD_PROGRESS:-auto}" # Set to 'plain' for full logs
|
||||
|
||||
compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose")
|
||||
compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose")
|
||||
|
||||
# Determine build flags
|
||||
build_flags=""
|
||||
|
|
|
|||
|
|
@ -35,12 +35,12 @@ print_error() {
|
|||
cleanup_docker() {
|
||||
print_header "Docker Compose Cleanup"
|
||||
|
||||
local compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose")
|
||||
local compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker-compose")
|
||||
|
||||
# Stop all project-related containers
|
||||
$compose_cmd -f deployments/docker-compose.dev.yml down --volumes --remove-orphans 2>/dev/null || true
|
||||
$compose_cmd -f deployments/docker-compose.local.yml down --volumes --remove-orphans 2>/dev/null || true
|
||||
$compose_cmd -f tests/e2e/docker-compose.logs-debug.yml down --volumes --remove-orphans 2>/dev/null || true
|
||||
$compose_cmd -f deployments/docker-compose.test.yml --profile debug down --volumes --remove-orphans 2>/dev/null || true
|
||||
$compose_cmd -f deployments/docker-compose.prod.yml --profile smoke down --volumes --remove-orphans 2>/dev/null || true
|
||||
|
||||
# Remove project-specific images (keep base images)
|
||||
docker images --filter "reference=fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/bash
|
||||
# Test script for native libraries with Redis via docker-compose
|
||||
# Test script for native libraries with Redis via docker compose
|
||||
# Usage: ./scripts/test-native-with-redis.sh
|
||||
|
||||
set -e
|
||||
|
|
@ -13,12 +13,12 @@ if [ ! -f "native/build/libqueue_index.so" ] && [ ! -f "native/build/libqueue_in
|
|||
make native-build
|
||||
fi
|
||||
|
||||
compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose")
|
||||
compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose")
|
||||
|
||||
# Start Redis via docker-compose
|
||||
# Start Redis via docker compose
|
||||
echo "Starting Redis..."
|
||||
cd deployments
|
||||
$compose_cmd -f docker-compose.dev.yml up -d redis
|
||||
$compose_cmd -f docker compose.dev.yml up -d redis
|
||||
cd ..
|
||||
|
||||
# Wait for Redis to be ready
|
||||
|
|
@ -49,7 +49,7 @@ fi
|
|||
echo ""
|
||||
echo "Stopping Redis..."
|
||||
cd deployments
|
||||
$compose_cmd -f docker-compose.dev.yml stop redis
|
||||
$compose_cmd -f docker compose.dev.yml stop redis
|
||||
cd ..
|
||||
|
||||
exit $TEST_EXIT
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ set -e
|
|||
|
||||
echo "Starting Full Production Test Environment with Podman and SQLite..."
|
||||
|
||||
compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose")
|
||||
compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose")
|
||||
|
||||
# Clean up any existing containers
|
||||
echo "Cleaning up existing containers..."
|
||||
$compose_cmd -f deployments/docker-compose.prod.yml down -v
|
||||
$compose_cmd -f deployments/docker compose.prod.yml down -v
|
||||
|
||||
# Create necessary directories
|
||||
echo "Creating directories..."
|
||||
|
|
@ -17,7 +17,7 @@ mkdir -p data logs
|
|||
|
||||
# Build and start services
|
||||
echo "Building and starting services..."
|
||||
$compose_cmd -f deployments/docker-compose.prod.yml up --build -d
|
||||
$compose_cmd -f deployments/docker compose.prod.yml up --build -d
|
||||
|
||||
# Wait for services to be healthy
|
||||
echo "Waiting for services to be healthy..."
|
||||
|
|
@ -25,7 +25,7 @@ sleep 15
|
|||
|
||||
# Check service health
|
||||
echo "Checking service health..."
|
||||
$compose_cmd -f deployments/docker-compose.prod.yml ps
|
||||
$compose_cmd -f deployments/docker compose.prod.yml ps
|
||||
|
||||
# Test API server
|
||||
echo "Testing API server..."
|
||||
|
|
@ -61,7 +61,7 @@ echo " ./cli/zig-out/bin/ml queue prod-test-job"
|
|||
echo " ./cli/zig-out/bin/ml status"
|
||||
echo ""
|
||||
echo "To view logs:"
|
||||
echo " $compose_cmd -f deployments/docker-compose.prod.yml logs -f worker"
|
||||
echo " $compose_cmd -f deployments/docker compose.prod.yml logs -f worker"
|
||||
echo ""
|
||||
echo "To stop:"
|
||||
echo " $compose_cmd -f deployments/docker-compose.prod.yml down"
|
||||
echo " $compose_cmd -f deployments/docker compose.prod.yml down"
|
||||
|
|
|
|||
|
|
@ -24,25 +24,25 @@ if [[ ! -f "$SSH_KEY" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Check if docker-compose services are running
|
||||
compose_cmd=$(command -v docker-compose >/dev/null 2>&1 && echo "docker-compose" || echo "docker compose")
|
||||
# Check if docker compose services are running
|
||||
compose_cmd=$(command -v docker compose >/dev/null 2>&1 && echo "docker compose" || echo "docker compose")
|
||||
|
||||
echo "=== Checking Docker Compose Services ==="
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if $compose_cmd -f deployments/docker-compose.prod.smoke.yml ps | grep -q "ml-smoke-caddy"; then
|
||||
if $compose_cmd -f deployments/docker compose.prod.smoke.yml ps | grep -q "ml-smoke-caddy"; then
|
||||
echo "Caddy container running"
|
||||
else
|
||||
echo "✗ Caddy container not running"
|
||||
echo "Start services: $compose_cmd -f deployments/docker-compose.prod.smoke.yml up -d"
|
||||
echo "Start services: $compose_cmd -f deployments/docker compose.prod.smoke.yml up -d"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if $compose_cmd -f deployments/docker-compose.prod.smoke.yml ps | grep -q "ml-ssh-test"; then
|
||||
if $compose_cmd -f deployments/docker compose.prod.smoke.yml ps | grep -q "ml-ssh-test"; then
|
||||
echo "SSH test container running"
|
||||
else
|
||||
echo "✗ SSH test container not running"
|
||||
echo "Start services: $compose_cmd -f deployments/docker-compose.prod.smoke.yml up -d"
|
||||
echo "Start services: $compose_cmd -f deployments/docker compose.prod.smoke.yml up -d"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
logsDebugComposeFile = "docker-compose.logs-debug.yml"
|
||||
logsDebugComposeFile = "../../deployments/docker-compose.test.yml"
|
||||
apiPort = "9102" // Use docker-compose port
|
||||
apiHost = "localhost"
|
||||
)
|
||||
|
|
|
|||
2
tests/fixtures/ssh_server.go
vendored
2
tests/fixtures/ssh_server.go
vendored
|
|
@ -67,7 +67,7 @@ func NewSSHTestServer(t *testing.T) *SSHTestServer {
|
|||
|
||||
// Verify container is running
|
||||
if err := server.waitForContainer(); err != nil {
|
||||
t.Skipf("SSH test container not available: %v. Run: docker-compose -f deployments/docker-compose.prod.smoke.yml up -d ssh-test-server", err)
|
||||
t.Skipf("SSH test container not available: %v. Run: docker compose -f deployments/docker-compose.prod.yml --profile smoke up -d ssh-test-server", err)
|
||||
}
|
||||
|
||||
// Verify SSH connectivity
|
||||
|
|
|
|||
Loading…
Reference in a new issue