diff --git a/deployments/docker-compose.dev.yml b/deployments/docker-compose.dev.yml index 9310820..213b8b9 100644 --- a/deployments/docker-compose.dev.yml +++ b/deployments/docker-compose.dev.yml @@ -11,8 +11,8 @@ services: - "8443:443" volumes: - ${FETCHML_REPO_ROOT:-..}/deployments/Caddyfile.dev:/etc/caddy/Caddyfile:ro - - ${FETCHML_REPO_ROOT:-..}/data/dev/caddy/data:/data - - ${FETCHML_REPO_ROOT:-..}/data/dev/caddy/config:/config + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/caddy/data:/data + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/caddy/config:/config depends_on: api-server: condition: service_healthy @@ -42,10 +42,10 @@ services: expose: - "9101" # API and health endpoints (internal; external access via Caddy) volumes: - - ${FETCHML_REPO_ROOT:-..}/data/dev/logs:/logs - - ${FETCHML_REPO_ROOT:-..}/data/dev/experiments:/data/experiments - - ${FETCHML_REPO_ROOT:-..}/data/dev/active:/data/active - - ${FETCHML_REPO_ROOT:-..}/data/dev/workspaces:/data/active/workspaces:delegated + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/logs:/logs + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/experiments:/data/experiments + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/active:/data/active + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/workspaces:/data/active/workspaces:delegated - ${FETCHML_REPO_ROOT:-..}/configs/api/dev.yaml:/app/configs/api/dev.yaml - ${FETCHML_REPO_ROOT:-..}/ssl:/app/ssl depends_on: @@ -71,7 +71,7 @@ services: - "9000:9000" - "9001:9001" volumes: - - ${FETCHML_REPO_ROOT:-..}/data/dev/minio:/data + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/minio:/data environment: - MINIO_ROOT_USER=minioadmin - MINIO_ROOT_PASSWORD=minioadmin123 @@ -133,10 +133,10 @@ services: ports: - "8888:8888" volumes: - - ${FETCHML_REPO_ROOT:-..}/data/dev/logs:/logs - - ${FETCHML_REPO_ROOT:-..}/data/dev/active:/data/active - - ${FETCHML_REPO_ROOT:-..}/data/dev/experiments:/data/experiments - - ${FETCHML_REPO_ROOT:-..}/data/dev/workspaces:/data/active/workspaces:delegated + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/logs:/logs + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/active:/data/active + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/experiments:/data/experiments + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/workspaces:/data/active/workspaces:delegated - ${FETCHML_REPO_ROOT:-..}/configs/workers/docker-dev.yaml:/app/configs/worker.yaml - /sys/fs/cgroup:/sys/fs/cgroup:rw depends_on: @@ -210,7 +210,7 @@ services: container_name: ml-experiments-promtail volumes: - ${FETCHML_REPO_ROOT:-..}/monitoring/promtail-config.yml:/etc/promtail/config.yml - - ${FETCHML_REPO_ROOT:-..}/data/dev/logs:/var/log/app + - ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/dev}/logs:/var/log/app - /var/lib/docker/containers:/var/lib/docker/containers:ro - /var/run/docker.sock:/var/run/docker.sock command: -config.file=/etc/promtail/config.yml diff --git a/scripts/dev/smoke-test.sh b/scripts/dev/smoke-test.sh index 9c4ef83..e81e1b8 100644 --- a/scripts/dev/smoke-test.sh +++ b/scripts/dev/smoke-test.sh @@ -104,42 +104,26 @@ api_wait_seconds=90 prometheus_wait_seconds=90 if [ "$env" = "dev" ]; then + # Use temp directory for smoke test data to avoid file sharing issues on macOS/Colima + SMOKE_TEST_DATA_DIR="${SMOKE_TEST_DATA_DIR:-$(mktemp -d /tmp/fetch_ml_smoke.XXXXXX)}" + echo "Using temp directory: $SMOKE_TEST_DATA_DIR" + mkdir -p \ - "$repo_root/data/dev/redis" \ - "$repo_root/data/dev/minio" \ - "$repo_root/data/dev/prometheus" \ - "$repo_root/data/dev/grafana" \ - "$repo_root/data/dev/loki" \ - "$repo_root/data/dev/logs" \ - "$repo_root/data/dev/experiments" \ - "$repo_root/data/dev/active" \ - "$repo_root/data/dev/workspaces" + "$SMOKE_TEST_DATA_DIR/redis" \ + "$SMOKE_TEST_DATA_DIR/minio" \ + "$SMOKE_TEST_DATA_DIR/prometheus" \ + "$SMOKE_TEST_DATA_DIR/grafana" \ + "$SMOKE_TEST_DATA_DIR/loki" \ + "$SMOKE_TEST_DATA_DIR/logs" \ + "$SMOKE_TEST_DATA_DIR/experiments" \ + "$SMOKE_TEST_DATA_DIR/active" \ + "$SMOKE_TEST_DATA_DIR/workspaces" - # Verify Docker can access the data directory (macOS file sharing check) - if ! docker run --rm -v "$repo_root/data/dev:/test-data:ro" alpine:3.19 ls /test-data >/dev/null 2>&1; then - echo "ERROR: Docker cannot access $repo_root/data/dev" - echo "" - if command -v colima >/dev/null 2>&1; then - echo "You are using Colima. To fix:" - echo "1. Stop Colima: colima stop" - echo "2. Start with host mounts enabled: colima start --mount-type=virtiofs" - echo " OR: colima start --mount-type=sshfs --mount-writable" - echo "3. Verify the mount: colima ssh -- ls $repo_root/data/dev" - echo "" - echo "If using Colima template, ensure your ~/.colima/default/colima.yaml has:" - echo " mounts:" - echo " - location: $repo_root" - echo " writable: true" - else - echo "This is a Docker Desktop file sharing issue. To fix:" - echo "1. Open Docker Desktop -> Settings -> Resources -> File sharing" - echo "2. Add or verify: $repo_root" - echo "3. Click 'Apply & Restart'" - fi - echo "" - echo "Alternatively, run: mkdir -p $repo_root/data/dev/{redis,minio,prometheus,grafana,loki,logs,experiments,active,workspaces}" - exit 1 - fi + # Export for docker-compose to use + export SMOKE_TEST_DATA_DIR + + # Update compose project args to include env file with the data dir + compose_project_args=("--project-directory" "$repo_root" "--env-file" <(echo "SMOKE_TEST_DATA_DIR=$SMOKE_TEST_DATA_DIR")) stack_name="dev" api_wait_seconds=180 @@ -151,13 +135,23 @@ if [ "$env" = "dev" ]; then fi prometheus_base="http://localhost:9090" else + # Use temp directory for prod smoke test too + SMOKE_TEST_DATA_DIR="${SMOKE_TEST_DATA_DIR:-$(mktemp -d /tmp/fetch_ml_smoke_prod.XXXXXX)}" + echo "Using temp directory: $SMOKE_TEST_DATA_DIR" + mkdir -p \ - "$repo_root/data/prod-smoke/caddy/data" \ - "$repo_root/data/prod-smoke/caddy/config" \ - "$repo_root/data/prod-smoke/redis" \ - "$repo_root/data/prod-smoke/logs" \ - "$repo_root/data/prod-smoke/experiments" \ - "$repo_root/data/prod-smoke/active" + "$SMOKE_TEST_DATA_DIR/caddy/data" \ + "$SMOKE_TEST_DATA_DIR/caddy/config" \ + "$SMOKE_TEST_DATA_DIR/redis" \ + "$SMOKE_TEST_DATA_DIR/logs" \ + "$SMOKE_TEST_DATA_DIR/experiments" \ + "$SMOKE_TEST_DATA_DIR/active" + + # Export for docker-compose to use + export SMOKE_TEST_DATA_DIR + + # Update compose project args to include env file with the data dir + compose_project_args=("--project-directory" "$repo_root" "--env-file" <(echo "SMOKE_TEST_DATA_DIR=$SMOKE_TEST_DATA_DIR")) stack_name="prod" compose_files=("-f" "$repo_root/deployments/docker-compose.prod.smoke.yml")