fetch_ml/deployments/docker-compose.dev.yml

---
# Development Docker Compose
# Includes: API, Redis, MinIO, Worker, Caddy
services:
  caddy:
    image: caddy:2-alpine
    container_name: ml-dev-caddy
    restart: unless-stopped
    ports:
      - "8080:80"
      - "8443:443"
    volumes:
      - ./Caddyfile.dev:/etc/caddy/Caddyfile:ro
      - ${DATA_DIR:-./data/smoke}/caddy/data:/data
      - ${DATA_DIR:-./data/smoke}/caddy/config:/config
    depends_on:
      api-server:
        condition: service_healthy
  redis:
    image: redis:7-alpine
    container_name: ml-experiments-redis
    user: "999:999"
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    restart: unless-stopped
    command: redis-server --appendonly yes
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 10s
      retries: 3
  api-server:
    build:
      context: ..
      dockerfile: build/docker/simple.Dockerfile
    container_name: ml-experiments-api
    user: "0:0"
    ports:
      - "9101:9101"
    expose:
      - "9101" # API and health endpoints (internal; external access via Caddy)
    volumes:
      - ${DATA_DIR:-./data/smoke}/logs:/logs
      - ${DATA_DIR:-./data/smoke}/experiments:/data/experiments
      - ${DATA_DIR:-./data/smoke}/active:/data/active
      - ${DATA_DIR:-./data/smoke}/workspaces:/data/active/workspaces:delegated
      - ${CONFIG_DIR:-../configs}/api/dev.yaml:/app/configs/api/dev.yaml:ro
      - ${DATA_DIR:-./data/smoke}/ssl:/app/ssl:ro
    depends_on:
      - redis
    restart: unless-stopped
    command: ["/bin/sh", "-c", "mkdir -p /data/experiments /data/active/datasets /data/active/snapshots && exec /usr/local/bin/api-server -config /app/configs/api/dev.yaml"]
    environment:
      - LOG_LEVEL=info
    # Native libs enabled via build tag: -tags native_libs
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9101/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    labels:
      job: "api-server"
  # MinIO for local development (single-node filesystem backend)
  minio:
    image: minio/minio:latest
    container_name: ml-dev-minio
    ports:
      - "9000:9000"
      - "9001:9001"
    volumes:
      - ${DATA_DIR:-./data/smoke}/minio:/data
    environment:
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin123
      - MINIO_BROWSER=on
    command: ["server", "/data", "--console-address", ":9001"]
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  # Initialize minio bucket (runs once)
  minio-init:
    image: minio/mc:latest
    container_name: ml-dev-minio-init
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        mc alias set local http://minio:9000 minioadmin minioadmin123 || exit 1
        mc mb -p local/fetchml-snapshots 2>/dev/null || echo "Bucket exists"
        echo "MinIO initialized"        
    restart: "no"
  worker:
    build:
      context: ..
      dockerfile: build/docker/simple.Dockerfile
    # Remove fixed container name to allow scaling
    # container_name: ml-experiments-worker
    user: "0:0"
    ports:
      - "8888-8891:8888"  # Port range for multiple workers
    volumes:
      - ${DATA_DIR:-./data/smoke}/logs:/logs
      - ${DATA_DIR:-./data/smoke}/active:/data/active
      - ${DATA_DIR:-./data/smoke}/experiments:/data/experiments
      - ${DATA_DIR:-./data/smoke}/workspaces:/data/active/workspaces:delegated
      - ${CONFIG_DIR:-../configs}/worker/docker-dev.yaml:/app/configs/worker.yaml:ro
      - ${DATA_DIR:-./data/smoke}/ssl:/app/ssl:ro
      - /sys/fs/cgroup:/sys/fs/cgroup:rw
    depends_on:
      redis:
        condition: service_healthy
      api-server:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    restart: unless-stopped
    environment:
      - LOG_LEVEL=info
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin123
      - FETCHML_JUPYTER_DEFAULT_IMAGE=quay.io/jupyter/minimal-notebook:latest
      - FETCHML_JUPYTER_CONDA_ENV=base
      - FETCHML_JUPYTER_KERNEL_NAME=python
      - FETCHML_PODMAN_CGROUPS=disabled
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8888/health"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    # Native libs enabled via build tag: -tags native_libs
    privileged: true
    command: ["/usr/local/bin/worker", "-config", "/app/configs/worker.yaml"]
    deploy:
      replicas: 2  # Scale to 2 workers for dev (adjust as needed)

  # Local profile: use pre-built images instead of building from source
  api-server-local:
    image: fetchml-api:latest
    profiles: ["local"]
    ports:
      - "9101:9101"
    volumes:
      - ${LOCAL_DATA_DIR:-./data/dev}/logs:/logs
      - ${LOCAL_DATA_DIR:-./data/dev}/experiments:/data/experiments
      - ${LOCAL_DATA_DIR:-./data/dev}/active:/data/active
      - ${LOCAL_DATA_DIR:-./data/dev}/workspaces:/data/active/workspaces:delegated
      - ${CONFIG_DIR:-../configs}/api/dev.yaml:/app/configs/api/dev.yaml:ro
    environment:
      - LOG_LEVEL=info
    depends_on:
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9101/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  worker-local:
    image: fetchml-worker:latest
    profiles: ["local"]
    privileged: true
    ports:
      - "8888:8888"
    volumes:
      - ${LOCAL_DATA_DIR:-./data/dev}/logs:/logs
      - ${LOCAL_DATA_DIR:-./data/dev}/active:/data/active
      - ${LOCAL_DATA_DIR:-./data/dev}/experiments:/data/experiments
      - ${LOCAL_DATA_DIR:-./data/dev}/workspaces:/data/active/workspaces:delegated
      - ${LOCAL_DATA_DIR:-./data/dev}/snapshots:/data/snapshots
      - ${CONFIG_DIR:-../configs}/worker/docker-dev.yaml:/app/configs/worker.yaml:ro
      - /sys/fs/cgroup:/sys/fs/cgroup:rw
    environment:
      - LOG_LEVEL=info
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin123
    depends_on:
      redis:
        condition: service_healthy
      api-server-local:
        condition: service_healthy

volumes:
  redis_data:
    driver: local