jfraeysd/fetch_ml
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix(deployments): add env var support for data directories

Browse source 
Update all docker-compose files to use environment variables for data paths:

- docker-compose.local.yml: Use LOCAL_DATA_DIR with fallback to ../data/dev
- docker-compose.prod.yml: Use PROD_DATA_DIR with fallback to data/prod
- docker-compose.prod.smoke.yml: Use SMOKE_TEST_DATA_DIR with fallback

This allows smoke tests and local development to use temp directories
instead of repo-relative paths, avoiding file sharing permission issues
on macOS with Docker Desktop or Colima.
This commit is contained in:
Jeremie Fraeys 2026-02-24 11:43:11 -05:00
parent ce4106a837
commit 5691b06876
No known key found for this signature in database
3 changed files with 21 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
deployments/docker-compose.local.yml
View file

@ -7,10 +7,10 @@ services:
ports:
- "9101:9101"
volumes:
- ../data/dev/logs:/logs
- ../data/dev/experiments:/data/experiments
- ../data/dev/active:/data/active
- ../data/dev/workspaces:/data/active/workspaces:delegated
- ${LOCAL_DATA_DIR:-../data/dev}/logs:/logs
- ${LOCAL_DATA_DIR:-../data/dev}/experiments:/data/experiments
- ${LOCAL_DATA_DIR:-../data/dev}/active:/data/active
- ${LOCAL_DATA_DIR:-../data/dev}/workspaces:/data/active/workspaces:delegated
- ../configs/api/dev.yaml:/app/configs/api/dev.yaml
environment:
- LOG_LEVEL=info
@ -30,10 +30,10 @@ services:
ports:
- "8888:8888"
volumes:
- ../data/dev/logs:/logs
- ../data/dev/active:/data/active
- ../data/dev/experiments:/data/experiments
- ../data/dev/workspaces:/data/active/workspaces:delegated
- ${LOCAL_DATA_DIR:-../data/dev}/logs:/logs
- ${LOCAL_DATA_DIR:-../data/dev}/active:/data/active
- ${LOCAL_DATA_DIR:-../data/dev}/experiments:/data/experiments
- ${LOCAL_DATA_DIR:-../data/dev}/workspaces:/data/active/workspaces:delegated
- ../configs/workers/docker-dev.yaml:/app/configs/worker.yaml
- /sys/fs/cgroup:/sys/fs/cgroup:rw
environment:

12
deployments/docker-compose.prod.smoke.yml
View file

@ -8,8 +8,8 @@ services:
- "8080:80"
volumes:
- ${FETCHML_REPO_ROOT:-..}/deployments/Caddyfile.prod.smoke:/etc/caddy/Caddyfile:ro
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/caddy/data:/data
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/caddy/config:/config
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/caddy/data:/data
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/caddy/config:/config
networks:
- default
depends_on:
@ -22,7 +22,7 @@ services:
expose:
- "6379"
volumes:
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/redis:/data
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/redis:/data
command: redis-server --appendonly yes
healthcheck:
test: [ "CMD", "redis-cli", "ping" ]
@ -42,9 +42,9 @@ services:
redis:
condition: service_healthy
volumes:
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/experiments:/data/experiments
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/active:/data/active
- ${FETCHML_REPO_ROOT:-..}/data/prod-smoke/logs:/logs
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/experiments:/data/experiments
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/active:/data/active
- ${SMOKE_TEST_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod-smoke}/logs:/logs
- ${FETCHML_REPO_ROOT:-..}/configs/api/dev.yaml:/app/configs/api/dev.yaml:ro
command: ["/bin/sh", "-c", "mkdir -p /data/experiments /data/active/datasets /data/active/snapshots && exec /usr/local/bin/api-server -config /app/configs/api/dev.yaml"]
environment:

14
deployments/docker-compose.prod.yml
View file

@ -7,7 +7,7 @@ services:
expose:
- "6379"
volumes:
- ${FETCHML_REPO_ROOT:-..}/data/prod/redis:/data
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/redis:/data
restart: unless-stopped
command: redis-server --appendonly yes
healthcheck:
@ -25,9 +25,9 @@ services:
- "9101"
- "2222"
volumes:
- ${FETCHML_REPO_ROOT:-..}/data/prod/experiments:/app/data/experiments
- ${FETCHML_REPO_ROOT:-..}/data/prod/active:/data/active
- ${FETCHML_REPO_ROOT:-..}/data/prod/logs:/logs
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/experiments:/app/data/experiments
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/active:/data/active
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/logs:/logs
- ${FETCHML_REPO_ROOT:-..}/configs/api/multi-user.yaml:/app/configs/api/prod.yaml
depends_on:
redis:
@ -59,9 +59,9 @@ services:
dockerfile: ${FETCHML_REPO_ROOT:-..}/build/docker/simple.Dockerfile
container_name: ml-prod-worker
volumes:
- ${FETCHML_REPO_ROOT:-..}/data/prod/experiments:/app/data/experiments
- ${FETCHML_REPO_ROOT:-..}/data/prod/active:/data/active
- ${FETCHML_REPO_ROOT:-..}/data/prod/logs:/logs
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/experiments:/app/data/experiments
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/active:/data/active
- ${PROD_DATA_DIR:-${FETCHML_REPO_ROOT:-..}/data/prod}/logs:/logs
- ${FETCHML_REPO_ROOT:-..}/configs/workers/docker-prod.yaml:/app/configs/worker.yaml
depends_on:
redis: