8271277dc3
Phase 2: Deterministic Manifests - Add manifest.Validator with required field checking - Support Validate() and ValidateStrict() modes - Integrate validation into worker executor before execution - Block execution if manifest missing commit_id or deps_manifest_sha256 Phase 5: Pinned Dependencies - Add hermetic.dockerfile template with pinned system deps - Frozen package versions: libblas3, libcudnn8, etc. - Support for deps_manifest.json and requirements.txt with hashes - Image tagging strategy: deps-<first-8-of-sha256> Phase 8: Tests as Specifications - Add queue_spec_test.go with executable scheduler specs - Document priority ordering (higher first) - Document FIFO tiebreaker for same priority - Test cases for negative/zero priorities Phase 10: Local Dev Parity - Create root-level docker-compose.dev.yml - Simplified from deployments/ for quick local dev - Redis + API server + Worker with hot reload volumes - Debug ports: 9101 (API), 6379 (Redis)
40 lines
1.5 KiB
Docker
40 lines
1.5 KiB
Docker
# Hermetic Dockerfile - Reproducible builds with pinned dependencies
|
|
# Tag image with: deps-<first-8-of-sha256>
|
|
# Example: docker build -t mylab/pytorch:deps-abc123 .
|
|
|
|
FROM pytorch/pytorch:2.0.1-cuda11.8-cudnn8-runtime
|
|
|
|
# Pin system dependencies to specific versions for reproducibility
|
|
# These versions are frozen - update only after testing
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
libblas3=3.9.0-1build1 \
|
|
liblapack3=3.9.0-1build1 \
|
|
libcudnn8=8.6.0.163-1+cuda11.8 \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install conda environment with pinned packages
|
|
COPY deps_manifest.json /tmp/deps_manifest.json
|
|
|
|
# If using conda environment file
|
|
RUN if [ -f /tmp/deps_manifest.json ]; then \
|
|
conda env update -n base -f /tmp/deps_manifest.json; \
|
|
fi
|
|
|
|
# If using requirements.txt with hashes
|
|
COPY requirements.txt /tmp/requirements.txt
|
|
RUN pip install --require-hashes --no-deps -r /tmp/requirements.txt \
|
|
|| echo "Warning: Some packages may not have hashes"
|
|
|
|
# Verify installation
|
|
RUN python -c "import torch; print(f'PyTorch: {torch.__version__}')" \
|
|
&& python -c "import numpy; print(f'NumPy: {numpy.__version__}')"
|
|
|
|
# Labels for provenance
|
|
LABEL org.opencontainers.image.title="Hermetic ML Environment" \
|
|
org.opencontainers.image.description="Reproducible ML training environment" \
|
|
org.fetchml.deps_manifest="/tmp/deps_manifest.json" \
|
|
org.fetchml.build_date="${BUILD_DATE}" \
|
|
org.fetchml.git_commit="${GIT_COMMIT}"
|
|
|
|
WORKDIR /workspace
|
|
CMD ["python", "--version"]
|