feat: initialize FetchML ML platform with core project structure

- Add comprehensive README with architecture overview and quick start guide - Set up Go module with production-ready dependencies - Configure build system with Makefile for development and production builds - Add Docker Compose for local development environment - Include project configuration files (linting, Python, etc.) This establishes the foundation for a production-ready ML experiment platform with task queuing, monitoring, and modern CLI/API interface.
2025-12-04 16:52:09 -05:00 · 2025-12-04 16:52:09 -05:00 · c5049a2fdf
commit c5049a2fdf
11 changed files with 1577 additions and 0 deletions
--- a/.flake8
+++ b/.flake8
@ -0,0 +1,47 @@
+[flake8]
+max-line-length = 80
+max-complexity = 10
+exclude = 
+    .git,
+    __pycache__,
+    *.pyc,
+    .pytest_cache,
+    .mypy_cache,
+    .venv,
+    venv,
+    build,
+    dist,
+    # ML experiment code - exclude from style checks
+    podman/workspace,
+    workspace,
+    tests/fixtures/examples,
+    tests/fixtures/podman,
+    results,
+    data,
+    logs,
+    secrets,
+    .agent
+
+ignore = 
+    E203,
+    E501,
+    W503,
+    W504,
+    F401,
+    E402,
+    C901
+
+per-file-ignores =
+    __init__.py:F401
+    tests/*:F401,F811
+    *_test.py:F401,F811
+    conftest.py:F401,F811
+
+# Additional Google Style Guide recommendations
+inline-quotes = single
+multiline-quotes = double
+docstring-quotes = double
+
+# Import ordering (Google style)
+import-order-style = google
+application-import-names = fetch_ml
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,224 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+# IDE files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Python cache and artifacts
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# Python lib directories
+**/site-packages/
+**/lib/python*/
+# Allow scripts/lib for shell libraries
+!scripts/lib/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# poetry
+poetry.lock
+
+# pdm
+.pdm.toml
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+.idea/
+
+# Project-specific
+ml_jobs/
+test_output/
+# Python build directory only
+**/build/lib/
+**/build/bdist*/
+**/build/scripts*/
+# Allow build/ directory for Docker configs
+!build/
+!build/**
+# But ignore Python build artifacts if they happen
+build/lib/
+build/bdist*/
+coverage.out
+coverage.html
+queue-coverage.out
+coverage/
+
+# Redis dump
+dump.rdb
+
+# Configuration files with secrets
+!config_*.yaml
+!config_local.yaml.example
+!config_prod.yaml.example
+
+# Database files
+db/*.db
+!db/.gitkeep
+
+# API keys and secrets
+*.key
+*.pem
+secrets/
+cli/src/assets/rsync_release.bin
+
+# Test files
+test_*.go
+*_test_output/
+
+# Build artifacts  
+bin/
+cli/zig-out/
+cli/.zig-cache/
+zig-out/
+.zig-cache/
+
+# Experiment data (local testing)
+experiments/
+data/
--- a/.golangci.yml
+++ b/.golangci.yml
@ -0,0 +1,79 @@
+---
+version: "2"
+run:
+  timeout: 5m
+  tests: true
+output:
+  format: colored-line-number
+linters-settings:
+  govet:
+    enable:
+      - shadow
+      - fieldalignment
+  gocyclo:
+    min-complexity: 15
+  dupl:
+    threshold: 100
+  goconst:
+    min-len: 3
+    min-occurrences: 3
+  misspell:
+    locale: US
+  lll:
+    line-length: 100
+  revive:
+    confidence: 0.8
+  depguard:
+    rules:
+      main:
+        allow:
+          - $gostd
+          - github.com/jfraeys/fetch_ml
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - depguard
+    - dogsled
+    - dupl
+    - errcheck
+    - exhaustive
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gocyclo
+    - goprintffuncname
+    - gosec
+    - govet
+    - ineffassign
+    - lll
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - staticcheck
+    - unconvert
+    - unparam
+    - unused
+    - whitespace
+    - revive
+issues:
+  exclude-rules:
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - dupl
+        - gosec
+        - lll
+    - text: "weak cryptographic primitive"
+      linters:
+        - gosec
+    - text: "Use of weak random number generator"
+      linters:
+        - gosec
+  max-issues-per-linter: 0
+  max-same-issues: 0
+severity:
+  default-severity: error
--- a/.pylintrc
+++ b/.pylintrc
@ -0,0 +1,307 @@
+[MASTER]
+# Google Python Style Guide Configuration
+# Based on https://google.github.io/styleguide/pyguide.html
+
+# Files or directories to be skipped over when performing analysis.
+ignore=
+    .git,
+    __pycache__,
+    *.pyc,
+    .pytest_cache,
+    .mypy_cache,
+    .venv,
+    venv,
+    # ML experiment code - exclude from style checks
+    podman/workspace,
+    workspace,
+    tests/fixtures/examples,
+    tests/fixtures/podman,
+    results,
+    data,
+    logs,
+    secrets,
+    .agent
+
+# Regular expression matching files that should be skipped entirely.
+ignore-patterns=
+    ^\.#,
+    ^setup\.py$,
+    ^conftest\.py$
+
+# Add files or directories matching the regex patterns to the ignore list.
+ignore-paths=
+
+# Files or directories to be added to the list of ignored regular expressions.
+ignore-paths=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Use multiple processes to speed up Pylint.
+jobs=1
+
+# List of plugins (as comma separated values of python module names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Load all enabled extensions.
+load-all-plugins=no
+
+# Control the amount of potential astroid warnings (0, 1 or 2).
+unsafe-load-any-extension=no
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code.
+extension-pkg-whitelist=
+
+[MESSAGES CONTROL]
+# Only show warnings with the listed confidence levels.
+confidence=
+
+# Disable the message, report, category or checker with the given id(s).
+disable=
+    # Google Style Guide allows some of these
+    missing-module-docstring,
+    missing-function-docstring,
+    too-many-locals,
+    too-many-arguments,
+    too-many-instance-attributes,
+    too-many-public-methods,
+    too-few-public-methods,
+    too-many-lines,
+    too-many-statements,
+    too-many-branches,
+    too-many-nested-blocks,
+    # Not relevant for ML projects
+    invalid-name,
+    C0114,  # missing-module-docstring
+    C0115,  # missing-class-docstring
+    C0116,  # missing-function-docstring
+    R0903,  # too-few-public-methods
+    R0902,  # too-many-instance-attributes
+    R0913,  # too-many-arguments
+    R0914,  # too-many-locals
+    R0915,  # too-many-statements
+    R0912,  # too-many-branches
+    R0911,  # too-many-return-statements
+    W0613,  # unused-argument (common in callbacks)
+
+# Enable the message, report, category or checker with the given id(s).
+enable=
+
+[REPORTS]
+# Set the output format.
+output-format=text
+
+# Tells whether to display a full report or only the messages.
+reports=no
+
+# Python expression which determines if a message should not be displayed.
+evaluation=
+
+# Template used to display messages.
+msg-template=
+
+[BASIC]
+# Good variable names which should always be accepted, separated by a comma.
+good-names=
+    i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused.
+bad-names=
+    foo,bar,baz,toto,tutu,tata
+
+# List of builtins function names that should not be used.
+bad-functions=
+
+# Regular expression for correct variable names.
+variable-rgx=^[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression for correct constant names.
+const-rgx=^(_?[A-Z][A-Z0-9_]{2,30}|__.*__)$
+
+# Regular expression for correct function names.
+function-rgx=^(?:(?P<camel_case>_?[A-Z][a-zA-Z0-9]{2,30})|(?P<snake_case>_?[a-z][a-z0-9_]{2,30}))$
+
+# Regular expression for correct class names.
+class-rgx=^_?[A-Z][a-zA-Z0-9]{2,30}$
+
+# Regular expression for correct attribute names.
+attr-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$
+
+# Regular expression for correct argument names.
+argument-rgx=^[a-z][a-z0-9_]{2,30}$
+
+# Regular expression for correct method resolution order names.
+method-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# Minimum line length for functions/classes that require docstrings.
+docstring-min-length=-1
+
+[TYPECHECK]
+# List of decorators that produce context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of member names which should be excluded from the access member check.
+exclude-members=
+
+# List of qualified names (objects) which require a disable protection.
+generated-members=
+
+[MISCELLANEOUS]
+# List of note tags to take in consideration.
+notes=
+
+[VARIABLES]
+# Tells whether we should check for unused imports.
+init-import=no
+
+# A regular expression matching the name of dummy variables.
+dummy-variable-rgx=^_|dummy
+
+# List of variable names that should not be used.
+additional-builtins=
+
+# List of variables which are ignored for the undefined-variable check.
+ignored-classes=
+
+# List of members which are skipped from the member access check.
+ignored-modules=
+
+# List of deprecated symbols which should not be used.
+deprecated-modules=
+
+# List of symbols that can be safely accessed.
+allowed-redefined-builtins=
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Regexp for a line that is allowed to be longer than the limit.
+long-line-regexp=^[\t ]*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test.
+single-line-if-stmt=no
+
+# List of optional parameters for which we don't want to call a linting
+# function.
+single-line-class-stmt=no
+
+# Maximum number of characters on a single line, when there are no multiline
+# strings.
+max-line-length-sigil-fallback=80
+
+[SIMILARITIES]
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+[LOGGING]
+# Logging modules to check that the string formatting arguments are in
+# logging function parameter format.
+logging-modules=logging
+
+[DESIGN]
+# Maximum number of arguments for function / method.
+max-args=5
+
+# Argument names that match this regular expression are ignored.
+ignored-argument-names=
+
+# Maximum number of locals for function / method body.
+max-locals=15
+
+# Maximum number of return statements for function / method body.
+max-returns=6
+
+# Maximum number of branch for function / method body.
+max-branches=12
+
+# Maximum number of statements in function / method body.
+max-statements=50
+
+# Maximum number of parents for a class.
+max-parents=7
+
+# Maximum number of attributes for a class.
+max-attributes=7
+
+# Minimum number of public methods for a class.
+min-public-methods=2
+
+# Maximum number of public methods for a class.
+max-public-methods=20
+
+# Maximum number of boolean expressions in a if statement.
+max-bool-expr=5
+
+[CLASSES]
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcls
+
+# List of member names which should be excluded from the access member check.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+[IMPORTS]
+# Deprecated modules which should not be used.
+deprecated-modules=
+
+# Create a graph of every module's dependencies.
+import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# current file.
+ext-import-graph=
+
+# Create a graph of external dependencies in the current file.
+int-import-graph=
+
+# Force import order to match a specific order.
+import-order-style=google
+
+# Add files or directories to the blacklist.
+ignore-on-import-order=no
+
+# List of modules that can be imported at any level from the specified
+# application modules.
+known-third-party=
+
+# Analyse import fallback blocks.
+analyse-fallback-blocks=no
+
+[EXCEPTIONS]
+# Exceptions that will emit a warning when being caught.
+overgeneral-exceptions=Exception,BaseException
+
+# Exceptions that will emit a warning when being returned.
+return-exceptions=no
+
+# Exceptions that will emit a warning when being instantiated.
+nonstandard-exceptions=no
+
+[REFACTORING]
+# Maximum number of nested blocks.
+max-nested-blocks=5
+
+# Complete name of the function that returns the YAPF version.
+yapf-version=
--- a/21
+++ b/21
@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Fetch ML
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/198
+++ b/198
@ -0,0 +1,198 @@
+.PHONY: all build clean clean-docs test test-unit test-integration test-e2e test-coverage lint install dev prod setup validate configlint ci-local docs
+
+# Default target
+all: build
+
+# Build all components
+build:
+	go build -o bin/api-server cmd/api-server/main.go
+	go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go
+	go build -o bin/tui ./cmd/tui
+	cd cli && zig build prod
+	@echo "✓ All components built"
+
+# Build production-optimized binaries
+prod:
+	go build -ldflags="-s -w" -o bin/api-server cmd/api-server/main.go
+	go build -ldflags="-s -w" -o bin/worker cmd/worker/worker_server.go
+	go build -ldflags="-s -w" -o bin/tui ./cmd/tui
+	cd cli && zig build prod && strip zig-out/prod/ml
+	@echo "✓ Production binaries built"
+
+# Development build (faster compilation)
+dev:
+	go build -o bin/api-server cmd/api-server/main.go
+	go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go
+	go build -o bin/tui ./cmd/tui
+	cd cli && zig build dev
+	@echo "✓ Development binaries built"
+
+# Clean build artifacts
+clean:
+	rm -rf bin/ coverage/
+	rm -rf cli/zig-out/
+	rm -rf cli/.zig-cache/
+	go clean
+	@echo "✓ Cleaned"
+
+clean-docs:
+	rm -rf docs/_site/
+	@echo "✓ Cleaned docs"
+
+# Run tests
+test:
+	go test ./...
+	cd cli && zig build test
+	@echo "✓ All tests passed"
+
+# Lint Go and Zig code
+lint:
+	gofmt -w ./cmd ./internal ./tests || true
+	go vet ./...
+	cd cli && zig fmt .
+	@echo "✓ Lint completed"
+
+# Install to system (requires sudo)
+install: prod
+	sudo cp bin/api-server /usr/local/bin/fetchml-api
+	sudo cp bin/worker /usr/local/bin/fetchml-worker
+	sudo cp bin/tui /usr/local/bin/fetchml-tui
+	sudo cp cli/zig-out/prod/ml /usr/local/bin/ml
+	@echo "✓ Installed"
+
+# Setup production environment
+setup:
+	@if [ "$(shell uname)" = "Linux" ]; then \
+		sudo ./scripts/setup-prod.sh; \
+	else \
+		echo "Production setup is for Linux only. You're on $(shell uname)."; \
+		echo "Use docker-compose for local development."; \
+	fi
+
+# Validate production configuration
+validate:
+	./scripts/validate-prod-config.sh configs/config-prod.yaml configs/worker-prod.toml
+
+# Validate YAML configs against JSON schema
+configlint:
+	go run ./cmd/configlint --schema configs/schema/config_schema.yaml \
+		configs/config-prod.yaml \
+		configs/config-no-tls.yaml \
+		configs/config-dev.yaml
+
+# Run a local approximation of the CI pipeline
+ci-local:
+	make test
+	make lint
+	make configlint
+	@echo "Running queue package tests with race detector..."
+	go test -v -race -coverprofile=coverage/queue-coverage.out ./internal/queue/...
+	@echo "Running coverage..."
+	make test-coverage
+
+# Docker targets
+docker-build:
+	docker build -f build/docker/simple.Dockerfile -t fetchml:latest .
+	@echo "✓ Docker image built"
+
+docker-run:
+	docker-compose up -d
+	@echo "✓ Services started"
+
+docker-stop:
+	docker-compose down
+	@echo "✓ Services stopped"
+
+docker-logs:
+	docker-compose logs -f
+
+# Monitoring setup (Linux only)
+setup-monitoring:
+	@if [ "$(shell uname)" = "Linux" ]; then \
+		sudo ./scripts/setup-monitoring-prod.sh; \
+	else \
+		echo "Monitoring setup is for Linux production. Use docker-compose for local development."; \
+	fi
+
+# Enhanced test targets
+test-unit:
+	go test -v -short ./...
+
+test-integration:
+	go test -v ./...
+
+test-e2e:
+	go test -v ./tests/e2e/...
+
+test-coverage:
+	go test -coverprofile=coverage/coverage.out ./...
+	go tool cover -html=coverage/coverage.out -o coverage/coverage.html
+	@echo "✓ Coverage report: coverage/coverage.html"
+
+# Documentation setup
+docs-setup:
+	@echo "Setting up MkDocs documentation..."
+	@if ! command -v mkdocs >/dev/null 2>&1; then \
+		echo "MkDocs not found. Please install it manually:"; \
+		echo "  macOS: brew install mkdocs mkdocs-material"; \
+		echo "  Linux: pip install mkdocs mkdocs-material"; \
+		echo "Or visit: https://www.mkdocs.org/user-guide/installation/"; \
+		exit 1; \
+	fi
+	@if ! python3 -c "import pymdownx.superfences" >/dev/null 2>&1; then \
+		echo "pymdown-extensions not found. Please install it manually:"; \
+		echo "  pip3 install --user pymdown-extensions"; \
+		echo "  Or use a virtual environment: python3 -m venv docs-env && source docs-env/bin/activate && pip install pymdown-extensions"; \
+		exit 1; \
+	fi
+	@echo "Documentation setup complete!"
+
+# Documentation
+docs: docs-setup docs-build
+	@echo "Starting MkDocs development server..."
+	cd docs && mkdocs serve
+
+# Build documentation
+docs-build:
+	@echo "Building static documentation..."
+	cd docs && mkdocs build
+
+# Help
+help:
+	@echo "FetchML Build System"
+	@echo ""
+	@echo "Build Targets:"
+	@echo "  make build            - Build all components (default)"
+	@echo "  make prod             - Build production-optimized binaries"
+	@echo "  make dev              - Build development binaries (faster)"
+	@echo "  make clean            - Remove build artifacts"
+	@echo ""
+	@echo "Docker Targets:"
+	@echo "  make docker-build     - Build Docker image"
+	@echo "  make docker-run       - Start services with docker-compose"
+	@echo "  make docker-stop      - Stop docker-compose services"  
+	@echo "  make docker-logs      - View docker-compose logs"
+	@echo ""
+	@echo "Test Targets:"
+	@echo "  make test             - Run all tests"
+	@echo "  make test-unit        - Run unit tests only"
+	@echo "  make test-integration - Run integration tests"
+	@echo "  make test-coverage    - Generate coverage report"
+	@echo "  make lint             - Run formatters and linters"
+	@echo "  make ci-local         - Run local CI dry-run (tests, lint, config validation, coverage)"
+	@echo "  make configlint       - Validate YAML configs against schema"
+	@echo ""
+	@echo "Setup Targets:"
+	@echo "  make install          - Install binaries to /usr/local/bin (requires sudo)"
+	@echo "  make setup            - Run production setup (Linux only)"
+	@echo "  make setup-monitoring - Setup monitoring stack (Linux only)"
+	@echo "  make validate         - Validate production configuration"
+	@echo ""
+	@echo "Documentation:"
+	@echo "  make docs-setup       - Install MkDocs and dependencies"
+	@echo "  make docs             - Start MkDocs development server with live reload"
+	@echo "  make docs-build       - Build static documentation for deployment"
+	@echo ""
+	@echo "Utility:"
+	@echo "  make size             - Show binary sizes"
+	@echo "  make help             - Show this help"
--- a/README.md
+++ b/README.md
@ -0,0 +1,200 @@
+# FetchML - Machine Learning Platform
+
+A production-ready ML experiment platform with task queuing, monitoring, and a modern CLI/API.
+
+## Features
+
+- **🚀 Production Resilience** - Task leasing, smart retries, dead-letter queues
+- **📊 Monitoring** - Grafana/Prometheus/Loki with auto-provisioned dashboards
+- **🔐 Security** - API key auth, TLS, rate limiting, IP whitelisting
+- **⚡ Performance** - Go API server + Zig CLI for speed
+- **📦 Easy Deployment** - Docker Compose (dev) or systemd (prod)
+
+## Quick Start
+
+### Development (macOS/Linux)
+
+```bash
+# Clone and start
+git clone <your-repo>
+cd fetch_ml
+docker-compose up -d
+
+# Access Grafana: http://localhost:3000 (admin/admin)
+```
+
+### Production (Linux)
+
+```bash
+# Setup application
+sudo ./scripts/setup-prod.sh
+
+# Setup monitoring  
+sudo ./scripts/setup-monitoring-prod.sh
+
+# Build and install
+make prod
+make install
+
+# Start services
+sudo systemctl start fetchml-api fetchml-worker
+sudo systemctl start prometheus grafana loki promtail
+```
+
+## Architecture
+
+```
+┌──────────────┐   WebSocket   ┌──────────────┐
+│  Zig CLI/TUI │◄─────────────►│  API Server  │
+└──────────────┘               │    (Go)      │
+                               └──────┬───────┘
+                                      │
+                        ┌─────────────┼─────────────┐
+                        │             │             │
+                   ┌────▼────┐   ┌───▼────┐   ┌───▼────┐
+                   │  Redis  │   │ Worker │   │  Loki  │
+                   │ (Queue) │   │  (Go)  │   │ (Logs) │
+                   └─────────┘   └────────┘   └────────┘
+```
+
+## Usage
+
+### API Server
+
+```bash
+# Development (stderr logging)
+go run cmd/api-server/main.go --config configs/config-dev.yaml
+
+# Production (file logging)
+go run cmd/api-server/main.go --config configs/config-no-tls.yaml
+```
+
+### CLI
+
+```bash
+# Build
+cd cli && zig build prod
+
+# Run experiment
+./cli/zig-out/bin/ml run --config config.toml
+
+# Check status  
+./cli/zig-out/bin/ml status
+```
+
+### Docker
+
+```bash
+make docker-run      # Start all services
+make docker-logs     # View logs
+make docker-stop     # Stop services
+```
+
+## Development
+
+### Prerequisites
+
+- Go 1.21+
+- Zig 0.11+
+- Redis
+- Docker (for local dev)
+
+### Build
+
+```bash
+make build           # All components
+make dev             # Fast dev build
+make prod            # Optimized production build
+```
+
+### Test
+
+```bash
+make test            # All tests
+make test-unit       # Unit tests only
+make test-coverage   # With coverage report
+```
+
+## Configuration
+
+### Development (`configs/config-dev.yaml`)
+```yaml
+logging:
+  level: "info"
+  file: ""  # stderr only
+
+redis:
+  url: "redis://localhost:6379"
+```
+
+### Production (`configs/config-no-tls.yaml`)
+```yaml
+logging:
+  level: "info"
+  file: "./logs/fetch_ml.log"  # file only
+
+redis:
+  url: "redis://redis:6379"
+```
+
+## Monitoring
+
+### Grafana Dashboards (Auto-Provisioned)
+
+- **ML Task Queue** - Queue depth, task duration, failure rates
+- **Application Logs** - Log streams, error tracking, search
+
+Access: `http://localhost:3000` (dev) or `http://YOUR_SERVER:3000` (prod)
+
+### Metrics
+
+- Queue depth and task processing rates
+- Retry attempts by error category
+- Dead letter queue size
+- Lease expirations
+
+## Documentation
+
+- **[Getting Started](docs/getting-started.md)** - Detailed setup guide
+- **[Production Deployment](docs/production-monitoring.md)** - Linux deployment
+- **[WebSocket API](docs/api/)** - Protocol documentation
+- **[Architecture](docs/architecture/)** - System design
+
+## Makefile Targets
+
+```bash
+# Build
+make build               # Build all components
+make prod                # Production build
+make clean               # Clean artifacts
+
+# Docker
+make docker-build        # Build image
+make docker-run          # Start services
+make docker-stop         # Stop services
+
+# Test
+make test                # All tests
+make test-coverage       # With coverage
+
+# Production (Linux only)
+make setup               # Setup app
+make setup-monitoring    # Setup monitoring
+make install             # Install binaries
+```
+
+## Security
+
+- **TLS/HTTPS** - End-to-end encryption
+- **API Keys** - Hashed with SHA256
+- **Rate Limiting** - Per-user quotas
+- **IP Whitelist** - Network restrictions
+- **Audit Logging** - All API access logged
+
+## License
+
+MIT - See [LICENSE](LICENSE)
+
+## Contributing
+
+Contributions welcome! This is a personal homelab project but PRs are appreciated.
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,128 @@
+# Homelab Docker Compose with Centralized Monitoring
+# Includes: API, Redis, Prometheus, Grafana, Loki
+
+services:
+  redis:
+    image: redis:7-alpine
+    container_name: ml-experiments-redis
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    restart: unless-stopped
+    command: redis-server --appendonly yes
+    healthcheck:
+      test: [ "CMD", "redis-cli", "ping" ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  api-server:
+    build:
+      context: .
+      dockerfile: build/docker/simple.Dockerfile
+    container_name: ml-experiments-api
+    ports:
+      - "9101:9101"
+      - "9100:9100" # Prometheus metrics endpoint
+    volumes:
+      - ./data:/data/experiments
+      - ./logs:/logs
+      - ./configs/config-no-tls.yaml:/app/configs/config.yaml
+    depends_on:
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - LOG_LEVEL=info
+    healthcheck:
+      test: [ "CMD", "curl", "http://localhost:9101/health" ]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    labels:
+      logging: "promtail"
+      job: "api-server"
+
+  # Prometheus - Metrics collection
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ml-experiments-prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--web.enable-lifecycle'
+    restart: unless-stopped
+
+  # Grafana - Visualization
+  grafana:
+    image: grafana/grafana:latest
+    container_name: ml-experiments-grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - ./monitoring/grafana-dashboard.json:/var/lib/grafana/dashboards/ml-queue.json
+      - ./monitoring/logs-dashboard.json:/var/lib/grafana/dashboards/logs.json
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_SERVER_ROOT_URL=http://localhost:3000
+      - GF_AUTH_ANONYMOUS_ENABLED=false
+    restart: unless-stopped
+    depends_on:
+      - prometheus
+      - loki
+
+  # Loki - Log aggregation
+  loki:
+    image: grafana/loki:latest
+    container_name: ml-experiments-loki
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./monitoring/loki-config.yml:/etc/loki/local-config.yaml
+      - loki_data:/loki
+    command: -config.file=/etc/loki/local-config.yaml
+    restart: unless-stopped
+
+  # Promtail - Log collector
+  promtail:
+    image: grafana/promtail:latest
+    container_name: ml-experiments-promtail
+    volumes:
+      - ./monitoring/promtail-config.yml:/etc/promtail/config.yml
+      - ./logs:/var/log/app
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/config.yml
+    restart: unless-stopped
+    depends_on:
+      - loki
+
+volumes:
+  redis_data:
+    driver: local
+  prometheus_data:
+    driver: local
+  grafana_data:
+    driver: local
+  loki_data:
+    driver: local
+
+networks:
+  default:
+    name: ml-experiments-network
+  backend:
+    name: ml-backend-network
+    internal: true # No external access
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,73 @@
+module github.com/jfraeys/fetch_ml
+
+go 1.25.0
+
+// Fetch ML - Secure Machine Learning Platform
+// Copyright (c) 2024 Fetch ML
+// Licensed under the MIT License
+
+require (
+	github.com/BurntSushi/toml v1.5.0
+	github.com/alicebob/miniredis/v2 v2.35.0
+	github.com/charmbracelet/bubbles v0.21.0
+	github.com/charmbracelet/bubbletea v1.3.10
+	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/go-redis/redis/v8 v8.11.5
+	github.com/google/uuid v1.6.0
+	github.com/gorilla/websocket v1.5.3
+	github.com/lib/pq v1.10.9
+	github.com/mattn/go-sqlite3 v1.14.32
+	github.com/prometheus/client_golang v1.23.2
+	github.com/redis/go-redis/v9 v9.17.2
+	github.com/stretchr/testify v1.11.1
+	github.com/xeipuuv/gojsonschema v1.2.0
+	github.com/zalando/go-keyring v0.2.6
+	golang.org/x/crypto v0.45.0
+	golang.org/x/time v0.14.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	al.essio.dev/pkg/shellescape v1.6.0 // indirect
+	github.com/atotto/clipboard v0.1.4 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/colorprofile v0.3.3 // indirect
+	github.com/charmbracelet/x/ansi v0.11.2 // indirect
+	github.com/charmbracelet/x/cellbuf v0.0.14 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.6.1 // indirect
+	github.com/clipperhouse/stringish v0.1.1 // indirect
+	github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
+	github.com/danieljoos/wincred v1.2.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/godbus/dbus/v5 v5.2.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.19 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.67.4 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sahilm/fuzzy v0.1.1 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/yuin/gopher-lua v1.1.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,168 @@
+al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
+al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
+github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
+github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
+github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
+github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
+github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
+github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
+github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg=
+github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
+github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
+github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=
+github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4=
+github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
+github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
+github.com/charmbracelet/x/ansi v0.11.2 h1:XAG3FSjiVtFvgEgGrNBkCNNYrsucAt8c6bfxHyROLLs=
+github.com/charmbracelet/x/ansi v0.11.2/go.mod h1:9tY2bzX5SiJCU0iWyskjBeI2BRQfvPqI+J760Mjf+Rg=
+github.com/charmbracelet/x/cellbuf v0.0.14 h1:iUEMryGyFTelKW3THW4+FfPgi4fkmKnnaLOXuc+/Kj4=
+github.com/charmbracelet/x/cellbuf v0.0.14/go.mod h1:P447lJl49ywBbil/KjCk2HexGh4tEY9LH0/1QrZZ9rA=
+github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ=
+github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
+github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
+github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
+github.com/clipperhouse/displaywidth v0.6.1 h1:/zMlAezfDzT2xy6acHBzwIfyu2ic0hgkT83UX5EY2gY=
+github.com/clipperhouse/displaywidth v0.6.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
+github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
+github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
+github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
+github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ=
+github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
+github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
+github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8=
+github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
+github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
+github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
+github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
+github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
+github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
+github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
+github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
+github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
+github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s=
+github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
+go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
+golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/pyproject.toml
+++ b/pyproject.toml
@ -0,0 +1,132 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+# Google Python Style Guide Configuration for Black
+line-length = 80
+target-version = ['py38']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  # ML experiment code - exclude from formatting
+  | podman/workspace
+  | workspace
+  | tests/fixtures/examples
+  | tests/fixtures/podman
+  | results
+  | data
+  | logs
+  | secrets
+  | .agent
+)/
+'''
+
+[tool.isort]
+# Google Python Style Guide Configuration for isort
+profile = "google"
+line_length = 80
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+split_on_trailing_comma = true
+known_first_party = ["fetch_ml"]
+skip_glob = [
+    "podman/workspace/*",
+    "workspace/*",
+    "tests/fixtures/examples/*",
+    "tests/fixtures/podman/*",
+]
+
+[tool.mypy]
+# Google Python Style Guide Configuration for mypy
+python_version = "3.8"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[[tool.mypy.overrides]]
+module = [
+    "torch.*",
+    "tensorflow.*", 
+    "sklearn.*",
+    "pandas.*",
+    "numpy.*",
+    "matplotlib.*",
+    "seaborn.*",
+    "scipy.*",
+    "joblib.*",
+]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+# Google Python Style Guide Configuration for pytest
+minversion = "6.0"
+addopts = "-ra -q --strict-markers --strict-config"
+testpaths = ["tests"]
+python_files = ["*_test.py", "test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+]
+
+[tool.coverage.run]
+source = ["."]
+omit = [
+    "tests/*",
+    "*/tests/*",
+    "test_*",
+    "*_test.py",
+    "setup.py",
+    "*/site-packages/*",
+    # ML experiment code
+    "podman/workspace/*",
+    "workspace/*",
+    "tests/fixtures/examples/*",
+    "tests/fixtures/podman/*",
+    "results/*",
+    "data/*",
+    "logs/*",
+    "secrets/*",
+    ".agent/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]