commit c5049a2fdf7e68b2249a23bdd2e1f93761e6efb9 Author: Jeremie Fraeys Date: Thu Dec 4 16:52:09 2025 -0500 feat: initialize FetchML ML platform with core project structure - Add comprehensive README with architecture overview and quick start guide - Set up Go module with production-ready dependencies - Configure build system with Makefile for development and production builds - Add Docker Compose for local development environment - Include project configuration files (linting, Python, etc.) This establishes the foundation for a production-ready ML experiment platform with task queuing, monitoring, and modern CLI/API interface. diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bdf162d --- /dev/null +++ b/.flake8 @@ -0,0 +1,47 @@ +[flake8] +max-line-length = 80 +max-complexity = 10 +exclude = + .git, + __pycache__, + *.pyc, + .pytest_cache, + .mypy_cache, + .venv, + venv, + build, + dist, + # ML experiment code - exclude from style checks + podman/workspace, + workspace, + tests/fixtures/examples, + tests/fixtures/podman, + results, + data, + logs, + secrets, + .agent + +ignore = + E203, + E501, + W503, + W504, + F401, + E402, + C901 + +per-file-ignores = + __init__.py:F401 + tests/*:F401,F811 + *_test.py:F401,F811 + conftest.py:F401,F811 + +# Additional Google Style Guide recommendations +inline-quotes = single +multiline-quotes = double +docstring-quotes = double + +# Import ordering (Google style) +import-order-style = google +application-import-names = fetch_ml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6da3c15 --- /dev/null +++ b/.gitignore @@ -0,0 +1,224 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Python cache and artifacts +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +# Python lib directories +**/site-packages/ +**/lib/python*/ +# Allow scripts/lib for shell libraries +!scripts/lib/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +.pdm.toml + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ + +# Project-specific +ml_jobs/ +test_output/ +# Python build directory only +**/build/lib/ +**/build/bdist*/ +**/build/scripts*/ +# Allow build/ directory for Docker configs +!build/ +!build/** +# But ignore Python build artifacts if they happen +build/lib/ +build/bdist*/ +coverage.out +coverage.html +queue-coverage.out +coverage/ + +# Redis dump +dump.rdb + +# Configuration files with secrets +!config_*.yaml +!config_local.yaml.example +!config_prod.yaml.example + +# Database files +db/*.db +!db/.gitkeep + +# API keys and secrets +*.key +*.pem +secrets/ +cli/src/assets/rsync_release.bin + +# Test files +test_*.go +*_test_output/ + +# Build artifacts +bin/ +cli/zig-out/ +cli/.zig-cache/ +zig-out/ +.zig-cache/ + +# Experiment data (local testing) +experiments/ +data/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..0585580 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,79 @@ +--- +version: "2" +run: + timeout: 5m + tests: true +output: + format: colored-line-number +linters-settings: + govet: + enable: + - shadow + - fieldalignment + gocyclo: + min-complexity: 15 + dupl: + threshold: 100 + goconst: + min-len: 3 + min-occurrences: 3 + misspell: + locale: US + lll: + line-length: 100 + revive: + confidence: 0.8 + depguard: + rules: + main: + allow: + - $gostd + - github.com/jfraeys/fetch_ml +linters: + disable-all: true + enable: + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - exhaustive + - gochecknoinits + - goconst + - gocritic + - gocyclo + - goprintffuncname + - gosec + - govet + - ineffassign + - lll + - misspell + - nakedret + - noctx + - nolintlint + - rowserrcheck + - staticcheck + - unconvert + - unparam + - unused + - whitespace + - revive +issues: + exclude-rules: + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + - lll + - text: "weak cryptographic primitive" + linters: + - gosec + - text: "Use of weak random number generator" + linters: + - gosec + max-issues-per-linter: 0 + max-same-issues: 0 +severity: + default-severity: error diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..dd3574c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,307 @@ +[MASTER] +# Google Python Style Guide Configuration +# Based on https://google.github.io/styleguide/pyguide.html + +# Files or directories to be skipped over when performing analysis. +ignore= + .git, + __pycache__, + *.pyc, + .pytest_cache, + .mypy_cache, + .venv, + venv, + # ML experiment code - exclude from style checks + podman/workspace, + workspace, + tests/fixtures/examples, + tests/fixtures/podman, + results, + data, + logs, + secrets, + .agent + +# Regular expression matching files that should be skipped entirely. +ignore-patterns= + ^\.#, + ^setup\.py$, + ^conftest\.py$ + +# Add files or directories matching the regex patterns to the ignore list. +ignore-paths= + +# Files or directories to be added to the list of ignored regular expressions. +ignore-paths= + +# Pickle collected data for later comparisons. +persistent=yes + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Load all enabled extensions. +load-all-plugins=no + +# Control the amount of potential astroid warnings (0, 1 or 2). +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +[MESSAGES CONTROL] +# Only show warnings with the listed confidence levels. +confidence= + +# Disable the message, report, category or checker with the given id(s). +disable= + # Google Style Guide allows some of these + missing-module-docstring, + missing-function-docstring, + too-many-locals, + too-many-arguments, + too-many-instance-attributes, + too-many-public-methods, + too-few-public-methods, + too-many-lines, + too-many-statements, + too-many-branches, + too-many-nested-blocks, + # Not relevant for ML projects + invalid-name, + C0114, # missing-module-docstring + C0115, # missing-class-docstring + C0116, # missing-function-docstring + R0903, # too-few-public-methods + R0902, # too-many-instance-attributes + R0913, # too-many-arguments + R0914, # too-many-locals + R0915, # too-many-statements + R0912, # too-many-branches + R0911, # too-many-return-statements + W0613, # unused-argument (common in callbacks) + +# Enable the message, report, category or checker with the given id(s). +enable= + +[REPORTS] +# Set the output format. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Python expression which determines if a message should not be displayed. +evaluation= + +# Template used to display messages. +msg-template= + +[BASIC] +# Good variable names which should always be accepted, separated by a comma. +good-names= + i,j,k,ex,Run,_ + +# Bad variable names which should always be refused. +bad-names= + foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used. +bad-functions= + +# Regular expression for correct variable names. +variable-rgx=^[a-z_][a-z0-9_]{2,30}$ + +# Regular expression for correct constant names. +const-rgx=^(_?[A-Z][A-Z0-9_]{2,30}|__.*__)$ + +# Regular expression for correct function names. +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]{2,30})|(?P_?[a-z][a-z0-9_]{2,30}))$ + +# Regular expression for correct class names. +class-rgx=^_?[A-Z][a-zA-Z0-9]{2,30}$ + +# Regular expression for correct attribute names. +attr-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$ + +# Regular expression for correct argument names. +argument-rgx=^[a-z][a-z0-9_]{2,30}$ + +# Regular expression for correct method resolution order names. +method-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings. +docstring-min-length=-1 + +[TYPECHECK] +# List of decorators that produce context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of member names which should be excluded from the access member check. +exclude-members= + +# List of qualified names (objects) which require a disable protection. +generated-members= + +[MISCELLANEOUS] +# List of note tags to take in consideration. +notes= + +[VARIABLES] +# Tells whether we should check for unused imports. +init-import=no + +# A regular expression matching the name of dummy variables. +dummy-variable-rgx=^_|dummy + +# List of variable names that should not be used. +additional-builtins= + +# List of variables which are ignored for the undefined-variable check. +ignored-classes= + +# List of members which are skipped from the member access check. +ignored-modules= + +# List of deprecated symbols which should not be used. +deprecated-modules= + +# List of symbols that can be safely accessed. +allowed-redefined-builtins= + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +long-line-regexp=^[\t ]*(# )??$ + +# Allow the body of an if to be on the same line as the test. +single-line-if-stmt=no + +# List of optional parameters for which we don't want to call a linting +# function. +single-line-class-stmt=no + +# Maximum number of characters on a single line, when there are no multiline +# strings. +max-line-length-sigil-fallback=80 + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +[LOGGING] +# Logging modules to check that the string formatting arguments are in +# logging function parameter format. +logging-modules=logging + +[DESIGN] +# Maximum number of arguments for function / method. +max-args=5 + +# Argument names that match this regular expression are ignored. +ignored-argument-names= + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of return statements for function / method body. +max-returns=6 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Maximum number of parents for a class. +max-parents=7 + +# Maximum number of attributes for a class. +max-attributes=7 + +# Minimum number of public methods for a class. +min-public-methods=2 + +# Maximum number of public methods for a class. +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement. +max-bool-expr=5 + +[CLASSES] +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcls + +# List of member names which should be excluded from the access member check. +exclude-protected=_asdict,_fields,_replace,_source,_make + +[IMPORTS] +# Deprecated modules which should not be used. +deprecated-modules= + +# Create a graph of every module's dependencies. +import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# current file. +ext-import-graph= + +# Create a graph of external dependencies in the current file. +int-import-graph= + +# Force import order to match a specific order. +import-order-style=google + +# Add files or directories to the blacklist. +ignore-on-import-order=no + +# List of modules that can be imported at any level from the specified +# application modules. +known-third-party= + +# Analyse import fallback blocks. +analyse-fallback-blocks=no + +[EXCEPTIONS] +# Exceptions that will emit a warning when being caught. +overgeneral-exceptions=Exception,BaseException + +# Exceptions that will emit a warning when being returned. +return-exceptions=no + +# Exceptions that will emit a warning when being instantiated. +nonstandard-exceptions=no + +[REFACTORING] +# Maximum number of nested blocks. +max-nested-blocks=5 + +# Complete name of the function that returns the YAPF version. +yapf-version= diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..00e9572 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Fetch ML + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4e292c6 --- /dev/null +++ b/Makefile @@ -0,0 +1,198 @@ +.PHONY: all build clean clean-docs test test-unit test-integration test-e2e test-coverage lint install dev prod setup validate configlint ci-local docs + +# Default target +all: build + +# Build all components +build: + go build -o bin/api-server cmd/api-server/main.go + go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go + go build -o bin/tui ./cmd/tui + cd cli && zig build prod + @echo "✓ All components built" + +# Build production-optimized binaries +prod: + go build -ldflags="-s -w" -o bin/api-server cmd/api-server/main.go + go build -ldflags="-s -w" -o bin/worker cmd/worker/worker_server.go + go build -ldflags="-s -w" -o bin/tui ./cmd/tui + cd cli && zig build prod && strip zig-out/prod/ml + @echo "✓ Production binaries built" + +# Development build (faster compilation) +dev: + go build -o bin/api-server cmd/api-server/main.go + go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go + go build -o bin/tui ./cmd/tui + cd cli && zig build dev + @echo "✓ Development binaries built" + +# Clean build artifacts +clean: + rm -rf bin/ coverage/ + rm -rf cli/zig-out/ + rm -rf cli/.zig-cache/ + go clean + @echo "✓ Cleaned" + +clean-docs: + rm -rf docs/_site/ + @echo "✓ Cleaned docs" + +# Run tests +test: + go test ./... + cd cli && zig build test + @echo "✓ All tests passed" + +# Lint Go and Zig code +lint: + gofmt -w ./cmd ./internal ./tests || true + go vet ./... + cd cli && zig fmt . + @echo "✓ Lint completed" + +# Install to system (requires sudo) +install: prod + sudo cp bin/api-server /usr/local/bin/fetchml-api + sudo cp bin/worker /usr/local/bin/fetchml-worker + sudo cp bin/tui /usr/local/bin/fetchml-tui + sudo cp cli/zig-out/prod/ml /usr/local/bin/ml + @echo "✓ Installed" + +# Setup production environment +setup: + @if [ "$(shell uname)" = "Linux" ]; then \ + sudo ./scripts/setup-prod.sh; \ + else \ + echo "Production setup is for Linux only. You're on $(shell uname)."; \ + echo "Use docker-compose for local development."; \ + fi + +# Validate production configuration +validate: + ./scripts/validate-prod-config.sh configs/config-prod.yaml configs/worker-prod.toml + +# Validate YAML configs against JSON schema +configlint: + go run ./cmd/configlint --schema configs/schema/config_schema.yaml \ + configs/config-prod.yaml \ + configs/config-no-tls.yaml \ + configs/config-dev.yaml + +# Run a local approximation of the CI pipeline +ci-local: + make test + make lint + make configlint + @echo "Running queue package tests with race detector..." + go test -v -race -coverprofile=coverage/queue-coverage.out ./internal/queue/... + @echo "Running coverage..." + make test-coverage + +# Docker targets +docker-build: + docker build -f build/docker/simple.Dockerfile -t fetchml:latest . + @echo "✓ Docker image built" + +docker-run: + docker-compose up -d + @echo "✓ Services started" + +docker-stop: + docker-compose down + @echo "✓ Services stopped" + +docker-logs: + docker-compose logs -f + +# Monitoring setup (Linux only) +setup-monitoring: + @if [ "$(shell uname)" = "Linux" ]; then \ + sudo ./scripts/setup-monitoring-prod.sh; \ + else \ + echo "Monitoring setup is for Linux production. Use docker-compose for local development."; \ + fi + +# Enhanced test targets +test-unit: + go test -v -short ./... + +test-integration: + go test -v ./... + +test-e2e: + go test -v ./tests/e2e/... + +test-coverage: + go test -coverprofile=coverage/coverage.out ./... + go tool cover -html=coverage/coverage.out -o coverage/coverage.html + @echo "✓ Coverage report: coverage/coverage.html" + +# Documentation setup +docs-setup: + @echo "Setting up MkDocs documentation..." + @if ! command -v mkdocs >/dev/null 2>&1; then \ + echo "MkDocs not found. Please install it manually:"; \ + echo " macOS: brew install mkdocs mkdocs-material"; \ + echo " Linux: pip install mkdocs mkdocs-material"; \ + echo "Or visit: https://www.mkdocs.org/user-guide/installation/"; \ + exit 1; \ + fi + @if ! python3 -c "import pymdownx.superfences" >/dev/null 2>&1; then \ + echo "pymdown-extensions not found. Please install it manually:"; \ + echo " pip3 install --user pymdown-extensions"; \ + echo " Or use a virtual environment: python3 -m venv docs-env && source docs-env/bin/activate && pip install pymdown-extensions"; \ + exit 1; \ + fi + @echo "Documentation setup complete!" + +# Documentation +docs: docs-setup docs-build + @echo "Starting MkDocs development server..." + cd docs && mkdocs serve + +# Build documentation +docs-build: + @echo "Building static documentation..." + cd docs && mkdocs build + +# Help +help: + @echo "FetchML Build System" + @echo "" + @echo "Build Targets:" + @echo " make build - Build all components (default)" + @echo " make prod - Build production-optimized binaries" + @echo " make dev - Build development binaries (faster)" + @echo " make clean - Remove build artifacts" + @echo "" + @echo "Docker Targets:" + @echo " make docker-build - Build Docker image" + @echo " make docker-run - Start services with docker-compose" + @echo " make docker-stop - Stop docker-compose services" + @echo " make docker-logs - View docker-compose logs" + @echo "" + @echo "Test Targets:" + @echo " make test - Run all tests" + @echo " make test-unit - Run unit tests only" + @echo " make test-integration - Run integration tests" + @echo " make test-coverage - Generate coverage report" + @echo " make lint - Run formatters and linters" + @echo " make ci-local - Run local CI dry-run (tests, lint, config validation, coverage)" + @echo " make configlint - Validate YAML configs against schema" + @echo "" + @echo "Setup Targets:" + @echo " make install - Install binaries to /usr/local/bin (requires sudo)" + @echo " make setup - Run production setup (Linux only)" + @echo " make setup-monitoring - Setup monitoring stack (Linux only)" + @echo " make validate - Validate production configuration" + @echo "" + @echo "Documentation:" + @echo " make docs-setup - Install MkDocs and dependencies" + @echo " make docs - Start MkDocs development server with live reload" + @echo " make docs-build - Build static documentation for deployment" + @echo "" + @echo "Utility:" + @echo " make size - Show binary sizes" + @echo " make help - Show this help" diff --git a/README.md b/README.md new file mode 100644 index 0000000..282293f --- /dev/null +++ b/README.md @@ -0,0 +1,200 @@ +# FetchML - Machine Learning Platform + +A production-ready ML experiment platform with task queuing, monitoring, and a modern CLI/API. + +## Features + +- **🚀 Production Resilience** - Task leasing, smart retries, dead-letter queues +- **📊 Monitoring** - Grafana/Prometheus/Loki with auto-provisioned dashboards +- **🔐 Security** - API key auth, TLS, rate limiting, IP whitelisting +- **⚡ Performance** - Go API server + Zig CLI for speed +- **📦 Easy Deployment** - Docker Compose (dev) or systemd (prod) + +## Quick Start + +### Development (macOS/Linux) + +```bash +# Clone and start +git clone +cd fetch_ml +docker-compose up -d + +# Access Grafana: http://localhost:3000 (admin/admin) +``` + +### Production (Linux) + +```bash +# Setup application +sudo ./scripts/setup-prod.sh + +# Setup monitoring +sudo ./scripts/setup-monitoring-prod.sh + +# Build and install +make prod +make install + +# Start services +sudo systemctl start fetchml-api fetchml-worker +sudo systemctl start prometheus grafana loki promtail +``` + +## Architecture + +``` +┌──────────────┐ WebSocket ┌──────────────┐ +│ Zig CLI/TUI │◄─────────────►│ API Server │ +└──────────────┘ │ (Go) │ + └──────┬───────┘ + │ + ┌─────────────┼─────────────┐ + │ │ │ + ┌────▼────┐ ┌───▼────┐ ┌───▼────┐ + │ Redis │ │ Worker │ │ Loki │ + │ (Queue) │ │ (Go) │ │ (Logs) │ + └─────────┘ └────────┘ └────────┘ +``` + +## Usage + +### API Server + +```bash +# Development (stderr logging) +go run cmd/api-server/main.go --config configs/config-dev.yaml + +# Production (file logging) +go run cmd/api-server/main.go --config configs/config-no-tls.yaml +``` + +### CLI + +```bash +# Build +cd cli && zig build prod + +# Run experiment +./cli/zig-out/bin/ml run --config config.toml + +# Check status +./cli/zig-out/bin/ml status +``` + +### Docker + +```bash +make docker-run # Start all services +make docker-logs # View logs +make docker-stop # Stop services +``` + +## Development + +### Prerequisites + +- Go 1.21+ +- Zig 0.11+ +- Redis +- Docker (for local dev) + +### Build + +```bash +make build # All components +make dev # Fast dev build +make prod # Optimized production build +``` + +### Test + +```bash +make test # All tests +make test-unit # Unit tests only +make test-coverage # With coverage report +``` + +## Configuration + +### Development (`configs/config-dev.yaml`) +```yaml +logging: + level: "info" + file: "" # stderr only + +redis: + url: "redis://localhost:6379" +``` + +### Production (`configs/config-no-tls.yaml`) +```yaml +logging: + level: "info" + file: "./logs/fetch_ml.log" # file only + +redis: + url: "redis://redis:6379" +``` + +## Monitoring + +### Grafana Dashboards (Auto-Provisioned) + +- **ML Task Queue** - Queue depth, task duration, failure rates +- **Application Logs** - Log streams, error tracking, search + +Access: `http://localhost:3000` (dev) or `http://YOUR_SERVER:3000` (prod) + +### Metrics + +- Queue depth and task processing rates +- Retry attempts by error category +- Dead letter queue size +- Lease expirations + +## Documentation + +- **[Getting Started](docs/getting-started.md)** - Detailed setup guide +- **[Production Deployment](docs/production-monitoring.md)** - Linux deployment +- **[WebSocket API](docs/api/)** - Protocol documentation +- **[Architecture](docs/architecture/)** - System design + +## Makefile Targets + +```bash +# Build +make build # Build all components +make prod # Production build +make clean # Clean artifacts + +# Docker +make docker-build # Build image +make docker-run # Start services +make docker-stop # Stop services + +# Test +make test # All tests +make test-coverage # With coverage + +# Production (Linux only) +make setup # Setup app +make setup-monitoring # Setup monitoring +make install # Install binaries +``` + +## Security + +- **TLS/HTTPS** - End-to-end encryption +- **API Keys** - Hashed with SHA256 +- **Rate Limiting** - Per-user quotas +- **IP Whitelist** - Network restrictions +- **Audit Logging** - All API access logged + +## License + +MIT - See [LICENSE](LICENSE) + +## Contributing + +Contributions welcome! This is a personal homelab project but PRs are appreciated. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0eee7f7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,128 @@ +# Homelab Docker Compose with Centralized Monitoring +# Includes: API, Redis, Prometheus, Grafana, Loki + +services: + redis: + image: redis:7-alpine + container_name: ml-experiments-redis + ports: + - "6379:6379" + volumes: + - redis_data:/data + restart: unless-stopped + command: redis-server --appendonly yes + healthcheck: + test: [ "CMD", "redis-cli", "ping" ] + interval: 30s + timeout: 10s + retries: 3 + + api-server: + build: + context: . + dockerfile: build/docker/simple.Dockerfile + container_name: ml-experiments-api + ports: + - "9101:9101" + - "9100:9100" # Prometheus metrics endpoint + volumes: + - ./data:/data/experiments + - ./logs:/logs + - ./configs/config-no-tls.yaml:/app/configs/config.yaml + depends_on: + redis: + condition: service_healthy + restart: unless-stopped + environment: + - REDIS_URL=redis://redis:6379 + - LOG_LEVEL=info + healthcheck: + test: [ "CMD", "curl", "http://localhost:9101/health" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + labels: + logging: "promtail" + job: "api-server" + + # Prometheus - Metrics collection + prometheus: + image: prom/prometheus:latest + container_name: ml-experiments-prometheus + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + restart: unless-stopped + + # Grafana - Visualization + grafana: + image: grafana/grafana:latest + container_name: ml-experiments-grafana + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana-dashboard.json:/var/lib/grafana/dashboards/ml-queue.json + - ./monitoring/logs-dashboard.json:/var/lib/grafana/dashboards/logs.json + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://localhost:3000 + - GF_AUTH_ANONYMOUS_ENABLED=false + restart: unless-stopped + depends_on: + - prometheus + - loki + + # Loki - Log aggregation + loki: + image: grafana/loki:latest + container_name: ml-experiments-loki + ports: + - "3100:3100" + volumes: + - ./monitoring/loki-config.yml:/etc/loki/local-config.yaml + - loki_data:/loki + command: -config.file=/etc/loki/local-config.yaml + restart: unless-stopped + + # Promtail - Log collector + promtail: + image: grafana/promtail:latest + container_name: ml-experiments-promtail + volumes: + - ./monitoring/promtail-config.yml:/etc/promtail/config.yml + - ./logs:/var/log/app + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + command: -config.file=/etc/promtail/config.yml + restart: unless-stopped + depends_on: + - loki + +volumes: + redis_data: + driver: local + prometheus_data: + driver: local + grafana_data: + driver: local + loki_data: + driver: local + +networks: + default: + name: ml-experiments-network + backend: + name: ml-backend-network + internal: true # No external access diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ae94fd1 --- /dev/null +++ b/go.mod @@ -0,0 +1,73 @@ +module github.com/jfraeys/fetch_ml + +go 1.25.0 + +// Fetch ML - Secure Machine Learning Platform +// Copyright (c) 2024 Fetch ML +// Licensed under the MIT License + +require ( + github.com/BurntSushi/toml v1.5.0 + github.com/alicebob/miniredis/v2 v2.35.0 + github.com/charmbracelet/bubbles v0.21.0 + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 + github.com/go-redis/redis/v8 v8.11.5 + github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 + github.com/lib/pq v1.10.9 + github.com/mattn/go-sqlite3 v1.14.32 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 + github.com/stretchr/testify v1.11.1 + github.com/xeipuuv/gojsonschema v1.2.0 + github.com/zalando/go-keyring v0.2.6 + golang.org/x/crypto v0.45.0 + golang.org/x/time v0.14.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + al.essio.dev/pkg/shellescape v1.6.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.3 // indirect + github.com/charmbracelet/x/ansi v0.11.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.14 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.6.1 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/danieljoos/wincred v1.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/godbus/dbus/v5 v5.2.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sahilm/fuzzy v0.1.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cbf4015 --- /dev/null +++ b/go.sum @@ -0,0 +1,168 @@ +al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA= +al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= +github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= +github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI= +github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.11.2 h1:XAG3FSjiVtFvgEgGrNBkCNNYrsucAt8c6bfxHyROLLs= +github.com/charmbracelet/x/ansi v0.11.2/go.mod h1:9tY2bzX5SiJCU0iWyskjBeI2BRQfvPqI+J760Mjf+Rg= +github.com/charmbracelet/x/cellbuf v0.0.14 h1:iUEMryGyFTelKW3THW4+FfPgi4fkmKnnaLOXuc+/Kj4= +github.com/charmbracelet/x/cellbuf v0.0.14/go.mod h1:P447lJl49ywBbil/KjCk2HexGh4tEY9LH0/1QrZZ9rA= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/clipperhouse/displaywidth v0.6.1 h1:/zMlAezfDzT2xy6acHBzwIfyu2ic0hgkT83UX5EY2gY= +github.com/clipperhouse/displaywidth v0.6.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= +github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= +github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= +github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= +github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s= +github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5cbafa0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,132 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.black] +# Google Python Style Guide Configuration for Black +line-length = 80 +target-version = ['py38'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | venv + | _build + | buck-out + | build + | dist + # ML experiment code - exclude from formatting + | podman/workspace + | workspace + | tests/fixtures/examples + | tests/fixtures/podman + | results + | data + | logs + | secrets + | .agent +)/ +''' + +[tool.isort] +# Google Python Style Guide Configuration for isort +profile = "google" +line_length = 80 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +split_on_trailing_comma = true +known_first_party = ["fetch_ml"] +skip_glob = [ + "podman/workspace/*", + "workspace/*", + "tests/fixtures/examples/*", + "tests/fixtures/podman/*", +] + +[tool.mypy] +# Google Python Style Guide Configuration for mypy +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[[tool.mypy.overrides]] +module = [ + "torch.*", + "tensorflow.*", + "sklearn.*", + "pandas.*", + "numpy.*", + "matplotlib.*", + "seaborn.*", + "scipy.*", + "joblib.*", +] +ignore_missing_imports = true + +[tool.pytest.ini_options] +# Google Python Style Guide Configuration for pytest +minversion = "6.0" +addopts = "-ra -q --strict-markers --strict-config" +testpaths = ["tests"] +python_files = ["*_test.py", "test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["."] +omit = [ + "tests/*", + "*/tests/*", + "test_*", + "*_test.py", + "setup.py", + "*/site-packages/*", + # ML experiment code + "podman/workspace/*", + "workspace/*", + "tests/fixtures/examples/*", + "tests/fixtures/podman/*", + "results/*", + "data/*", + "logs/*", + "secrets/*", + ".agent/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +]