From c5049a2fdf7e68b2249a23bdd2e1f93761e6efb9 Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Thu, 4 Dec 2025 16:52:09 -0500 Subject: [PATCH] feat: initialize FetchML ML platform with core project structure - Add comprehensive README with architecture overview and quick start guide - Set up Go module with production-ready dependencies - Configure build system with Makefile for development and production builds - Add Docker Compose for local development environment - Include project configuration files (linting, Python, etc.) This establishes the foundation for a production-ready ML experiment platform with task queuing, monitoring, and modern CLI/API interface. --- .flake8 | 47 +++++++ .gitignore | 224 +++++++++++++++++++++++++++++++++ .golangci.yml | 79 ++++++++++++ .pylintrc | 307 +++++++++++++++++++++++++++++++++++++++++++++ LICENSE | 21 ++++ Makefile | 198 +++++++++++++++++++++++++++++ README.md | 200 +++++++++++++++++++++++++++++ docker-compose.yml | 128 +++++++++++++++++++ go.mod | 73 +++++++++++ go.sum | 168 +++++++++++++++++++++++++ pyproject.toml | 132 +++++++++++++++++++ 11 files changed, 1577 insertions(+) create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .pylintrc create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bdf162d --- /dev/null +++ b/.flake8 @@ -0,0 +1,47 @@ +[flake8] +max-line-length = 80 +max-complexity = 10 +exclude = + .git, + __pycache__, + *.pyc, + .pytest_cache, + .mypy_cache, + .venv, + venv, + build, + dist, + # ML experiment code - exclude from style checks + podman/workspace, + workspace, + tests/fixtures/examples, + tests/fixtures/podman, + results, + data, + logs, + secrets, + .agent + +ignore = + E203, + E501, + W503, + W504, + F401, + E402, + C901 + +per-file-ignores = + __init__.py:F401 + tests/*:F401,F811 + *_test.py:F401,F811 + conftest.py:F401,F811 + +# Additional Google Style Guide recommendations +inline-quotes = single +multiline-quotes = double +docstring-quotes = double + +# Import ordering (Google style) +import-order-style = google +application-import-names = fetch_ml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6da3c15 --- /dev/null +++ b/.gitignore @@ -0,0 +1,224 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Python cache and artifacts +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +# Python lib directories +**/site-packages/ +**/lib/python*/ +# Allow scripts/lib for shell libraries +!scripts/lib/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +.pdm.toml + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ + +# Project-specific +ml_jobs/ +test_output/ +# Python build directory only +**/build/lib/ +**/build/bdist*/ +**/build/scripts*/ +# Allow build/ directory for Docker configs +!build/ +!build/** +# But ignore Python build artifacts if they happen +build/lib/ +build/bdist*/ +coverage.out +coverage.html +queue-coverage.out +coverage/ + +# Redis dump +dump.rdb + +# Configuration files with secrets +!config_*.yaml +!config_local.yaml.example +!config_prod.yaml.example + +# Database files +db/*.db +!db/.gitkeep + +# API keys and secrets +*.key +*.pem +secrets/ +cli/src/assets/rsync_release.bin + +# Test files +test_*.go +*_test_output/ + +# Build artifacts +bin/ +cli/zig-out/ +cli/.zig-cache/ +zig-out/ +.zig-cache/ + +# Experiment data (local testing) +experiments/ +data/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..0585580 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,79 @@ +--- +version: "2" +run: + timeout: 5m + tests: true +output: + format: colored-line-number +linters-settings: + govet: + enable: + - shadow + - fieldalignment + gocyclo: + min-complexity: 15 + dupl: + threshold: 100 + goconst: + min-len: 3 + min-occurrences: 3 + misspell: + locale: US + lll: + line-length: 100 + revive: + confidence: 0.8 + depguard: + rules: + main: + allow: + - $gostd + - github.com/jfraeys/fetch_ml +linters: + disable-all: true + enable: + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - exhaustive + - gochecknoinits + - goconst + - gocritic + - gocyclo + - goprintffuncname + - gosec + - govet + - ineffassign + - lll + - misspell + - nakedret + - noctx + - nolintlint + - rowserrcheck + - staticcheck + - unconvert + - unparam + - unused + - whitespace + - revive +issues: + exclude-rules: + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + - lll + - text: "weak cryptographic primitive" + linters: + - gosec + - text: "Use of weak random number generator" + linters: + - gosec + max-issues-per-linter: 0 + max-same-issues: 0 +severity: + default-severity: error diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..dd3574c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,307 @@ +[MASTER] +# Google Python Style Guide Configuration +# Based on https://google.github.io/styleguide/pyguide.html + +# Files or directories to be skipped over when performing analysis. +ignore= + .git, + __pycache__, + *.pyc, + .pytest_cache, + .mypy_cache, + .venv, + venv, + # ML experiment code - exclude from style checks + podman/workspace, + workspace, + tests/fixtures/examples, + tests/fixtures/podman, + results, + data, + logs, + secrets, + .agent + +# Regular expression matching files that should be skipped entirely. +ignore-patterns= + ^\.#, + ^setup\.py$, + ^conftest\.py$ + +# Add files or directories matching the regex patterns to the ignore list. +ignore-paths= + +# Files or directories to be added to the list of ignored regular expressions. +ignore-paths= + +# Pickle collected data for later comparisons. +persistent=yes + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Load all enabled extensions. +load-all-plugins=no + +# Control the amount of potential astroid warnings (0, 1 or 2). +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +[MESSAGES CONTROL] +# Only show warnings with the listed confidence levels. +confidence= + +# Disable the message, report, category or checker with the given id(s). +disable= + # Google Style Guide allows some of these + missing-module-docstring, + missing-function-docstring, + too-many-locals, + too-many-arguments, + too-many-instance-attributes, + too-many-public-methods, + too-few-public-methods, + too-many-lines, + too-many-statements, + too-many-branches, + too-many-nested-blocks, + # Not relevant for ML projects + invalid-name, + C0114, # missing-module-docstring + C0115, # missing-class-docstring + C0116, # missing-function-docstring + R0903, # too-few-public-methods + R0902, # too-many-instance-attributes + R0913, # too-many-arguments + R0914, # too-many-locals + R0915, # too-many-statements + R0912, # too-many-branches + R0911, # too-many-return-statements + W0613, # unused-argument (common in callbacks) + +# Enable the message, report, category or checker with the given id(s). +enable= + +[REPORTS] +# Set the output format. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Python expression which determines if a message should not be displayed. +evaluation= + +# Template used to display messages. +msg-template= + +[BASIC] +# Good variable names which should always be accepted, separated by a comma. +good-names= + i,j,k,ex,Run,_ + +# Bad variable names which should always be refused. +bad-names= + foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used. +bad-functions= + +# Regular expression for correct variable names. +variable-rgx=^[a-z_][a-z0-9_]{2,30}$ + +# Regular expression for correct constant names. +const-rgx=^(_?[A-Z][A-Z0-9_]{2,30}|__.*__)$ + +# Regular expression for correct function names. +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]{2,30})|(?P_?[a-z][a-z0-9_]{2,30}))$ + +# Regular expression for correct class names. +class-rgx=^_?[A-Z][a-zA-Z0-9]{2,30}$ + +# Regular expression for correct attribute names. +attr-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$ + +# Regular expression for correct argument names. +argument-rgx=^[a-z][a-z0-9_]{2,30}$ + +# Regular expression for correct method resolution order names. +method-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings. +docstring-min-length=-1 + +[TYPECHECK] +# List of decorators that produce context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of member names which should be excluded from the access member check. +exclude-members= + +# List of qualified names (objects) which require a disable protection. +generated-members= + +[MISCELLANEOUS] +# List of note tags to take in consideration. +notes= + +[VARIABLES] +# Tells whether we should check for unused imports. +init-import=no + +# A regular expression matching the name of dummy variables. +dummy-variable-rgx=^_|dummy + +# List of variable names that should not be used. +additional-builtins= + +# List of variables which are ignored for the undefined-variable check. +ignored-classes= + +# List of members which are skipped from the member access check. +ignored-modules= + +# List of deprecated symbols which should not be used. +deprecated-modules= + +# List of symbols that can be safely accessed. +allowed-redefined-builtins= + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +long-line-regexp=^[\t ]*(# )??$ + +# Allow the body of an if to be on the same line as the test. +single-line-if-stmt=no + +# List of optional parameters for which we don't want to call a linting +# function. +single-line-class-stmt=no + +# Maximum number of characters on a single line, when there are no multiline +# strings. +max-line-length-sigil-fallback=80 + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +[LOGGING] +# Logging modules to check that the string formatting arguments are in +# logging function parameter format. +logging-modules=logging + +[DESIGN] +# Maximum number of arguments for function / method. +max-args=5 + +# Argument names that match this regular expression are ignored. +ignored-argument-names= + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of return statements for function / method body. +max-returns=6 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Maximum number of parents for a class. +max-parents=7 + +# Maximum number of attributes for a class. +max-attributes=7 + +# Minimum number of public methods for a class. +min-public-methods=2 + +# Maximum number of public methods for a class. +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement. +max-bool-expr=5 + +[CLASSES] +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcls + +# List of member names which should be excluded from the access member check. +exclude-protected=_asdict,_fields,_replace,_source,_make + +[IMPORTS] +# Deprecated modules which should not be used. +deprecated-modules= + +# Create a graph of every module's dependencies. +import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# current file. +ext-import-graph= + +# Create a graph of external dependencies in the current file. +int-import-graph= + +# Force import order to match a specific order. +import-order-style=google + +# Add files or directories to the blacklist. +ignore-on-import-order=no + +# List of modules that can be imported at any level from the specified +# application modules. +known-third-party= + +# Analyse import fallback blocks. +analyse-fallback-blocks=no + +[EXCEPTIONS] +# Exceptions that will emit a warning when being caught. +overgeneral-exceptions=Exception,BaseException + +# Exceptions that will emit a warning when being returned. +return-exceptions=no + +# Exceptions that will emit a warning when being instantiated. +nonstandard-exceptions=no + +[REFACTORING] +# Maximum number of nested blocks. +max-nested-blocks=5 + +# Complete name of the function that returns the YAPF version. +yapf-version= diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..00e9572 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Fetch ML + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4e292c6 --- /dev/null +++ b/Makefile @@ -0,0 +1,198 @@ +.PHONY: all build clean clean-docs test test-unit test-integration test-e2e test-coverage lint install dev prod setup validate configlint ci-local docs + +# Default target +all: build + +# Build all components +build: + go build -o bin/api-server cmd/api-server/main.go + go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go + go build -o bin/tui ./cmd/tui + cd cli && zig build prod + @echo "✓ All components built" + +# Build production-optimized binaries +prod: + go build -ldflags="-s -w" -o bin/api-server cmd/api-server/main.go + go build -ldflags="-s -w" -o bin/worker cmd/worker/worker_server.go + go build -ldflags="-s -w" -o bin/tui ./cmd/tui + cd cli && zig build prod && strip zig-out/prod/ml + @echo "✓ Production binaries built" + +# Development build (faster compilation) +dev: + go build -o bin/api-server cmd/api-server/main.go + go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go + go build -o bin/tui ./cmd/tui + cd cli && zig build dev + @echo "✓ Development binaries built" + +# Clean build artifacts +clean: + rm -rf bin/ coverage/ + rm -rf cli/zig-out/ + rm -rf cli/.zig-cache/ + go clean + @echo "✓ Cleaned" + +clean-docs: + rm -rf docs/_site/ + @echo "✓ Cleaned docs" + +# Run tests +test: + go test ./... + cd cli && zig build test + @echo "✓ All tests passed" + +# Lint Go and Zig code +lint: + gofmt -w ./cmd ./internal ./tests || true + go vet ./... + cd cli && zig fmt . + @echo "✓ Lint completed" + +# Install to system (requires sudo) +install: prod + sudo cp bin/api-server /usr/local/bin/fetchml-api + sudo cp bin/worker /usr/local/bin/fetchml-worker + sudo cp bin/tui /usr/local/bin/fetchml-tui + sudo cp cli/zig-out/prod/ml /usr/local/bin/ml + @echo "✓ Installed" + +# Setup production environment +setup: + @if [ "$(shell uname)" = "Linux" ]; then \ + sudo ./scripts/setup-prod.sh; \ + else \ + echo "Production setup is for Linux only. You're on $(shell uname)."; \ + echo "Use docker-compose for local development."; \ + fi + +# Validate production configuration +validate: + ./scripts/validate-prod-config.sh configs/config-prod.yaml configs/worker-prod.toml + +# Validate YAML configs against JSON schema +configlint: + go run ./cmd/configlint --schema configs/schema/config_schema.yaml \ + configs/config-prod.yaml \ + configs/config-no-tls.yaml \ + configs/config-dev.yaml + +# Run a local approximation of the CI pipeline +ci-local: + make test + make lint + make configlint + @echo "Running queue package tests with race detector..." + go test -v -race -coverprofile=coverage/queue-coverage.out ./internal/queue/... + @echo "Running coverage..." + make test-coverage + +# Docker targets +docker-build: + docker build -f build/docker/simple.Dockerfile -t fetchml:latest . + @echo "✓ Docker image built" + +docker-run: + docker-compose up -d + @echo "✓ Services started" + +docker-stop: + docker-compose down + @echo "✓ Services stopped" + +docker-logs: + docker-compose logs -f + +# Monitoring setup (Linux only) +setup-monitoring: + @if [ "$(shell uname)" = "Linux" ]; then \ + sudo ./scripts/setup-monitoring-prod.sh; \ + else \ + echo "Monitoring setup is for Linux production. Use docker-compose for local development."; \ + fi + +# Enhanced test targets +test-unit: + go test -v -short ./... + +test-integration: + go test -v ./... + +test-e2e: + go test -v ./tests/e2e/... + +test-coverage: + go test -coverprofile=coverage/coverage.out ./... + go tool cover -html=coverage/coverage.out -o coverage/coverage.html + @echo "✓ Coverage report: coverage/coverage.html" + +# Documentation setup +docs-setup: + @echo "Setting up MkDocs documentation..." + @if ! command -v mkdocs >/dev/null 2>&1; then \ + echo "MkDocs not found. Please install it manually:"; \ + echo " macOS: brew install mkdocs mkdocs-material"; \ + echo " Linux: pip install mkdocs mkdocs-material"; \ + echo "Or visit: https://www.mkdocs.org/user-guide/installation/"; \ + exit 1; \ + fi + @if ! python3 -c "import pymdownx.superfences" >/dev/null 2>&1; then \ + echo "pymdown-extensions not found. Please install it manually:"; \ + echo " pip3 install --user pymdown-extensions"; \ + echo " Or use a virtual environment: python3 -m venv docs-env && source docs-env/bin/activate && pip install pymdown-extensions"; \ + exit 1; \ + fi + @echo "Documentation setup complete!" + +# Documentation +docs: docs-setup docs-build + @echo "Starting MkDocs development server..." + cd docs && mkdocs serve + +# Build documentation +docs-build: + @echo "Building static documentation..." + cd docs && mkdocs build + +# Help +help: + @echo "FetchML Build System" + @echo "" + @echo "Build Targets:" + @echo " make build - Build all components (default)" + @echo " make prod - Build production-optimized binaries" + @echo " make dev - Build development binaries (faster)" + @echo " make clean - Remove build artifacts" + @echo "" + @echo "Docker Targets:" + @echo " make docker-build - Build Docker image" + @echo " make docker-run - Start services with docker-compose" + @echo " make docker-stop - Stop docker-compose services" + @echo " make docker-logs - View docker-compose logs" + @echo "" + @echo "Test Targets:" + @echo " make test - Run all tests" + @echo " make test-unit - Run unit tests only" + @echo " make test-integration - Run integration tests" + @echo " make test-coverage - Generate coverage report" + @echo " make lint - Run formatters and linters" + @echo " make ci-local - Run local CI dry-run (tests, lint, config validation, coverage)" + @echo " make configlint - Validate YAML configs against schema" + @echo "" + @echo "Setup Targets:" + @echo " make install - Install binaries to /usr/local/bin (requires sudo)" + @echo " make setup - Run production setup (Linux only)" + @echo " make setup-monitoring - Setup monitoring stack (Linux only)" + @echo " make validate - Validate production configuration" + @echo "" + @echo "Documentation:" + @echo " make docs-setup - Install MkDocs and dependencies" + @echo " make docs - Start MkDocs development server with live reload" + @echo " make docs-build - Build static documentation for deployment" + @echo "" + @echo "Utility:" + @echo " make size - Show binary sizes" + @echo " make help - Show this help" diff --git a/README.md b/README.md new file mode 100644 index 0000000..282293f --- /dev/null +++ b/README.md @@ -0,0 +1,200 @@ +# FetchML - Machine Learning Platform + +A production-ready ML experiment platform with task queuing, monitoring, and a modern CLI/API. + +## Features + +- **🚀 Production Resilience** - Task leasing, smart retries, dead-letter queues +- **📊 Monitoring** - Grafana/Prometheus/Loki with auto-provisioned dashboards +- **🔐 Security** - API key auth, TLS, rate limiting, IP whitelisting +- **⚡ Performance** - Go API server + Zig CLI for speed +- **📦 Easy Deployment** - Docker Compose (dev) or systemd (prod) + +## Quick Start + +### Development (macOS/Linux) + +```bash +# Clone and start +git clone +cd fetch_ml +docker-compose up -d + +# Access Grafana: http://localhost:3000 (admin/admin) +``` + +### Production (Linux) + +```bash +# Setup application +sudo ./scripts/setup-prod.sh + +# Setup monitoring +sudo ./scripts/setup-monitoring-prod.sh + +# Build and install +make prod +make install + +# Start services +sudo systemctl start fetchml-api fetchml-worker +sudo systemctl start prometheus grafana loki promtail +``` + +## Architecture + +``` +┌──────────────┐ WebSocket ┌──────────────┐ +│ Zig CLI/TUI │◄─────────────►│ API Server │ +└──────────────┘ │ (Go) │ + └──────┬───────┘ + │ + ┌─────────────┼─────────────┐ + │ │ │ + ┌────▼────┐ ┌───▼────┐ ┌───▼────┐ + │ Redis │ │ Worker │ │ Loki │ + │ (Queue) │ │ (Go) │ │ (Logs) │ + └─────────┘ └────────┘ └────────┘ +``` + +## Usage + +### API Server + +```bash +# Development (stderr logging) +go run cmd/api-server/main.go --config configs/config-dev.yaml + +# Production (file logging) +go run cmd/api-server/main.go --config configs/config-no-tls.yaml +``` + +### CLI + +```bash +# Build +cd cli && zig build prod + +# Run experiment +./cli/zig-out/bin/ml run --config config.toml + +# Check status +./cli/zig-out/bin/ml status +``` + +### Docker + +```bash +make docker-run # Start all services +make docker-logs # View logs +make docker-stop # Stop services +``` + +## Development + +### Prerequisites + +- Go 1.21+ +- Zig 0.11+ +- Redis +- Docker (for local dev) + +### Build + +```bash +make build # All components +make dev # Fast dev build +make prod # Optimized production build +``` + +### Test + +```bash +make test # All tests +make test-unit # Unit tests only +make test-coverage # With coverage report +``` + +## Configuration + +### Development (`configs/config-dev.yaml`) +```yaml +logging: + level: "info" + file: "" # stderr only + +redis: + url: "redis://localhost:6379" +``` + +### Production (`configs/config-no-tls.yaml`) +```yaml +logging: + level: "info" + file: "./logs/fetch_ml.log" # file only + +redis: + url: "redis://redis:6379" +``` + +## Monitoring + +### Grafana Dashboards (Auto-Provisioned) + +- **ML Task Queue** - Queue depth, task duration, failure rates +- **Application Logs** - Log streams, error tracking, search + +Access: `http://localhost:3000` (dev) or `http://YOUR_SERVER:3000` (prod) + +### Metrics + +- Queue depth and task processing rates +- Retry attempts by error category +- Dead letter queue size +- Lease expirations + +## Documentation + +- **[Getting Started](docs/getting-started.md)** - Detailed setup guide +- **[Production Deployment](docs/production-monitoring.md)** - Linux deployment +- **[WebSocket API](docs/api/)** - Protocol documentation +- **[Architecture](docs/architecture/)** - System design + +## Makefile Targets + +```bash +# Build +make build # Build all components +make prod # Production build +make clean # Clean artifacts + +# Docker +make docker-build # Build image +make docker-run # Start services +make docker-stop # Stop services + +# Test +make test # All tests +make test-coverage # With coverage + +# Production (Linux only) +make setup # Setup app +make setup-monitoring # Setup monitoring +make install # Install binaries +``` + +## Security + +- **TLS/HTTPS** - End-to-end encryption +- **API Keys** - Hashed with SHA256 +- **Rate Limiting** - Per-user quotas +- **IP Whitelist** - Network restrictions +- **Audit Logging** - All API access logged + +## License + +MIT - See [LICENSE](LICENSE) + +## Contributing + +Contributions welcome! This is a personal homelab project but PRs are appreciated. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0eee7f7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,128 @@ +# Homelab Docker Compose with Centralized Monitoring +# Includes: API, Redis, Prometheus, Grafana, Loki + +services: + redis: + image: redis:7-alpine + container_name: ml-experiments-redis + ports: + - "6379:6379" + volumes: + - redis_data:/data + restart: unless-stopped + command: redis-server --appendonly yes + healthcheck: + test: [ "CMD", "redis-cli", "ping" ] + interval: 30s + timeout: 10s + retries: 3 + + api-server: + build: + context: . + dockerfile: build/docker/simple.Dockerfile + container_name: ml-experiments-api + ports: + - "9101:9101" + - "9100:9100" # Prometheus metrics endpoint + volumes: + - ./data:/data/experiments + - ./logs:/logs + - ./configs/config-no-tls.yaml:/app/configs/config.yaml + depends_on: + redis: + condition: service_healthy + restart: unless-stopped + environment: + - REDIS_URL=redis://redis:6379 + - LOG_LEVEL=info + healthcheck: + test: [ "CMD", "curl", "http://localhost:9101/health" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + labels: + logging: "promtail" + job: "api-server" + + # Prometheus - Metrics collection + prometheus: + image: prom/prometheus:latest + container_name: ml-experiments-prometheus + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + restart: unless-stopped + + # Grafana - Visualization + grafana: + image: grafana/grafana:latest + container_name: ml-experiments-grafana + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana-dashboard.json:/var/lib/grafana/dashboards/ml-queue.json + - ./monitoring/logs-dashboard.json:/var/lib/grafana/dashboards/logs.json + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://localhost:3000 + - GF_AUTH_ANONYMOUS_ENABLED=false + restart: unless-stopped + depends_on: + - prometheus + - loki + + # Loki - Log aggregation + loki: + image: grafana/loki:latest + container_name: ml-experiments-loki + ports: + - "3100:3100" + volumes: + - ./monitoring/loki-config.yml:/etc/loki/local-config.yaml + - loki_data:/loki + command: -config.file=/etc/loki/local-config.yaml + restart: unless-stopped + + # Promtail - Log collector + promtail: + image: grafana/promtail:latest + container_name: ml-experiments-promtail + volumes: + - ./monitoring/promtail-config.yml:/etc/promtail/config.yml + - ./logs:/var/log/app + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + command: -config.file=/etc/promtail/config.yml + restart: unless-stopped + depends_on: + - loki + +volumes: + redis_data: + driver: local + prometheus_data: + driver: local + grafana_data: + driver: local + loki_data: + driver: local + +networks: + default: + name: ml-experiments-network + backend: + name: ml-backend-network + internal: true # No external access diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ae94fd1 --- /dev/null +++ b/go.mod @@ -0,0 +1,73 @@ +module github.com/jfraeys/fetch_ml + +go 1.25.0 + +// Fetch ML - Secure Machine Learning Platform +// Copyright (c) 2024 Fetch ML +// Licensed under the MIT License + +require ( + github.com/BurntSushi/toml v1.5.0 + github.com/alicebob/miniredis/v2 v2.35.0 + github.com/charmbracelet/bubbles v0.21.0 + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 + github.com/go-redis/redis/v8 v8.11.5 + github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 + github.com/lib/pq v1.10.9 + github.com/mattn/go-sqlite3 v1.14.32 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 + github.com/stretchr/testify v1.11.1 + github.com/xeipuuv/gojsonschema v1.2.0 + github.com/zalando/go-keyring v0.2.6 + golang.org/x/crypto v0.45.0 + golang.org/x/time v0.14.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + al.essio.dev/pkg/shellescape v1.6.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.3 // indirect + github.com/charmbracelet/x/ansi v0.11.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.14 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.6.1 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/danieljoos/wincred v1.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/godbus/dbus/v5 v5.2.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.4 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sahilm/fuzzy v0.1.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cbf4015 --- /dev/null +++ b/go.sum @@ -0,0 +1,168 @@ +al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA= +al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= +github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= +github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI= +github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.11.2 h1:XAG3FSjiVtFvgEgGrNBkCNNYrsucAt8c6bfxHyROLLs= +github.com/charmbracelet/x/ansi v0.11.2/go.mod h1:9tY2bzX5SiJCU0iWyskjBeI2BRQfvPqI+J760Mjf+Rg= +github.com/charmbracelet/x/cellbuf v0.0.14 h1:iUEMryGyFTelKW3THW4+FfPgi4fkmKnnaLOXuc+/Kj4= +github.com/charmbracelet/x/cellbuf v0.0.14/go.mod h1:P447lJl49ywBbil/KjCk2HexGh4tEY9LH0/1QrZZ9rA= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/clipperhouse/displaywidth v0.6.1 h1:/zMlAezfDzT2xy6acHBzwIfyu2ic0hgkT83UX5EY2gY= +github.com/clipperhouse/displaywidth v0.6.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= +github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= +github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= +github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc= +github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= +github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s= +github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5cbafa0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,132 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.black] +# Google Python Style Guide Configuration for Black +line-length = 80 +target-version = ['py38'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | venv + | _build + | buck-out + | build + | dist + # ML experiment code - exclude from formatting + | podman/workspace + | workspace + | tests/fixtures/examples + | tests/fixtures/podman + | results + | data + | logs + | secrets + | .agent +)/ +''' + +[tool.isort] +# Google Python Style Guide Configuration for isort +profile = "google" +line_length = 80 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +split_on_trailing_comma = true +known_first_party = ["fetch_ml"] +skip_glob = [ + "podman/workspace/*", + "workspace/*", + "tests/fixtures/examples/*", + "tests/fixtures/podman/*", +] + +[tool.mypy] +# Google Python Style Guide Configuration for mypy +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[[tool.mypy.overrides]] +module = [ + "torch.*", + "tensorflow.*", + "sklearn.*", + "pandas.*", + "numpy.*", + "matplotlib.*", + "seaborn.*", + "scipy.*", + "joblib.*", +] +ignore_missing_imports = true + +[tool.pytest.ini_options] +# Google Python Style Guide Configuration for pytest +minversion = "6.0" +addopts = "-ra -q --strict-markers --strict-config" +testpaths = ["tests"] +python_files = ["*_test.py", "test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", +] + +[tool.coverage.run] +source = ["."] +omit = [ + "tests/*", + "*/tests/*", + "test_*", + "*_test.py", + "setup.py", + "*/site-packages/*", + # ML experiment code + "podman/workspace/*", + "workspace/*", + "tests/fixtures/examples/*", + "tests/fixtures/podman/*", + "results/*", + "data/*", + "logs/*", + "secrets/*", + ".agent/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +]