jfraeysd/fetch_ml
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: initialize FetchML ML platform with core project structure

Browse source 
- Add comprehensive README with architecture overview and quick start guide
- Set up Go module with production-ready dependencies
- Configure build system with Makefile for development and production builds
- Add Docker Compose for local development environment
- Include project configuration files (linting, Python, etc.)

This establishes the foundation for a production-ready ML experiment platform
with task queuing, monitoring, and modern CLI/API interface.
This commit is contained in:
Jeremie Fraeys 2025-12-04 16:52:09 -05:00
commit c5049a2fdf
11 changed files with 1577 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
.flake8 Normal file
View file

@ -0,0 +1,47 @@
[flake8]
max-line-length = 80
max-complexity = 10
exclude =
.git,
__pycache__,
*.pyc,
.pytest_cache,
.mypy_cache,
.venv,
venv,
build,
dist,
# ML experiment code - exclude from style checks
podman/workspace,
workspace,
tests/fixtures/examples,
tests/fixtures/podman,
results,
data,
logs,
secrets,
.agent
ignore =
E203,
E501,
W503,
W504,
F401,
E402,
C901
per-file-ignores =
__init__.py:F401
tests/*:F401,F811
*_test.py:F401,F811
conftest.py:F401,F811
# Additional Google Style Guide recommendations
inline-quotes = single
multiline-quotes = double
docstring-quotes = double
# Import ordering (Google style)
import-order-style = google
application-import-names = fetch_ml

224
.gitignore vendored Normal file
View file

@ -0,0 +1,224 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
# Go workspace file
go.work
# IDE files
.vscode/
.idea/
*.swp
*.swo
*~
# OS generated files
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db
# Python cache and artifacts
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
# Python lib directories
**/site-packages/
**/lib/python*/
# Allow scripts/lib for shell libraries
!scripts/lib/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
Pipfile.lock
# poetry
poetry.lock
# pdm
.pdm.toml
# PEP 582
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
.idea/
# Project-specific
ml_jobs/
test_output/
# Python build directory only
**/build/lib/
**/build/bdist*/
**/build/scripts*/
# Allow build/ directory for Docker configs
!build/
!build/**
# But ignore Python build artifacts if they happen
build/lib/
build/bdist*/
coverage.out
coverage.html
queue-coverage.out
coverage/
# Redis dump
dump.rdb
# Configuration files with secrets
!config_*.yaml
!config_local.yaml.example
!config_prod.yaml.example
# Database files
db/*.db
!db/.gitkeep
# API keys and secrets
*.key
*.pem
secrets/
cli/src/assets/rsync_release.bin
# Test files
test_*.go
*_test_output/
# Build artifacts
bin/
cli/zig-out/
cli/.zig-cache/
zig-out/
.zig-cache/
# Experiment data (local testing)
experiments/
data/

79
.golangci.yml Normal file
View file

@ -0,0 +1,79 @@
---
version: "2"
run:
timeout: 5m
tests: true
output:
format: colored-line-number
linters-settings:
govet:
enable:
- shadow
- fieldalignment
gocyclo:
min-complexity: 15
dupl:
threshold: 100
goconst:
min-len: 3
min-occurrences: 3
misspell:
locale: US
lll:
line-length: 100
revive:
confidence: 0.8
depguard:
rules:
main:
allow:
- $gostd
- github.com/jfraeys/fetch_ml
linters:
disable-all: true
enable:
- bodyclose
- depguard
- dogsled
- dupl
- errcheck
- exhaustive
- gochecknoinits
- goconst
- gocritic
- gocyclo
- goprintffuncname
- gosec
- govet
- ineffassign
- lll
- misspell
- nakedret
- noctx
- nolintlint
- rowserrcheck
- staticcheck
- unconvert
- unparam
- unused
- whitespace
- revive
issues:
exclude-rules:
- path: _test\.go
linters:
- gocyclo
- errcheck
- dupl
- gosec
- lll
- text: "weak cryptographic primitive"
linters:
- gosec
- text: "Use of weak random number generator"
linters:
- gosec
max-issues-per-linter: 0
max-same-issues: 0
severity:
default-severity: error

307
.pylintrc Normal file
View file

@ -0,0 +1,307 @@
[MASTER]
# Google Python Style Guide Configuration
# Based on https://google.github.io/styleguide/pyguide.html
# Files or directories to be skipped over when performing analysis.
ignore=
.git,
__pycache__,
*.pyc,
.pytest_cache,
.mypy_cache,
.venv,
venv,
# ML experiment code - exclude from style checks
podman/workspace,
workspace,
tests/fixtures/examples,
tests/fixtures/podman,
results,
data,
logs,
secrets,
.agent
# Regular expression matching files that should be skipped entirely.
ignore-patterns=
^\.#,
^setup\.py$,
^conftest\.py$
# Add files or directories matching the regex patterns to the ignore list.
ignore-paths=
# Files or directories to be added to the list of ignored regular expressions.
ignore-paths=
# Pickle collected data for later comparisons.
persistent=yes
# Use multiple processes to speed up Pylint.
jobs=1
# List of plugins (as comma separated values of python module names) to load,
# usually to register additional checkers.
load-plugins=
# Load all enabled extensions.
load-all-plugins=no
# Control the amount of potential astroid warnings (0, 1 or 2).
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code.
extension-pkg-whitelist=
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels.
confidence=
# Disable the message, report, category or checker with the given id(s).
disable=
# Google Style Guide allows some of these
missing-module-docstring,
missing-function-docstring,
too-many-locals,
too-many-arguments,
too-many-instance-attributes,
too-many-public-methods,
too-few-public-methods,
too-many-lines,
too-many-statements,
too-many-branches,
too-many-nested-blocks,
# Not relevant for ML projects
invalid-name,
C0114, # missing-module-docstring
C0115, # missing-class-docstring
C0116, # missing-function-docstring
R0903, # too-few-public-methods
R0902, # too-many-instance-attributes
R0913, # too-many-arguments
R0914, # too-many-locals
R0915, # too-many-statements
R0912, # too-many-branches
R0911, # too-many-return-statements
W0613, # unused-argument (common in callbacks)
# Enable the message, report, category or checker with the given id(s).
enable=
[REPORTS]
# Set the output format.
output-format=text
# Tells whether to display a full report or only the messages.
reports=no
# Python expression which determines if a message should not be displayed.
evaluation=
# Template used to display messages.
msg-template=
[BASIC]
# Good variable names which should always be accepted, separated by a comma.
good-names=
i,j,k,ex,Run,_
# Bad variable names which should always be refused.
bad-names=
foo,bar,baz,toto,tutu,tata
# List of builtins function names that should not be used.
bad-functions=
# Regular expression for correct variable names.
variable-rgx=^[a-z_][a-z0-9_]{2,30}$
# Regular expression for correct constant names.
const-rgx=^(_?[A-Z][A-Z0-9_]{2,30}|__.*__)$
# Regular expression for correct function names.
function-rgx=^(?:(?P<camel_case>_?[A-Z][a-zA-Z0-9]{2,30})|(?P<snake_case>_?[a-z][a-z0-9_]{2,30}))$
# Regular expression for correct class names.
class-rgx=^_?[A-Z][a-zA-Z0-9]{2,30}$
# Regular expression for correct attribute names.
attr-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$
# Regular expression for correct argument names.
argument-rgx=^[a-z][a-z0-9_]{2,30}$
# Regular expression for correct method resolution order names.
method-rgx=^_{0,2}[a-z][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings.
docstring-min-length=-1
[TYPECHECK]
# List of decorators that produce context managers.
contextmanager-decorators=contextlib.contextmanager
# List of member names which should be excluded from the access member check.
exclude-members=
# List of qualified names (objects) which require a disable protection.
generated-members=
[MISCELLANEOUS]
# List of note tags to take in consideration.
notes=
[VARIABLES]
# Tells whether we should check for unused imports.
init-import=no
# A regular expression matching the name of dummy variables.
dummy-variable-rgx=^_|dummy
# List of variable names that should not be used.
additional-builtins=
# List of variables which are ignored for the undefined-variable check.
ignored-classes=
# List of members which are skipped from the member access check.
ignored-modules=
# List of deprecated symbols which should not be used.
deprecated-modules=
# List of symbols that can be safely accessed.
allowed-redefined-builtins=
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=80
# Regexp for a line that is allowed to be longer than the limit.
long-line-regexp=^[\t ]*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test.
single-line-if-stmt=no
# List of optional parameters for which we don't want to call a linting
# function.
single-line-class-stmt=no
# Maximum number of characters on a single line, when there are no multiline
# strings.
max-line-length-sigil-fallback=80
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[LOGGING]
# Logging modules to check that the string formatting arguments are in
# logging function parameter format.
logging-modules=logging
[DESIGN]
# Maximum number of arguments for function / method.
max-args=5
# Argument names that match this regular expression are ignored.
ignored-argument-names=
# Maximum number of locals for function / method body.
max-locals=15
# Maximum number of return statements for function / method body.
max-returns=6
# Maximum number of branch for function / method body.
max-branches=12
# Maximum number of statements in function / method body.
max-statements=50
# Maximum number of parents for a class.
max-parents=7
# Maximum number of attributes for a class.
max-attributes=7
# Minimum number of public methods for a class.
min-public-methods=2
# Maximum number of public methods for a class.
max-public-methods=20
# Maximum number of boolean expressions in a if statement.
max-bool-expr=5
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcls
# List of member names which should be excluded from the access member check.
exclude-protected=_asdict,_fields,_replace,_source,_make
[IMPORTS]
# Deprecated modules which should not be used.
deprecated-modules=
# Create a graph of every module's dependencies.
import-graph=
# Create a graph of every (i.e. internal and external) dependencies in the
# current file.
ext-import-graph=
# Create a graph of external dependencies in the current file.
int-import-graph=
# Force import order to match a specific order.
import-order-style=google
# Add files or directories to the blacklist.
ignore-on-import-order=no
# List of modules that can be imported at any level from the specified
# application modules.
known-third-party=
# Analyse import fallback blocks.
analyse-fallback-blocks=no
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught.
overgeneral-exceptions=Exception,BaseException
# Exceptions that will emit a warning when being returned.
return-exceptions=no
# Exceptions that will emit a warning when being instantiated.
nonstandard-exceptions=no
[REFACTORING]
# Maximum number of nested blocks.
max-nested-blocks=5
# Complete name of the function that returns the YAPF version.
yapf-version=

21
LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Fetch ML
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

198
Makefile Normal file
View file

@ -0,0 +1,198 @@
.PHONY: all build clean clean-docs test test-unit test-integration test-e2e test-coverage lint install dev prod setup validate configlint ci-local docs
# Default target
all: build
# Build all components
build:
go build -o bin/api-server cmd/api-server/main.go
go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go
go build -o bin/tui ./cmd/tui
cd cli && zig build prod
@echo "✓ All components built"
# Build production-optimized binaries
prod:
go build -ldflags="-s -w" -o bin/api-server cmd/api-server/main.go
go build -ldflags="-s -w" -o bin/worker cmd/worker/worker_server.go
go build -ldflags="-s -w" -o bin/tui ./cmd/tui
cd cli && zig build prod && strip zig-out/prod/ml
@echo "✓ Production binaries built"
# Development build (faster compilation)
dev:
go build -o bin/api-server cmd/api-server/main.go
go build -o bin/worker cmd/worker/worker_server.go cmd/worker/worker_config.go
go build -o bin/tui ./cmd/tui
cd cli && zig build dev
@echo "✓ Development binaries built"
# Clean build artifacts
clean:
rm -rf bin/ coverage/
rm -rf cli/zig-out/
rm -rf cli/.zig-cache/
go clean
@echo "✓ Cleaned"
clean-docs:
rm -rf docs/_site/
@echo "✓ Cleaned docs"
# Run tests
test:
go test ./...
cd cli && zig build test
@echo "✓ All tests passed"
# Lint Go and Zig code
lint:
gofmt -w ./cmd ./internal ./tests || true
go vet ./...
cd cli && zig fmt .
@echo "✓ Lint completed"
# Install to system (requires sudo)
install: prod
sudo cp bin/api-server /usr/local/bin/fetchml-api
sudo cp bin/worker /usr/local/bin/fetchml-worker
sudo cp bin/tui /usr/local/bin/fetchml-tui
sudo cp cli/zig-out/prod/ml /usr/local/bin/ml
@echo "✓ Installed"
# Setup production environment
setup:
@if [ "$(shell uname)" = "Linux" ]; then \
sudo ./scripts/setup-prod.sh; \
else \
echo "Production setup is for Linux only. You're on $(shell uname)."; \
echo "Use docker-compose for local development."; \
fi
# Validate production configuration
validate:
./scripts/validate-prod-config.sh configs/config-prod.yaml configs/worker-prod.toml
# Validate YAML configs against JSON schema
configlint:
go run ./cmd/configlint --schema configs/schema/config_schema.yaml \
configs/config-prod.yaml \
configs/config-no-tls.yaml \
configs/config-dev.yaml
# Run a local approximation of the CI pipeline
ci-local:
make test
make lint
make configlint
@echo "Running queue package tests with race detector..."
go test -v -race -coverprofile=coverage/queue-coverage.out ./internal/queue/...
@echo "Running coverage..."
make test-coverage
# Docker targets
docker-build:
docker build -f build/docker/simple.Dockerfile -t fetchml:latest .
@echo "✓ Docker image built"
docker-run:
docker-compose up -d
@echo "✓ Services started"
docker-stop:
docker-compose down
@echo "✓ Services stopped"
docker-logs:
docker-compose logs -f
# Monitoring setup (Linux only)
setup-monitoring:
@if [ "$(shell uname)" = "Linux" ]; then \
sudo ./scripts/setup-monitoring-prod.sh; \
else \
echo "Monitoring setup is for Linux production. Use docker-compose for local development."; \
fi
# Enhanced test targets
test-unit:
go test -v -short ./...
test-integration:
go test -v ./...
test-e2e:
go test -v ./tests/e2e/...
test-coverage:
go test -coverprofile=coverage/coverage.out ./...
go tool cover -html=coverage/coverage.out -o coverage/coverage.html
@echo "✓ Coverage report: coverage/coverage.html"
# Documentation setup
docs-setup:
@echo "Setting up MkDocs documentation..."
@if ! command -v mkdocs >/dev/null 2>&1; then \
echo "MkDocs not found. Please install it manually:"; \
echo " macOS: brew install mkdocs mkdocs-material"; \
echo " Linux: pip install mkdocs mkdocs-material"; \
echo "Or visit: https://www.mkdocs.org/user-guide/installation/"; \
exit 1; \
fi
@if ! python3 -c "import pymdownx.superfences" >/dev/null 2>&1; then \
echo "pymdown-extensions not found. Please install it manually:"; \
echo " pip3 install --user pymdown-extensions"; \
echo " Or use a virtual environment: python3 -m venv docs-env && source docs-env/bin/activate && pip install pymdown-extensions"; \
exit 1; \
fi
@echo "Documentation setup complete!"
# Documentation
docs: docs-setup docs-build
@echo "Starting MkDocs development server..."
cd docs && mkdocs serve
# Build documentation
docs-build:
@echo "Building static documentation..."
cd docs && mkdocs build
# Help
help:
@echo "FetchML Build System"
@echo ""
@echo "Build Targets:"
@echo " make build - Build all components (default)"
@echo " make prod - Build production-optimized binaries"
@echo " make dev - Build development binaries (faster)"
@echo " make clean - Remove build artifacts"
@echo ""
@echo "Docker Targets:"
@echo " make docker-build - Build Docker image"
@echo " make docker-run - Start services with docker-compose"
@echo " make docker-stop - Stop docker-compose services"
@echo " make docker-logs - View docker-compose logs"
@echo ""
@echo "Test Targets:"
@echo " make test - Run all tests"
@echo " make test-unit - Run unit tests only"
@echo " make test-integration - Run integration tests"
@echo " make test-coverage - Generate coverage report"
@echo " make lint - Run formatters and linters"
@echo " make ci-local - Run local CI dry-run (tests, lint, config validation, coverage)"
@echo " make configlint - Validate YAML configs against schema"
@echo ""
@echo "Setup Targets:"
@echo " make install - Install binaries to /usr/local/bin (requires sudo)"
@echo " make setup - Run production setup (Linux only)"
@echo " make setup-monitoring - Setup monitoring stack (Linux only)"
@echo " make validate - Validate production configuration"
@echo ""
@echo "Documentation:"
@echo " make docs-setup - Install MkDocs and dependencies"
@echo " make docs - Start MkDocs development server with live reload"
@echo " make docs-build - Build static documentation for deployment"
@echo ""
@echo "Utility:"
@echo " make size - Show binary sizes"
@echo " make help - Show this help"

200
README.md Normal file
View file

@ -0,0 +1,200 @@
# FetchML - Machine Learning Platform
A production-ready ML experiment platform with task queuing, monitoring, and a modern CLI/API.
## Features
- **🚀 Production Resilience** - Task leasing, smart retries, dead-letter queues
- **📊 Monitoring** - Grafana/Prometheus/Loki with auto-provisioned dashboards
- **🔐 Security** - API key auth, TLS, rate limiting, IP whitelisting
- **⚡ Performance** - Go API server + Zig CLI for speed
- **📦 Easy Deployment** - Docker Compose (dev) or systemd (prod)
## Quick Start
### Development (macOS/Linux)
```bash
# Clone and start
git clone <your-repo>
cd fetch_ml
docker-compose up -d
# Access Grafana: http://localhost:3000 (admin/admin)
```
### Production (Linux)
```bash
# Setup application
sudo ./scripts/setup-prod.sh
# Setup monitoring
sudo ./scripts/setup-monitoring-prod.sh
# Build and install
make prod
make install
# Start services
sudo systemctl start fetchml-api fetchml-worker
sudo systemctl start prometheus grafana loki promtail
```
## Architecture
```
┌──────────────┐ WebSocket ┌──────────────┐
│ Zig CLI/TUI │◄─────────────►│ API Server │
└──────────────┘ │ (Go) │
└──────┬───────┘
│
┌─────────────┼─────────────┐
│ │ │
┌────▼────┐ ┌───▼────┐ ┌───▼────┐
│ Redis │ │ Worker │ │ Loki │
│ (Queue) │ │ (Go) │ │ (Logs) │
└─────────┘ └────────┘ └────────┘
```
## Usage
### API Server
```bash
# Development (stderr logging)
go run cmd/api-server/main.go --config configs/config-dev.yaml
# Production (file logging)
go run cmd/api-server/main.go --config configs/config-no-tls.yaml
```
### CLI
```bash
# Build
cd cli && zig build prod
# Run experiment
./cli/zig-out/bin/ml run --config config.toml
# Check status
./cli/zig-out/bin/ml status
```
### Docker
```bash
make docker-run # Start all services
make docker-logs # View logs
make docker-stop # Stop services
```
## Development
### Prerequisites
- Go 1.21+
- Zig 0.11+
- Redis
- Docker (for local dev)
### Build
```bash
make build # All components
make dev # Fast dev build
make prod # Optimized production build
```
### Test
```bash
make test # All tests
make test-unit # Unit tests only
make test-coverage # With coverage report
```
## Configuration
### Development (`configs/config-dev.yaml`)
```yaml
logging:
level: "info"
file: "" # stderr only
redis:
url: "redis://localhost:6379"
```
### Production (`configs/config-no-tls.yaml`)
```yaml
logging:
level: "info"
file: "./logs/fetch_ml.log" # file only
redis:
url: "redis://redis:6379"
```
## Monitoring
### Grafana Dashboards (Auto-Provisioned)
- **ML Task Queue** - Queue depth, task duration, failure rates
- **Application Logs** - Log streams, error tracking, search
Access: `http://localhost:3000` (dev) or `http://YOUR_SERVER:3000` (prod)
### Metrics
- Queue depth and task processing rates
- Retry attempts by error category
- Dead letter queue size
- Lease expirations
## Documentation
- **[Getting Started](docs/getting-started.md)** - Detailed setup guide
- **[Production Deployment](docs/production-monitoring.md)** - Linux deployment
- **[WebSocket API](docs/api/)** - Protocol documentation
- **[Architecture](docs/architecture/)** - System design
## Makefile Targets
```bash
# Build
make build # Build all components
make prod # Production build
make clean # Clean artifacts
# Docker
make docker-build # Build image
make docker-run # Start services
make docker-stop # Stop services
# Test
make test # All tests
make test-coverage # With coverage
# Production (Linux only)
make setup # Setup app
make setup-monitoring # Setup monitoring
make install # Install binaries
```
## Security
- **TLS/HTTPS** - End-to-end encryption
- **API Keys** - Hashed with SHA256
- **Rate Limiting** - Per-user quotas
- **IP Whitelist** - Network restrictions
- **Audit Logging** - All API access logged
## License
MIT - See [LICENSE](LICENSE)
## Contributing
Contributions welcome! This is a personal homelab project but PRs are appreciated.

128
docker-compose.yml Normal file
View file

@ -0,0 +1,128 @@
# Homelab Docker Compose with Centralized Monitoring
# Includes: API, Redis, Prometheus, Grafana, Loki
services:
redis:
image: redis:7-alpine
container_name: ml-experiments-redis
ports:
- "6379:6379"
volumes:
- redis_data:/data
restart: unless-stopped
command: redis-server --appendonly yes
healthcheck:
test: [ "CMD", "redis-cli", "ping" ]
interval: 30s
timeout: 10s
retries: 3
api-server:
build:
context: .
dockerfile: build/docker/simple.Dockerfile
container_name: ml-experiments-api
ports:
- "9101:9101"
- "9100:9100" # Prometheus metrics endpoint
volumes:
- ./data:/data/experiments
- ./logs:/logs
- ./configs/config-no-tls.yaml:/app/configs/config.yaml
depends_on:
redis:
condition: service_healthy
restart: unless-stopped
environment:
- REDIS_URL=redis://redis:6379
- LOG_LEVEL=info
healthcheck:
test: [ "CMD", "curl", "http://localhost:9101/health" ]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
labels:
logging: "promtail"
job: "api-server"
# Prometheus - Metrics collection
prometheus:
image: prom/prometheus:latest
container_name: ml-experiments-prometheus
ports:
- "9090:9090"
volumes:
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--web.enable-lifecycle'
restart: unless-stopped
# Grafana - Visualization
grafana:
image: grafana/grafana:latest
container_name: ml-experiments-grafana
ports:
- "3000:3000"
volumes:
- grafana_data:/var/lib/grafana
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning
- ./monitoring/grafana-dashboard.json:/var/lib/grafana/dashboards/ml-queue.json
- ./monitoring/logs-dashboard.json:/var/lib/grafana/dashboards/logs.json
environment:
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
- GF_SERVER_ROOT_URL=http://localhost:3000
- GF_AUTH_ANONYMOUS_ENABLED=false
restart: unless-stopped
depends_on:
- prometheus
- loki
# Loki - Log aggregation
loki:
image: grafana/loki:latest
container_name: ml-experiments-loki
ports:
- "3100:3100"
volumes:
- ./monitoring/loki-config.yml:/etc/loki/local-config.yaml
- loki_data:/loki
command: -config.file=/etc/loki/local-config.yaml
restart: unless-stopped
# Promtail - Log collector
promtail:
image: grafana/promtail:latest
container_name: ml-experiments-promtail
volumes:
- ./monitoring/promtail-config.yml:/etc/promtail/config.yml
- ./logs:/var/log/app
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock
command: -config.file=/etc/promtail/config.yml
restart: unless-stopped
depends_on:
- loki
volumes:
redis_data:
driver: local
prometheus_data:
driver: local
grafana_data:
driver: local
loki_data:
driver: local
networks:
default:
name: ml-experiments-network
backend:
name: ml-backend-network
internal: true # No external access

73
go.mod Normal file
View file

@ -0,0 +1,73 @@
module github.com/jfraeys/fetch_ml
go 1.25.0
// Fetch ML - Secure Machine Learning Platform
// Copyright (c) 2024 Fetch ML
// Licensed under the MIT License
require (
github.com/BurntSushi/toml v1.5.0
github.com/alicebob/miniredis/v2 v2.35.0
github.com/charmbracelet/bubbles v0.21.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/go-redis/redis/v8 v8.11.5
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/lib/pq v1.10.9
github.com/mattn/go-sqlite3 v1.14.32
github.com/prometheus/client_golang v1.23.2
github.com/redis/go-redis/v9 v9.17.2
github.com/stretchr/testify v1.11.1
github.com/xeipuuv/gojsonschema v1.2.0
github.com/zalando/go-keyring v0.2.6
golang.org/x/crypto v0.45.0
golang.org/x/time v0.14.0
gopkg.in/yaml.v3 v3.0.1
)
require (
al.essio.dev/pkg/shellescape v1.6.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/colorprofile v0.3.3 // indirect
github.com/charmbracelet/x/ansi v0.11.2 // indirect
github.com/charmbracelet/x/cellbuf v0.0.14 // indirect
github.com/charmbracelet/x/term v0.2.2 // indirect
github.com/clipperhouse/displaywidth v0.6.1 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/danieljoos/wincred v1.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/godbus/dbus/v5 v5.2.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.4 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sahilm/fuzzy v0.1.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
)

168
go.sum Normal file
View file

@ -0,0 +1,168 @@
al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg=
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
github.com/charmbracelet/colorprofile v0.3.3 h1:DjJzJtLP6/NZ8p7Cgjno0CKGr7wwRJGxWUwh2IyhfAI=
github.com/charmbracelet/colorprofile v0.3.3/go.mod h1:nB1FugsAbzq284eJcjfah2nhdSLppN2NqvfotkfRYP4=
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
github.com/charmbracelet/x/ansi v0.11.2 h1:XAG3FSjiVtFvgEgGrNBkCNNYrsucAt8c6bfxHyROLLs=
github.com/charmbracelet/x/ansi v0.11.2/go.mod h1:9tY2bzX5SiJCU0iWyskjBeI2BRQfvPqI+J760Mjf+Rg=
github.com/charmbracelet/x/cellbuf v0.0.14 h1:iUEMryGyFTelKW3THW4+FfPgi4fkmKnnaLOXuc+/Kj4=
github.com/charmbracelet/x/cellbuf v0.0.14/go.mod h1:P447lJl49ywBbil/KjCk2HexGh4tEY9LH0/1QrZZ9rA=
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ=
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
github.com/clipperhouse/displaywidth v0.6.1 h1:/zMlAezfDzT2xy6acHBzwIfyu2ic0hgkT83UX5EY2gY=
github.com/clipperhouse/displaywidth v0.6.1/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ=
github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8=
github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+Lvsc=
github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s=
github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

132
pyproject.toml Normal file
View file

@ -0,0 +1,132 @@
[build-system]
requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
build-backend = "setuptools.build_meta"
[tool.black]
# Google Python Style Guide Configuration for Black
line-length = 80
target-version = ['py38']
include = '\.pyi?$'
extend-exclude = '''
/(
# directories
\.eggs
| \.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| venv
| _build
| buck-out
| build
| dist
# ML experiment code - exclude from formatting
| podman/workspace
| workspace
| tests/fixtures/examples
| tests/fixtures/podman
| results
| data
| logs
| secrets
| .agent
)/
'''
[tool.isort]
# Google Python Style Guide Configuration for isort
profile = "google"
line_length = 80
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
ensure_newline_before_comments = true
split_on_trailing_comma = true
known_first_party = ["fetch_ml"]
skip_glob = [
"podman/workspace/*",
"workspace/*",
"tests/fixtures/examples/*",
"tests/fixtures/podman/*",
]
[tool.mypy]
# Google Python Style Guide Configuration for mypy
python_version = "3.8"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
check_untyped_defs = true
disallow_untyped_decorators = true
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
strict_equality = true
[[tool.mypy.overrides]]
module = [
"torch.*",
"tensorflow.*",
"sklearn.*",
"pandas.*",
"numpy.*",
"matplotlib.*",
"seaborn.*",
"scipy.*",
"joblib.*",
]
ignore_missing_imports = true
[tool.pytest.ini_options]
# Google Python Style Guide Configuration for pytest
minversion = "6.0"
addopts = "-ra -q --strict-markers --strict-config"
testpaths = ["tests"]
python_files = ["*_test.py", "test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"integration: marks tests as integration tests",
"unit: marks tests as unit tests",
]
[tool.coverage.run]
source = ["."]
omit = [
"tests/*",
"*/tests/*",
"test_*",
"*_test.py",
"setup.py",
"*/site-packages/*",
# ML experiment code
"podman/workspace/*",
"workspace/*",
"tests/fixtures/examples/*",
"tests/fixtures/podman/*",
"results/*",
"data/*",
"logs/*",
"secrets/*",
".agent/*",
]
[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"def __repr__",
"if self.debug:",
"if settings.DEBUG",
"raise AssertionError",
"raise NotImplementedError",
"if 0:",
"if __name__ == .__main__.:",
"class .*\\bProtocol\\):",
"@(abc\\.)?abstractmethod",
]