fetch_ml/Makefile

.PHONY: all build prod prod-with-native native-release native-build native-debug native-test native-smoke native-clean dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install configlint worker-configlint ci-local docs docs-setup docs-check-port docs-stop docs-build docs-build-prod benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status size load-test chaos-test profile-load profile-load-norate profile-ws-queue profile-tools detect-regressions tech-excellence docker-build dev-smoke prod-smoke native-smoke self-cleanup test-full test-auth deploy-up deploy-down deploy-status deploy-clean dev-up dev-down dev-status dev-logs prod-up prod-down prod-status prod-logs
OK = ✓
DOCS_PORT ?= 1313
DOCS_BIND ?= 127.0.0.1
DOCS_BASEURL ?= /
DOCS_PROD_BASEURL ?= $(DOCS_BASEURL)
# Default target
all: build

# Build all components (Go binaries + optimized CLI)
build:
	go build -o bin/api-server ./cmd/api-server/main.go
	go build -o bin/worker ./cmd/worker/worker_server.go
	go build -o bin/data_manager ./cmd/data_manager
	go build -o bin/user_manager ./cmd/user_manager
	go build -o bin/tui ./cmd/tui
	$(MAKE) -C ./cli all
	@echo "${OK} All components built"

# Build native C++ libraries for production (optimized, stripped)
native-release:
	@mkdir -p native/build
	@cd native/build && cmake .. -DCMAKE_BUILD_TYPE=Release \
		-DCMAKE_C_FLAGS="-O3 -DNDEBUG -fomit-frame-pointer" \
		-DCMAKE_CXX_FLAGS="-O3 -DNDEBUG -fomit-frame-pointer" && \
		make -j$(shell nproc 2>/dev/null || sysctl -n hw.ncpu)
	@echo "${OK} Native libraries built (release)"

# Build Go binaries with native library support
prod-with-native: native-release
	@mkdir -p bin
	@cp native/build/lib*.so native/build/lib*.dylib bin/ 2>/dev/null || true
	@go build -ldflags="-s -w" -o bin/api-server ./cmd/api-server/main.go
	@go build -ldflags="-s -w" -o bin/worker ./cmd/worker/worker_server.go
	@echo "${OK} Production binaries built (with native libs)"
	@echo "Copy native libraries from bin/ alongside your binaries"

# Build native C++ libraries for performance optimization
native-build:
	@mkdir -p native/build
	@cd native/build && cmake .. -DCMAKE_BUILD_TYPE=Release && make -j$(shell nproc 2>/dev/null || sysctl -n hw.ncpu)
	@echo "${OK} Native libraries built"

# Build native libraries with AddressSanitizer (for testing)
native-debug:
	@mkdir -p native/build
	@cd native/build && cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON && make -j$(shell nproc 2>/dev/null || sysctl -n hw.ncpu)
	@echo "${OK} Native libraries built (debug mode)"

# Clean native build artifacts
native-clean:
	@rm -rf native/build
	@echo "${OK} Native build cleaned"

# Run native library tests
native-test: native-build
	@cd native/build && ctest --output-on-failure
	@echo "${OK} Native tests passed"

# Run native libraries smoke test (builds + C++ tests + Go integration)
native-smoke:
	@bash ./scripts/smoke-test-native.sh
	@echo "${OK} Native smoke test passed"

# Build production-optimized binaries
prod:
	go build -ldflags="-s -w" -o bin/api-server cmd/api-server/main.go
	go build -ldflags="-s -w" -o bin/worker cmd/worker/worker_server.go
	go build -ldflags="-s -w" -o bin/data_manager ./cmd/data_manager
	go build -ldflags="-s -w" -o bin/user_manager ./cmd/user_manager
	go build -ldflags="-s -w" -o bin/tui ./cmd/tui
	$(MAKE) -C cli prod
	@echo "${OK} Production binaries built"

cross-platform:
	@rm -rf dist
	@mkdir -p dist
	@set -e; \
	  LDFLAGS='-s -w -buildid='; \
	  for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64; do \
	    goos=$${target%/*}; \
	    goarch=$${target#*/}; \
	    ext=''; \
	    if [ "$$goos" = "windows" ]; then ext='.exe'; fi; \
	    echo "Building $$goos/$$goarch..."; \
	    CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch go build -trimpath -buildvcs=false -ldflags="$$LDFLAGS" -o dist/fetch_ml_api-server_$${goos}_$${goarch}$${ext} cmd/api-server/main.go; \
	    CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch go build -trimpath -buildvcs=false -ldflags="$$LDFLAGS" -o dist/fetch_ml_worker_$${goos}_$${goarch}$${ext} cmd/worker/worker_server.go; \
	    CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch go build -trimpath -buildvcs=false -ldflags="$$LDFLAGS" -o dist/fetch_ml_data_manager_$${goos}_$${goarch}$${ext} ./cmd/data_manager; \
	    CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch go build -trimpath -buildvcs=false -ldflags="$$LDFLAGS" -o dist/fetch_ml_user_manager_$${goos}_$${goarch}$${ext} ./cmd/user_manager; \
	    CGO_ENABLED=0 GOOS=$$goos GOARCH=$$goarch go build -trimpath -buildvcs=false -ldflags="$$LDFLAGS" -o dist/fetch_ml_tui_$${goos}_$${goarch}$${ext} ./cmd/tui; \
	  done
	@echo "${OK} Cross-platform binaries built in dist/"

# Development build (faster compilation)
dev:
	go build -buildvcs=false -o bin/api-server cmd/api-server/main.go
	go build -buildvcs=false -o bin/worker cmd/worker/worker_server.go
	go build -buildvcs=false -o bin/data_manager ./cmd/data_manager
	go build -buildvcs=false -o bin/user_manager ./cmd/user_manager
	go build -buildvcs=false -o bin/tui ./cmd/tui
	$(MAKE) -C cli dev
	@echo "${OK} Development binaries built"

# Clean build artifacts (Go + Zig + test outputs)
clean:
	rm -rf bin/ coverage/ tests/bin/
	rm -rf cli/zig-out/ cli/.zig-cache/ .zig-cache/
	rm -rf dist/
	go clean
	@echo "${OK} Cleaned"

clean-docs:
	rm -rf docs/_site/
	@echo "${OK} Cleaned docs"

# Run tests
test:
	go test ./tests/...
	cd cli && zig build test
	@echo "${OK} All tests passed"

# Lint Go and Zig code
lint:
	gofmt -w ./cmd ./internal ./tests || true
	go vet ./...
	cd cli && zig fmt .
	@echo "${OK} Lint completed"

# Install to system (requires sudo)
install: prod
	sudo cp bin/api-server /usr/local/bin/fetchml-api
	sudo cp bin/worker /usr/local/bin/fetchml-worker
	sudo cp bin/tui /usr/local/bin/fetchml-tui
	sudo cp cli/zig-out/bin/ml /usr/local/bin/ml
	@echo "${OK} Installed"

# Validate YAML configs against JSON schema
configlint:
	go run ./cmd/configlint --schema configs/schema/api_server_config.yaml \
		configs/api/dev.yaml \
		configs/api/homelab-secure.yaml \
		configs/api/multi-user.yaml \
		configs/api/prod.yaml

worker-configlint:
	go run ./cmd/configlint --schema configs/schema/worker_config_schema.yaml \
		configs/workers/worker-prod.toml \
		configs/workers/docker.yaml \
		configs/workers/docker-dev.yaml \
		configs/workers/docker-prod.yaml \
		configs/workers/homelab-secure.yaml

dev-smoke:
	bash ./scripts/smoke-test.sh dev
	@echo "dev smoke: OK"

prod-smoke:
	bash ./scripts/smoke-test.sh prod
	@echo "prod smoke: OK"

# Run maintainability CI checks
ci-checks:
	@bash ./scripts/ci-checks.sh

# Run a local approximation of the CI pipeline
ci-local: ci-checks test lint configlint worker-configlint
	@mkdir -p coverage
	@echo "Running queue package tests with race detector..."
	go test -v -race -coverprofile=coverage/queue-coverage.out ./internal/queue/...
	@echo "Running coverage..."
	make test-coverage

# Docker image build (no direct docker-compose run; use deploy-* targets instead)
docker-build:
	docker build -f build/docker/simple.Dockerfile -t fetchml:latest .
	@echo "${OK} Docker image built"

# Enhanced test targets
test-unit:
	go test -v -short ./tests/unit/...
	cd cli && zig build test

test-integration:
	go test -v ./tests/integration/... ./tests
	cd cli && zig build test

test-e2e:
	go test -v ./tests/e2e/...

test-coverage:
	@mkdir -p coverage
	go test -coverprofile=coverage/coverage.out ./...
	go tool cover -html=coverage/coverage.out -o coverage/coverage.html
	@echo "${OK} Coverage report: coverage/coverage.html"

# Documentation setup
docs-setup:
	@echo "Setting up Hugo documentation..."
	@if ! command -v hugo >/dev/null 2>&1; then \
		echo "Hugo not found. Please install it manually:"; \
		echo "  macOS: brew install hugo"; \
		echo "  Linux: See https://gohugo.io/installation/"; \
		exit 1; \
	fi
	@echo "Documentation setup complete!"

# Documentation
docs: docs-setup docs-check-port
	@echo "Starting Hugo development server..."
	cd docs && hugo server --bind "$(DOCS_BIND)" --port "$(DOCS_PORT)" --baseURL "$(DOCS_BASEURL)"

# Build documentation
docs-build:
	@echo "Building static documentation..."
	cd docs && hugo

docs-build-prod:
	@echo "Building production docs..."
	cd docs && hugo --minify --baseURL "$(DOCS_PROD_BASEURL)"

# Performance benchmarking tools
benchmark:
	@echo "Running performance benchmarks..."
	go test -bench=. -benchmem ./tests/benchmarks/...

# Run benchmarks locally with artifact management
benchmark-local:
	@echo "Running benchmarks locally with full workflow..."
	./scripts/benchmarks/run-benchmarks-local.sh

# Manage benchmark artifacts
artifacts:
	@echo "Managing benchmark artifacts..."
	./scripts/manage-artifacts.sh help

# Clean benchmark artifacts (keep last 10)
clean-benchmarks:
	@echo "Cleaning benchmark artifacts..."
	./scripts/maintenance/cleanup-benchmarks.sh benchmarks

# Comprehensive cleanup (keep last 5 runs)
clean-all:
	@echo "Running comprehensive cleanup..."
	./scripts/maintenance/cleanup-benchmarks.sh all

# Aggressive cleanup (removes more data)
clean-aggressive:
	@echo "Running aggressive cleanup..."
	./scripts/maintenance/cleanup-benchmarks.sh aggressive

# Show disk usage status
status:
	@echo "Checking disk usage..."
	./scripts/maintenance/cleanup-benchmarks.sh status

size:
	@echo "Binary sizes:"
	@ls -lh bin/* cli/zig-out/bin/ml 2>/dev/null || true

# Load testing
load-test:
	@echo "Running load tests..."
	go test -v ./tests/load/...

# CPU profiling for HTTP LoadTestSuite (MediumLoad only for speed)
profile-load:
	@echo "CPU profiling MediumLoad HTTP load test..."
	go test ./tests/load -run TestLoadProfile_Medium -count=1 -cpuprofile tests/bin/cpu_load.out
	@echo "${OK} CPU profile written to cpu_load.out (inspect with: go tool pprof tests/bin/cpu_load.out)"

profile-load-norate:
	@echo "CPU profiling MediumLoad HTTP load test (no rate limiting)..."
	go test ./tests/load -run TestLoadProfile_Medium -count=1 -cpuprofile tests/bin/cpu_load.out -v -args -profile-norate
	@echo "${OK} CPU profile written to cpu_load.out (inspect with: go tool pprof tests/bin/cpu_load.out)"

# CPU profiling for WebSocket → Redis queue → worker path
profile-ws-queue:
	@echo "CPU profiling WebSocket queue integration test..."
	go test ./tests/integration -run WebSocketQueue -count=5 -cpuprofile tests/bin/cpu_ws.out
	@echo "${OK} CPU profile written to cpu_ws.out (inspect with: go tool pprof tests/bin/cpu_ws.out)"

# Chaos engineering tests
chaos-test:
	@echo "Running chaos engineering tests..."
	go test -v ./tests/chaos/...

# Performance profiling tools
profile-tools:
	@echo "Building profiling tools..."
	go build -o bin/performance-regression-detector ./cmd/performance-regression-detector
	go build -o bin/profiler ./cmd/profiler

# Performance regression detection threshold (percentage)
REGRESSION_THRESHOLD ?= 10.0

# Performance regression detection
detect-regressions:
	@echo "Detecting performance regressions..."
	@mkdir -p tests/bin
	@if [ ! -f "tests/bin/baseline.bench.txt" ]; then \
		echo "Creating baseline performance metrics..."; \
		go test -bench=. -benchmem ./tests/benchmarks/... | tee tests/bin/baseline.bench.txt; \
	fi
	@echo "Analyzing current performance against baseline..."
	go test -bench=. -benchmem ./tests/benchmarks/... | tee tests/bin/current.bench.txt
	@$(MAKE) profile-tools
	@./bin/performance-regression-detector -baseline tests/bin/baseline.bench.txt -current tests/bin/current.bench.txt -threshold $(REGRESSION_THRESHOLD)

# Technical excellence suite (runs all performance tests)
complete-suite: benchmark load-test chaos-test profile-tools
	@echo "Technical excellence test suite completed"
	@echo "Results summary:"
	@echo "  - Benchmarks: See test output above"
	@echo "  - Load tests: See test output above"
	@echo "  - Chaos tests: See test output above"
	@echo "  - Profiling tools: Built in bin/"
	@echo "  - Regression detection: Run 'make detect-regressions'"

# Help
help:
	@echo "FetchML Build System"
	@echo ""
	@echo "Build Targets:"
	@echo "  make build            - Build all components (default)"
	@echo "  make prod             - Build production-optimized binaries"
	@echo "  make prod-with-native - Build production binaries with native C++ libs"
	@echo "  make dev              - Build development binaries (faster)"
	@echo "  make clean            - Remove build artifacts"
	@echo ""
	@echo "Native Library Targets:"
	@echo "  make native-build     - Build native C++ libraries"
	@echo "  make native-release   - Build native libs (release optimized)"
	@echo "  make native-debug     - Build native libs (debug with ASan)"
	@echo "  make native-test      - Run native library tests"
	@echo "  make native-smoke     - Run native smoke test (C++ + Go integration)"
	@echo "  make native-clean     - Clean native build artifacts"
	@echo ""
	@echo "Docker Targets:"
	@echo "  make docker-build     - Build Docker image"
	@echo ""
	@echo "Test Targets:"
	@echo "  make test             - Run all tests"
	@echo "  make test-unit        - Run unit tests only"
	@echo "  make test-integration - Run integration tests"
	@echo "  make test-e2e         - Run end-to-end tests (Podman test is opt-in via FETCH_ML_E2E_PODMAN=1)"
	@echo "  make test-coverage    - Generate coverage report"
	@echo "  make lint             - Run formatters and linters"
	@echo "  make ci-checks        - Run maintainability CI checks"
	@echo "  make ci-local         - Run local CI dry-run (tests, lint, config validation, coverage)"
	@echo "  make configlint       - Validate YAML configs against schema"
	@echo "  make worker-configlint - Validate worker configs against schema"
	@echo ""
	@echo "Setup Targets:"
	@echo "  make install          - Install binaries to /usr/local/bin (requires sudo)"
	@echo ""
	@echo "Performance Testing:"
	@echo "  make benchmark         - Run performance benchmarks"
	@echo "  make benchmark-local   - Run benchmarks locally with artifact management"
	@echo "  make artifacts         - Manage benchmark artifacts (list, clean, compare, export)"
	@echo "  make clean-benchmarks  - Clean benchmark artifacts (keep last 10)"
	@echo "  make clean-all         - Comprehensive cleanup (keep last 5 runs)"
	@echo "  make clean-aggressive  - Aggressive cleanup (removes more data)"
	@echo "  make status            - Show disk usage status"
	@echo "  make load-test         - Run load testing suite"
	@echo "  make profile-load      - CPU profile MediumLoad HTTP test suite"
	@echo "  make profile-ws-queue  - CPU profile WebSocket→queue→worker path"
	@echo "  make chaos-test        - Run chaos engineering tests"
	@echo "  make profile-tools     - Build performance profiling tools"
	@echo "  make detect-regressions - Detect performance regressions"
	@echo "  make complete-suite   	- Run complete technical suite"
	@echo ""
	@echo "Documentation:"
	@echo "  make docs-setup       - Validate Hugo is installed"
	@echo "  make docs             - Start Hugo dev server (checks DOCS_PORT is free)"
	@echo "  make docs-check-port  - Check if DOCS_PORT is already in use"
	@echo "  make docs-stop        - Stop process listening on DOCS_PORT (use with care)"
	@echo "  make docs-build       - Build static documentation (local)"
	@echo "  make docs-build-prod  - Build static documentation (prod flags; set DOCS_PROD_BASEURL)"
	@echo ""
	@echo "Utility:"
	@echo "  make size             - Show binary sizes"
	@echo "  make self-cleanup     - Clean up Docker resources"
	@echo "  make test-full        - Run complete test suite"
	@echo "  make test-auth        - Test multi-user authentication"
	@echo "  make help             - Show this help"

# Self-cleaning for Docker resources
self-cleanup:
	@echo "Running self-cleanup..."
	@./scripts/maintenance/cleanup.sh

# Run full test suite
test-full:
	@echo "Running full test suite..."
	@$(MAKE) ci-local

# Quick authentication test
test-auth:
	@echo "Testing multi-user authentication..."
	@echo "Testing admin user..." && cp ~/.ml/config-admin.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status
	@echo "Testing researcher user..." && cp ~/.ml/config-researcher.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status
	@echo "Testing analyst user..." && cp ~/.ml/config-analyst.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status

# Deployment management (using organized docker-compose files)
deploy-up:
	@echo "Starting development environment..."
	@./deployments/deploy.sh dev up

deploy-down:
	@echo "Stopping development environment..."
	@./deployments/deploy.sh dev down

deploy-status:
	@echo "Checking deployment status..."
	@./deployments/deploy.sh dev status

deploy-clean:
	@echo "Cleaning all deployments..."
	@cd deployments && make clean

dev-up:
	@./deployments/deploy.sh dev up

dev-down:
	@./deployments/deploy.sh dev down

dev-status:
	@./deployments/deploy.sh dev status

dev-logs:
	@./deployments/deploy.sh dev logs

prod-up:
	@./deployments/deploy.sh prod up

prod-down:
	@./deployments/deploy.sh prod down

prod-status:
	@./deployments/deploy.sh prod status

prod-logs:
	@./deployments/deploy.sh prod logs