diff --git a/Makefile b/Makefile index 151dedd..3f4707a 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,27 @@ clean: go clean @echo "${OK} Cleaned" +# Thorough cleanup for release +clean-release: clean + @echo "Release cleanup..." + rm -rf bin/ cli/zig-out/ cli/.zig-cache/ + rm -rf dist/ coverage/ tests/bin/ native/build/ + go clean -cache -testcache + find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + find . -name "*.pyc" -delete 2>/dev/null || true + @./scripts/release/cleanup-testdata.sh 2>/dev/null || true + @./scripts/release/cleanup-logs.sh 2>/dev/null || true + @./scripts/release/cleanup-state.sh 2>/dev/null || true + @echo "${OK} Release cleanup complete" + +# Run release verification checks +release-check: + @./scripts/release/verify-release.sh + +# Full release preparation +prepare-release: clean-release release-check + @echo "${OK} Release preparation complete" + clean-docs: rm -rf docs/_site/ @echo "${OK} Cleaned docs" @@ -360,6 +381,8 @@ help: @echo " make prod-with-native - Build production binaries with native C++ libs" @echo " make dev - Build development binaries (faster)" @echo " make clean - Remove build artifacts" + @echo " make clean-release - Thorough cleanup for release" + @echo " make prepare-release - Full release preparation (cleanup + verify)" @echo "" @echo "Native Library Targets:" @echo " make native-build - Build native C++ libraries" diff --git a/api-server b/api-server deleted file mode 100755 index 2a285e2..0000000 Binary files a/api-server and /dev/null differ diff --git a/scripts/release/cleanup-docker.sh b/scripts/release/cleanup-docker.sh new file mode 100755 index 0000000..4634c8c --- /dev/null +++ b/scripts/release/cleanup-docker.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Docker Compose Cleanup ===" + +# Stop all project-related containers +docker-compose -f deployments/docker-compose.dev.yml down --volumes --remove-orphans 2>/dev/null || true +docker-compose -f deployments/docker-compose.local.yml down --volumes --remove-orphans 2>/dev/null || true +docker-compose -f tests/e2e/docker-compose.logs-debug.yml down --volumes --remove-orphans 2>/dev/null || true + +# Remove project-specific images (keep base images) +docker images --filter "reference=fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true +docker images --filter "reference=*/fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true + +# Remove dangling volumes +docker volume ls -q --filter dangling=true | xargs -r docker volume rm 2>/dev/null || true + +# Prune build cache (keep for 24h) +docker builder prune --keep-duration 24h --force 2>/dev/null || true + +echo "✓ Docker cleanup complete" diff --git a/scripts/release/cleanup-logs.sh b/scripts/release/cleanup-logs.sh new file mode 100755 index 0000000..df06d2c --- /dev/null +++ b/scripts/release/cleanup-logs.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Log Cleanup ===" + +# Remove old logs (keep last 30 days) +find . -name "*.log" -mtime +30 -delete 2>/dev/null || true +find . -name "audit*.log" -mtime +30 -delete 2>/dev/null || true + +# Truncate current logs (keep file, clear content) +find . -name "fetchml*.log" -size +100M -exec sh -c '> {}' \; 2>/dev/null || true + +# Remove crash dumps +rm -f core.* 2>/dev/null || true +rm -f *.prof 2>/dev/null || true +rm -f /tmp/fetchml_*.prof 2>/dev/null || true + +echo "✓ Log cleanup complete" diff --git a/scripts/release/cleanup-podman.sh b/scripts/release/cleanup-podman.sh new file mode 100755 index 0000000..ea5c4a2 --- /dev/null +++ b/scripts/release/cleanup-podman.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Podman Cleanup ===" + +# Stop all fetchml containers +podman ps -a --filter "name=fetchml" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true +podman ps -a --filter "name=ml-*" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true + +# Remove stopped containers +podman ps -a --filter "name=fetchml" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true +podman ps -a --filter "name=ml-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true +podman ps -a --filter "name=jupyter-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true + +# Remove project images +podman images --filter "reference=fetchml*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rmi -f 2>/dev/null || true + +# Remove unused volumes +podman volume ls -q 2>/dev/null | grep -E "(fetchml|jupyter|ml-)" | xargs -r podman volume rm 2>/dev/null || true + +# Clean up pods +podman pod ls --filter "name=ml-" --format "{{.Name}}" 2>/dev/null | xargs -r podman pod rm -f 2>/dev/null || true + +echo "✓ Podman cleanup complete" diff --git a/scripts/release/cleanup-secrets.sh b/scripts/release/cleanup-secrets.sh new file mode 100755 index 0000000..d7b009d --- /dev/null +++ b/scripts/release/cleanup-secrets.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Secret Cleanup ===" + +# Remove any Podman secrets created during testing +podman secret ls --format "{{.Name}}" 2>/dev/null | grep -E "(fetchml|test|dev)" | xargs -r podman secret rm 2>/dev/null || true + +# Clear temporary credential files +rm -f /tmp/fetchml_*_key 2>/dev/null || true +rm -f /tmp/test_*_secret 2>/dev/null || true +rm -f /tmp/*.pem 2>/dev/null || true + +# Reset example config permissions (ensure they're not world-readable) +find configs/ -name "*.yaml" -o -name "*.yml" -o -name "*.toml" 2>/dev/null | while read f; do + if [[ "$f" != *example* ]]; then + chmod 600 "$f" 2>/dev/null || true + fi +done + +echo "✓ Secret cleanup complete" diff --git a/scripts/release/cleanup-state.sh b/scripts/release/cleanup-state.sh new file mode 100755 index 0000000..e953a70 --- /dev/null +++ b/scripts/release/cleanup-state.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +echo "=== State File Cleanup ===" + +# Remove local state files +rm -f data/active/jupyter/fetch_ml_jupyter_*.json 2>/dev/null || true +rm -f data/active/jupyter/fetch_ml_jupyter_workspaces.json 2>/dev/null || true +rm -rf data/active/jupyter/trash/* 2>/dev/null || true + +# Clean queue state +rm -f data/queue/*.tmp 2>/dev/null || true +rm -f data/queue/.*.lock 2>/dev/null || true + +# Remove pid files +rm -f /tmp/fetchml*.pid 2>/dev/null || true + +echo "✓ State cleanup complete" diff --git a/scripts/release/cleanup-testdata.sh b/scripts/release/cleanup-testdata.sh new file mode 100755 index 0000000..bfbdc61 --- /dev/null +++ b/scripts/release/cleanup-testdata.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Test Data Cleanup ===" + +# Remove test experiment data +rm -rf data/experiments/test_* 2>/dev/null || true +rm -rf data/active/workspaces/test_* 2>/dev/null || true + +# Clean up benchmark artifacts +if [ -f ./scripts/maintenance/cleanup-benchmarks.sh ]; then + ./scripts/maintenance/cleanup-benchmarks.sh all 2>/dev/null || true +fi + +# Remove temporary test databases +rm -f /tmp/fetchml_test_*.sqlite 2>/dev/null || true +rm -f /tmp/fetchml_test_*.db 2>/dev/null || true + +# Clean Jupyter service test data +rm -rf /tmp/jupyter-test-* 2>/dev/null || true +rm -f /tmp/fetch_ml_jupyter_*.json 2>/dev/null || true + +echo "✓ Test data cleanup complete" diff --git a/scripts/release/prepare-release.sh b/scripts/release/prepare-release.sh new file mode 100755 index 0000000..8db20f4 --- /dev/null +++ b/scripts/release/prepare-release.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +cd "$PROJECT_ROOT" + +echo "========================================" +echo "FetchML Release Preparation" +echo "========================================" +echo "" + +# Run all cleanup phases +"${SCRIPT_DIR}/cleanup-docker.sh" +"${SCRIPT_DIR}/cleanup-podman.sh" +"${SCRIPT_DIR}/cleanup-secrets.sh" +"${SCRIPT_DIR}/sanitize-configs.sh" +"${SCRIPT_DIR}/cleanup-testdata.sh" +"${SCRIPT_DIR}/cleanup-logs.sh" +"${SCRIPT_DIR}/cleanup-state.sh" + +# Run verification +echo "" +"${SCRIPT_DIR}/verify-release.sh" + +echo "" +echo "========================================" +echo "Release preparation complete!" +echo "========================================" diff --git a/scripts/release/sanitize-configs.sh b/scripts/release/sanitize-configs.sh new file mode 100755 index 0000000..68d2fbc --- /dev/null +++ b/scripts/release/sanitize-configs.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Config File Sanitization ===" + +# Remove any accidentally committed passwords/keys (check only, don't auto-fix) +echo "Checking for potential passwords in configs..." +if grep -r "password:.*[^*]" configs/ --include="*.yaml" --include="*.yml" 2>/dev/null | grep -v "example\|dummy\|changeme\|your_\|\[REDACTED\]"; then + echo "WARNING: Potential passwords found in configs (review above)" +fi + +# Ensure all non-example configs have secure permissions +find configs/ -type f \( -name "*.yaml" -o -name "*.yml" -o -name "*.toml" \) ! -name "*example*" ! -name "*schema*" -exec chmod 600 {} \; 2>/dev/null || true + +# Remove temp config files +rm -f configs/.tmp.* 2>/dev/null || true +rm -f configs/api/.local.* 2>/dev/null || true + +# Validate no real credentials in examples +if grep -rE "(sk-[a-zA-Z0-9]{20,}|password: [^\"'*]+[^*])" configs/examples/ 2>/dev/null | grep -v "example\|dummy\|changeme\|your_"; then + echo "WARNING: Potential real credentials found in example configs!" +fi + +echo "✓ Config sanitization complete" diff --git a/scripts/release/verify-release.sh b/scripts/release/verify-release.sh new file mode 100755 index 0000000..2c685ac --- /dev/null +++ b/scripts/release/verify-release.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +FAILED=0 +echo "=== Release Verification ===" + +# Check 1: No real credentials in configs (allow empty strings) +echo "Checking for credentials in configs..." +if grep -r "password:.*[^*\"' ]" configs/ --include="*.yaml" --include="*.yml" 2>/dev/null | grep -v "example\|schema\|changeme\|your_\|\[REDACTED\]\|password: \"\"\|password: ''"; then + echo "✗ FAIL: Potential passwords found in configs" + FAILED=1 +fi + +# Check 2: Config file permissions +echo "Checking config permissions..." +find configs/ -name "*.yaml" ! -name "*example*" ! -name "*schema*" -print0 2>/dev/null | while IFS= read -r -d '' f; do + PERM=$(stat -c %a "$f" 2>/dev/null || stat -f %A "$f") + if [ "$PERM" != "600" ]; then + echo "✗ FAIL: $f has permissions $PERM (expected 600)" + FAILED=1 + fi +done + +# Check 3: No uncommitted changes in configs +echo "Checking for uncommitted config changes..." +if git diff --name-only 2>/dev/null | grep -q "configs/"; then + echo "WARNING: Uncommitted changes in configs/" +fi + +# Check 4: Docker containers stopped +echo "Checking Docker containers..." +if docker ps --filter "name=fetchml" --format "{{.Names}}" 2>/dev/null | grep -q .; then + echo "WARNING: Running FetchML Docker containers detected" +fi + +# Check 5: Podman containers stopped +echo "Checking Podman containers..." +if podman ps --filter "name=fetchml" --format "{{.Names}}" 2>/dev/null | grep -q .; then + echo "WARNING: Running FetchML Podman containers detected" +fi + +# Check 6: No .env files committed +echo "Checking for .env files in git..." +if git ls-files 2>/dev/null | grep -E "^\.env" | grep -v "example"; then + echo "✗ FAIL: .env files found in git" + FAILED=1 +fi + +# Check 7: Binary is not committed +echo "Checking for committed binaries..." +if git ls-files 2>/dev/null | grep -E "^(api-server|worker|bin/)"; then + echo "✗ FAIL: Binaries found in git" + FAILED=1 +fi + +# Check 8: Security audit passes +echo "Running security audit..." +if [ -f ./api-server ]; then + ./api-server --security-audit 2>&1 | grep -q "All security checks passed" || { + echo "✗ FAIL: Security audit did not pass" + FAILED=1 + } +fi + +if [ $FAILED -eq 0 ]; then + echo "✓ All release checks passed" + exit 0 +else + echo "✗ Release checks failed" + exit 1 +fi