diff --git a/Makefile b/Makefile index 3f4707a..f668bfd 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ native-test: native-build # Run native libraries smoke test (builds + C++ tests + Go integration) native-smoke: - @bash ./scripts/smoke-test-native.sh + @bash ./scripts/dev/smoke-test.sh --native @echo "${OK} Native smoke test passed" # Build production-optimized binaries @@ -133,14 +133,14 @@ clean-release: clean go clean -cache -testcache find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true find . -name "*.pyc" -delete 2>/dev/null || true - @./scripts/release/cleanup-testdata.sh 2>/dev/null || true - @./scripts/release/cleanup-logs.sh 2>/dev/null || true - @./scripts/release/cleanup-state.sh 2>/dev/null || true + @./scripts/release/cleanup.sh testdata 2>/dev/null || true + @./scripts/release/cleanup.sh logs 2>/dev/null || true + @./scripts/release/cleanup.sh state 2>/dev/null || true @echo "${OK} Release cleanup complete" # Run release verification checks release-check: - @./scripts/release/verify-release.sh + @./scripts/release/verify.sh # Full release preparation prepare-release: clean-release release-check @@ -193,16 +193,16 @@ worker-configlint: configs/workers/homelab-secure.yaml dev-smoke: - bash ./scripts/smoke-test.sh dev + bash ./scripts/dev/smoke-test.sh dev @echo "dev smoke: OK" prod-smoke: - bash ./scripts/smoke-test.sh prod + bash ./scripts/dev/smoke-test.sh prod @echo "prod smoke: OK" # Run maintainability CI checks ci-checks: - @bash ./scripts/ci-checks.sh + @bash ./scripts/ci/checks.sh # Run a local approximation of the CI pipeline ci-local: ci-checks test lint configlint worker-configlint @@ -286,7 +286,7 @@ benchmark-compare: # Manage benchmark artifacts artifacts: @echo "Managing benchmark artifacts..." - ./scripts/manage-artifacts.sh help + ./scripts/dev/manage-artifacts.sh help # Clean benchmark artifacts (keep last 10) clean-benchmarks: diff --git a/scripts/README.md b/scripts/README.md index 4ced778..c31606f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,77 +1,123 @@ # Scripts Directory -This directory contains setup and utility scripts for FetchML. +Organized utility scripts for FetchML development, testing, and release management. -## Production Scripts +## Development Scripts (`dev/`) -### `setup-prod.sh` -**Purpose**: Automated production setup for Rocky Linux bare metal deployment -**Usage**: `sudo ./scripts/setup-prod.sh [base_path] [user] [group]` -**What it does**: -- Creates system user and groups -- Sets up directory structure (`/data/ml-experiments/*`) -- Installs dependencies (Go, Podman, Redis) -- Configures GPU support for Podman -- Creates systemd service files -- Sets up log rotation +### `smoke-test.sh` +**Purpose**: End-to-end smoke testing for dev/prod environments +**Usage**: `./scripts/dev/smoke-test.sh [dev|prod] [--native]` -**Example**: +Tests full stack health by starting docker-compose services and verifying: +- API health endpoints +- Prometheus metrics ingestion (dev mode) +- Native library integration (with `--native` flag) + +**Examples**: ```bash -sudo ./scripts/setup-prod.sh /data/ml-experiments ml-user ml-group +./scripts/dev/smoke-test.sh dev # Standard smoke test +./scripts/dev/smoke-test.sh prod --native # Prod with native libs ``` -### Configuration validation -Validate configs using the built-in config lint targets: +### `manage-artifacts.sh` +**Purpose**: Local benchmark artifact management +**Usage**: `./scripts/dev/manage-artifacts.sh [list|clean|compare|export]` -```bash -make configlint -make worker-configlint -``` +Manages benchmark run artifacts in `.local-artifacts/`. +--- -### Cleanup Recommendation -These legacy scripts can be removed or archived. The current production setup only needs: -- `setup-prod.sh` +## CI Scripts (`ci/`) -## Usage Workflow +### `checks.sh` +**Purpose**: Maintainability enforcement +**Usage**: `./scripts/ci/checks.sh` or `make ci-checks` -### First-Time Production Setup -```bash -# 1. Run production setup -sudo ./scripts/setup-prod.sh +### `test.sh` +**Purpose**: CI test runner +**Usage**: `./scripts/ci/test.sh` -# 2. Copy and configure -sudo cp configs/api/prod.yaml /etc/fetch_ml/config.yaml -sudo cp configs/workers/worker-prod.toml /etc/fetch_ml/worker.toml -sudo vim /etc/fetch_ml/config.yaml # Update API keys, etc. +### `verify-paths.sh` +**Purpose**: Path convention verification +**Usage**: `./scripts/ci/verify-paths.sh` -# 3. Build and install -make prod -sudo make install +--- -# 4. Validate -./bin/configlint --schema configs/schema/api_server_config.yaml /etc/fetch_ml/config.yaml -./bin/configlint --schema configs/schema/worker_config_schema.yaml /etc/fetch_ml/worker.toml +## Release Scripts (`release/`) -# 5. Start services -sudo systemctl start fetchml-api fetchml-worker -sudo systemctl enable fetchml-api fetchml-worker -``` +### `cleanup.sh` +**Purpose**: Unified cleanup utility +**Usage**: `./scripts/release/cleanup.sh [all|docker|podman|logs|secrets|state|testdata]` -### Development Setup (macOS) -```bash -# Use docker-compose for local development -docker-compose up -d +### `prepare.sh` +**Purpose**: Release preparation +**Usage**: `./scripts/release/prepare.sh` -# Or run components directly -make dev -./bin/api-server -config configs/api/dev.yaml -``` +### `sanitize.sh` +**Purpose**: Config sanitization +**Usage**: `./scripts/release/sanitize.sh` -## Script Maintenance +### `verify.sh` +**Purpose**: Pre-release verification +**Usage**: `./scripts/release/verify.sh` or `make release-check` -When adding new scripts: -1. Add executable permission: `chmod +x scripts/new-script.sh` -2. Add header comment with purpose and usage -3. Update this README -4. Use consistent error handling and logging +--- + +## Testing Scripts (`testing/`) + +### `gen-ssh-test-keys.sh` +**Purpose**: Generate SSH keys for TUI testing +**Usage**: `./scripts/testing/gen-ssh-test-keys.sh` + +### `tui-ssh-test.sh` +**Purpose**: Manual TUI SSH testing +**Usage**: `./scripts/testing/tui-ssh-test.sh` + +### `test-prod.sh` +**Purpose**: Production validation +**Usage**: `./scripts/testing/test-prod.sh` + +--- + +## Benchmark Scripts (`benchmarks/`) + +### `run-benchmarks-local.sh` +**Purpose**: Local benchmark execution +**Usage**: `./scripts/benchmarks/run-benchmarks-local.sh` + +--- + +## Maintenance Scripts (`maintenance/`) + +### `cleanup.sh` +**Purpose**: Archive cleanup +**Usage**: `./scripts/maintenance/cleanup.sh [all|dry-run]` + +### `cleanup-benchmarks.sh` +**Purpose**: Benchmark lifecycle management +**Usage**: `./scripts/maintenance/cleanup-benchmarks.sh [benchmarks|all|aggressive|status]` + +--- + +## Shared Library (`lib/`) + +### `common.sh` +**Purpose**: Common shell functions +**Usage**: `source "$(dirname "$0")/lib/common.sh"` + +--- + +## Legacy Consolidation + +| Old Script | New Location | +|------------|--------------| +| `ci-checks.sh` | `ci/checks.sh` | +| `ci-test.sh` | `ci/test.sh` | +| `smoke-test.sh` | `dev/smoke-test.sh` | +| `smoke-test-native.sh` | *merged into dev/smoke-test.sh --native* | +| `manage-artifacts.sh` | `dev/manage-artifacts.sh` | +| `verify-paths.sh` | `ci/verify-paths.sh` | +| `release/verify-release.sh` | `release/verify.sh` | +| `release/prepare-release.sh` | `release/prepare.sh` | +| `release/sanitize-configs.sh` | `release/sanitize.sh` | +| `release/cleanup-*.sh` (6 files) | `release/cleanup.sh` | diff --git a/scripts/track_performance.sh b/scripts/benchmarks/track-performance.sh similarity index 100% rename from scripts/track_performance.sh rename to scripts/benchmarks/track-performance.sh diff --git a/scripts/ci-checks.sh b/scripts/ci/checks.sh similarity index 100% rename from scripts/ci-checks.sh rename to scripts/ci/checks.sh diff --git a/scripts/ci-test.sh b/scripts/ci/test.sh similarity index 100% rename from scripts/ci-test.sh rename to scripts/ci/test.sh diff --git a/scripts/verify-paths.sh b/scripts/ci/verify-paths.sh similarity index 100% rename from scripts/verify-paths.sh rename to scripts/ci/verify-paths.sh diff --git a/scripts/manage-artifacts.sh b/scripts/dev/manage-artifacts.sh similarity index 100% rename from scripts/manage-artifacts.sh rename to scripts/dev/manage-artifacts.sh diff --git a/scripts/smoke-test.sh b/scripts/dev/smoke-test.sh similarity index 64% rename from scripts/smoke-test.sh rename to scripts/dev/smoke-test.sh index 760fffb..517df63 100644 --- a/scripts/smoke-test.sh +++ b/scripts/dev/smoke-test.sh @@ -1,13 +1,73 @@ #!/usr/bin/env bash set -euo pipefail; -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" export FETCHML_REPO_ROOT="$repo_root" -env="${1:-dev}"; -if [ "$env" != "dev" ] && [ "$env" != "prod" ]; then - echo "usage: $0 [dev|prod]" >&2 - exit 2 +# Parse arguments +env="dev" +native_mode=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --native) + native_mode=true + shift + ;; + dev|prod) + env="$1" + shift + ;; + --help|-h) + echo "Usage: $0 [dev|prod] [--native]" + echo "" + echo "Options:" + echo " dev|prod Environment to test (default: dev)" + echo " --native Also test native libraries (C++ integration)" + echo " --help Show this help" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: $0 [dev|prod] [--native]" >&2 + exit 2 + ;; + esac +done + +# Native library smoke test (merged from smoke-test-native.sh) +if [[ "$native_mode" == true ]]; then + echo "=== FetchML Native Libraries Smoke Test ===" + echo "" + + cd "$repo_root" + + # Build native libraries + echo "1. Building native libraries..." + if [[ -d native/build ]]; then + cd native/build + cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_ASAN=OFF >/dev/null 2>&1 || true + make -j4 2>&1 | grep -E "(Built|Error|error)" || true + cd ../.. + echo " ✓ Native libraries built" + else + echo " ⚠ native/build not found, skipping native build" + fi + echo "" + + # Run C++ unit tests + if [[ -f ./native/build/queue_index/test_storage ]]; then + echo "2. Running C++ smoke tests..." + ./native/build/queue_index/test_storage 2>/dev/null || echo " ⚠ C++ tests skipped" + echo "" + fi + + # Build Go with native libs + echo "3. Building Go applications with native libs..." + FETCHML_NATIVE_LIBS=1 go build -o /dev/null ./cmd/api-server 2>&1 | grep -v "ignoring duplicate" || true + echo " ✓ api-server builds" + FETCHML_NATIVE_LIBS=1 go build -o /dev/null ./cmd/worker 2>&1 | grep -v "ignoring duplicate" || true 2>/dev/null || echo " (worker optional)" + echo "" fi probe_https_health_openssl() { diff --git a/scripts/release/cleanup-docker.sh b/scripts/release/cleanup-docker.sh deleted file mode 100755 index 4634c8c..0000000 --- a/scripts/release/cleanup-docker.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== Docker Compose Cleanup ===" - -# Stop all project-related containers -docker-compose -f deployments/docker-compose.dev.yml down --volumes --remove-orphans 2>/dev/null || true -docker-compose -f deployments/docker-compose.local.yml down --volumes --remove-orphans 2>/dev/null || true -docker-compose -f tests/e2e/docker-compose.logs-debug.yml down --volumes --remove-orphans 2>/dev/null || true - -# Remove project-specific images (keep base images) -docker images --filter "reference=fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true -docker images --filter "reference=*/fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true - -# Remove dangling volumes -docker volume ls -q --filter dangling=true | xargs -r docker volume rm 2>/dev/null || true - -# Prune build cache (keep for 24h) -docker builder prune --keep-duration 24h --force 2>/dev/null || true - -echo "✓ Docker cleanup complete" diff --git a/scripts/release/cleanup-logs.sh b/scripts/release/cleanup-logs.sh deleted file mode 100755 index df06d2c..0000000 --- a/scripts/release/cleanup-logs.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== Log Cleanup ===" - -# Remove old logs (keep last 30 days) -find . -name "*.log" -mtime +30 -delete 2>/dev/null || true -find . -name "audit*.log" -mtime +30 -delete 2>/dev/null || true - -# Truncate current logs (keep file, clear content) -find . -name "fetchml*.log" -size +100M -exec sh -c '> {}' \; 2>/dev/null || true - -# Remove crash dumps -rm -f core.* 2>/dev/null || true -rm -f *.prof 2>/dev/null || true -rm -f /tmp/fetchml_*.prof 2>/dev/null || true - -echo "✓ Log cleanup complete" diff --git a/scripts/release/cleanup-podman.sh b/scripts/release/cleanup-podman.sh deleted file mode 100755 index ea5c4a2..0000000 --- a/scripts/release/cleanup-podman.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== Podman Cleanup ===" - -# Stop all fetchml containers -podman ps -a --filter "name=fetchml" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true -podman ps -a --filter "name=ml-*" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true - -# Remove stopped containers -podman ps -a --filter "name=fetchml" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true -podman ps -a --filter "name=ml-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true -podman ps -a --filter "name=jupyter-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true - -# Remove project images -podman images --filter "reference=fetchml*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rmi -f 2>/dev/null || true - -# Remove unused volumes -podman volume ls -q 2>/dev/null | grep -E "(fetchml|jupyter|ml-)" | xargs -r podman volume rm 2>/dev/null || true - -# Clean up pods -podman pod ls --filter "name=ml-" --format "{{.Name}}" 2>/dev/null | xargs -r podman pod rm -f 2>/dev/null || true - -echo "✓ Podman cleanup complete" diff --git a/scripts/release/cleanup-secrets.sh b/scripts/release/cleanup-secrets.sh deleted file mode 100755 index d7b009d..0000000 --- a/scripts/release/cleanup-secrets.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== Secret Cleanup ===" - -# Remove any Podman secrets created during testing -podman secret ls --format "{{.Name}}" 2>/dev/null | grep -E "(fetchml|test|dev)" | xargs -r podman secret rm 2>/dev/null || true - -# Clear temporary credential files -rm -f /tmp/fetchml_*_key 2>/dev/null || true -rm -f /tmp/test_*_secret 2>/dev/null || true -rm -f /tmp/*.pem 2>/dev/null || true - -# Reset example config permissions (ensure they're not world-readable) -find configs/ -name "*.yaml" -o -name "*.yml" -o -name "*.toml" 2>/dev/null | while read f; do - if [[ "$f" != *example* ]]; then - chmod 600 "$f" 2>/dev/null || true - fi -done - -echo "✓ Secret cleanup complete" diff --git a/scripts/release/cleanup-state.sh b/scripts/release/cleanup-state.sh deleted file mode 100755 index e953a70..0000000 --- a/scripts/release/cleanup-state.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== State File Cleanup ===" - -# Remove local state files -rm -f data/active/jupyter/fetch_ml_jupyter_*.json 2>/dev/null || true -rm -f data/active/jupyter/fetch_ml_jupyter_workspaces.json 2>/dev/null || true -rm -rf data/active/jupyter/trash/* 2>/dev/null || true - -# Clean queue state -rm -f data/queue/*.tmp 2>/dev/null || true -rm -f data/queue/.*.lock 2>/dev/null || true - -# Remove pid files -rm -f /tmp/fetchml*.pid 2>/dev/null || true - -echo "✓ State cleanup complete" diff --git a/scripts/release/cleanup-testdata.sh b/scripts/release/cleanup-testdata.sh deleted file mode 100755 index bfbdc61..0000000 --- a/scripts/release/cleanup-testdata.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail - -echo "=== Test Data Cleanup ===" - -# Remove test experiment data -rm -rf data/experiments/test_* 2>/dev/null || true -rm -rf data/active/workspaces/test_* 2>/dev/null || true - -# Clean up benchmark artifacts -if [ -f ./scripts/maintenance/cleanup-benchmarks.sh ]; then - ./scripts/maintenance/cleanup-benchmarks.sh all 2>/dev/null || true -fi - -# Remove temporary test databases -rm -f /tmp/fetchml_test_*.sqlite 2>/dev/null || true -rm -f /tmp/fetchml_test_*.db 2>/dev/null || true - -# Clean Jupyter service test data -rm -rf /tmp/jupyter-test-* 2>/dev/null || true -rm -f /tmp/fetch_ml_jupyter_*.json 2>/dev/null || true - -echo "✓ Test data cleanup complete" diff --git a/scripts/release/cleanup.sh b/scripts/release/cleanup.sh new file mode 100755 index 0000000..ba64a85 --- /dev/null +++ b/scripts/release/cleanup.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# Unified cleanup script for release preparation +# Usage: ./scripts/release/cleanup.sh [all|docker|podman|logs|secrets|state|testdata|list] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "$REPO_ROOT" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_header() { + echo -e "${BLUE}=== $1 ===${NC}" +} + +print_success() { + echo -e "${GREEN}✓${NC} $1" +} + +print_warn() { + echo -e "${YELLOW}⚠${NC} $1" +} + +print_error() { + echo -e "${RED}✗${NC} $1" +} + +# Docker cleanup +cleanup_docker() { + print_header "Docker Compose Cleanup" + + # Stop all project-related containers + docker-compose -f deployments/docker-compose.dev.yml down --volumes --remove-orphans 2>/dev/null || true + docker-compose -f deployments/docker-compose.local.yml down --volumes --remove-orphans 2>/dev/null || true + docker-compose -f tests/e2e/docker-compose.logs-debug.yml down --volumes --remove-orphans 2>/dev/null || true + + # Remove project-specific images (keep base images) + docker images --filter "reference=fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true + docker images --filter "reference=*/fetchml*" --format "{{.ID}}" | xargs -r docker rmi -f 2>/dev/null || true + + # Remove dangling volumes + docker volume ls -q --filter dangling=true | xargs -r docker volume rm 2>/dev/null || true + + # Prune build cache (keep for 24h) + docker builder prune --keep-duration 24h --force 2>/dev/null || true + + print_success "Docker cleanup complete" +} + +# Podman cleanup +cleanup_podman() { + print_header "Podman Cleanup" + + # Stop all fetchml containers + podman ps -a --filter "name=fetchml" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true + podman ps -a --filter "name=ml-*" --format "{{.Names}}" 2>/dev/null | xargs -r podman stop 2>/dev/null || true + + # Remove stopped containers + podman ps -a --filter "name=fetchml" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true + podman ps -a --filter "name=ml-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true + podman ps -a --filter "name=jupyter-*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rm 2>/dev/null || true + + # Remove project images + podman images --filter "reference=fetchml*" --format "{{.ID}}" 2>/dev/null | xargs -r podman rmi -f 2>/dev/null || true + + # Remove unused volumes + podman volume ls -q 2>/dev/null | grep -E "(fetchml|jupyter|ml-)" | xargs -r podman volume rm 2>/dev/null || true + + # Clean up pods + podman pod ls --filter "name=ml-" --format "{{.Name}}" 2>/dev/null | xargs -r podman pod rm -f 2>/dev/null || true + + print_success "Podman cleanup complete" +} + +# Log cleanup +cleanup_logs() { + print_header "Log Cleanup" + + # Remove old logs (keep last 30 days) + find . -name "*.log" -mtime +30 -delete 2>/dev/null || true + find . -name "audit*.log" -mtime +30 -delete 2>/dev/null || true + + # Truncate current logs (keep file, clear content) + find . -name "fetchml*.log" -size +100M -exec sh -c '> {}' \; 2>/dev/null || true + + # Remove crash dumps + rm -f core.* 2>/dev/null || true + rm -f *.prof 2>/dev/null || true + rm -f /tmp/fetchml_*.prof 2>/dev/null || true + + print_success "Log cleanup complete" +} + +# Secret cleanup +cleanup_secrets() { + print_header "Secret Cleanup" + + # Remove any Podman secrets created during testing + podman secret ls --format "{{.Name}}" 2>/dev/null | grep -E "(fetchml|test|dev)" | xargs -r podman secret rm 2>/dev/null || true + + # Clear temporary credential files + rm -f /tmp/fetchml_*_key 2>/dev/null || true + rm -f /tmp/test_*_secret 2>/dev/null || true + rm -f /tmp/*.pem 2>/dev/null || true + + # Reset example config permissions (ensure they're not world-readable) + find configs/ -name "*.yaml" -o -name "*.yml" -o -name "*.toml" 2>/dev/null | while read f; do + if [[ "$f" != *example* ]]; then + chmod 600 "$f" 2>/dev/null || true + fi + done + + print_success "Secret cleanup complete" +} + +# State cleanup +cleanup_state() { + print_header "State File Cleanup" + + # Remove local state files + rm -f data/active/jupyter/fetch_ml_jupyter_*.json 2>/dev/null || true + rm -f data/active/jupyter/fetch_ml_jupyter_workspaces.json 2>/dev/null || true + rm -rf data/active/jupyter/trash/* 2>/dev/null || true + + # Clean queue state + rm -f data/queue/*.tmp 2>/dev/null || true + rm -f data/queue/.*.lock 2>/dev/null || true + + # Remove pid files + rm -f /tmp/fetchml*.pid 2>/dev/null || true + + print_success "State cleanup complete" +} + +# Test data cleanup +cleanup_testdata() { + print_header "Test Data Cleanup" + + # Remove test experiment data + rm -rf data/experiments/test_* 2>/dev/null || true + rm -rf data/active/workspaces/test_* 2>/dev/null || true + + # Clean up benchmark artifacts + if [ -f ./scripts/maintenance/cleanup-benchmarks.sh ]; then + ./scripts/maintenance/cleanup-benchmarks.sh all 2>/dev/null || true + fi + + # Remove temporary test databases + rm -f /tmp/fetchml_test_*.sqlite 2>/dev/null || true + rm -f /tmp/fetchml_test_*.db 2>/dev/null || true + + # Clean Jupyter service test data + rm -rf /tmp/jupyter-test-* 2>/dev/null || true + rm -f /tmp/fetch_ml_jupyter_*.json 2>/dev/null || true + + print_success "Test data cleanup complete" +} + +# List available cleanup targets +list_targets() { + echo "Available cleanup targets:" + echo " all - Run all cleanup operations" + echo " docker - Clean Docker containers, images, and volumes" + echo " podman - Clean Podman containers, images, and pods" + echo " logs - Remove old logs and crash dumps" + echo " secrets - Remove temporary credentials and reset permissions" + echo " state - Clean runtime state files and PID files" + echo " testdata - Remove test data and temporary databases" +} + +# Show usage +usage() { + echo "Usage: $0 [TARGET]" + echo "" + list_targets + echo "" + echo "Examples:" + echo " $0 all # Clean everything (default)" + echo " $0 docker # Clean only Docker resources" + echo " $0 list # Show available targets" +} + +# Main +main() { + local target="${1:-all}" + + case "$target" in + all) + cleanup_docker + cleanup_podman + cleanup_logs + cleanup_secrets + cleanup_state + cleanup_testdata + echo "" + print_header "All cleanup operations complete" + ;; + docker) + cleanup_docker + ;; + podman) + cleanup_podman + ;; + logs) + cleanup_logs + ;; + secrets) + cleanup_secrets + ;; + state) + cleanup_state + ;; + testdata) + cleanup_testdata + ;; + list|--list|-l) + list_targets + ;; + help|--help|-h) + usage + exit 0 + ;; + *) + print_error "Unknown target: $target" + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/release/prepare-release.sh b/scripts/release/prepare.sh similarity index 100% rename from scripts/release/prepare-release.sh rename to scripts/release/prepare.sh diff --git a/scripts/release/sanitize-configs.sh b/scripts/release/sanitize.sh similarity index 100% rename from scripts/release/sanitize-configs.sh rename to scripts/release/sanitize.sh diff --git a/scripts/verify_release.sh b/scripts/release/verify-checksums.sh similarity index 100% rename from scripts/verify_release.sh rename to scripts/release/verify-checksums.sh diff --git a/scripts/release/verify-release.sh b/scripts/release/verify.sh similarity index 100% rename from scripts/release/verify-release.sh rename to scripts/release/verify.sh diff --git a/scripts/smoke-test-native.sh b/scripts/smoke-test-native.sh deleted file mode 100644 index c0568aa..0000000 --- a/scripts/smoke-test-native.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "=== FetchML Native Libraries Smoke Test ===" -echo "" - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$repo_root" - -# Build native libraries -echo "1. Building native libraries..." -cd native/build -cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_ASAN=OFF >/dev/null 2>&1 -make -j4 2>&1 | grep -E "(Built|Error|error)" || true -cd ../.. -echo " ✓ Native libraries built" -echo "" - -# Run C++ unit tests -echo "2. Running C++ smoke tests..." -./native/build/queue_index/test_storage -echo "" - -# Build Go with native libs -echo "3. Building Go applications with native libs..." -FETCHML_NATIVE_LIBS=1 go build -o /dev/null ./cmd/api-server 2>&1 | grep -v "ignoring duplicate" || true -echo " ✓ api-server builds" -FETCHML_NATIVE_LIBS=1 go build -o /dev/null ./cmd/worker 2>&1 | grep -v "ignoring duplicate" || true 2>/dev/null || echo " (worker optional)" -echo "" - -# Run Go native queue benchmark -echo "4. Running native queue benchmark..." -FETCHML_NATIVE_LIBS=1 go test -bench=BenchmarkNativeQueueBasic -benchtime=1s ./tests/benchmarks/native_queue_basic_test.go 2>&1 | grep -E "(BenchmarkNative|PASS|ns/op)" -echo "" - -# Run E2E tests -echo "5. Running E2E smoke tests..." -FETCHML_NATIVE_LIBS=1 go test -v -run "TestExample" ./tests/e2e/... 2>&1 | grep -E "(RUN|PASS|FAIL)" | head -15 -echo "" - -echo "=== All Smoke Tests Passed ===" -echo "Native libraries: ENABLED (SIMD: ARMv8)" -echo "Integration: WORKING"