From c80e01b752854a7b06c2339acf79ea9650663edf Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Sat, 6 Dec 2025 12:40:35 -0500 Subject: [PATCH] Add comprehensive self-cleaning system - Add cleanup.sh script with dry-run, force, and all options - Add auto-cleanup service setup for macOS (launchd) and Linux (systemd) - Add cleanup-status.sh for monitoring Docker resources - Add Makefile targets: self-cleanup, auto-cleanup - Features colored output, confirmation prompts, and detailed logging - Auto-cleanup runs daily to keep system clean - Status monitoring shows resources and service state --- Makefile | 14 ++- scripts/auto-cleanup.service | 15 +++ scripts/auto-cleanup.timer | 11 ++ scripts/cleanup-status.sh | 96 +++++++++++++++ scripts/cleanup.sh | 219 ++++++++++++++++++++++++++++++++++ scripts/setup-auto-cleanup.sh | 90 ++++++++++++++ 6 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 scripts/auto-cleanup.service create mode 100644 scripts/auto-cleanup.timer create mode 100755 scripts/cleanup-status.sh create mode 100755 scripts/cleanup.sh create mode 100755 scripts/setup-auto-cleanup.sh diff --git a/Makefile b/Makefile index 45501f1..5abd2a2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance +.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance self-cleanup auto-cleanup # Default target all: build @@ -325,4 +325,16 @@ help: @echo "" @echo "Utility:" @echo " make size - Show binary sizes" + @echo " make self-cleanup - Clean up Docker resources" + @echo " make auto-cleanup - Setup daily auto-cleanup service" @echo " make help - Show this help" + +# Self-cleaning for Docker resources +self-cleanup: + @echo "Running self-cleanup..." + @./scripts/cleanup.sh + +# Setup auto-cleanup service +auto-cleanup: + @echo "Setting up auto-cleanup service..." + @./scripts/setup-auto-cleanup.sh diff --git a/scripts/auto-cleanup.service b/scripts/auto-cleanup.service new file mode 100644 index 0000000..f6694c9 --- /dev/null +++ b/scripts/auto-cleanup.service @@ -0,0 +1,15 @@ +[Unit] +Description=FetchML Auto Cleanup Service +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +ExecStart=/Users/jfraeys/Documents/dev/fetch_ml/scripts/cleanup.sh --force +User=jfraeys +Group=staff +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=timers.target diff --git a/scripts/auto-cleanup.timer b/scripts/auto-cleanup.timer new file mode 100644 index 0000000..b7b5bde --- /dev/null +++ b/scripts/auto-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run FetchML Auto Cleanup daily +Requires=auto-cleanup.service + +[Timer] +OnCalendar=daily +Persistent=true +RandomizedDelaySec=1h + +[Install] +WantedBy=timers.target diff --git a/scripts/cleanup-status.sh b/scripts/cleanup-status.sh new file mode 100755 index 0000000..39a6744 --- /dev/null +++ b/scripts/cleanup-status.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# Check status of Docker resources and auto-cleanup service + +set -euo pipefail + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +echo "=== FetchML Cleanup Status ===" +echo "" + +# Docker resources +log_info "Docker Resources:" +containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l | tr -d ' ') +images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l | tr -d ' ') +networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ') +volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ') + +echo " Containers: $containers" +echo " Images: $images" +echo " Networks: $networks" +echo " Volumes: $volumes" + +if [ "$containers" -gt 0 ]; then + echo "" + log_info "Active containers:" + docker ps --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" +fi + +# Auto-cleanup service status +echo "" +log_info "Auto-cleanup Service:" + +if [[ "$OSTYPE" == "darwin"* ]]; then + if launchctl list | grep -q "com.fetchml.cleanup"; then + log_success "Auto-cleanup service is running" + echo " Logs: /tmp/fetchml-cleanup.log" + echo " To check logs: tail -f /tmp/fetchml-cleanup.log" + else + log_warning "Auto-cleanup service is not installed" + echo " To install: make auto-cleanup" + fi +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + if systemctl is-active --quiet auto-cleanup.timer; then + log_success "Auto-cleanup timer is active" + echo " Status: systemctl status auto-cleanup.timer" + echo " Logs: journalctl -u auto-cleanup.service" + else + log_warning "Auto-cleanup timer is not active" + echo " To install: make auto-cleanup" + fi +fi + +# Disk usage +echo "" +log_info "Docker Disk Usage:" +docker system df --format "table {{.Type}}\t{{.TotalCount}}\t{{.Size}}\t{{.Reclaimable}}" + +# Quick cleanup suggestion +echo "" +if [ "$containers" -gt 0 ] || [ "$images" -gt 0 ] || [ "$networks" -gt 0 ] || [ "$volumes" -gt 0 ]; then + log_warning "Resources found that can be cleaned up" + echo " Quick cleanup: make self-cleanup" + echo " Force cleanup: make self-cleanup --force" + echo " Full cleanup: make self-cleanup --all" +else + log_success "No fetch_ml resources found - system is clean!" +fi + +echo "" +echo "=== Cleanup Commands ===" +echo " make self-cleanup - Interactive cleanup" +echo " ./scripts/cleanup.sh --dry-run - Preview what would be cleaned" +echo " ./scripts/cleanup.sh --force - Force cleanup without prompts" +echo " ./scripts/cleanup.sh --all - Clean everything including images" diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh new file mode 100755 index 0000000..ed7dbd5 --- /dev/null +++ b/scripts/cleanup.sh @@ -0,0 +1,219 @@ +#!/bin/bash + +# Self-cleaning script for fetch_ml Docker resources +# Usage: ./scripts/cleanup.sh [--dry-run] [--force] [--all] + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Default options +DRY_RUN=false +FORCE=false +ALL=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN=true + shift + ;; + --force) + FORCE=true + shift + ;; + --all) + ALL=true + shift + ;; + --help|-h) + echo "Usage: $0 [--dry-run] [--force] [--all]" + echo "" + echo "Options:" + echo " --dry-run Show what would be cleaned without actually doing it" + echo " --force Skip confirmation prompts" + echo " --all Clean everything including images" + echo "" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Helper functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Docker check +if ! command -v docker &> /dev/null; then + log_error "Docker is not installed or not in PATH" + exit 1 +fi + +# Check if Docker is running +if ! docker info &> /dev/null; then + log_error "Docker daemon is not running" + exit 1 +fi + +log_info "Starting cleanup of fetch_ml Docker resources..." + +# Function to count resources +count_resources() { + local containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l) + local images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l) + local networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l) + local volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l) + + echo "Containers: $containers, Images: $images, Networks: $networks, Volumes: $volumes" +} + +# Show current state +log_info "Current resources: $(count_resources)" + +# Dry run mode +if [ "$DRY_RUN" = true ]; then + log_info "DRY RUN MODE - No actual cleanup will be performed" + + echo "" + log_info "Containers that would be stopped and removed:" + docker ps -a --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || echo "No containers found" + + echo "" + log_info "Images that would be removed:" + if [ "$ALL" = true ]; then + docker images --filter "reference=fetch_ml-*" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" || echo "No images found" + else + echo "Use --all to remove images" + fi + + echo "" + log_info "Networks that would be removed:" + docker network ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No networks found" + + echo "" + log_info "Volumes that would be removed:" + docker volume ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No volumes found" + + exit 0 +fi + +# Confirmation prompt +if [ "$FORCE" = false ]; then + echo "" + read -p "This will stop and remove all fetch_ml containers. Continue? (y/N): " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Cleanup cancelled" + exit 0 + fi +fi + +# Stop containers +log_info "Stopping containers..." +containers=$(docker ps -q --filter "name=ml-") +if [ -n "$containers" ]; then + if [ "$DRY_RUN" = false ]; then + echo "$containers" | xargs docker stop + log_success "Containers stopped" + fi +else + log_info "No running containers found" +fi + +# Remove containers +log_info "Removing containers..." +containers=$(docker ps -aq --filter "name=ml-") +if [ -n "$containers" ]; then + if [ "$DRY_RUN" = false ]; then + echo "$containers" | xargs docker rm -f + log_success "Containers removed" + fi +else + log_info "No containers found" +fi + +# Remove networks +log_info "Removing networks..." +networks=$(docker network ls -q --filter "name=ml-") +if [ -n "$networks" ]; then + if [ "$DRY_RUN" = false ]; then + echo "$networks" | xargs docker network rm + log_success "Networks removed" + fi +else + log_info "No networks found" +fi + +# Remove volumes (with caution) +log_warning "Removing volumes (this will delete data)..." +if [ "$FORCE" = true ] || [ "$ALL" = true ]; then + volumes=$(docker volume ls -q --filter "name=ml-") + if [ -n "$volumes" ]; then + if [ "$DRY_RUN" = false ]; then + echo "$volumes" | xargs docker volume rm + log_success "Volumes removed" + fi + else + log_info "No volumes found" + fi +else + log_info "Skipping volumes (use --force or --all to remove them)" +fi + +# Remove images if requested +if [ "$ALL" = true ]; then + log_info "Removing images..." + images=$(docker images -q --filter "reference=fetch_ml-*") + if [ -n "$images" ]; then + if [ "$DRY_RUN" = false ]; then + echo "$images" | xargs docker rmi -f + log_success "Images removed" + fi + else + log_info "No images found" + fi +else + log_info "Skipping images (use --all to remove them)" +fi + +# General Docker cleanup +log_info "Running general Docker cleanup..." +if [ "$DRY_RUN" = false ]; then + docker system prune -f + log_success "General cleanup completed" +fi + +# Show final state +log_info "Final resources: $(count_resources)" + +# Space reclaimed +if [ "$DRY_RUN" = false ]; then + log_info "Docker system info:" + docker system df +fi + +log_success "Cleanup completed!" diff --git a/scripts/setup-auto-cleanup.sh b/scripts/setup-auto-cleanup.sh new file mode 100755 index 0000000..34bb285 --- /dev/null +++ b/scripts/setup-auto-cleanup.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Setup auto-cleanup service for fetch_ml +# This creates a systemd timer that runs cleanup daily + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +# Colors +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_info "Setting up auto-cleanup service..." + +# Check if running on macOS or Linux +if [[ "$OSTYPE" == "darwin"* ]]; then + log_info "Detected macOS - setting up launchd agent" + + # Create launchd plist + cat > ~/Library/LaunchAgents/com.fetchml.cleanup.plist << EOF + + + + + Label + com.fetchml.cleanup + ProgramArguments + + $PROJECT_DIR/scripts/cleanup.sh + --force + + StartInterval + 86400 + RunAtLoad + + StandardOutPath + /tmp/fetchml-cleanup.log + StandardErrorPath + /tmp/fetchml-cleanup.error.log + + +EOF + + # Load the launchd agent + launchctl load ~/Library/LaunchAgents/com.fetchml.cleanup.plist + + log_success "Auto-cleanup service installed for macOS" + log_info "Logs will be in /tmp/fetchml-cleanup.log" + +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + log_info "Detected Linux - setting up systemd timer" + + # Copy service files + sudo cp "$SCRIPT_DIR/auto-cleanup.service" /etc/systemd/system/ + sudo cp "$SCRIPT_DIR/auto-cleanup.timer" /etc/systemd/system/ + + # Reload systemd and enable timer + sudo systemctl daemon-reload + sudo systemctl enable auto-cleanup.timer + sudo systemctl start auto-cleanup.timer + + log_success "Auto-cleanup service installed for Linux" + log_info "Check status with: systemctl status auto-cleanup.timer" + +else + echo "Unsupported OS: $OSTYPE" + exit 1 +fi + +log_info "Auto-cleanup will run daily" +log_info "To uninstall:" +if [[ "$OSTYPE" == "darwin"* ]]; then + echo " launchctl unload ~/Library/LaunchAgents/com.fetchml.cleanup.plist" + echo " rm ~/Library/LaunchAgents/com.fetchml.cleanup.plist" +else + echo " sudo systemctl stop auto-cleanup.timer" + echo " sudo systemctl disable auto-cleanup.timer" + echo " sudo rm /etc/systemd/system/auto-cleanup.*" +fi