Add comprehensive self-cleaning system
- Add cleanup.sh script with dry-run, force, and all options - Add auto-cleanup service setup for macOS (launchd) and Linux (systemd) - Add cleanup-status.sh for monitoring Docker resources - Add Makefile targets: self-cleanup, auto-cleanup - Features colored output, confirmation prompts, and detailed logging - Auto-cleanup runs daily to keep system clean - Status monitoring shows resources and service state
This commit is contained in:
Jeremie Fraeys
parent
ea15af1833
commit
c80e01b752
6 changed files with 444 additions and 1 deletions
14
Makefile
14
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance
|
||||
.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance self-cleanup auto-cleanup
|
||||
|
||||
# Default target
|
||||
all: build
|
||||
|
|
@ -325,4 +325,16 @@ help:
|
|||
@echo ""
|
||||
@echo "Utility:"
|
||||
@echo " make size - Show binary sizes"
|
||||
@echo " make self-cleanup - Clean up Docker resources"
|
||||
@echo " make auto-cleanup - Setup daily auto-cleanup service"
|
||||
@echo " make help - Show this help"
|
||||
|
||||
# Self-cleaning for Docker resources
|
||||
self-cleanup:
|
||||
@echo "Running self-cleanup..."
|
||||
@./scripts/cleanup.sh
|
||||
|
||||
# Setup auto-cleanup service
|
||||
auto-cleanup:
|
||||
@echo "Setting up auto-cleanup service..."
|
||||
@./scripts/setup-auto-cleanup.sh
|
||||
|
|
|
|||
15
scripts/auto-cleanup.service
Normal file
15
scripts/auto-cleanup.service
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=FetchML Auto Cleanup Service
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/Users/jfraeys/Documents/dev/fetch_ml/scripts/cleanup.sh --force
|
||||
User=jfraeys
|
||||
Group=staff
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
scripts/auto-cleanup.timer
Normal file
11
scripts/auto-cleanup.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Run FetchML Auto Cleanup daily
|
||||
Requires=auto-cleanup.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
Persistent=true
|
||||
RandomizedDelaySec=1h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
96
scripts/cleanup-status.sh
Executable file
96
scripts/cleanup-status.sh
Executable file
|
|
@ -0,0 +1,96 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Check status of Docker resources and auto-cleanup service
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
echo "=== FetchML Cleanup Status ==="
|
||||
echo ""
|
||||
|
||||
# Docker resources
|
||||
log_info "Docker Resources:"
|
||||
containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l | tr -d ' ')
|
||||
images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l | tr -d ' ')
|
||||
networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
|
||||
echo " Containers: $containers"
|
||||
echo " Images: $images"
|
||||
echo " Networks: $networks"
|
||||
echo " Volumes: $volumes"
|
||||
|
||||
if [ "$containers" -gt 0 ]; then
|
||||
echo ""
|
||||
log_info "Active containers:"
|
||||
docker ps --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
fi
|
||||
|
||||
# Auto-cleanup service status
|
||||
echo ""
|
||||
log_info "Auto-cleanup Service:"
|
||||
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
if launchctl list | grep -q "com.fetchml.cleanup"; then
|
||||
log_success "Auto-cleanup service is running"
|
||||
echo " Logs: /tmp/fetchml-cleanup.log"
|
||||
echo " To check logs: tail -f /tmp/fetchml-cleanup.log"
|
||||
else
|
||||
log_warning "Auto-cleanup service is not installed"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
if systemctl is-active --quiet auto-cleanup.timer; then
|
||||
log_success "Auto-cleanup timer is active"
|
||||
echo " Status: systemctl status auto-cleanup.timer"
|
||||
echo " Logs: journalctl -u auto-cleanup.service"
|
||||
else
|
||||
log_warning "Auto-cleanup timer is not active"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Disk usage
|
||||
echo ""
|
||||
log_info "Docker Disk Usage:"
|
||||
docker system df --format "table {{.Type}}\t{{.TotalCount}}\t{{.Size}}\t{{.Reclaimable}}"
|
||||
|
||||
# Quick cleanup suggestion
|
||||
echo ""
|
||||
if [ "$containers" -gt 0 ] || [ "$images" -gt 0 ] || [ "$networks" -gt 0 ] || [ "$volumes" -gt 0 ]; then
|
||||
log_warning "Resources found that can be cleaned up"
|
||||
echo " Quick cleanup: make self-cleanup"
|
||||
echo " Force cleanup: make self-cleanup --force"
|
||||
echo " Full cleanup: make self-cleanup --all"
|
||||
else
|
||||
log_success "No fetch_ml resources found - system is clean!"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Cleanup Commands ==="
|
||||
echo " make self-cleanup - Interactive cleanup"
|
||||
echo " ./scripts/cleanup.sh --dry-run - Preview what would be cleaned"
|
||||
echo " ./scripts/cleanup.sh --force - Force cleanup without prompts"
|
||||
echo " ./scripts/cleanup.sh --all - Clean everything including images"
|
||||
219
scripts/cleanup.sh
Executable file
219
scripts/cleanup.sh
Executable file
|
|
@ -0,0 +1,219 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Self-cleaning script for fetch_ml Docker resources
|
||||
# Usage: ./scripts/cleanup.sh [--dry-run] [--force] [--all]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default options
|
||||
DRY_RUN=false
|
||||
FORCE=false
|
||||
ALL=false
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--force)
|
||||
FORCE=true
|
||||
shift
|
||||
;;
|
||||
--all)
|
||||
ALL=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--dry-run] [--force] [--all]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --dry-run Show what would be cleaned without actually doing it"
|
||||
echo " --force Skip confirmation prompts"
|
||||
echo " --all Clean everything including images"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Helper functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Docker check
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Docker is running
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker daemon is not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Starting cleanup of fetch_ml Docker resources..."
|
||||
|
||||
# Function to count resources
|
||||
count_resources() {
|
||||
local containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l)
|
||||
local images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l)
|
||||
local networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
local volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
|
||||
echo "Containers: $containers, Images: $images, Networks: $networks, Volumes: $volumes"
|
||||
}
|
||||
|
||||
# Show current state
|
||||
log_info "Current resources: $(count_resources)"
|
||||
|
||||
# Dry run mode
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
log_info "DRY RUN MODE - No actual cleanup will be performed"
|
||||
|
||||
echo ""
|
||||
log_info "Containers that would be stopped and removed:"
|
||||
docker ps -a --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || echo "No containers found"
|
||||
|
||||
echo ""
|
||||
log_info "Images that would be removed:"
|
||||
if [ "$ALL" = true ]; then
|
||||
docker images --filter "reference=fetch_ml-*" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" || echo "No images found"
|
||||
else
|
||||
echo "Use --all to remove images"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Networks that would be removed:"
|
||||
docker network ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No networks found"
|
||||
|
||||
echo ""
|
||||
log_info "Volumes that would be removed:"
|
||||
docker volume ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No volumes found"
|
||||
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Confirmation prompt
|
||||
if [ "$FORCE" = false ]; then
|
||||
echo ""
|
||||
read -p "This will stop and remove all fetch_ml containers. Continue? (y/N): " -n 1 -r
|
||||
echo ""
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log_info "Cleanup cancelled"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Stop containers
|
||||
log_info "Stopping containers..."
|
||||
containers=$(docker ps -q --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker stop
|
||||
log_success "Containers stopped"
|
||||
fi
|
||||
else
|
||||
log_info "No running containers found"
|
||||
fi
|
||||
|
||||
# Remove containers
|
||||
log_info "Removing containers..."
|
||||
containers=$(docker ps -aq --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker rm -f
|
||||
log_success "Containers removed"
|
||||
fi
|
||||
else
|
||||
log_info "No containers found"
|
||||
fi
|
||||
|
||||
# Remove networks
|
||||
log_info "Removing networks..."
|
||||
networks=$(docker network ls -q --filter "name=ml-")
|
||||
if [ -n "$networks" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$networks" | xargs docker network rm
|
||||
log_success "Networks removed"
|
||||
fi
|
||||
else
|
||||
log_info "No networks found"
|
||||
fi
|
||||
|
||||
# Remove volumes (with caution)
|
||||
log_warning "Removing volumes (this will delete data)..."
|
||||
if [ "$FORCE" = true ] || [ "$ALL" = true ]; then
|
||||
volumes=$(docker volume ls -q --filter "name=ml-")
|
||||
if [ -n "$volumes" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$volumes" | xargs docker volume rm
|
||||
log_success "Volumes removed"
|
||||
fi
|
||||
else
|
||||
log_info "No volumes found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping volumes (use --force or --all to remove them)"
|
||||
fi
|
||||
|
||||
# Remove images if requested
|
||||
if [ "$ALL" = true ]; then
|
||||
log_info "Removing images..."
|
||||
images=$(docker images -q --filter "reference=fetch_ml-*")
|
||||
if [ -n "$images" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$images" | xargs docker rmi -f
|
||||
log_success "Images removed"
|
||||
fi
|
||||
else
|
||||
log_info "No images found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping images (use --all to remove them)"
|
||||
fi
|
||||
|
||||
# General Docker cleanup
|
||||
log_info "Running general Docker cleanup..."
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
docker system prune -f
|
||||
log_success "General cleanup completed"
|
||||
fi
|
||||
|
||||
# Show final state
|
||||
log_info "Final resources: $(count_resources)"
|
||||
|
||||
# Space reclaimed
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
log_info "Docker system info:"
|
||||
docker system df
|
||||
fi
|
||||
|
||||
log_success "Cleanup completed!"
|
||||
90
scripts/setup-auto-cleanup.sh
Executable file
90
scripts/setup-auto-cleanup.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Setup auto-cleanup service for fetch_ml
|
||||
# This creates a systemd timer that runs cleanup daily
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_info "Setting up auto-cleanup service..."
|
||||
|
||||
# Check if running on macOS or Linux
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
log_info "Detected macOS - setting up launchd agent"
|
||||
|
||||
# Create launchd plist
|
||||
cat > ~/Library/LaunchAgents/com.fetchml.cleanup.plist << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.fetchml.cleanup</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$PROJECT_DIR/scripts/cleanup.sh</string>
|
||||
<string>--force</string>
|
||||
</array>
|
||||
<key>StartInterval</key>
|
||||
<integer>86400</integer>
|
||||
<key>RunAtLoad</key>
|
||||
<false/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/fetchml-cleanup.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/fetchml-cleanup.error.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
# Load the launchd agent
|
||||
launchctl load ~/Library/LaunchAgents/com.fetchml.cleanup.plist
|
||||
|
||||
log_success "Auto-cleanup service installed for macOS"
|
||||
log_info "Logs will be in /tmp/fetchml-cleanup.log"
|
||||
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
log_info "Detected Linux - setting up systemd timer"
|
||||
|
||||
# Copy service files
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.service" /etc/systemd/system/
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.timer" /etc/systemd/system/
|
||||
|
||||
# Reload systemd and enable timer
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable auto-cleanup.timer
|
||||
sudo systemctl start auto-cleanup.timer
|
||||
|
||||
log_success "Auto-cleanup service installed for Linux"
|
||||
log_info "Check status with: systemctl status auto-cleanup.timer"
|
||||
|
||||
else
|
||||
echo "Unsupported OS: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Auto-cleanup will run daily"
|
||||
log_info "To uninstall:"
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo " launchctl unload ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
echo " rm ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
else
|
||||
echo " sudo systemctl stop auto-cleanup.timer"
|
||||
echo " sudo systemctl disable auto-cleanup.timer"
|
||||
echo " sudo rm /etc/systemd/system/auto-cleanup.*"
|
||||
fi
|
||||
Loading…
Reference in a new issue