Organize configs and scripts, create testing protocol
- Reorganize configs into environments/, workers/, deprecated/ folders - Reorganize scripts into testing/, deployment/, maintenance/, benchmarks/ folders - Add comprehensive testing guide documentation - Add new Makefile targets: test-full, test-auth, test-status - Update script paths in Makefile to match new organization - Create testing protocol documentation - Add cleanup status checking functionality Testing framework now includes: - Quick authentication tests (make test-auth) - Full test suite runner (make test-full) - Cleanup status monitoring (make test-status) - Comprehensive documentation and troubleshooting guides
This commit is contained in:
Jeremie Fraeys
parent
c80e01b752
commit
5a19358d00
32 changed files with 873 additions and 518 deletions
26
Makefile
26
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance self-cleanup auto-cleanup
|
||||
.PHONY: all build prod dev clean clean-docs test test-unit test-integration test-e2e test-coverage lint install setup validate configlint ci-local docs benchmark benchmark-local artifacts clean-benchmarks clean-all clean-aggressive status load-test chaos-test profile-tools detect-regressions tech-excellence docker-build docker-run docker-stop docker-logs monitoring-performance monitoring-performance-stop dashboard-performance self-cleanup auto-cleanup test-full test-auth test-status
|
||||
|
||||
# Default target
|
||||
all: build
|
||||
|
|
@ -327,14 +327,34 @@ help:
|
|||
@echo " make size - Show binary sizes"
|
||||
@echo " make self-cleanup - Clean up Docker resources"
|
||||
@echo " make auto-cleanup - Setup daily auto-cleanup service"
|
||||
@echo " make test-full - Run complete test suite"
|
||||
@echo " make test-auth - Test multi-user authentication"
|
||||
@echo " make test-status - Check cleanup status"
|
||||
@echo " make help - Show this help"
|
||||
|
||||
# Self-cleaning for Docker resources
|
||||
self-cleanup:
|
||||
@echo "Running self-cleanup..."
|
||||
@./scripts/cleanup.sh
|
||||
@./scripts/maintenance/cleanup.sh
|
||||
|
||||
# Setup auto-cleanup service
|
||||
auto-cleanup:
|
||||
@echo "Setting up auto-cleanup service..."
|
||||
@./scripts/setup-auto-cleanup.sh
|
||||
@./scripts/deployment/setup-auto-cleanup.sh
|
||||
|
||||
# Run full test suite
|
||||
test-full:
|
||||
@echo "Running full test suite..."
|
||||
@./scripts/testing/run-full-test-suite.sh
|
||||
|
||||
# Quick authentication test
|
||||
test-auth:
|
||||
@echo "Testing multi-user authentication..."
|
||||
@echo "Testing admin user..." && cp ~/.ml/config-admin.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status
|
||||
@echo "Testing researcher user..." && cp ~/.ml/config-researcher.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status
|
||||
@echo "Testing analyst user..." && cp ~/.ml/config-analyst.toml ~/.ml/config.toml && ./cli/zig-out/bin/ml status
|
||||
|
||||
# Test cleanup status
|
||||
test-status:
|
||||
@echo "Checking cleanup status..."
|
||||
@./scripts/maintenance/cleanup-status.sh
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
base_path: "/app/data/experiments"
|
||||
|
||||
auth:
|
||||
enabled: true
|
||||
api_keys:
|
||||
admin_user:
|
||||
hash: "5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8" # "password"
|
||||
admin: true
|
||||
roles: ["user", "admin"]
|
||||
permissions:
|
||||
read: true
|
||||
write: true
|
||||
delete: true
|
||||
researcher1:
|
||||
hash: "ef92b778ba7a6c8f2150019a5678047b6a9a2b95cef8189518f9b35c54d2e3ae" # "research123"
|
||||
admin: false
|
||||
roles: ["user", "researcher"]
|
||||
permissions:
|
||||
jobs:read: true
|
||||
jobs:create: true
|
||||
jobs:update: true
|
||||
jobs:delete: false
|
||||
analyst1:
|
||||
hash: "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3" # "analyst123"
|
||||
admin: false
|
||||
roles: ["user", "analyst"]
|
||||
permissions:
|
||||
jobs:read: true
|
||||
jobs:create: false
|
||||
jobs:update: false
|
||||
jobs:delete: false
|
||||
|
||||
server:
|
||||
address: ":9101"
|
||||
tls:
|
||||
enabled: false
|
||||
|
||||
security:
|
||||
rate_limit:
|
||||
enabled: true
|
||||
requests_per_minute: 60
|
||||
burst_size: 20
|
||||
ip_whitelist: []
|
||||
cors:
|
||||
enabled: true
|
||||
allowed_origins: ["https://localhost:9103", "https://localhost:3000"]
|
||||
allowed_methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
|
||||
allowed_headers: ["Content-Type", "Authorization"]
|
||||
|
||||
database:
|
||||
type: "sqlite"
|
||||
connection: "/app/data/experiments/fetch_ml.db"
|
||||
max_connections: 20
|
||||
connection_timeout: "30s"
|
||||
|
||||
redis:
|
||||
url: "redis://redis:6379"
|
||||
max_connections: 15
|
||||
connection_timeout: "10s"
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
file: "/app/logs/app.log"
|
||||
max_size: "100MB"
|
||||
max_backups: 5
|
||||
compress: true
|
||||
|
||||
resources:
|
||||
max_workers: 3
|
||||
desired_rps_per_worker: 3
|
||||
podman_cpus: "2"
|
||||
podman_memory: "4g"
|
||||
job_timeout: "30m"
|
||||
|
||||
monitoring:
|
||||
enabled: true
|
||||
metrics_path: "/metrics"
|
||||
health_check_interval: "30s"
|
||||
|
|
@ -37,9 +37,3 @@ logging:
|
|||
level: "info"
|
||||
file: "/app/logs/app.log"
|
||||
audit_file: "/app/logs/audit.log"
|
||||
|
||||
resources:
|
||||
max_workers: 1
|
||||
desired_rps_per_worker: 2
|
||||
podman_cpus: "2"
|
||||
podman_memory: "8g"
|
||||
33
configs/environments/config-local.yaml
Normal file
33
configs/environments/config-local.yaml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
auth:
|
||||
enabled: true
|
||||
apikeys:
|
||||
dev_user:
|
||||
hash: 2baf1f40105d9501fe319a8ec463fdf4325a2a5df445adf3f572f626253678c9
|
||||
admin: true
|
||||
roles:
|
||||
- admin
|
||||
permissions:
|
||||
'*': true
|
||||
|
||||
server:
|
||||
address: ":9101"
|
||||
tls:
|
||||
enabled: false
|
||||
|
||||
security:
|
||||
rate_limit:
|
||||
enabled: false
|
||||
ip_whitelist:
|
||||
- "127.0.0.1"
|
||||
- "::1"
|
||||
- "localhost"
|
||||
- "10.0.0.0/8"
|
||||
- "192.168.0.0/16"
|
||||
- "172.16.0.0/12"
|
||||
|
||||
# Prometheus metrics
|
||||
metrics:
|
||||
enabled: true
|
||||
listen_addr: ":9100"
|
||||
tls:
|
||||
enabled: false
|
||||
78
configs/environments/config-multi-user.yaml
Normal file
78
configs/environments/config-multi-user.yaml
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
base_path: "/app/data/experiments"
|
||||
|
||||
auth:
|
||||
enabled: true
|
||||
api_keys:
|
||||
admin_user:
|
||||
hash: "5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8" # "password"
|
||||
admin: true
|
||||
roles: ["user", "admin"]
|
||||
permissions:
|
||||
read: true
|
||||
write: true
|
||||
delete: true
|
||||
researcher1:
|
||||
hash: "ef92b778ba7a6c8f2150019a5678047b6a9a2b95cef8189518f9b35c54d2e3ae" # "research123"
|
||||
admin: false
|
||||
roles: ["user", "researcher"]
|
||||
permissions:
|
||||
jobs:read: true
|
||||
jobs:create: true
|
||||
jobs:update: true
|
||||
jobs:delete: false
|
||||
analyst1:
|
||||
hash: "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3" # "analyst123"
|
||||
admin: false
|
||||
roles: ["user", "analyst"]
|
||||
permissions:
|
||||
jobs:read: true
|
||||
jobs:create: false
|
||||
jobs:update: false
|
||||
jobs:delete: false
|
||||
|
||||
server:
|
||||
address: ":9101"
|
||||
tls:
|
||||
enabled: false
|
||||
|
||||
security:
|
||||
rate_limit:
|
||||
enabled: true
|
||||
requests_per_minute: 60
|
||||
burst_size: 20
|
||||
ip_whitelist: []
|
||||
cors:
|
||||
enabled: true
|
||||
allowed_origins: ["https://localhost:9103", "https://localhost:3000"]
|
||||
allowed_methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"]
|
||||
allowed_headers: ["Content-Type", "Authorization"]
|
||||
|
||||
database:
|
||||
type: "sqlite"
|
||||
connection: "/app/data/experiments/fetch_ml.db"
|
||||
max_connections: 20
|
||||
connection_timeout: "30s"
|
||||
|
||||
redis:
|
||||
url: "redis://redis:6379"
|
||||
max_connections: 15
|
||||
connection_timeout: "10s"
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
file: "/app/logs/app.log"
|
||||
max_size: "100MB"
|
||||
max_backups: 5
|
||||
compress: true
|
||||
|
||||
resources:
|
||||
max_workers: 3
|
||||
desired_rps_per_worker: 3
|
||||
podman_cpus: "2"
|
||||
podman_memory: "4g"
|
||||
job_timeout: "30m"
|
||||
|
||||
monitoring:
|
||||
enabled: true
|
||||
metrics_path: "/metrics"
|
||||
health_check_interval: "30s"
|
||||
50
docs/src/testing-guide.md
Normal file
50
docs/src/testing-guide.md
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# Testing Guide
|
||||
|
||||
## Quick Start
|
||||
|
||||
The FetchML project includes comprehensive testing tools.
|
||||
|
||||
## Testing Commands
|
||||
|
||||
### Quick Tests
|
||||
```bash
|
||||
make test-auth # Test multi-user authentication
|
||||
make test-status # Check cleanup status
|
||||
make self-cleanup # Clean environment
|
||||
```
|
||||
|
||||
### Full Test Suite
|
||||
```bash
|
||||
make test-full # Run complete test suite
|
||||
```
|
||||
|
||||
## Expected Results
|
||||
|
||||
### Admin User
|
||||
Status retrieved for user: admin_user (admin: true)
|
||||
Tasks: X total, X queued, X running, X failed, X completed
|
||||
|
||||
### Researcher User
|
||||
Status retrieved for user: researcher1 (admin: false)
|
||||
Tasks: X total, X queued, X running, X failed, X completed
|
||||
|
||||
### Analyst User
|
||||
Status retrieved for user: analyst1 (admin: false)
|
||||
Tasks: X total, X queued, X running, X failed, X completed
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Authentication Failures
|
||||
- Check API key in ~/.ml/config.toml
|
||||
- Verify server is running with auth enabled
|
||||
|
||||
### Container Issues
|
||||
- Check Docker daemon is running
|
||||
- Verify ports 9100, 9103 are available
|
||||
- Review logs: docker logs ml-prod-api
|
||||
|
||||
## Cleanup
|
||||
```bash
|
||||
make self-cleanup # Interactive cleanup
|
||||
make auto-cleanup # Setup daily auto-cleanup
|
||||
```
|
||||
258
docs/src/testing-protocol.md
Normal file
258
docs/src/testing-protocol.md
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
# Testing Protocol
|
||||
|
||||
This document outlines the comprehensive testing protocol for the FetchML project.
|
||||
|
||||
## Overview
|
||||
|
||||
The testing protocol is designed to ensure:
|
||||
- Multi-user authentication works correctly
|
||||
- API functionality is reliable
|
||||
- CLI commands function properly
|
||||
- Docker containers run as expected
|
||||
- Performance meets requirements
|
||||
|
||||
## Test Categories
|
||||
|
||||
### 1. Authentication Tests
|
||||
|
||||
#### 1.1 Multi-User Authentication
|
||||
```bash
|
||||
# Test admin user
|
||||
cp ~/.ml/config-admin.toml ~/.ml/config.toml
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Shows admin status and all jobs
|
||||
|
||||
# Test researcher user
|
||||
cp ~/.ml/config-researcher.toml ~/.ml/config.toml
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Shows researcher status and own jobs only
|
||||
|
||||
# Test analyst user
|
||||
cp ~/.ml/config-analyst.toml ~/.ml/config.toml
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Shows analyst status, read-only access
|
||||
```
|
||||
|
||||
#### 1.2 API Key Validation
|
||||
```bash
|
||||
# Test invalid API key
|
||||
echo "invalid_key" > ~/.ml/config.toml
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Authentication failed error
|
||||
|
||||
# Test missing API key
|
||||
rm ~/.ml/config.toml
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: API key not configured error
|
||||
```
|
||||
|
||||
### 2. CLI Functionality Tests
|
||||
|
||||
#### 2.1 Job Queueing
|
||||
```bash
|
||||
# Test job queueing with different users
|
||||
cp ~/.ml/config-admin.toml ~/.ml/config.toml
|
||||
echo "test job" | ./cli/zig-out/bin/ml queue test-job
|
||||
|
||||
cp ~/.ml/config-researcher.toml ~/.ml/config.toml
|
||||
echo "research job" | ./cli/zig-out/bin/ml queue research-job
|
||||
|
||||
cp ~/.ml/config-analyst.toml ~/.ml/config.toml
|
||||
echo "analysis job" | ./cli/zig-out/bin/ml queue analysis-job
|
||||
# Expected: Admin and researcher can queue, analyst cannot
|
||||
```
|
||||
|
||||
#### 2.2 Status Checking
|
||||
```bash
|
||||
# Check status after job queueing
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Shows jobs based on user permissions
|
||||
```
|
||||
|
||||
### 3. Docker Container Tests
|
||||
|
||||
#### 3.1 Container Startup
|
||||
```bash
|
||||
# Start production environment
|
||||
docker-compose -f docker-compose.prod.yml up -d
|
||||
|
||||
# Check container status
|
||||
docker ps --filter "name=ml-"
|
||||
# Expected: All containers running and healthy
|
||||
```
|
||||
|
||||
#### 3.2 Port Accessibility
|
||||
```bash
|
||||
# Test API server port
|
||||
curl -I http://localhost:9103/health
|
||||
# Expected: 200 OK response
|
||||
|
||||
# Test metrics port
|
||||
curl -I http://localhost:9100/metrics
|
||||
# Expected: 200 OK response
|
||||
```
|
||||
|
||||
#### 3.3 Container Cleanup
|
||||
```bash
|
||||
# Test cleanup script
|
||||
./scripts/maintenance/cleanup.sh --dry-run
|
||||
./scripts/maintenance/cleanup.sh --force
|
||||
# Expected: Containers stopped and removed
|
||||
```
|
||||
|
||||
### 4. Performance Tests
|
||||
|
||||
#### 4.1 API Performance
|
||||
```bash
|
||||
# Run API benchmarks
|
||||
./scripts/benchmarks/run-benchmarks-local.sh
|
||||
# Expected: Response times under 100ms for basic operations
|
||||
```
|
||||
|
||||
#### 4.2 Load Testing
|
||||
```bash
|
||||
# Run load tests
|
||||
go test -v ./tests/load/...
|
||||
# Expected: System handles concurrent requests without degradation
|
||||
```
|
||||
|
||||
### 5. Integration Tests
|
||||
|
||||
#### 5.1 End-to-End Workflow
|
||||
```bash
|
||||
# Complete workflow test
|
||||
cp ~/.ml/config-admin.toml ~/.ml/config.toml
|
||||
|
||||
# Queue job
|
||||
echo "integration test" | ./cli/zig-out/bin/ml queue integration-test
|
||||
|
||||
# Check status
|
||||
./cli/zig-out/bin/ml status
|
||||
|
||||
# Verify job appears in queue
|
||||
# Expected: Job queued and visible in status
|
||||
```
|
||||
|
||||
#### 5.2 WebSocket Communication
|
||||
```bash
|
||||
# Test WebSocket handshake
|
||||
./cli/zig-out/bin/ml status
|
||||
# Expected: Successful WebSocket upgrade and response
|
||||
```
|
||||
|
||||
## Test Execution Order
|
||||
|
||||
### Phase 1: Environment Setup
|
||||
1. Clean up any existing containers
|
||||
2. Start fresh Docker environment
|
||||
3. Verify all services are running
|
||||
|
||||
### Phase 2: Authentication Testing
|
||||
1. Test all user roles (admin, researcher, analyst)
|
||||
2. Test invalid authentication scenarios
|
||||
3. Verify role-based permissions
|
||||
|
||||
### Phase 3: Functional Testing
|
||||
1. Test CLI commands (queue, status)
|
||||
2. Test API endpoints
|
||||
3. Test WebSocket communication
|
||||
|
||||
### Phase 4: Integration Testing
|
||||
1. Test complete workflows
|
||||
2. Test error scenarios
|
||||
3. Test cleanup procedures
|
||||
|
||||
### Phase 5: Performance Testing
|
||||
1. Run benchmarks
|
||||
2. Perform load testing
|
||||
3. Validate performance metrics
|
||||
|
||||
## Automated Testing
|
||||
|
||||
### Continuous Integration Tests
|
||||
```bash
|
||||
# Run all tests
|
||||
make test
|
||||
|
||||
# Run specific test categories
|
||||
make test-unit
|
||||
make test-integration
|
||||
make test-e2e
|
||||
```
|
||||
|
||||
### Pre-deployment Checklist
|
||||
```bash
|
||||
# Complete test suite
|
||||
./scripts/testing/run-full-test-suite.sh
|
||||
|
||||
# Performance validation
|
||||
./scripts/benchmarks/run-benchmarks-local.sh
|
||||
|
||||
# Security validation
|
||||
./scripts/security/security-scan.sh
|
||||
```
|
||||
|
||||
## Test Data Management
|
||||
|
||||
### Test Users
|
||||
- **admin_user**: Full access, can see all jobs
|
||||
- **researcher1**: Can create and view own jobs
|
||||
- **analyst1**: Read-only access, cannot create jobs
|
||||
|
||||
### Test Jobs
|
||||
- **test-job**: Basic job for testing
|
||||
- **research-job**: Research-specific job
|
||||
- **analysis-job**: Analysis-specific job
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
#### Authentication Failures
|
||||
- Check API key configuration
|
||||
- Verify server is running with auth enabled
|
||||
- Check YAML config syntax
|
||||
|
||||
#### Container Issues
|
||||
- Verify Docker daemon is running
|
||||
- Check port conflicts
|
||||
- Review container logs
|
||||
|
||||
#### Performance Issues
|
||||
- Monitor resource usage
|
||||
- Check for memory leaks
|
||||
- Verify database connections
|
||||
|
||||
### Debug Commands
|
||||
```bash
|
||||
# Check container logs
|
||||
docker logs ml-prod-api
|
||||
|
||||
# Check system resources
|
||||
docker stats
|
||||
|
||||
# Verify network connectivity
|
||||
docker network ls
|
||||
```
|
||||
|
||||
## Test Results Documentation
|
||||
|
||||
All test results should be documented in:
|
||||
- `test-results/` directory
|
||||
- Performance benchmarks
|
||||
- Integration test reports
|
||||
- Security scan results
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Regular Tasks
|
||||
- Update test data periodically
|
||||
- Review and update test cases
|
||||
- Maintain test infrastructure
|
||||
- Monitor test performance
|
||||
|
||||
### Test Environment
|
||||
- Keep test environment isolated
|
||||
- Use consistent test data
|
||||
- Regular cleanup of test artifacts
|
||||
- Monitor test resource usage
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
[Unit]
|
||||
Description=FetchML Auto Cleanup Service
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/Users/jfraeys/Documents/dev/fetch_ml/scripts/cleanup.sh --force
|
||||
User=jfraeys
|
||||
Group=staff
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
[Unit]
|
||||
Description=Run FetchML Auto Cleanup daily
|
||||
Requires=auto-cleanup.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
Persistent=true
|
||||
RandomizedDelaySec=1h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
96
scripts/cleanup-status.sh
Executable file → Normal file
96
scripts/cleanup-status.sh
Executable file → Normal file
|
|
@ -1,96 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Check status of Docker resources and auto-cleanup service
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
echo "=== FetchML Cleanup Status ==="
|
||||
echo ""
|
||||
|
||||
# Docker resources
|
||||
log_info "Docker Resources:"
|
||||
containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l | tr -d ' ')
|
||||
images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l | tr -d ' ')
|
||||
networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
|
||||
echo " Containers: $containers"
|
||||
echo " Images: $images"
|
||||
echo " Networks: $networks"
|
||||
echo " Volumes: $volumes"
|
||||
|
||||
if [ "$containers" -gt 0 ]; then
|
||||
echo ""
|
||||
log_info "Active containers:"
|
||||
docker ps --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
fi
|
||||
|
||||
# Auto-cleanup service status
|
||||
echo ""
|
||||
log_info "Auto-cleanup Service:"
|
||||
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
if launchctl list | grep -q "com.fetchml.cleanup"; then
|
||||
log_success "Auto-cleanup service is running"
|
||||
echo " Logs: /tmp/fetchml-cleanup.log"
|
||||
echo " To check logs: tail -f /tmp/fetchml-cleanup.log"
|
||||
else
|
||||
log_warning "Auto-cleanup service is not installed"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
if systemctl is-active --quiet auto-cleanup.timer; then
|
||||
log_success "Auto-cleanup timer is active"
|
||||
echo " Status: systemctl status auto-cleanup.timer"
|
||||
echo " Logs: journalctl -u auto-cleanup.service"
|
||||
else
|
||||
log_warning "Auto-cleanup timer is not active"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Disk usage
|
||||
echo ""
|
||||
log_info "Docker Disk Usage:"
|
||||
docker system df --format "table {{.Type}}\t{{.TotalCount}}\t{{.Size}}\t{{.Reclaimable}}"
|
||||
|
||||
# Quick cleanup suggestion
|
||||
echo ""
|
||||
if [ "$containers" -gt 0 ] || [ "$images" -gt 0 ] || [ "$networks" -gt 0 ] || [ "$volumes" -gt 0 ]; then
|
||||
log_warning "Resources found that can be cleaned up"
|
||||
echo " Quick cleanup: make self-cleanup"
|
||||
echo " Force cleanup: make self-cleanup --force"
|
||||
echo " Full cleanup: make self-cleanup --all"
|
||||
else
|
||||
log_success "No fetch_ml resources found - system is clean!"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Cleanup Commands ==="
|
||||
echo " make self-cleanup - Interactive cleanup"
|
||||
echo " ./scripts/cleanup.sh --dry-run - Preview what would be cleaned"
|
||||
echo " ./scripts/cleanup.sh --force - Force cleanup without prompts"
|
||||
echo " ./scripts/cleanup.sh --all - Clean everything including images"
|
||||
219
scripts/cleanup.sh
Executable file → Normal file
219
scripts/cleanup.sh
Executable file → Normal file
|
|
@ -1,219 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Self-cleaning script for fetch_ml Docker resources
|
||||
# Usage: ./scripts/cleanup.sh [--dry-run] [--force] [--all]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default options
|
||||
DRY_RUN=false
|
||||
FORCE=false
|
||||
ALL=false
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--force)
|
||||
FORCE=true
|
||||
shift
|
||||
;;
|
||||
--all)
|
||||
ALL=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--dry-run] [--force] [--all]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --dry-run Show what would be cleaned without actually doing it"
|
||||
echo " --force Skip confirmation prompts"
|
||||
echo " --all Clean everything including images"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Helper functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Docker check
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Docker is running
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker daemon is not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Starting cleanup of fetch_ml Docker resources..."
|
||||
|
||||
# Function to count resources
|
||||
count_resources() {
|
||||
local containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l)
|
||||
local images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l)
|
||||
local networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
local volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
|
||||
echo "Containers: $containers, Images: $images, Networks: $networks, Volumes: $volumes"
|
||||
}
|
||||
|
||||
# Show current state
|
||||
log_info "Current resources: $(count_resources)"
|
||||
|
||||
# Dry run mode
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
log_info "DRY RUN MODE - No actual cleanup will be performed"
|
||||
|
||||
echo ""
|
||||
log_info "Containers that would be stopped and removed:"
|
||||
docker ps -a --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || echo "No containers found"
|
||||
|
||||
echo ""
|
||||
log_info "Images that would be removed:"
|
||||
if [ "$ALL" = true ]; then
|
||||
docker images --filter "reference=fetch_ml-*" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" || echo "No images found"
|
||||
else
|
||||
echo "Use --all to remove images"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Networks that would be removed:"
|
||||
docker network ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No networks found"
|
||||
|
||||
echo ""
|
||||
log_info "Volumes that would be removed:"
|
||||
docker volume ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No volumes found"
|
||||
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Confirmation prompt
|
||||
if [ "$FORCE" = false ]; then
|
||||
echo ""
|
||||
read -p "This will stop and remove all fetch_ml containers. Continue? (y/N): " -n 1 -r
|
||||
echo ""
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log_info "Cleanup cancelled"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Stop containers
|
||||
log_info "Stopping containers..."
|
||||
containers=$(docker ps -q --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker stop
|
||||
log_success "Containers stopped"
|
||||
fi
|
||||
else
|
||||
log_info "No running containers found"
|
||||
fi
|
||||
|
||||
# Remove containers
|
||||
log_info "Removing containers..."
|
||||
containers=$(docker ps -aq --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker rm -f
|
||||
log_success "Containers removed"
|
||||
fi
|
||||
else
|
||||
log_info "No containers found"
|
||||
fi
|
||||
|
||||
# Remove networks
|
||||
log_info "Removing networks..."
|
||||
networks=$(docker network ls -q --filter "name=ml-")
|
||||
if [ -n "$networks" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$networks" | xargs docker network rm
|
||||
log_success "Networks removed"
|
||||
fi
|
||||
else
|
||||
log_info "No networks found"
|
||||
fi
|
||||
|
||||
# Remove volumes (with caution)
|
||||
log_warning "Removing volumes (this will delete data)..."
|
||||
if [ "$FORCE" = true ] || [ "$ALL" = true ]; then
|
||||
volumes=$(docker volume ls -q --filter "name=ml-")
|
||||
if [ -n "$volumes" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$volumes" | xargs docker volume rm
|
||||
log_success "Volumes removed"
|
||||
fi
|
||||
else
|
||||
log_info "No volumes found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping volumes (use --force or --all to remove them)"
|
||||
fi
|
||||
|
||||
# Remove images if requested
|
||||
if [ "$ALL" = true ]; then
|
||||
log_info "Removing images..."
|
||||
images=$(docker images -q --filter "reference=fetch_ml-*")
|
||||
if [ -n "$images" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$images" | xargs docker rmi -f
|
||||
log_success "Images removed"
|
||||
fi
|
||||
else
|
||||
log_info "No images found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping images (use --all to remove them)"
|
||||
fi
|
||||
|
||||
# General Docker cleanup
|
||||
log_info "Running general Docker cleanup..."
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
docker system prune -f
|
||||
log_success "General cleanup completed"
|
||||
fi
|
||||
|
||||
# Show final state
|
||||
log_info "Final resources: $(count_resources)"
|
||||
|
||||
# Space reclaimed
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
log_info "Docker system info:"
|
||||
docker system df
|
||||
fi
|
||||
|
||||
log_success "Cleanup completed!"
|
||||
90
scripts/deployment/setup-auto-cleanup.sh
Executable file
90
scripts/deployment/setup-auto-cleanup.sh
Executable file
|
|
@ -0,0 +1,90 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Setup auto-cleanup service for fetch_ml
|
||||
# This creates a systemd timer that runs cleanup daily
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_info "Setting up auto-cleanup service..."
|
||||
|
||||
# Check if running on macOS or Linux
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
log_info "Detected macOS - setting up launchd agent"
|
||||
|
||||
# Create launchd plist
|
||||
cat > ~/Library/LaunchAgents/com.fetchml.cleanup.plist << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.fetchml.cleanup</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$PROJECT_DIR/scripts/cleanup.sh</string>
|
||||
<string>--force</string>
|
||||
</array>
|
||||
<key>StartInterval</key>
|
||||
<integer>86400</integer>
|
||||
<key>RunAtLoad</key>
|
||||
<false/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/fetchml-cleanup.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/fetchml-cleanup.error.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
# Load the launchd agent
|
||||
launchctl load ~/Library/LaunchAgents/com.fetchml.cleanup.plist
|
||||
|
||||
log_success "Auto-cleanup service installed for macOS"
|
||||
log_info "Logs will be in /tmp/fetchml-cleanup.log"
|
||||
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
log_info "Detected Linux - setting up systemd timer"
|
||||
|
||||
# Copy service files
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.service" /etc/systemd/system/
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.timer" /etc/systemd/system/
|
||||
|
||||
# Reload systemd and enable timer
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable auto-cleanup.timer
|
||||
sudo systemctl start auto-cleanup.timer
|
||||
|
||||
log_success "Auto-cleanup service installed for Linux"
|
||||
log_info "Check status with: systemctl status auto-cleanup.timer"
|
||||
|
||||
else
|
||||
echo "Unsupported OS: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Auto-cleanup will run daily"
|
||||
log_info "To uninstall:"
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo " launchctl unload ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
echo " rm ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
else
|
||||
echo " sudo systemctl stop auto-cleanup.timer"
|
||||
echo " sudo systemctl disable auto-cleanup.timer"
|
||||
echo " sudo rm /etc/systemd/system/auto-cleanup.*"
|
||||
fi
|
||||
15
scripts/maintenance/auto-cleanup.service
Normal file
15
scripts/maintenance/auto-cleanup.service
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=FetchML Auto Cleanup Service
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/Users/jfraeys/Documents/dev/fetch_ml/scripts/cleanup.sh --force
|
||||
User=jfraeys
|
||||
Group=staff
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
11
scripts/maintenance/auto-cleanup.timer
Normal file
11
scripts/maintenance/auto-cleanup.timer
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Run FetchML Auto Cleanup daily
|
||||
Requires=auto-cleanup.service
|
||||
|
||||
[Timer]
|
||||
OnCalendar=daily
|
||||
Persistent=true
|
||||
RandomizedDelaySec=1h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
96
scripts/maintenance/cleanup-status.sh
Executable file
96
scripts/maintenance/cleanup-status.sh
Executable file
|
|
@ -0,0 +1,96 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Check status of Docker resources and auto-cleanup service
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
echo "=== FetchML Cleanup Status ==="
|
||||
echo ""
|
||||
|
||||
# Docker resources
|
||||
log_info "Docker Resources:"
|
||||
containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l | tr -d ' ')
|
||||
images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l | tr -d ' ')
|
||||
networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l | tr -d ' ')
|
||||
|
||||
echo " Containers: $containers"
|
||||
echo " Images: $images"
|
||||
echo " Networks: $networks"
|
||||
echo " Volumes: $volumes"
|
||||
|
||||
if [ "$containers" -gt 0 ]; then
|
||||
echo ""
|
||||
log_info "Active containers:"
|
||||
docker ps --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
fi
|
||||
|
||||
# Auto-cleanup service status
|
||||
echo ""
|
||||
log_info "Auto-cleanup Service:"
|
||||
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
if launchctl list | grep -q "com.fetchml.cleanup"; then
|
||||
log_success "Auto-cleanup service is running"
|
||||
echo " Logs: /tmp/fetchml-cleanup.log"
|
||||
echo " To check logs: tail -f /tmp/fetchml-cleanup.log"
|
||||
else
|
||||
log_warning "Auto-cleanup service is not installed"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
if systemctl is-active --quiet auto-cleanup.timer; then
|
||||
log_success "Auto-cleanup timer is active"
|
||||
echo " Status: systemctl status auto-cleanup.timer"
|
||||
echo " Logs: journalctl -u auto-cleanup.service"
|
||||
else
|
||||
log_warning "Auto-cleanup timer is not active"
|
||||
echo " To install: make auto-cleanup"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Disk usage
|
||||
echo ""
|
||||
log_info "Docker Disk Usage:"
|
||||
docker system df --format "table {{.Type}}\t{{.TotalCount}}\t{{.Size}}\t{{.Reclaimable}}"
|
||||
|
||||
# Quick cleanup suggestion
|
||||
echo ""
|
||||
if [ "$containers" -gt 0 ] || [ "$images" -gt 0 ] || [ "$networks" -gt 0 ] || [ "$volumes" -gt 0 ]; then
|
||||
log_warning "Resources found that can be cleaned up"
|
||||
echo " Quick cleanup: make self-cleanup"
|
||||
echo " Force cleanup: make self-cleanup --force"
|
||||
echo " Full cleanup: make self-cleanup --all"
|
||||
else
|
||||
log_success "No fetch_ml resources found - system is clean!"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Cleanup Commands ==="
|
||||
echo " make self-cleanup - Interactive cleanup"
|
||||
echo " ./scripts/cleanup.sh --dry-run - Preview what would be cleaned"
|
||||
echo " ./scripts/cleanup.sh --force - Force cleanup without prompts"
|
||||
echo " ./scripts/cleanup.sh --all - Clean everything including images"
|
||||
219
scripts/maintenance/cleanup.sh
Executable file
219
scripts/maintenance/cleanup.sh
Executable file
|
|
@ -0,0 +1,219 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Self-cleaning script for fetch_ml Docker resources
|
||||
# Usage: ./scripts/cleanup.sh [--dry-run] [--force] [--all]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default options
|
||||
DRY_RUN=false
|
||||
FORCE=false
|
||||
ALL=false
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--force)
|
||||
FORCE=true
|
||||
shift
|
||||
;;
|
||||
--all)
|
||||
ALL=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--dry-run] [--force] [--all]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --dry-run Show what would be cleaned without actually doing it"
|
||||
echo " --force Skip confirmation prompts"
|
||||
echo " --all Clean everything including images"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Helper functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Docker check
|
||||
if ! command -v docker &> /dev/null; then
|
||||
log_error "Docker is not installed or not in PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Docker is running
|
||||
if ! docker info &> /dev/null; then
|
||||
log_error "Docker daemon is not running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Starting cleanup of fetch_ml Docker resources..."
|
||||
|
||||
# Function to count resources
|
||||
count_resources() {
|
||||
local containers=$(docker ps -a --filter "name=ml-" --format "{{.Names}}" | wc -l)
|
||||
local images=$(docker images --filter "reference=fetch_ml-*" --format "{{.Repository}}" | wc -l)
|
||||
local networks=$(docker network ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
local volumes=$(docker volume ls --filter "name=ml-" --format "{{.Name}}" | wc -l)
|
||||
|
||||
echo "Containers: $containers, Images: $images, Networks: $networks, Volumes: $volumes"
|
||||
}
|
||||
|
||||
# Show current state
|
||||
log_info "Current resources: $(count_resources)"
|
||||
|
||||
# Dry run mode
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
log_info "DRY RUN MODE - No actual cleanup will be performed"
|
||||
|
||||
echo ""
|
||||
log_info "Containers that would be stopped and removed:"
|
||||
docker ps -a --filter "name=ml-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || echo "No containers found"
|
||||
|
||||
echo ""
|
||||
log_info "Images that would be removed:"
|
||||
if [ "$ALL" = true ]; then
|
||||
docker images --filter "reference=fetch_ml-*" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" || echo "No images found"
|
||||
else
|
||||
echo "Use --all to remove images"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Networks that would be removed:"
|
||||
docker network ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No networks found"
|
||||
|
||||
echo ""
|
||||
log_info "Volumes that would be removed:"
|
||||
docker volume ls --filter "name=ml-" --format "table {{.Name}}\t{{.Driver}}" || echo "No volumes found"
|
||||
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Confirmation prompt
|
||||
if [ "$FORCE" = false ]; then
|
||||
echo ""
|
||||
read -p "This will stop and remove all fetch_ml containers. Continue? (y/N): " -n 1 -r
|
||||
echo ""
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
log_info "Cleanup cancelled"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Stop containers
|
||||
log_info "Stopping containers..."
|
||||
containers=$(docker ps -q --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker stop
|
||||
log_success "Containers stopped"
|
||||
fi
|
||||
else
|
||||
log_info "No running containers found"
|
||||
fi
|
||||
|
||||
# Remove containers
|
||||
log_info "Removing containers..."
|
||||
containers=$(docker ps -aq --filter "name=ml-")
|
||||
if [ -n "$containers" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$containers" | xargs docker rm -f
|
||||
log_success "Containers removed"
|
||||
fi
|
||||
else
|
||||
log_info "No containers found"
|
||||
fi
|
||||
|
||||
# Remove networks
|
||||
log_info "Removing networks..."
|
||||
networks=$(docker network ls -q --filter "name=ml-")
|
||||
if [ -n "$networks" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$networks" | xargs docker network rm
|
||||
log_success "Networks removed"
|
||||
fi
|
||||
else
|
||||
log_info "No networks found"
|
||||
fi
|
||||
|
||||
# Remove volumes (with caution)
|
||||
log_warning "Removing volumes (this will delete data)..."
|
||||
if [ "$FORCE" = true ] || [ "$ALL" = true ]; then
|
||||
volumes=$(docker volume ls -q --filter "name=ml-")
|
||||
if [ -n "$volumes" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$volumes" | xargs docker volume rm
|
||||
log_success "Volumes removed"
|
||||
fi
|
||||
else
|
||||
log_info "No volumes found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping volumes (use --force or --all to remove them)"
|
||||
fi
|
||||
|
||||
# Remove images if requested
|
||||
if [ "$ALL" = true ]; then
|
||||
log_info "Removing images..."
|
||||
images=$(docker images -q --filter "reference=fetch_ml-*")
|
||||
if [ -n "$images" ]; then
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
echo "$images" | xargs docker rmi -f
|
||||
log_success "Images removed"
|
||||
fi
|
||||
else
|
||||
log_info "No images found"
|
||||
fi
|
||||
else
|
||||
log_info "Skipping images (use --all to remove them)"
|
||||
fi
|
||||
|
||||
# General Docker cleanup
|
||||
log_info "Running general Docker cleanup..."
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
docker system prune -f
|
||||
log_success "General cleanup completed"
|
||||
fi
|
||||
|
||||
# Show final state
|
||||
log_info "Final resources: $(count_resources)"
|
||||
|
||||
# Space reclaimed
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
log_info "Docker system info:"
|
||||
docker system df
|
||||
fi
|
||||
|
||||
log_success "Cleanup completed!"
|
||||
90
scripts/setup-auto-cleanup.sh
Executable file → Normal file
90
scripts/setup-auto-cleanup.sh
Executable file → Normal file
|
|
@ -1,90 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Setup auto-cleanup service for fetch_ml
|
||||
# This creates a systemd timer that runs cleanup daily
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_info "Setting up auto-cleanup service..."
|
||||
|
||||
# Check if running on macOS or Linux
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
log_info "Detected macOS - setting up launchd agent"
|
||||
|
||||
# Create launchd plist
|
||||
cat > ~/Library/LaunchAgents/com.fetchml.cleanup.plist << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.fetchml.cleanup</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$PROJECT_DIR/scripts/cleanup.sh</string>
|
||||
<string>--force</string>
|
||||
</array>
|
||||
<key>StartInterval</key>
|
||||
<integer>86400</integer>
|
||||
<key>RunAtLoad</key>
|
||||
<false/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/fetchml-cleanup.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/fetchml-cleanup.error.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
# Load the launchd agent
|
||||
launchctl load ~/Library/LaunchAgents/com.fetchml.cleanup.plist
|
||||
|
||||
log_success "Auto-cleanup service installed for macOS"
|
||||
log_info "Logs will be in /tmp/fetchml-cleanup.log"
|
||||
|
||||
elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
log_info "Detected Linux - setting up systemd timer"
|
||||
|
||||
# Copy service files
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.service" /etc/systemd/system/
|
||||
sudo cp "$SCRIPT_DIR/auto-cleanup.timer" /etc/systemd/system/
|
||||
|
||||
# Reload systemd and enable timer
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable auto-cleanup.timer
|
||||
sudo systemctl start auto-cleanup.timer
|
||||
|
||||
log_success "Auto-cleanup service installed for Linux"
|
||||
log_info "Check status with: systemctl status auto-cleanup.timer"
|
||||
|
||||
else
|
||||
echo "Unsupported OS: $OSTYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Auto-cleanup will run daily"
|
||||
log_info "To uninstall:"
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
echo " launchctl unload ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
echo " rm ~/Library/LaunchAgents/com.fetchml.cleanup.plist"
|
||||
else
|
||||
echo " sudo systemctl stop auto-cleanup.timer"
|
||||
echo " sudo systemctl disable auto-cleanup.timer"
|
||||
echo " sudo rm /etc/systemd/system/auto-cleanup.*"
|
||||
fi
|
||||
0
scripts/testing/run-full-test-suite.sh
Executable file
0
scripts/testing/run-full-test-suite.sh
Executable file
Loading…
Reference in a new issue