bb25743b0f
- Add production setup scripts for automated deployment - Include monitoring setup and configuration validation - Add legacy setup scripts for various Linux distributions - Implement Bitwarden integration for secure credential management - Add development and production environment setup - Include comprehensive management tools and utilities - Add shell script library with common functions Provides complete automation for setup, deployment, and management of FetchML platform in development and production environments.
314 lines
8.1 KiB
Bash
Executable file
314 lines
8.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
# Fetch ML Quick Start Script with Security
|
|
# Sets up development environment with security features and creates test user
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
print_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
print_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
print_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
}
|
|
|
|
print_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
check_prerequisites() {
|
|
print_info "Checking prerequisites..."
|
|
|
|
# Check Go
|
|
if ! command -v go &> /dev/null; then
|
|
print_error "Go is not installed. Please install Go 1.25 or later."
|
|
exit 1
|
|
fi
|
|
|
|
local go_version=$(go version | awk '{print $3}' | sed 's/go//')
|
|
print_info "Go version: $go_version"
|
|
|
|
# Check Zig
|
|
if ! command -v zig &> /dev/null; then
|
|
print_warning "Zig is not installed. CLI features will not be available."
|
|
else
|
|
local zig_version=$(zig version)
|
|
print_info "Zig version: $zig_version"
|
|
fi
|
|
|
|
# Check Docker
|
|
if ! command -v docker &> /dev/null; then
|
|
print_warning "Docker is not installed. Container features will not work."
|
|
fi
|
|
|
|
# Check Redis
|
|
if ! command -v redis-server &> /dev/null && ! command -v redis-cli &> /dev/null; then
|
|
print_warning "Redis is not installed. Starting local Redis..."
|
|
fi
|
|
|
|
# Check OpenSSL for certificates
|
|
if ! command -v openssl &> /dev/null; then
|
|
print_warning "OpenSSL is not installed. TLS certificates will not be generated."
|
|
fi
|
|
|
|
print_success "Prerequisites checked"
|
|
}
|
|
|
|
setup_project() {
|
|
print_info "Setting up Fetch ML project..."
|
|
|
|
# Create directories
|
|
mkdir -p bin
|
|
mkdir -p data
|
|
mkdir -p logs
|
|
mkdir -p db
|
|
mkdir -p ssl
|
|
mkdir -p configs
|
|
|
|
print_success "Project directories created"
|
|
}
|
|
|
|
build_project() {
|
|
print_info "Building Fetch ML..."
|
|
|
|
# Build Go binaries
|
|
make build
|
|
|
|
# Build Zig CLI if available
|
|
if command -v zig &> /dev/null; then
|
|
make cli-build
|
|
print_success "Zig CLI built"
|
|
fi
|
|
|
|
print_success "Build completed"
|
|
}
|
|
|
|
generate_ssl_certificates() {
|
|
print_info "Generating SSL certificates..."
|
|
|
|
if command -v openssl &> /dev/null; then
|
|
# Generate self-signed certificate for development
|
|
openssl req -x509 -newkey rsa:4096 -keyout ssl/key.pem -out ssl/cert.pem \
|
|
-days 365 -nodes -subj "/C=US/ST=State/L=City/O=Organization/CN=localhost" \
|
|
-addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null || {
|
|
print_warning "Failed to generate SSL certificates"
|
|
return 1
|
|
}
|
|
|
|
print_success "SSL certificates generated in ssl/"
|
|
print_info "Certificates are self-signed (development only)"
|
|
else
|
|
print_warning "OpenSSL not available, skipping SSL certificates"
|
|
fi
|
|
}
|
|
|
|
setup_redis() {
|
|
print_info "Setting up Redis..."
|
|
|
|
if command -v redis-server &> /dev/null; then
|
|
if ! pgrep -f "redis-server" > /dev/null; then
|
|
redis-server --daemonize yes --port 6379
|
|
print_success "Redis started"
|
|
else
|
|
print_info "Redis already running"
|
|
fi
|
|
else
|
|
print_warning "Redis not available, some features may be limited"
|
|
fi
|
|
}
|
|
|
|
create_secure_config() {
|
|
print_info "Creating secure development configuration..."
|
|
|
|
# Generate secure passwords and secrets
|
|
local redis_password=$(openssl rand -base64 32 2>/dev/null || echo "dev_redis_password_123")
|
|
local jwt_secret=$(openssl rand -base64 64 2>/dev/null || echo "dev_jwt_secret_1234567890123456789012345678901234567890123456789012345678901234")
|
|
|
|
# Create development config
|
|
cat > configs/config.yaml << EOF
|
|
base_path: "/data/ml-experiments"
|
|
|
|
auth:
|
|
enabled: true
|
|
api_keys:
|
|
test_user:
|
|
hash: "$(echo -n "dev_test_api_key_12345" | sha256sum | cut -d' ' -f1)"
|
|
admin: true
|
|
roles: ["data_scientist", "admin"]
|
|
permissions:
|
|
read: true
|
|
write: true
|
|
delete: true
|
|
|
|
server:
|
|
address: ":9101"
|
|
tls:
|
|
enabled: true
|
|
cert_file: "./ssl/cert.pem"
|
|
key_file: "./ssl/key.pem"
|
|
min_version: "1.3"
|
|
|
|
security:
|
|
rate_limit:
|
|
enabled: true
|
|
requests_per_minute: 60
|
|
burst_size: 10
|
|
ip_whitelist:
|
|
- "127.0.0.1"
|
|
- "::1"
|
|
- "10.0.0.0/8"
|
|
- "192.168.0.0/16"
|
|
- "172.16.0.0/12"
|
|
failed_login_lockout:
|
|
enabled: true
|
|
max_attempts: 5
|
|
lockout_duration: "15m"
|
|
|
|
redis:
|
|
url: "redis://localhost:6379"
|
|
password: "${redis_password}"
|
|
|
|
logging:
|
|
level: "info"
|
|
file: "logs/fetch_ml.log"
|
|
audit_log: "logs/audit.log"
|
|
EOF
|
|
|
|
# Create environment file
|
|
cat > .env.dev << EOF
|
|
# Development environment variables
|
|
REDIS_PASSWORD=${redis_password}
|
|
JWT_SECRET=${jwt_secret}
|
|
GRAFANA_USER=admin
|
|
GRAFANA_PASSWORD=$(openssl rand -base64 16 2>/dev/null || echo "dev_grafana_password")
|
|
EOF
|
|
|
|
print_success "Secure configuration created"
|
|
print_warning "Using development certificates and passwords"
|
|
}
|
|
|
|
create_test_user() {
|
|
print_info "Creating test user..."
|
|
|
|
# Generate API key for test user
|
|
local api_key="dev_test_api_key_12345"
|
|
local api_key_hash=$(echo -n "$api_key" | sha256sum | cut -d' ' -f1)
|
|
|
|
print_success "Test user created successfully"
|
|
echo "Username: test_user"
|
|
echo "API Key: $api_key"
|
|
echo "API Key Hash: $api_key_hash"
|
|
echo "Store this key safely!"
|
|
echo ""
|
|
echo "Environment variables in .env.dev"
|
|
echo "Run: source .env.dev"
|
|
}
|
|
|
|
test_setup() {
|
|
print_info "Testing setup..."
|
|
|
|
# Test Go binaries
|
|
if [[ -f "bin/api-server" ]]; then
|
|
./bin/api-server --help > /dev/null 2>&1 || true
|
|
print_success "API server binary OK"
|
|
fi
|
|
|
|
if [[ -f "bin/worker" ]]; then
|
|
./bin/worker --help > /dev/null 2>&1 || true
|
|
print_success "Worker binary OK"
|
|
fi
|
|
|
|
# Test Zig CLI
|
|
if [[ -f "cli/zig-out/bin/ml" ]]; then
|
|
./cli/zig-out/bin/ml --help > /dev/null 2>&1 || true
|
|
print_success "Zig CLI binary OK"
|
|
fi
|
|
|
|
# Test Redis connection
|
|
if command -v redis-cli &> /dev/null; then
|
|
if redis-cli ping > /dev/null 2>&1; then
|
|
print_success "Redis connection OK"
|
|
else
|
|
print_warning "Redis not responding"
|
|
fi
|
|
fi
|
|
|
|
# Test SSL certificates
|
|
if [[ -f "ssl/cert.pem" && -f "ssl/key.pem" ]]; then
|
|
if openssl x509 -in ssl/cert.pem -noout -checkend 86400 > /dev/null 2>&1; then
|
|
print_success "SSL certificates valid"
|
|
else
|
|
print_warning "SSL certificates expired or invalid"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
show_next_steps() {
|
|
print_success "Secure quick start completed!"
|
|
echo
|
|
echo "Next steps:"
|
|
echo "1. Load environment variables:"
|
|
echo " source .env.dev"
|
|
echo
|
|
echo "2. Start API server:"
|
|
echo " ./bin/api-server -config configs/config.yaml"
|
|
echo
|
|
echo "3. Test Zig CLI:"
|
|
echo " ./cli/zig-out/bin/ml --help"
|
|
echo
|
|
echo "4. Test with curl (HTTPS):"
|
|
echo " curl -k -H 'X-API-Key: dev_test_api_key_12345' https://localhost:9101/health"
|
|
echo
|
|
echo "5. Deploy with Docker:"
|
|
echo " docker-compose up -d"
|
|
echo
|
|
echo "Features Enabled:"
|
|
echo " ✅ HTTPS/TLS encryption"
|
|
echo " ✅ API key authentication"
|
|
echo " ✅ Rate limiting"
|
|
echo " ✅ IP whitelisting"
|
|
echo " ✅ Security headers"
|
|
echo " ✅ Audit logging"
|
|
echo
|
|
echo "Configuration Files:"
|
|
echo " configs/config.yaml # Main configuration"
|
|
echo " .env.dev # Environment variables"
|
|
echo " ssl/cert.pem, ssl/key.pem # TLS certificates"
|
|
echo
|
|
echo "Documentation:"
|
|
echo " docs/DEPLOYMENT.md # Deployment guide"
|
|
echo ""
|
|
print_success "Ready to run ML experiments!"
|
|
}
|
|
|
|
# Main function
|
|
main() {
|
|
echo "Fetch ML Quick Start Script (with Security & Zig CLI)"
|
|
echo "===================================================="
|
|
echo ""
|
|
|
|
check_prerequisites
|
|
setup_project
|
|
build_project
|
|
generate_ssl_certificates
|
|
setup_redis
|
|
create_secure_config
|
|
create_test_user
|
|
test_setup
|
|
show_next_steps
|
|
}
|
|
|
|
# Run main function
|
|
main "$@"
|