fetch_ml/internal/api/server_config.go
Jeremie Fraeys 420de879ff
feat(api): integrate scheduler protocol and WebSocket enhancements
Update API layer for scheduler integration:
- WebSocket handlers with scheduler protocol support
- Jobs WebSocket endpoint with priority queue integration
- Validation middleware for scheduler messages
- Server configuration with security hardening
- Protocol definitions for worker-scheduler communication
- Dataset handlers with tenant isolation checks
- Response helpers with audit context
- OpenAPI spec updates for new endpoints
2026-02-26 12:05:57 -05:00

208 lines
5.9 KiB
Go

package api
import (
"fmt"
"log"
"path/filepath"
"strings"
"github.com/jfraeys/fetch_ml/internal/auth"
"github.com/jfraeys/fetch_ml/internal/config"
"github.com/jfraeys/fetch_ml/internal/fileutil"
"github.com/jfraeys/fetch_ml/internal/logging"
"github.com/jfraeys/fetch_ml/internal/storage"
"gopkg.in/yaml.v3"
)
type QueueConfig struct {
Backend string `yaml:"backend"`
SQLitePath string `yaml:"sqlite_path"`
FilesystemPath string `yaml:"filesystem_path"`
FallbackToFilesystem bool `yaml:"fallback_to_filesystem"`
}
// ServerConfig holds all server configuration
type ServerConfig struct {
Logging logging.Config `yaml:"logging"`
BasePath string `yaml:"base_path"`
DataDir string `yaml:"data_dir"`
Auth auth.Config `yaml:"auth"`
Database DatabaseConfig `yaml:"database"`
Server ServerSection `yaml:"server"`
Monitoring MonitoringConfig `yaml:"monitoring"`
Queue QueueConfig `yaml:"queue"`
Redis RedisConfig `yaml:"redis"`
Resources config.ResourceConfig `yaml:"resources"`
Security SecurityConfig `yaml:"security"`
}
// ServerSection holds server-specific configuration
type ServerSection struct {
Address string `yaml:"address"`
TLS TLSConfig `yaml:"tls"`
}
// TLSConfig holds TLS configuration
type TLSConfig struct {
CertFile string `yaml:"cert_file"`
KeyFile string `yaml:"key_file"`
Enabled bool `yaml:"enabled"`
}
// SecurityConfig holds security-related configuration
type SecurityConfig struct {
AuditLogging AuditLog `yaml:"audit_logging"`
AllowedOrigins []string `yaml:"allowed_origins"`
IPWhitelist []string `yaml:"ip_whitelist"`
FailedLockout LockoutConfig `yaml:"failed_login_lockout"`
RateLimit RateLimitConfig `yaml:"rate_limit"`
APIKeyRotationDays int `yaml:"api_key_rotation_days"`
ProductionMode bool `yaml:"production_mode"`
}
// AuditLog holds audit logging configuration
type AuditLog struct {
LogPath string `yaml:"log_path"`
Enabled bool `yaml:"enabled"`
}
// RateLimitConfig holds rate limiting configuration
type RateLimitConfig struct {
Enabled bool `yaml:"enabled"`
RequestsPerMinute int `yaml:"requests_per_minute"`
BurstSize int `yaml:"burst_size"`
}
// LockoutConfig holds failed login lockout configuration
type LockoutConfig struct {
LockoutDuration string `yaml:"lockout_duration"`
MaxAttempts int `yaml:"max_attempts"`
Enabled bool `yaml:"enabled"`
}
// RedisConfig holds Redis connection configuration
type RedisConfig struct {
Addr string `yaml:"addr"`
Password string `yaml:"password"`
URL string `yaml:"url"`
DB int `yaml:"db"`
}
// DatabaseConfig holds database connection configuration
type DatabaseConfig struct {
Type string `yaml:"type"`
Connection string `yaml:"connection"`
Host string `yaml:"host"`
Username string `yaml:"username"`
Password string `yaml:"password"`
Database string `yaml:"database"`
Port int `yaml:"port"`
}
// LoadServerConfig loads and validates server configuration
func LoadServerConfig(path string) (*ServerConfig, error) {
resolvedConfig, err := config.ResolveConfigPath(path)
if err != nil {
return nil, err
}
cfg, err := loadConfigFromFile(resolvedConfig)
if err != nil {
return nil, err
}
cfg.Resources.ApplyDefaults()
return cfg, nil
}
// loadConfigFromFile loads configuration from a YAML file
func loadConfigFromFile(path string) (*ServerConfig, error) {
data, err := secureFileRead(path)
if err != nil {
return nil, err
}
var cfg ServerConfig
if err := yaml.Unmarshal(data, &cfg); err != nil {
return nil, err
}
return &cfg, nil
}
// secureFileRead safely reads a file
func secureFileRead(path string) ([]byte, error) {
return fileutil.SecureFileRead(path)
}
// EnsureLogDirectory creates the log directory if needed using PathRegistry
func (c *ServerConfig) EnsureLogDirectory() error {
if c.Logging.File == "" {
return nil
}
// Use PathRegistry for consistent log directory management
paths := config.FromEnv()
logDir := paths.LogDir()
if err := paths.EnsureDir(logDir); err != nil {
return fmt.Errorf("failed to create log directory: %w", err)
}
return nil
}
// BuildAuthConfig creates the auth configuration
func (c *ServerConfig) BuildAuthConfig() *auth.Config {
if !c.Auth.Enabled {
return nil
}
log.Printf("Authentication enabled with %d API keys", len(c.Auth.APIKeys))
return &c.Auth
}
// Validate performs basic configuration validation
func (c *ServerConfig) Validate() error {
// Add validation logic here
if c.Server.Address == "" {
c.Server.Address = ":8080"
}
// Use PathRegistry for consistent path management
paths := config.FromEnv()
if c.BasePath == "" {
c.BasePath = paths.ExperimentsDir()
}
if c.DataDir == "" {
c.DataDir = paths.DataDir()
}
backend := strings.ToLower(strings.TrimSpace(c.Queue.Backend))
if backend == "" {
backend = "redis"
c.Queue.Backend = backend
}
if backend != "redis" && backend != "sqlite" && backend != "filesystem" {
return fmt.Errorf("queue.backend must be one of 'redis', 'sqlite', or 'filesystem'")
}
if backend == "sqlite" {
if strings.TrimSpace(c.Queue.SQLitePath) == "" {
c.Queue.SQLitePath = filepath.Join(c.DataDir, "queue.db")
}
c.Queue.SQLitePath = storage.ExpandPath(c.Queue.SQLitePath)
if !filepath.IsAbs(c.Queue.SQLitePath) {
c.Queue.SQLitePath = filepath.Join(config.DefaultLocalDataDir, c.Queue.SQLitePath)
}
}
if backend == "filesystem" || c.Queue.FallbackToFilesystem {
if strings.TrimSpace(c.Queue.FilesystemPath) == "" {
c.Queue.FilesystemPath = filepath.Join(c.DataDir, "queue-fs")
}
c.Queue.FilesystemPath = storage.ExpandPath(c.Queue.FilesystemPath)
if !filepath.IsAbs(c.Queue.FilesystemPath) {
c.Queue.FilesystemPath = filepath.Join(config.DefaultLocalDataDir, c.Queue.FilesystemPath)
}
}
return nil
}