fetch_ml/internal/fileutil/secure.go
Jeremie Fraeys 92aab06d76
feat(security): implement comprehensive security hardening phases 1-5,7
Implements defense-in-depth security for HIPAA and multi-tenant requirements:

**Phase 1 - File Ingestion Security:**
- SecurePathValidator with symlink resolution and path boundary enforcement
  in internal/fileutil/secure.go
- Magic bytes validation for ML artifacts (safetensors, GGUF, HDF5, numpy)
  in internal/fileutil/filetype.go
- Dangerous extension blocking (.pt, .pkl, .exe, .sh, .zip)
- Upload limits (10GB size, 100MB/s rate, 10 uploads/min)

**Phase 2 - Sandbox Hardening:**
- ApplySecurityDefaults() with secure-by-default principle
  - network_mode: none, read_only_root: true, no_new_privileges: true
  - drop_all_caps: true, user_ns: true, run_as_uid/gid: 1000
- PodmanSecurityConfig and BuildSecurityArgs() in internal/container/podman.go
- BuildPodmanCommand now accepts full security configuration
- Container executor passes SandboxConfig to Podman command builder
- configs/seccomp/default-hardened.json blocks dangerous syscalls
  (ptrace, mount, reboot, kexec_load, open_by_handle_at)

**Phase 3 - Secrets Management:**
- expandSecrets() for environment variable expansion using ${VAR} syntax
- validateNoPlaintextSecrets() with entropy-based detection
- Pattern matching for AWS, GitHub, GitLab, OpenAI, Stripe tokens
- Shannon entropy calculation (>4 bits/char triggers detection)
- Secrets expanded during LoadConfig() before validation

**Phase 5 - HIPAA Audit Logging:**
- Tamper-evident chain hashing with SHA-256 in internal/audit/audit.go
- Event struct extended with PrevHash, EventHash, SequenceNum
- File access event types: EventFileRead, EventFileWrite, EventFileDelete
- LogFileAccess() helper for HIPAA compliance
- VerifyChain() function for tamper detection

**Supporting Changes:**
- Add DeleteJob() and DeleteJobsByPrefix() to storage package
- Integrate SecurePathValidator in artifact scanning
2026-02-23 18:00:33 -05:00

135 lines
4.4 KiB
Go

// Package fileutil provides secure file operation utilities to prevent path traversal attacks.
package fileutil
import (
"crypto/rand"
"encoding/base64"
"fmt"
"os"
"path/filepath"
"strings"
)
// SecurePathValidator provides path traversal protection with symlink resolution.
type SecurePathValidator struct {
BasePath string
}
// NewSecurePathValidator creates a new path validator for a base directory.
func NewSecurePathValidator(basePath string) *SecurePathValidator {
return &SecurePathValidator{BasePath: basePath}
}
// ValidatePath ensures resolved path is within base directory.
// It resolves symlinks and returns the canonical absolute path.
func (v *SecurePathValidator) ValidatePath(inputPath string) (string, error) {
if v.BasePath == "" {
return "", fmt.Errorf("base path not set")
}
// Clean the path to remove . and ..
cleaned := filepath.Clean(inputPath)
// Get absolute base path and resolve any symlinks (critical for macOS /tmp -> /private/tmp)
baseAbs, err := filepath.Abs(v.BasePath)
if err != nil {
return "", fmt.Errorf("failed to get absolute base path: %w", err)
}
// Resolve symlinks in base path for accurate comparison
baseResolved, err := filepath.EvalSymlinks(baseAbs)
if err != nil {
// Base path may not exist yet, use as-is
baseResolved = baseAbs
}
// If cleaned is already absolute, check if it's within base
var absPath string
if filepath.IsAbs(cleaned) {
absPath = cleaned
} else {
// Join with base path if relative
absPath = filepath.Join(baseAbs, cleaned)
}
// Resolve symlinks - critical for security
resolved, err := filepath.EvalSymlinks(absPath)
if err != nil {
// If the file doesn't exist, we still need to check the directory path
// Try to resolve the parent directory
dir := filepath.Dir(absPath)
resolvedDir, dirErr := filepath.EvalSymlinks(dir)
if dirErr != nil {
return "", fmt.Errorf("path resolution failed: %w", err)
}
// Reconstruct the path with resolved directory
base := filepath.Base(absPath)
resolved = filepath.Join(resolvedDir, base)
}
// Get absolute resolved path
resolvedAbs, err := filepath.Abs(resolved)
if err != nil {
return "", fmt.Errorf("failed to get absolute resolved path: %w", err)
}
// Verify within base directory (must have path separator after base to prevent prefix match issues)
baseWithSep := baseResolved + string(filepath.Separator)
if resolvedAbs != baseResolved && !strings.HasPrefix(resolvedAbs+string(filepath.Separator), baseWithSep) {
return "", fmt.Errorf("path escapes base directory: %s (resolved to %s, base is %s)", inputPath, resolvedAbs, baseResolved)
}
return resolvedAbs, nil
}
// SecureFileRead securely reads a file after cleaning the path to prevent path traversal.
func SecureFileRead(path string) ([]byte, error) {
return os.ReadFile(filepath.Clean(path))
}
// SecureFileWrite securely writes a file after cleaning the path to prevent path traversal.
func SecureFileWrite(path string, data []byte, perm os.FileMode) error {
return os.WriteFile(filepath.Clean(path), data, perm)
}
// SecureOpenFile securely opens a file after cleaning the path to prevent path traversal.
func SecureOpenFile(path string, flag int, perm os.FileMode) (*os.File, error) {
return os.OpenFile(filepath.Clean(path), flag, perm)
}
// SecureReadDir reads directory contents with path validation.
func (v *SecurePathValidator) SecureReadDir(dirPath string) ([]os.DirEntry, error) {
validatedPath, err := v.ValidatePath(dirPath)
if err != nil {
return nil, fmt.Errorf("directory path validation failed: %w", err)
}
return os.ReadDir(validatedPath)
}
// SecureCreateTemp creates a temporary file within the base directory.
func (v *SecurePathValidator) SecureCreateTemp(pattern string) (*os.File, string, error) {
validatedPath, err := v.ValidatePath("")
if err != nil {
return nil, "", fmt.Errorf("base directory validation failed: %w", err)
}
// Generate secure random suffix
randomBytes := make([]byte, 16)
if _, err := rand.Read(randomBytes); err != nil {
return nil, "", fmt.Errorf("failed to generate random bytes: %w", err)
}
randomSuffix := base64.URLEncoding.EncodeToString(randomBytes)
// Create temp file
if pattern == "" {
pattern = "tmp"
}
fileName := fmt.Sprintf("%s_%s", pattern, randomSuffix)
fullPath := filepath.Join(validatedPath, fileName)
file, err := os.Create(fullPath)
if err != nil {
return nil, "", fmt.Errorf("failed to create temp file: %w", err)
}
return file, fullPath, nil
}