fetch_ml/internal/audit/rotation.go
Jeremie Fraeys 1c7205c0a0
feat(audit): add HTTP audit middleware and tamper-evident logging
Comprehensive audit system for security and compliance:

- middleware/audit.go: HTTP request/response auditing middleware
  * Captures request details, user identity, response status
  * Chains audit events with cryptographic hashes for tamper detection
  * Configurable filtering for sensitive data redaction

- audit/chain.go: Blockchain-style audit log chaining
  * Each entry includes hash of previous entry
  * Tamper detection through hash verification
  * Supports incremental verification without full scan

- checkpoint.go: Periodic integrity checkpoints
  * Creates signed checkpoints for fast verification
  * Configurable checkpoint intervals
  * Recovery from last known good checkpoint

- rotation.go: Automatic log rotation and archival
  * Size-based and time-based rotation policies
  * Compressed archival with integrity seals
  * Retention policy enforcement

- sealed.go: Cryptographic sealing of audit logs
  * Digital signatures for log integrity
  * HSM support preparation
  * Exportable sealed bundles for external auditors

- verifier.go: Log verification and forensic analysis
  * Complete chain verification from genesis to latest
  * Detects gaps, tampering, unauthorized modifications
  * Forensic export for incident response
2026-03-08 13:03:02 -04:00

301 lines
7.9 KiB
Go

// Package audit provides tamper-evident audit logging with hash chaining
package audit
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/jfraeys/fetch_ml/internal/fileutil"
"github.com/jfraeys/fetch_ml/internal/logging"
)
// AnchorFile represents the anchor for a rotated log file
type AnchorFile struct {
Date string `json:"date"`
LastHash string `json:"last_hash"`
LastSeq uint64 `json:"last_seq"`
FileHash string `json:"file_hash"` // SHA256 of entire rotated file
}
// RotatingLogger extends Logger with daily rotation capabilities
// and maintains cross-file chain integrity using anchor files
type RotatingLogger struct {
*Logger
basePath string
anchorDir string
currentDate string
logger *logging.Logger
}
// NewRotatingLogger creates a new rotating audit logger
func NewRotatingLogger(enabled bool, basePath, anchorDir string, logger *logging.Logger) (*RotatingLogger, error) {
if !enabled {
return &RotatingLogger{
Logger: &Logger{enabled: false},
basePath: basePath,
anchorDir: anchorDir,
logger: logger,
}, nil
}
// Ensure anchor directory exists
if err := os.MkdirAll(anchorDir, 0o750); err != nil {
return nil, fmt.Errorf("create anchor directory: %w", err)
}
currentDate := time.Now().UTC().Format("2006-01-02")
fullPath := filepath.Join(basePath, fmt.Sprintf("audit-%s.log", currentDate))
// Create base directory if needed
dir := filepath.Dir(fullPath)
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, fmt.Errorf("create audit directory: %w", err)
}
// Open the log file for current date
// #nosec G304 -- fullPath is internally constructed from basePath and currentDate
file, err := os.OpenFile(fullPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
if err != nil {
return nil, fmt.Errorf("open audit log file: %w", err)
}
al := &Logger{
enabled: true,
filePath: fullPath,
file: file,
sequenceNum: 0,
lastHash: "",
logger: logger,
}
// Resume from file if it exists
if err := al.resumeFromFile(); err != nil {
if closeErr := file.Close(); closeErr != nil {
return nil, fmt.Errorf("resume audit chain: %w, close: %v", err, closeErr)
}
return nil, fmt.Errorf("resume audit chain: %w", err)
}
rl := &RotatingLogger{
Logger: al,
basePath: basePath,
anchorDir: anchorDir,
currentDate: currentDate,
logger: logger,
}
// Check if we need to rotate (different date from file)
if al.sequenceNum > 0 {
// File has entries, check if we crossed date boundary
stat, err := os.Stat(fullPath)
if err == nil {
modTime := stat.ModTime().UTC()
if modTime.Format("2006-01-02") != currentDate {
// File was last modified on a different date, should rotate
if err := rl.Rotate(); err != nil && logger != nil {
logger.Warn("failed to rotate audit log on startup", "error", err)
}
}
}
}
return rl, nil
}
// Rotate performs log rotation and creates an anchor file
// This should be called when the date changes or when the log reaches size limit
func (rl *RotatingLogger) Rotate() error {
if !rl.enabled {
return nil
}
oldPath := rl.filePath
oldDate := rl.currentDate
// Sync and close current file
if err := rl.file.Sync(); err != nil {
return fmt.Errorf("sync before rotation: %w", err)
}
if err := rl.file.Close(); err != nil {
return fmt.Errorf("close file before rotation: %w", err)
}
// Hash the rotated file for integrity
fileHash, err := sha256File(oldPath)
if err != nil {
return fmt.Errorf("hash rotated file: %w", err)
}
// Create anchor file with last hash
if rl.sequenceNum < 0 {
return fmt.Errorf("sequence number cannot be negative: %d", rl.sequenceNum)
}
anchor := AnchorFile{
Date: oldDate,
LastHash: rl.lastHash,
LastSeq: uint64(rl.sequenceNum),
FileHash: fileHash,
}
anchorPath := filepath.Join(rl.anchorDir, fmt.Sprintf("%s.anchor", oldDate))
if err := writeAnchorFile(anchorPath, anchor); err != nil {
return err
}
// Open new file for new day
rl.currentDate = time.Now().UTC().Format("2006-01-02")
newPath := filepath.Join(rl.basePath, fmt.Sprintf("audit-%s.log", rl.currentDate))
// #nosec G304 -- newPath is internally constructed from basePath and currentDate
f, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
rl.file = f
rl.filePath = newPath
// First event in new file links back to previous anchor hash
rl.Log(Event{
EventType: "rotation_marker",
Metadata: map[string]any{
"previous_anchor_hash": anchor.LastHash,
"previous_date": oldDate,
},
})
if rl.logger != nil {
rl.logger.Info("audit log rotated",
"previous_date", oldDate,
"new_date", rl.currentDate,
"anchor", anchorPath,
)
}
return nil
}
// CheckRotation checks if rotation is needed based on date
func (rl *RotatingLogger) CheckRotation() error {
if !rl.enabled {
return nil
}
newDate := time.Now().UTC().Format("2006-01-02")
if newDate != rl.currentDate {
return rl.Rotate()
}
return nil
}
// writeAnchorFile writes the anchor file to disk with crash safety (fsync)
func writeAnchorFile(path string, anchor AnchorFile) error {
data, err := json.Marshal(anchor)
if err != nil {
return fmt.Errorf("marshal anchor: %w", err)
}
// SECURITY: Write with fsync for crash safety
if err := fileutil.WriteFileSafe(path, data, 0o600); err != nil {
return fmt.Errorf("write anchor file: %w", err)
}
return nil
}
// readAnchorFile reads an anchor file from disk
func readAnchorFile(path string) (*AnchorFile, error) {
// #nosec G304 -- path is an internally controlled anchor file path
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read anchor file: %w", err)
}
var anchor AnchorFile
if err := json.Unmarshal(data, &anchor); err != nil {
return nil, fmt.Errorf("unmarshal anchor: %w", err)
}
return &anchor, nil
}
// sha256File computes the SHA256 hash of a file
func sha256File(path string) (string, error) {
// #nosec G304 -- path is an internally controlled audit log file
data, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read file: %w", err)
}
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:]), nil
}
// VerifyRotationIntegrity verifies that a rotated file matches its anchor
func VerifyRotationIntegrity(logPath, anchorPath string) error {
anchor, err := readAnchorFile(anchorPath)
if err != nil {
return err
}
// Verify file hash
actualFileHash, err := sha256File(logPath)
if err != nil {
return err
}
if !strings.EqualFold(actualFileHash, anchor.FileHash) {
return fmt.Errorf("TAMPERING DETECTED: file hash mismatch: expected=%s, got=%s",
anchor.FileHash, actualFileHash)
}
// Verify chain ends with anchor's last hash
lastSeq, lastHash, err := getLastEventFromFile(logPath)
if err != nil {
return err
}
if lastSeq < 0 {
return fmt.Errorf("sequence number cannot be negative: %d", lastSeq)
}
if uint64(lastSeq) != anchor.LastSeq || lastHash != anchor.LastHash {
return fmt.Errorf("TAMPERING DETECTED: chain mismatch: expected(seq=%d,hash=%s), got(seq=%d,hash=%s)",
anchor.LastSeq, anchor.LastHash, lastSeq, lastHash)
}
return nil
}
// getLastEventFromFile returns the last event's sequence and hash from a file
func getLastEventFromFile(path string) (int64, string, error) {
// #nosec G304 -- path is an internally controlled audit log file
file, err := os.Open(path)
if err != nil {
return 0, "", err
}
defer file.Close()
var lastLine string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if line != "" {
lastLine = line
}
}
if err := scanner.Err(); err != nil {
return 0, "", err
}
if lastLine == "" {
return 0, "", fmt.Errorf("no events in file")
}
var event Event
if err := json.Unmarshal([]byte(lastLine), &event); err != nil {
return 0, "", fmt.Errorf("parse last event: %w", err)
}
return event.SequenceNum, event.EventHash, nil
}