refactor: migrate from env var to build tags for native libs

Replace FETCHML_NATIVE_LIBS=1 environment variable with -tags native_libs:

Changes:
- internal/queue/native_queue.go: UseNativeQueue is now const true
- internal/queue/native_queue_stub.go: UseNativeQueue is now const false
- build/docker/simple.Dockerfile: Add -tags native_libs to go build
- deployments/docker-compose.dev.yml: Remove FETCHML_NATIVE_LIBS env var
- native/README.md: Update documentation for build tags
- scripts/test-native-with-redis.sh: New test script with Redis via docker-compose

Benefits:
- Compile-time enforcement (no runtime checks needed)
- Cleaner deployment (no env var management)
- Type safety (const vs var)
- Simpler testing with docker-compose Redis integration

build/docker/simple.Dockerfile

@@ -22,9 +22,9 @@ RUN rm -rf native/build && cd native && mkdir -p build && cd build && \
     cmake .. -DCMAKE_BUILD_TYPE=Release && \
     make -j$(nproc)
 
-# Build Go binaries with native libs
-RUN CGO_ENABLED=1 go build -o bin/api-server cmd/api-server/main.go && \
-    CGO_ENABLED=1 go build -o bin/worker ./cmd/worker
+# Build Go binaries with native libs enabled via build tag
+RUN CGO_ENABLED=1 go build -tags native_libs -o bin/api-server cmd/api-server/main.go && \
+    CGO_ENABLED=1 go build -tags native_libs -o bin/worker ./cmd/worker
 
 # Final stage
 FROM alpine:3.19

deployments/docker-compose.dev.yml

@@ -54,7 +54,7 @@ services:
     command: ["/bin/sh", "-c", "mkdir -p /data/experiments /data/active/datasets /data/active/snapshots && exec /usr/local/bin/api-server -config /app/configs/api/dev.yaml"]
     environment:
       - LOG_LEVEL=info
-      - FETCHML_NATIVE_LIBS=1
+    # Native libs enabled via build tag: -tags native_libs
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9101/health"]
       interval: 30s
@@ -155,7 +155,7 @@ services:
       - FETCHML_JUPYTER_CONDA_ENV=base
       - FETCHML_JUPYTER_KERNEL_NAME=python
       - FETCHML_PODMAN_CGROUPS=disabled
-      - FETCHML_NATIVE_LIBS=1
+    # Native libs enabled via build tag: -tags native_libs
     privileged: true
     command: ["/usr/local/bin/worker", "-config", "/app/configs/worker.yaml"]
     # # Prometheus - Metrics collection

internal/queue/native_queue.go

@@ -17,9 +17,8 @@ import (
 	"github.com/jfraeys/fetch_ml/internal/domain"
 )
 
-// UseNativeQueue controls whether to use C++ binary queue index.
-// Set FETCHML_NATIVE_LIBS=1 to enable native libraries.
-var UseNativeQueue = os.Getenv("FETCHML_NATIVE_LIBS") == "1"
+// UseNativeQueue is always true when this file is compiled (build tag: native_libs)
+const UseNativeQueue = true
 
 // NativeQueue wraps the C++ binary queue index for high-performance
 // task operations. It replaces JSON-based filesystem operations with

internal/queue/native_queue_stub.go

@@ -6,7 +6,7 @@ package queue
 import "errors"
 
 // UseNativeQueue is always false without native_libs build tag.
-var UseNativeQueue = false
+const UseNativeQueue = false
 
 // NativeQueue is not available without native_libs build tag.
 type NativeQueue struct{}

native/README.md

@@ -64,11 +64,11 @@ These exclusions were conscious design choices for security (symlink attack prev
 # Build all native libraries
 make native-build
 
-# Run with native libraries enabled
-FETCHML_NATIVE_LIBS=1 go run ./...
+# Run with native libraries enabled (using -tags native_libs)
+go build -tags native_libs ./...
 
 # Run benchmarks
-FETCHML_NATIVE_LIBS=1 go test -bench=. ./tests/benchmarks/
+go test -tags native_libs -bench=. ./tests/benchmarks/
 ```
 
 ## Test Coverage
@@ -154,7 +154,14 @@ import "C"
 **Build verification:**
 ```bash
 make native-build
-FETCHML_NATIVE_LIBS=1 make test
+
+# Test with native libs (using build tag)
+go test -tags native_libs ./tests/...
+
+# Or use the test script with Redis
+docker-compose -f deployments/docker-compose.dev.yml up -d redis
+sleep 2
+go test -tags native_libs ./tests/...
 ```
 
 **Adding new library:**

scripts/test-native-with-redis.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Test script for native libraries with Redis via docker-compose
+# Usage: ./scripts/test-native-with-redis.sh
+
+set -e
+
+echo "=== FetchML Native Library Test with Redis ==="
+echo ""
+
+# Check if native libraries are built
+if [ ! -f "native/build/libqueue_index.so" ] && [ ! -f "native/build/libqueue_index.dylib" ]; then
+    echo "Building native libraries..."
+    make native-build
+fi
+
+# Start Redis via docker-compose
+echo "Starting Redis..."
+cd deployments
+docker-compose -f docker-compose.dev.yml up -d redis
+cd ..
+
+# Wait for Redis to be ready
+echo "Waiting for Redis to be ready..."
+sleep 2
+until docker exec ml-experiments-redis redis-cli ping | grep -q PONG; do
+    echo "  Redis not ready yet, waiting..."
+    sleep 1
+done
+echo "  Redis is ready!"
+echo ""
+
+# Run tests with native_libs build tag
+echo "Running tests with -tags native_libs..."
+CGO_ENABLED=1 go test -tags native_libs -v ./tests/... 2>&1 | head -100
+
+TEST_EXIT=${PIPESTATUS[0]}
+
+echo ""
+echo "=== Test Summary ==="
+if [ $TEST_EXIT -eq 0 ]; then
+    echo "✅ All tests passed with native libraries enabled"
+else
+    echo "❌ Tests failed with exit code: $TEST_EXIT"
+fi
+
+# Cleanup
+echo ""
+echo "Stopping Redis..."
+cd deployments
+docker-compose -f docker-compose.dev.yml stop redis
+cd ..
+
+exit $TEST_EXIT