From c89d970210222f5469a2d45684a298e4f2061c7d Mon Sep 17 00:00:00 2001 From: Jeremie Fraeys Date: Sat, 21 Feb 2026 13:43:58 -0500 Subject: [PATCH] refactor: migrate from env var to build tags for native libs Replace FETCHML_NATIVE_LIBS=1 environment variable with -tags native_libs: Changes: - internal/queue/native_queue.go: UseNativeQueue is now const true - internal/queue/native_queue_stub.go: UseNativeQueue is now const false - build/docker/simple.Dockerfile: Add -tags native_libs to go build - deployments/docker-compose.dev.yml: Remove FETCHML_NATIVE_LIBS env var - native/README.md: Update documentation for build tags - scripts/test-native-with-redis.sh: New test script with Redis via docker-compose Benefits: - Compile-time enforcement (no runtime checks needed) - Cleaner deployment (no env var management) - Type safety (const vs var) - Simpler testing with docker-compose Redis integration --- build/docker/simple.Dockerfile | 6 ++-- deployments/docker-compose.dev.yml | 4 +-- internal/queue/native_queue.go | 5 ++- internal/queue/native_queue_stub.go | 2 +- native/README.md | 15 +++++--- scripts/test-native-with-redis.sh | 53 +++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 13 deletions(-) create mode 100755 scripts/test-native-with-redis.sh diff --git a/build/docker/simple.Dockerfile b/build/docker/simple.Dockerfile index fac9004..e2c4feb 100644 --- a/build/docker/simple.Dockerfile +++ b/build/docker/simple.Dockerfile @@ -22,9 +22,9 @@ RUN rm -rf native/build && cd native && mkdir -p build && cd build && \ cmake .. -DCMAKE_BUILD_TYPE=Release && \ make -j$(nproc) -# Build Go binaries with native libs -RUN CGO_ENABLED=1 go build -o bin/api-server cmd/api-server/main.go && \ - CGO_ENABLED=1 go build -o bin/worker ./cmd/worker +# Build Go binaries with native libs enabled via build tag +RUN CGO_ENABLED=1 go build -tags native_libs -o bin/api-server cmd/api-server/main.go && \ + CGO_ENABLED=1 go build -tags native_libs -o bin/worker ./cmd/worker # Final stage FROM alpine:3.19 diff --git a/deployments/docker-compose.dev.yml b/deployments/docker-compose.dev.yml index 651fcd4..58cd4d9 100644 --- a/deployments/docker-compose.dev.yml +++ b/deployments/docker-compose.dev.yml @@ -54,7 +54,7 @@ services: command: ["/bin/sh", "-c", "mkdir -p /data/experiments /data/active/datasets /data/active/snapshots && exec /usr/local/bin/api-server -config /app/configs/api/dev.yaml"] environment: - LOG_LEVEL=info - - FETCHML_NATIVE_LIBS=1 + # Native libs enabled via build tag: -tags native_libs healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9101/health"] interval: 30s @@ -155,7 +155,7 @@ services: - FETCHML_JUPYTER_CONDA_ENV=base - FETCHML_JUPYTER_KERNEL_NAME=python - FETCHML_PODMAN_CGROUPS=disabled - - FETCHML_NATIVE_LIBS=1 + # Native libs enabled via build tag: -tags native_libs privileged: true command: ["/usr/local/bin/worker", "-config", "/app/configs/worker.yaml"] # # Prometheus - Metrics collection diff --git a/internal/queue/native_queue.go b/internal/queue/native_queue.go index 4c08144..5090c78 100644 --- a/internal/queue/native_queue.go +++ b/internal/queue/native_queue.go @@ -17,9 +17,8 @@ import ( "github.com/jfraeys/fetch_ml/internal/domain" ) -// UseNativeQueue controls whether to use C++ binary queue index. -// Set FETCHML_NATIVE_LIBS=1 to enable native libraries. -var UseNativeQueue = os.Getenv("FETCHML_NATIVE_LIBS") == "1" +// UseNativeQueue is always true when this file is compiled (build tag: native_libs) +const UseNativeQueue = true // NativeQueue wraps the C++ binary queue index for high-performance // task operations. It replaces JSON-based filesystem operations with diff --git a/internal/queue/native_queue_stub.go b/internal/queue/native_queue_stub.go index fd50319..6a58b8c 100644 --- a/internal/queue/native_queue_stub.go +++ b/internal/queue/native_queue_stub.go @@ -6,7 +6,7 @@ package queue import "errors" // UseNativeQueue is always false without native_libs build tag. -var UseNativeQueue = false +const UseNativeQueue = false // NativeQueue is not available without native_libs build tag. type NativeQueue struct{} diff --git a/native/README.md b/native/README.md index dd0f872..1e23990 100644 --- a/native/README.md +++ b/native/README.md @@ -64,11 +64,11 @@ These exclusions were conscious design choices for security (symlink attack prev # Build all native libraries make native-build -# Run with native libraries enabled -FETCHML_NATIVE_LIBS=1 go run ./... +# Run with native libraries enabled (using -tags native_libs) +go build -tags native_libs ./... # Run benchmarks -FETCHML_NATIVE_LIBS=1 go test -bench=. ./tests/benchmarks/ +go test -tags native_libs -bench=. ./tests/benchmarks/ ``` ## Test Coverage @@ -154,7 +154,14 @@ import "C" **Build verification:** ```bash make native-build -FETCHML_NATIVE_LIBS=1 make test + +# Test with native libs (using build tag) +go test -tags native_libs ./tests/... + +# Or use the test script with Redis +docker-compose -f deployments/docker-compose.dev.yml up -d redis +sleep 2 +go test -tags native_libs ./tests/... ``` **Adding new library:** diff --git a/scripts/test-native-with-redis.sh b/scripts/test-native-with-redis.sh new file mode 100755 index 0000000..a66e889 --- /dev/null +++ b/scripts/test-native-with-redis.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Test script for native libraries with Redis via docker-compose +# Usage: ./scripts/test-native-with-redis.sh + +set -e + +echo "=== FetchML Native Library Test with Redis ===" +echo "" + +# Check if native libraries are built +if [ ! -f "native/build/libqueue_index.so" ] && [ ! -f "native/build/libqueue_index.dylib" ]; then + echo "Building native libraries..." + make native-build +fi + +# Start Redis via docker-compose +echo "Starting Redis..." +cd deployments +docker-compose -f docker-compose.dev.yml up -d redis +cd .. + +# Wait for Redis to be ready +echo "Waiting for Redis to be ready..." +sleep 2 +until docker exec ml-experiments-redis redis-cli ping | grep -q PONG; do + echo " Redis not ready yet, waiting..." + sleep 1 +done +echo " Redis is ready!" +echo "" + +# Run tests with native_libs build tag +echo "Running tests with -tags native_libs..." +CGO_ENABLED=1 go test -tags native_libs -v ./tests/... 2>&1 | head -100 + +TEST_EXIT=${PIPESTATUS[0]} + +echo "" +echo "=== Test Summary ===" +if [ $TEST_EXIT -eq 0 ]; then + echo "✅ All tests passed with native libraries enabled" +else + echo "❌ Tests failed with exit code: $TEST_EXIT" +fi + +# Cleanup +echo "" +echo "Stopping Redis..." +cd deployments +docker-compose -f docker-compose.dev.yml stop redis +cd .. + +exit $TEST_EXIT