fetch_ml/.forgejo/workflows/ci.yml

name: CI/CD Pipeline

on:
  workflow_dispatch:
  push:
    paths-ignore:
      - 'docs/**'
      - 'README.md'
      - 'CHANGELOG.md'
      - '.forgejo/ISSUE_TEMPLATE/**'
      - '**/*.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  security-events: write
  actions: read
  packages: write

env:
  GO_VERSION: '1.25.0'
  ZIG_VERSION: '0.15.2'
  RSYNC_VERSION: '3.3.0'

jobs:
  test:
    name: Test
    runs-on: self-hosted
    timeout-minutes: 30

    services:
      redis:
        image: redis:7
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5          

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      run: |
        REQUIRED_GO="1.25.0"
        if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
          echo "Go ${REQUIRED_GO} already installed - skipping download"
        else
          echo "Installing Go ${REQUIRED_GO}..."
          curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
          export PATH="/usr/local/go/bin:$PATH"
          echo "/usr/local/go/bin" >> $GITHUB_PATH
          echo "Go ${REQUIRED_GO} installed"
        fi
        go version        

    - name: Set up Zig
      run: |
        ZIG_VERSION="${{ env.ZIG_VERSION }}"
        if command -v zig &> /dev/null && zig version | grep -q "${ZIG_VERSION}"; then
          echo "Zig ${ZIG_VERSION} already installed - skipping download"
        else
          echo "Installing Zig ${ZIG_VERSION}..."
          ZIG_DIR="/usr/local/zig-${ZIG_VERSION}"
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl -fsSL --retry 3 "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
            sudo mkdir -p "${ZIG_DIR}"
            sudo tar -C "${ZIG_DIR}" --strip-components=1 -xJf /tmp/zig.tar.xz
            sudo ln -sf "${ZIG_DIR}/zig" /usr/local/bin/zig
          elif [[ "$OSTYPE" == "darwin"* ]]; then
            curl -fsSL --retry 3 "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
            sudo mkdir -p "${ZIG_DIR}"
            sudo tar -C "${ZIG_DIR}" --strip-components=1 -xJf /tmp/zig.tar.xz
            sudo ln -sf "${ZIG_DIR}/zig" /usr/local/bin/zig
          fi
          rm -f /tmp/zig.tar.xz
          echo "Zig ${ZIG_VERSION} installed"
        fi
        zig version        

    - name: Install dependencies
      run: |
        go mod download
        sudo apt-get update
        sudo apt-get install -y podman redis-tools build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source (for CLI tests)
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Verify dependencies
      run: go mod verify

    - name: Run tests
      run: make test

    - name: Test internal/queue package
      run: go test -v -race -coverprofile=queue-coverage.out ./internal/queue/...

    - name: Run comprehensive tests
      run: make test-full

    - name: Run linters
      run: make lint

    - name: Generate coverage report
      run: make test-coverage

  dev-smoke:
    name: Dev Compose Smoke Test
    runs-on: self-hosted
    needs: test
    timeout-minutes: 20

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Run dev smoke test
      run: make dev-smoke

  build:
    name: Build
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      run: |
        REQUIRED_GO="1.25.0"
        if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
          echo "Go ${REQUIRED_GO} already installed - skipping download"
        else
          echo "Installing Go ${REQUIRED_GO}..."
          curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
          export PATH="/usr/local/go/bin:$PATH"
          echo "/usr/local/go/bin" >> $GITHUB_PATH
          echo "Go ${REQUIRED_GO} installed"
        fi
        go version        

    - name: Set up Zig
      run: |
        ZIG_VERSION="${{ env.ZIG_VERSION }}"
        if command -v zig &> /dev/null && zig version | grep -q "${ZIG_VERSION}"; then
          echo "Zig ${ZIG_VERSION} already installed - skipping download"
        else
          echo "Installing Zig ${ZIG_VERSION}..."
          ZIG_DIR="/usr/local/zig-${ZIG_VERSION}"
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl -fsSL --retry 3 "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
            sudo mkdir -p "${ZIG_DIR}"
            sudo tar -C "${ZIG_DIR}" --strip-components=1 -xJf /tmp/zig.tar.xz
            sudo ln -sf "${ZIG_DIR}/zig" /usr/local/bin/zig
          elif [[ "$OSTYPE" == "darwin"* ]]; then
            curl -fsSL --retry 3 "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" -o /tmp/zig.tar.xz
            sudo mkdir -p "${ZIG_DIR}"
            sudo tar -C "${ZIG_DIR}" --strip-components=1 -xJf /tmp/zig.tar.xz
            sudo ln -sf "${ZIG_DIR}/zig" /usr/local/bin/zig
          fi
          rm -f /tmp/zig.tar.xz
          echo "Zig ${ZIG_VERSION} installed"
        fi
        zig version        

    - name: Install build dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Build SQLite for CLI
      run: |
        make -C cli build-sqlite        

    - name: Build CLI binary
      run: |
        cd cli && make tiny        

    - name: Build Go binaries
      run: |
        make build        

    - name: Test binaries
      run: |
        ./bin/user_manager --help
        ./bin/worker --help
        ./bin/tui --help
        ./bin/data_manager --help
        ./cli/zig-out/bin/ml --help
        ls -lh ./cli/zig-out/bin/ml        

    - name: Upload build artifacts
      uses: actions/upload-artifact@v4
      with:
        name: fetch_ml_binaries
        path: |
          bin/
          cli/zig-out/
          dist/          
        retention-days: 30

  test-scripts:
    name: Test Scripts
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman redis-tools bats        

    - name: Test scripts
      run: |
        chmod +x scripts/*.sh || true
        chmod +x scripts/maintenance/*.sh || true

        ./scripts/verify_release.sh --help
        ./scripts/manage-artifacts.sh help
        ./scripts/track_performance.sh --help
        ./scripts/smoke-test.sh --help        

  test-native:
    name: Test Native Libraries
    runs-on: self-hosted
    needs: test
    timeout-minutes: 30

    services:
      redis:
        image: redis:7-alpine
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 5s
          --health-timeout 3s
          --health-retries 3          

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Install cmake and build tools
        run: |
          echo "Installing cmake and build dependencies..."
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            if command -v apt-get &> /dev/null; then
              sudo apt-get update
              sudo apt-get install -y cmake zlib1g-dev build-essential
            elif command -v yum &> /dev/null; then
              sudo yum install -y cmake zlib-devel gcc-c++
            fi
          elif [[ "$OSTYPE" == "darwin"* ]]; then
            brew install cmake zlib
          fi
          which cmake          

      - name: Setup Go
        run: |
          REQUIRED_GO="1.25.0"
          if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
            echo "Go ${REQUIRED_GO} already installed - skipping download"
          else
            echo "Installing Go ${REQUIRED_GO}..."
            if [[ "$OSTYPE" == "linux-gnu"* ]]; then
              curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
            elif [[ "$OSTYPE" == "darwin"* ]]; then
              curl -sL "https://go.dev/dl/go${REQUIRED_GO}.darwin-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
            fi
            export PATH="/usr/local/go/bin:$PATH"
            echo "/usr/local/go/bin" >> $GITHUB_PATH
            echo "Go ${REQUIRED_GO} installed"
          fi
          go version          

      - name: Build Native Libraries
        run: |
          echo "Building native C++ libraries..."
          make native-build 2>&1 || {
            echo ""
            echo "Native build failed!"
            echo ""
            echo "Common causes:"
            echo "  1. Missing cmake: Install with 'apt-get install cmake'"
            echo "  2. Missing C++ compiler: Install with 'apt-get install build-essential'"
            echo "  3. Missing zlib: Install with 'apt-get install zlib1g-dev'"
            echo "  4. CMakeLists.txt not found: Ensure native/CMakeLists.txt exists"
            echo ""
            exit 1
          }
          echo "Native libraries built successfully"          

      - name: Test with Native Libraries
        run: |
          echo "Running tests WITH native libraries enabled..."
          CGO_ENABLED=1 go test -tags native_libs -v ./tests/... || true          

      - name: Native Smoke Test
        run: |
          echo "Running native libraries smoke test..."
          CGO_ENABLED=1 go test -tags native_libs ./tests/benchmarks/... -run TestNative || true          

      - name: Test Fallback (Go only)
        run: |
          echo "Running tests WITHOUT native libraries (Go fallback)..."
          go test -v ./tests/... || true          

      - name: Run Benchmarks
        run: |
          echo "Running performance benchmarks..."
          echo "=== Go Implementation ==="
          go test -bench=. ./tests/benchmarks/ -benchmem || true
          echo ""
          echo "=== Native Implementation ==="
          CGO_ENABLED=1 go test -tags native_libs -bench=. ./tests/benchmarks/ -benchmem || true          

  test-gpu-matrix:
    name: GPU Golden Test Matrix
    runs-on: self-hosted
    needs: test-native
    timeout-minutes: 15
    strategy:
      matrix:
        build_config: [cgo-native, cgo-only, nocgo]
      fail-fast: false

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Setup Go
        run: |
          REQUIRED_GO="1.25.0"
          if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
            echo "Go ${REQUIRED_GO} already installed"
          else
            echo "Installing Go ${REQUIRED_GO}..."
            curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
            export PATH="/usr/local/go/bin:$PATH"
            echo "/usr/local/go/bin" >> $GITHUB_PATH
          fi
          go version          

      - name: Build Native Libraries (for cgo-native config)
        if: matrix.build_config == 'cgo-native'
        run: |
          sudo apt-get update
          sudo apt-get install -y cmake zlib1g-dev build-essential
          make native-build || echo "Native build skipped (may fail without proper deps)"          

      - name: Run GPU Tests - cgo+native_libs
        if: matrix.build_config == 'cgo-native'
        run: |
          echo "=== Testing cgo + native_libs build ==="
          CGO_ENABLED=1 go test -tags native_libs -v ./tests/unit/gpu/ -run TestGoldenGPUStatus
          CGO_ENABLED=1 go test -tags native_libs -v ./tests/unit/gpu/ -run TestBuildTagMatrix          

      - name: Run GPU Tests - cgo only (no native_libs)
        if: matrix.build_config == 'cgo-only'
        run: |
          echo "=== Testing cgo without native_libs build ==="
          CGO_ENABLED=1 go test -v ./tests/unit/gpu/ -run TestGoldenGPUStatus
          CGO_ENABLED=1 go test -v ./tests/unit/gpu/ -run TestBuildTagMatrix          

      - name: Run GPU Tests - nocgo
        if: matrix.build_config == 'nocgo'
        run: |
          echo "=== Testing !cgo build ==="
          CGO_ENABLED=0 go test -v ./tests/unit/gpu/ -run TestGoldenGPUStatus
          CGO_ENABLED=0 go test -v ./tests/unit/gpu/ -run TestBuildTagMatrix          

  docker-build:
    name: Docker Build
    runs-on: self-hosted
    needs: [test, test-native, build, test-scripts]
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    timeout-minutes: 30

    steps:
    - name: Check Docker registry secret
      run: |
        if [ -z "${{ secrets.GHCR_TOKEN }}" ]; then
          echo "GHCR_TOKEN not set, skipping Docker build"
          exit 0
        fi        

    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
      with:
        driver-opts: |
          image=moby/buildkit:master          

    - name: Login to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ secrets.GHCR_USERNAME }}
        password: ${{ secrets.GHCR_TOKEN }}

    - name: Build and push Docker image
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          ghcr.io/${{ github.repository }}:latest
          ghcr.io/${{ github.repository }}:${{ github.sha }}