fetch_ml/.forgejo/workflows/ci.yml

name: CI/CD Pipeline

on:
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  security-events: write
  actions: read
  packages: write

env:
  GO_VERSION: '1.25.0'
  ZIG_VERSION: '0.15.2'
  RSYNC_VERSION: '3.3.0'

jobs:
  test:
    name: Test
    runs-on: self-hosted
    timeout-minutes: 30

    services:
      redis:
        image: redis:7
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5          

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version: ${{ env.GO_VERSION }}

    - name: Cache Go modules
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod          
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }}
        restore-keys: |
          ${{ runner.os }}-go-          

    - name: Set up Zig
      uses: goto-bus-stop/setup-zig@v2
      with:
        version: ${{ env.ZIG_VERSION }}

    - name: Cache Zig build
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/zig
          cli/zig-cache
          cli/zig-out          
        key: ${{ runner.os }}-zig-${{ hashFiles('cli/**') }}
        restore-keys: |
          ${{ runner.os }}-zig-          

    - name: Install dependencies
      run: |
        go mod download
        sudo apt-get update
        sudo apt-get install -y podman redis-tools build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source (for CLI tests)
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Verify dependencies
      run: go mod verify

    - name: Run tests
      run: make test

    - name: Test internal/queue package
      run: go test -v -race -coverprofile=queue-coverage.out ./internal/queue/...

    - name: Run comprehensive tests
      run: make test-full

    - name: Run linters
      run: make lint

    - name: Generate coverage report
      run: make test-coverage

    - name: Upload coverage to Codecov
      if: secrets.CODECOV_TOKEN != ''
      uses: codecov/codecov-action@v4
      with:
        file: ./coverage/coverage.out
        flags: unittests
        name: codecov-umbrella
        token: ${{ secrets.CODECOV_TOKEN }}

  dev-smoke:
    name: Dev Compose Smoke Test
    runs-on: self-hosted
    needs: test
    timeout-minutes: 20

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Run dev smoke test
      run: make dev-smoke

  build:
    name: Build
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version: ${{ env.GO_VERSION }}

    - name: Set up Zig
      uses: goto-bus-stop/setup-zig@v2
      with:
        version: ${{ env.ZIG_VERSION }}

    - name: Cache Go modules
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod          
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }}
        restore-keys: |
          ${{ runner.os }}-go-          

    - name: Install build dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Build binaries
      run: |
        make build        

    - name: Test binaries
      run: |
        ./bin/user_manager --help
        ./bin/worker --help
        ./bin/tui --help
        ./bin/data_manager --help
        ./cli/zig-out/bin/ml --help
        ls -lh ./cli/zig-out/bin/ml        

    - name: Upload build artifacts
      uses: actions/upload-artifact@v4
      with:
        name: fetch_ml_binaries
        path: |
          bin/
          cli/zig-out/
          dist/          
        retention-days: 30

  test-scripts:
    name: Test Scripts
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman redis-tools bats        

    - name: Test scripts
      run: |
        chmod +x scripts/*.sh || true
        chmod +x scripts/maintenance/*.sh || true

        ./scripts/verify_release.sh --help
        ./scripts/manage-artifacts.sh help
        ./scripts/track_performance.sh --help
        ./scripts/smoke-test.sh --help        

  security-scan:
    name: Security Scan
    runs-on: self-hosted
    timeout-minutes: 20

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        scan-ref: '.'
        format: 'sarif'
        output: 'trivy-results.sarif'

    - name: Upload Trivy scan results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: trivy-results
        path: trivy-results.sarif
        retention-days: 30

    - name: Gosec Security Scanner
      run: |
        go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest
        gosec ./...        

  docker-build:
    name: Docker Build
    runs-on: self-hosted
    needs: [test, build, test-scripts]
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    timeout-minutes: 30

    steps:
    - name: Check Docker registry secret
      run: |
        if [ -z "${{ secrets.GHCR_TOKEN }}" ]; then
          echo "GHCR_TOKEN not set, skipping Docker build"
          exit 0
        fi        

    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: Login to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ secrets.GHCR_USERNAME }}
        password: ${{ secrets.GHCR_TOKEN }}

    - name: Build and push Docker image
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          ghcr.io/${{ github.repository }}:latest
          ghcr.io/${{ github.repository }}:${{ github.sha }}          
        cache-from: type=gha
        cache-to: type=gha,mode=max