fetch_ml/.forgejo/workflows/ci.yml

name: CI/CD Pipeline

on:
  workflow_dispatch:
  push:
    paths-ignore:
      - 'docs/**'
      - 'README.md'
      - 'CHANGELOG.md'
      - '.forgejo/ISSUE_TEMPLATE/**'
      - '**/*.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  security-events: write
  actions: read
  packages: write

env:
  GO_VERSION: '1.25.0'
  ZIG_VERSION: '0.15.2'
  RSYNC_VERSION: '3.3.0'

jobs:
  test:
    name: Test
    runs-on: self-hosted
    timeout-minutes: 30

    services:
      redis:
        image: redis:7
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5          

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      run: |
        REQUIRED_GO="1.25.0"
        if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
          echo "Go ${REQUIRED_GO} already installed - skipping download"
        else
          echo "Installing Go ${REQUIRED_GO}..."
          curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
          export PATH="/usr/local/go/bin:$PATH"
          echo "/usr/local/go/bin" >> $GITHUB_PATH
          echo "Go ${REQUIRED_GO} installed"
        fi
        go version        

    - name: Cache Go modules
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod          
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }}
        restore-keys: |
          ${{ runner.os }}-go-          

    - name: Set up Zig
      run: |
        ZIG_VERSION="${{ env.ZIG_VERSION }}"
        if command -v zig &> /dev/null && zig version | grep -q "${ZIG_VERSION}"; then
          echo "Zig ${ZIG_VERSION} already installed - skipping download"
        else
          echo "Installing Zig ${ZIG_VERSION}..."
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl -sL "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz" | sudo tar -C /usr/local -xJf -
            sudo ln -sf "/usr/local/zig-linux-x86_64-${ZIG_VERSION}/zig" /usr/local/bin/zig
          elif [[ "$OSTYPE" == "darwin"* ]]; then
            curl -sL "https://ziglang.org/download/${ZIG_VERSION}/zig-macos-x86_64-${ZIG_VERSION}.tar.xz" | sudo tar -C /usr/local -xJf -
            sudo ln -sf "/usr/local/zig-macos-x86_64-${ZIG_VERSION}/zig" /usr/local/bin/zig
          fi
          echo "Zig ${ZIG_VERSION} installed"
        fi
        zig version        

    - name: Cache Zig build
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/zig
          cli/zig-cache
          cli/zig-out          
        key: ${{ runner.os }}-zig-${{ hashFiles('cli/**') }}
        restore-keys: |
          ${{ runner.os }}-zig-          

    - name: Cache native libraries
      uses: actions/cache@v4
      with:
        path: |
          native/build/          
        key: ${{ runner.os }}-native-${{ hashFiles('native/**/*.cpp', 'native/**/*.h', 'native/**/CMakeLists.txt') }}
        restore-keys: |
          ${{ runner.os }}-native-          

    - name: Cache apt packages
      uses: actions/cache@v4
      with:
        path: /var/cache/apt/archives
        key: ${{ runner.os }}-apt-${{ hashFiles('.forgejo/workflows/*.yml') }}
        restore-keys: |
          ${{ runner.os }}-apt-          

    - name: Install dependencies
      run: |
        go mod download
        sudo apt-get update
        sudo apt-get install -y podman redis-tools build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source (for CLI tests)
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Verify dependencies
      run: go mod verify

    - name: Run tests
      run: make test

    - name: Test internal/queue package
      run: go test -v -race -coverprofile=queue-coverage.out ./internal/queue/...

    - name: Run comprehensive tests
      run: make test-full

    - name: Run linters
      run: make lint

    - name: Generate coverage report
      run: make test-coverage

    - name: Upload coverage to Codecov
      if: secrets.CODECOV_TOKEN != ''
      uses: codecov/codecov-action@v4
      with:
        file: ./coverage/coverage.out
        flags: unittests
        name: codecov-umbrella
        token: ${{ secrets.CODECOV_TOKEN }}

  dev-smoke:
    name: Dev Compose Smoke Test
    runs-on: self-hosted
    needs: test
    timeout-minutes: 20

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Run dev smoke test
      run: make dev-smoke

  build:
    name: Build
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Go
      run: |
        REQUIRED_GO="1.25.0"
        if command -v go &> /dev/null && go version | grep -q "go${REQUIRED_GO}"; then
          echo "Go ${REQUIRED_GO} already installed - skipping download"
        else
          echo "Installing Go ${REQUIRED_GO}..."
          curl -sL "https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xzf -
          export PATH="/usr/local/go/bin:$PATH"
          echo "/usr/local/go/bin" >> $GITHUB_PATH
          echo "Go ${REQUIRED_GO} installed"
        fi
        go version        

    - name: Set up Zig
      run: |
        ZIG_VERSION="${{ env.ZIG_VERSION }}"
        if command -v zig &> /dev/null && zig version | grep -q "${ZIG_VERSION}"; then
          echo "Zig ${ZIG_VERSION} already installed - skipping download"
        else
          echo "Installing Zig ${ZIG_VERSION}..."
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
            curl -sL "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz" | sudo tar -C /usr/local -xJf -
            sudo ln -sf "/usr/local/zig-linux-x86_64-${ZIG_VERSION}/zig" /usr/local/bin/zig
          elif [[ "$OSTYPE" == "darwin"* ]]; then
            curl -sL "https://ziglang.org/download/${ZIG_VERSION}/zig-macos-x86_64-${ZIG_VERSION}.tar.xz" | sudo tar -C /usr/local -xJf -
            sudo ln -sf "/usr/local/zig-macos-x86_64-${ZIG_VERSION}/zig" /usr/local/bin/zig
          fi
          echo "Zig ${ZIG_VERSION} installed"
        fi
        zig version        

    - name: Cache Go modules
      uses: actions/cache@v4
      with:
        path: |
          ~/.cache/go-build
          ~/go/pkg/mod          
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }}
        restore-keys: |
          ${{ runner.os }}-go-          

    - name: Install build dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman build-essential autoconf automake libtool pkg-config musl-tools        

    - name: Build pinned rsync from official source
      run: |
        make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }}        

    - name: Build binaries
      run: |
        make build        

    - name: Test binaries
      run: |
        ./bin/user_manager --help
        ./bin/worker --help
        ./bin/tui --help
        ./bin/data_manager --help
        ./cli/zig-out/bin/ml --help
        ls -lh ./cli/zig-out/bin/ml        

    - name: Upload build artifacts
      uses: actions/upload-artifact@v4
      with:
        name: fetch_ml_binaries
        path: |
          bin/
          cli/zig-out/
          dist/          
        retention-days: 30

  test-scripts:
    name: Test Scripts
    runs-on: self-hosted
    needs: test
    timeout-minutes: 15

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y podman redis-tools bats        

    - name: Test scripts
      run: |
        chmod +x scripts/*.sh || true
        chmod +x scripts/maintenance/*.sh || true

        ./scripts/verify_release.sh --help
        ./scripts/manage-artifacts.sh help
        ./scripts/track_performance.sh --help
        ./scripts/smoke-test.sh --help        

  security-scan:
    name: Security Scan
    runs-on: self-hosted
    timeout-minutes: 20

    steps:
    - name: Checkout code
      uses: actions/checkout@v5

    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        scan-ref: '.'
        format: 'sarif'
        output: 'trivy-results.sarif'

    - name: Upload Trivy scan results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: trivy-results
        path: trivy-results.sarif
        retention-days: 30

    - name: Gosec Security Scanner
      run: |
        go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest
        gosec ./...        

  docker-build:
    name: Docker Build
    runs-on: self-hosted
    needs: [test, build, test-scripts]
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    timeout-minutes: 30

    steps:
    - name: Check Docker registry secret
      run: |
        if [ -z "${{ secrets.GHCR_TOKEN }}" ]; then
          echo "GHCR_TOKEN not set, skipping Docker build"
          exit 0
        fi        

    - name: Checkout code
      uses: actions/checkout@v5

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
      with:
        driver-opts: |
          image=moby/buildkit:master          

    - name: Cache Docker layers
      uses: actions/cache@v4
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-${{ github.sha }}
        restore-keys: |
          ${{ runner.os }}-buildx-          

    - name: Login to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ secrets.GHCR_USERNAME }}
        password: ${{ secrets.GHCR_TOKEN }}

    - name: Build and push Docker image
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          ghcr.io/${{ github.repository }}:latest
          ghcr.io/${{ github.repository }}:${{ github.sha }}          
        cache-from: type=local,src=/tmp/.buildx-cache
        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max