name: CI/CD Pipeline on: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read security-events: write actions: read packages: write env: GO_VERSION: '1.25.0' ZIG_VERSION: '0.15.2' RSYNC_VERSION: '3.3.0' jobs: test: name: Test runs-on: self-hosted timeout-minutes: 30 services: redis: image: redis:7 ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Checkout code uses: actions/checkout@v5 - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - name: Cache Go modules uses: actions/cache@v4 with: path: | ~/.cache/go-build ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }} restore-keys: | ${{ runner.os }}-go- - name: Set up Zig uses: goto-bus-stop/setup-zig@v2 with: version: ${{ env.ZIG_VERSION }} - name: Cache Zig build uses: actions/cache@v4 with: path: | ~/.cache/zig cli/zig-cache cli/zig-out key: ${{ runner.os }}-zig-${{ hashFiles('cli/**') }} restore-keys: | ${{ runner.os }}-zig- - name: Install dependencies run: | go mod download sudo apt-get update sudo apt-get install -y podman redis-tools build-essential autoconf automake libtool pkg-config musl-tools - name: Build pinned rsync from official source (for CLI tests) run: | make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }} - name: Verify dependencies run: go mod verify - name: Run tests run: make test - name: Test internal/queue package run: go test -v -race -coverprofile=queue-coverage.out ./internal/queue/... - name: Run comprehensive tests run: make test-full - name: Run linters run: make lint - name: Generate coverage report run: make test-coverage - name: Upload coverage to Codecov if: secrets.CODECOV_TOKEN != '' uses: codecov/codecov-action@v4 with: file: ./coverage/coverage.out flags: unittests name: codecov-umbrella token: ${{ secrets.CODECOV_TOKEN }} dev-smoke: name: Dev Compose Smoke Test runs-on: self-hosted needs: test timeout-minutes: 20 steps: - name: Checkout code uses: actions/checkout@v5 - name: Run dev smoke test run: make dev-smoke build: name: Build runs-on: self-hosted needs: test timeout-minutes: 15 steps: - name: Checkout code uses: actions/checkout@v5 - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - name: Set up Zig uses: goto-bus-stop/setup-zig@v2 with: version: ${{ env.ZIG_VERSION }} - name: Cache Go modules uses: actions/cache@v4 with: path: | ~/.cache/go-build ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum', '**/go.mod') }} restore-keys: | ${{ runner.os }}-go- - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y podman build-essential autoconf automake libtool pkg-config musl-tools - name: Build pinned rsync from official source run: | make -C cli build-rsync RSYNC_VERSION=${{ env.RSYNC_VERSION }} - name: Build binaries run: | make build - name: Test binaries run: | ./bin/user_manager --help ./bin/worker --help ./bin/tui --help ./bin/data_manager --help ./cli/zig-out/bin/ml --help ls -lh ./cli/zig-out/bin/ml - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: fetch_ml_binaries path: | bin/ cli/zig-out/ dist/ retention-days: 30 test-scripts: name: Test Scripts runs-on: self-hosted needs: test timeout-minutes: 15 steps: - name: Checkout code uses: actions/checkout@v5 - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y podman redis-tools bats - name: Test scripts run: | chmod +x scripts/*.sh || true chmod +x scripts/maintenance/*.sh || true ./scripts/verify_release.sh --help ./scripts/manage-artifacts.sh help ./scripts/track_performance.sh --help ./scripts/smoke-test.sh --help security-scan: name: Security Scan runs-on: self-hosted timeout-minutes: 20 steps: - name: Checkout code uses: actions/checkout@v5 - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: scan-type: 'fs' scan-ref: '.' format: 'sarif' output: 'trivy-results.sarif' - name: Upload Trivy scan results uses: actions/upload-artifact@v4 if: always() with: name: trivy-results path: trivy-results.sarif retention-days: 30 - name: Gosec Security Scanner run: | go install github.com/securecodewarrior/gosec/v2/cmd/gosec@latest gosec ./... docker-build: name: Docker Build runs-on: self-hosted needs: [test, build, test-scripts] if: github.event_name == 'push' && github.ref == 'refs/heads/main' timeout-minutes: 30 steps: - name: Check Docker registry secret run: | if [ -z "${{ secrets.GHCR_TOKEN }}" ]; then echo "GHCR_TOKEN not set, skipping Docker build" exit 0 fi - name: Checkout code uses: actions/checkout@v5 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build and push Docker image uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/${{ github.repository }}:latest ghcr.io/${{ github.repository }}:${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max