From ac3c9288530e2e26d1f589eba141e8ddf2ad75c6 Mon Sep 17 00:00:00 2001 From: Illia Polosukhin Date: Wed, 4 Mar 2026 06:23:53 +0000 Subject: [PATCH] ci: enhance coverage with feature matrix, postgres, and E2E (#523) * ci: enhance coverage workflow with feature matrix, postgres, and E2E Replace single-config coverage job with a multi-job pipeline: - Mirror test.yml's 3-config feature matrix (all-features, default, libsql-only) - Add PostgreSQL service (pgvector/pgvector:pg16) with migrations for postgres configs so integration tests actually run instead of skipping - Add E2E coverage job using cargo-llvm-cov instrumented binary with Playwright browser tests - Add coverage-gate roll-up job for branch protection - Upload per-config flags to Codecov (all-features, default, libsql-only, e2e) - Forward LLVM coverage env vars in E2E conftest.py so profraw data lands where cargo-llvm-cov report expects it [skip-regression-check] Co-Authored-By: Claude Opus 4.6 * fix: address PR review feedback on coverage workflow - Avoid setting DATABASE_URL to empty string for libsql-only config; use $GITHUB_ENV conditional step so the var is unset entirely - Add set -euo pipefail and psql -v ON_ERROR_STOP=1 to migrations so SQL errors fail the job immediately [skip-regression-check] Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .github/workflows/coverage.yml | 140 ++++++++++++++++++++++++++++++++- tests/e2e/conftest.py | 7 ++ 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 19e75340..1b08c60c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -9,24 +9,158 @@ permissions: jobs: coverage: - name: Coverage + name: Coverage (${{ matrix.name }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--all-features" + has_postgres: true + - name: default + flags: "" + has_postgres: true + - name: libsql-only + flags: "--no-default-features --features libsql" + has_postgres: false + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: ironclaw_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable with: components: llvm-tools-preview + - uses: Swatinem/rust-cache@v2 with: - key: coverage + key: coverage-${{ matrix.name }} + - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov + + - name: Run database migrations + if: matrix.has_postgres + run: | + set -euo pipefail + for f in migrations/V*.sql; do + echo "Applying $f..." + psql -v ON_ERROR_STOP=1 -f "$f" + done + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: ironclaw_test + + - name: Set DATABASE_URL for postgres configs + if: matrix.has_postgres + run: echo "DATABASE_URL=postgres://postgres:postgres@localhost/ironclaw_test" >> "$GITHUB_ENV" + - name: Generate coverage - run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + run: cargo llvm-cov ${{ matrix.flags }} --workspace --lcov --output-path lcov.info + - name: Upload to Codecov uses: codecov/codecov-action@v5 with: files: lcov.info + flags: ${{ matrix.name }} disable_search: true use_oidc: true fail_ci_if_error: true + + e2e-coverage: + name: E2E Coverage + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + + - uses: Swatinem/rust-cache@v2 + with: + key: e2e-coverage + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Set up coverage instrumentation + run: | + source <(cargo llvm-cov show-env --export-prefix) + # Persist env vars for subsequent steps + echo "RUSTFLAGS=${RUSTFLAGS}" >> "$GITHUB_ENV" + echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" >> "$GITHUB_ENV" + echo "CARGO_LLVM_COV=1" >> "$GITHUB_ENV" + echo "CARGO_LLVM_COV_SHOW_ENV=1" >> "$GITHUB_ENV" + echo "CARGO_LLVM_COV_TARGET_DIR=${CARGO_LLVM_COV_TARGET_DIR}" >> "$GITHUB_ENV" + cargo llvm-cov clean --workspace + + - name: Build instrumented binary + run: cargo build --no-default-features --features libsql + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run E2E tests + run: | + pytest tests/e2e/ -v -x --timeout=120 + env: + RUST_LOG: ironclaw=info + RUST_BACKTRACE: "1" + + - name: Generate coverage report + if: always() + run: cargo llvm-cov report --lcov --output-path e2e-coverage.info + + - name: Upload to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: e2e-coverage.info + flags: e2e + disable_search: true + use_oidc: true + fail_ci_if_error: true + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-screenshots + path: tests/e2e/screenshots/ + if-no-files-found: ignore + + coverage-gate: + name: Coverage + runs-on: ubuntu-latest + if: always() + needs: [coverage, e2e-coverage] + steps: + - run: | + if [[ "${{ needs.coverage.result }}" != "success" || "${{ needs.e2e-coverage.result }}" != "success" ]]; then + echo "One or more coverage jobs failed" + exit 1 + fi diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 84aed459..af885fb7 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -98,6 +98,13 @@ async def ironclaw_server(ironclaw_binary, mock_llm_server): # Prevent onboarding wizard from triggering "ONBOARD_COMPLETED": "true", } + # Forward LLVM coverage instrumentation env vars when present + # (allows cargo-llvm-cov to collect profraw data from E2E runs) + for key in ("LLVM_PROFILE_FILE", "CARGO_LLVM_COV", "CARGO_LLVM_COV_SHOW_ENV", + "CARGO_LLVM_COV_TARGET_DIR"): + val = os.environ.get(key) + if val is not None: + env[key] = val proc = await asyncio.create_subprocess_exec( ironclaw_binary, "--no-onboard", stdin=asyncio.subprocess.DEVNULL,