From c474972a494d2feedf8706095a3a4666a55a1c27 Mon Sep 17 00:00:00 2001 From: Henry Park Date: Mon, 9 Mar 2026 13:55:46 -0700 Subject: [PATCH] feat(ci): chained promotion PRs with multi-agent Claude review [skip-regression-check] Staging CI workflow with batched promotion PRs: - Creates staging-promote/ branches per batch - Chains PRs onto previous promotion branch (incremental diffs) - Claude Code reviews only the incremental changes per batch - Blocked PRs stay open as records of findings - staging-tested tag advances regardless of gate outcome - Runs every 60 min on cron + manual dispatch Multi-agent Claude review (Sonnet orchestrator + Haiku agents): - 4 parallel Sonnet review agents (security, architecture, bugs, performance) - Haiku agents for severity/confidence scoring - [SEVERITY:CONFIDENCE] output format - Severity/confidence matrix for issue creation and gate blocking: CRITICAL: always create issue, block if confidence >=80 HIGH: create issue if confidence >=50 MEDIUM/LOW: create issue if confidence >=80 Gate waits 15 min for Claude review then processes comments. Validated end-to-end in nearai/ironclaw-ci-test. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/claude-review.yml | 95 +++++++ .github/workflows/staging-ci.yml | 422 ++++++++++++++++++++++++++++ 2 files changed, 517 insertions(+) create mode 100644 .github/workflows/claude-review.yml create mode 100644 .github/workflows/staging-ci.yml diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml new file mode 100644 index 00000000..125404ae --- /dev/null +++ b/.github/workflows/claude-review.yml @@ -0,0 +1,95 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened] + +permissions: + contents: read + pull-requests: write + issues: write + id-token: write + +jobs: + review: + name: Claude Code Review + if: contains(github.event.pull_request.labels.*.name, 'staging-promotion') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Run Claude Code review + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + claude_args: "--max-turns 50 --model claude-sonnet-4-6 --allowedTools 'Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*),Bash(gh issue view:*),Bash(gh issue list:*),Bash(gh search:*),Bash(git blame:*),Bash(git log:*),Bash(git diff:*)'" + prompt: | + Code review this pull request. Follow these steps precisely: + + 1. Use a Haiku agent to find relevant CLAUDE.md files: the root CLAUDE.md + and any CLAUDE.md files in directories whose files this PR modifies. + + 2. Use a Haiku agent to summarize the PR change (use `gh pr diff`). + + 3. Launch 4 parallel Sonnet agents to review the change. Each agent should + read the PR diff with `gh pr diff` and the full source files for changed + code, then return a list of issues found: + + Agent 1 — Security & Safety + Check for: command injection, path traversal, SSRF, XSS, auth bypass, + secrets in logs, .unwrap()/.expect() in production code (not tests), + race conditions, TOCTOU, unsafe blocks, panics in async, unbounded allocations. + + Agent 2 — Architecture & Patterns + Check for: extensible design (traits/enums over nested conditionals), + clean abstractions, proper error types (thiserror), CLAUDE.md compliance, + type-driven design over stringly-typed code, DRY violations. + + Agent 3 — Bug Scan + Shallow diff-only scan for obvious bugs: logic errors, off-by-one, + missing error handling, division by zero, incorrect return values. + Ignore nitpicks and likely false positives. Do NOT read extra context + beyond the diff — focus only on the changes. + + Agent 4 — Performance & Production + Check for: blocking in async, N+1 queries, unbounded loops, missing + timeouts, resource leaks (file handles, connections), large allocations + in hot paths. + + 4. For each issue found, launch a parallel Haiku agent to: + a. Assign a severity: + - CRITICAL: security vulns, panics in prod (.unwrap/.expect), data exfiltration, race conditions + - HIGH: logic bugs, missing error handling, breaking API/schema changes + - MEDIUM: missing tests, unnecessary complexity, performance issues + - LOW: documentation gaps, naming suggestions + b. Score confidence 0-100 (give this rubric verbatim): + 0: False positive, doesn't stand up to scrutiny, or pre-existing issue. + 25: Might be real, but may be false positive. Stylistic issues not in CLAUDE.md. + 50: Real issue but nitpick or rare in practice. Not very important. + 75: Verified real issue, will be hit in practice. Directly impacts functionality + or explicitly mentioned in CLAUDE.md. + 100: Certain, confirmed, will happen frequently. Evidence directly confirms. + + 5. Post a single comment on the PR using `gh pr comment` with this format. + If no issues were found, post "No issues found." instead: + + ### Code review + + Found N issues: + + 1. [SEVERITY:CONFIDENCE] + + + + Example: [CRITICAL:92] `.unwrap()` can panic in production when config is missing + + You MUST use the full git SHA in links (not HEAD or branch name). + Provide 1 line of context before and after each linked range. + + Notes: + - Use `gh` for all GitHub interactions, not web fetch + - Do NOT check build signal or attempt to build/test the code + - Ignore pre-existing issues not introduced by this PR + - Ignore issues a linter/compiler would catch (formatting, imports, types) diff --git a/.github/workflows/staging-ci.yml b/.github/workflows/staging-ci.yml new file mode 100644 index 00000000..f08b9995 --- /dev/null +++ b/.github/workflows/staging-ci.yml @@ -0,0 +1,422 @@ +name: Staging CI (Batched) + +on: + schedule: + - cron: "0 * * * *" # Every 60 minutes + workflow_dispatch: + inputs: + force: + description: "Force run even if no new commits" + type: boolean + default: false + skip_claude_gate: + description: "Skip Claude review gate (bypass blocking findings)" + type: boolean + default: false + +permissions: + contents: write + issues: write + pull-requests: write + +concurrency: + group: staging-ci + cancel-in-progress: false # Let running suites finish + +jobs: + # ── Check for new commits ────────────────────────────────────── + check-changes: + name: Check for new commits + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.check.outputs.has_changes }} + current_head: ${{ steps.check.outputs.current_head }} + diff_range: ${{ steps.check.outputs.diff_range }} + steps: + - uses: actions/checkout@v6 + with: + ref: staging + fetch-depth: 0 + + - name: Check for changes since last tested + id: check + run: | + CURRENT_HEAD=$(git rev-parse HEAD) + echo "current_head=${CURRENT_HEAD}" >> "$GITHUB_OUTPUT" + + if git rev-parse staging-tested >/dev/null 2>&1; then + LAST_TESTED=$(git rev-parse staging-tested) + else + LAST_TESTED="" + fi + + DIFF_RANGE="" + if [ -n "$LAST_TESTED" ] && [ "$LAST_TESTED" = "$CURRENT_HEAD" ]; then + echo "No new commits since last tested (${CURRENT_HEAD})" + HAS_CHANGES=false + else + HAS_CHANGES=true + if [ -n "$LAST_TESTED" ]; then + COMMIT_COUNT=$(git rev-list --count "${LAST_TESTED}..HEAD") + echo "Found ${COMMIT_COUNT} new commit(s) since last tested" + DIFF_RANGE="${LAST_TESTED}..${CURRENT_HEAD}" + else + git fetch origin main + MERGE_BASE=$(git merge-base origin/main HEAD) + echo "First run -- reviewing from merge-base ${MERGE_BASE}" + DIFF_RANGE="${MERGE_BASE}..${CURRENT_HEAD}" + fi + fi + + # Force override from workflow_dispatch + if [ "${{ inputs.force }}" = "true" ]; then + echo "Force run requested" + HAS_CHANGES=true + if [ -z "$DIFF_RANGE" ]; then + DIFF_RANGE="${CURRENT_HEAD}..${CURRENT_HEAD}" + fi + fi + + echo "has_changes=${HAS_CHANGES}" >> "$GITHUB_OUTPUT" + echo "diff_range=${DIFF_RANGE}" >> "$GITHUB_OUTPUT" + + # ── Run full test suite ────────────────────────────────────────── + tests: + name: Test Suite + needs: check-changes + if: needs.check-changes.outputs.has_changes == 'true' + uses: ./.github/workflows/test.yml + + # ── Run E2E browser tests ──────────────────────────────────────── + e2e: + name: E2E Browser Tests + needs: check-changes + if: needs.check-changes.outputs.has_changes == 'true' + uses: ./.github/workflows/e2e.yml + + # ── Create promotion PR (triggers claude-review.yml on the PR) ── + create-promotion-pr: + name: Create Promotion PR + needs: check-changes + if: needs.check-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + outputs: + pr_number: ${{ steps.create-pr.outputs.pr_number }} + promotion_branch: ${{ steps.branch.outputs.branch }} + steps: + - uses: actions/checkout@v6 + with: + ref: staging + fetch-depth: 0 + + - name: Generate GitHub App token + id: app-token + if: ${{ secrets.GH_RELEASES_MANAGER_APP_ID != '' }} + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }} + private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }} + + - name: Set token + id: token + run: | + if [ -n "${{ steps.app-token.outputs.token }}" ]; then + echo "token=${{ steps.app-token.outputs.token }}" >> "$GITHUB_OUTPUT" + else + echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT" + fi + + - name: Check if staging is ahead of main + id: ahead-check + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + run: | + git fetch origin main + AHEAD=$(git rev-list --count origin/main..origin/staging) + echo "commits_ahead=${AHEAD}" >> "$GITHUB_OUTPUT" + if [ "$AHEAD" -eq 0 ]; then + echo "Staging is not ahead of main. Nothing to promote." + else + echo "Staging is ${AHEAD} commits ahead of main." + fi + + - name: Create promotion branch + id: branch + if: steps.ahead-check.outputs.commits_ahead != '0' + run: | + SHORT_SHA=$(echo "${{ needs.check-changes.outputs.current_head }}" | cut -c1-8) + BRANCH="staging-promote/${SHORT_SHA}" + git checkout -b "$BRANCH" + git push origin "$BRANCH" + echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT" + echo "Created promotion branch: ${BRANCH}" + + - name: Find base branch + id: find-base + if: steps.ahead-check.outputs.commits_ahead != '0' + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + run: | + # Find the newest open promotion PR with a staging-promote/* head branch + LATEST=$(gh pr list --label staging-promotion --state open \ + --json headRefName,createdAt \ + --jq '[.[] | select(.headRefName | startswith("staging-promote/"))] | sort_by(.createdAt) | last | .headRefName // empty') + if [ -n "$LATEST" ]; then + echo "base=${LATEST}" >> "$GITHUB_OUTPUT" + echo "Chaining onto existing promotion branch: ${LATEST}" + else + echo "base=main" >> "$GITHUB_OUTPUT" + echo "No existing promotion PR — targeting main" + fi + + - name: Create promotion PR + id: create-pr + if: steps.ahead-check.outputs.commits_ahead != '0' + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + run: | + RANGE="${{ needs.check-changes.outputs.diff_range }}" + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") + BRANCH="${{ steps.branch.outputs.branch }}" + BASE="${{ steps.find-base.outputs.base }}" + + PR_URL=$(gh pr create \ + --base "$BASE" \ + --head "$BRANCH" \ + --title "chore: promote staging to main (${TIMESTAMP})" \ + --body "## Auto-promotion from staging CI + + **Batch range:** \`${RANGE}\` + **Promotion branch:** \`${BRANCH}\` + **Base:** \`${BASE}\` + **Triggered by:** Staging CI batch at ${TIMESTAMP} + + Waiting for gates: + - Tests: pending + - E2E: pending + - Claude Code review: pending (will post comments on this PR) + + --- + *Auto-created by staging-ci workflow*" \ + --label "staging-promotion") + + PR_NUM=$(echo "$PR_URL" | grep -oE '[0-9]+$') + echo "pr_number=${PR_NUM}" >> "$GITHUB_OUTPUT" + echo "Created promotion PR #${PR_NUM}" + + # ── Gate: wait for review, process findings, merge or block ───── + gate: + name: Staging Gate + needs: [check-changes, tests, e2e, create-promotion-pr] + if: > + always() && + needs.check-changes.outputs.has_changes == 'true' && + needs.tests.result == 'success' && + needs.e2e.result == 'success' && + needs.create-promotion-pr.result == 'success' + runs-on: ubuntu-latest + outputs: + gate_passed: ${{ steps.evaluate.outputs.passed }} + steps: + - uses: actions/checkout@v6 + with: + ref: staging + fetch-depth: 1 + + - name: Generate GitHub App token + id: app-token + if: ${{ secrets.GH_RELEASES_MANAGER_APP_ID != '' }} + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }} + private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }} + + - name: Set token + id: token + run: | + if [ -n "${{ steps.app-token.outputs.token }}" ]; then + echo "token=${{ steps.app-token.outputs.token }}" >> "$GITHUB_OUTPUT" + else + echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT" + fi + + - name: Wait for Claude review + run: | + PR_NUMBER="${{ needs.create-promotion-pr.outputs.pr_number }}" + if [ -z "$PR_NUMBER" ]; then + echo "No PR number — skipping wait" + else + echo "Waiting 15 minutes for Claude review on PR #${PR_NUMBER}..." + sleep 900 + fi + + - name: Process Claude review comments and create issues + id: process-findings + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }} + REPO: ${{ github.repository }} + run: | + HAS_BLOCKING=false + ISSUES_CREATED=0 + + if [ -z "$PR_NUMBER" ]; then + echo "No PR — skipping finding processing" + echo "has_blocking=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Get the last Claude comment that contains findings + JQ_FILTER='[.[] | select(.user.login == "claude[bot]") | select(.body | test("Found [0-9]+ issue"))] | last' + BODY=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq "${JQ_FILTER} | .body // empty" 2>/dev/null || echo "") + COMMENT_URL=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq "${JQ_FILTER} | .html_url // empty" 2>/dev/null || echo "") + + if [ -z "$BODY" ]; then + echo "No actionable findings from Claude review on PR #${PR_NUMBER}" + echo "has_blocking=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Parse [SEVERITY:CONFIDENCE] tags from each numbered finding + # Matrix: CRITICAL always→issue, ≥80→block. HIGH ≥50→issue. MEDIUM ≥80→issue. LOW ≥80→issue. + echo "$BODY" | grep -oE '\[(CRITICAL|HIGH|MEDIUM|LOW):[0-9]+\].*' | while read -r line; do + TAG=$(echo "$line" | grep -oE '^\[(CRITICAL|HIGH|MEDIUM|LOW):[0-9]+\]') + SEVERITY=$(echo "$TAG" | sed 's/\[\(.*\):\(.*\)\]/\1/') + CONFIDENCE=$(echo "$TAG" | sed 's/\[\(.*\):\(.*\)\]/\2/') + DESC=$(echo "$line" | sed "s/\[${SEVERITY}:${CONFIDENCE}\] *//" | head -1) + + echo "Found: [${SEVERITY}:${CONFIDENCE}] ${DESC}" + + # Check if blocking (CRITICAL ≥80) + if [ "$SEVERITY" = "CRITICAL" ] && [ "$CONFIDENCE" -ge 80 ]; then + echo "blocking=true" >> /tmp/staging-ci-blocking + fi + + # Determine if this should create an issue + CREATE_ISSUE=false + case "$SEVERITY" in + CRITICAL) CREATE_ISSUE=true ;; + HIGH) [ "$CONFIDENCE" -ge 50 ] && CREATE_ISSUE=true ;; + MEDIUM) [ "$CONFIDENCE" -ge 80 ] && CREATE_ISSUE=true ;; + LOW) [ "$CONFIDENCE" -ge 80 ] && CREATE_ISSUE=true ;; + esac + + if [ "$CREATE_ISSUE" = "true" ]; then + case "$SEVERITY" in + CRITICAL) LABELS="bug,risk: high,staging-ci-review" ;; + HIGH) LABELS="bug,risk: medium,staging-ci-review" ;; + MEDIUM) LABELS="risk: medium,staging-ci-review" ;; + LOW) LABELS="risk: low,staging-ci-review" ;; + esac + + TITLE=$(echo "$DESC" | cut -c1-80) + { + echo "## [${SEVERITY}:${CONFIDENCE}] Issue Found by Staging CI Review" + echo "" + echo "**Severity:** ${SEVERITY}" + echo "**Confidence:** ${CONFIDENCE}/100" + echo "**PR comment:** ${COMMENT_URL}" + echo "" + echo "### Description" + echo "$DESC" + echo "" + echo "---" + echo "*Auto-created by staging-ci Claude Code review*" + } > /tmp/issue-body.md + + gh issue create \ + --title "[${SEVERITY}] ${TITLE}" \ + --body-file /tmp/issue-body.md \ + --label "${LABELS}" || echo "::warning::Failed to create issue for ${SEVERITY} finding" + ISSUES_CREATED=$((ISSUES_CREATED + 1)) + fi + done + + # Check if any finding was blocking (written by subshell via file) + if [ -f /tmp/staging-ci-blocking ]; then + HAS_BLOCKING=true + fi + + echo "Created ${ISSUES_CREATED} issues" + echo "has_blocking=${HAS_BLOCKING}" >> "$GITHUB_OUTPUT" + + - name: Evaluate gate + id: evaluate + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }} + run: | + HAS_BLOCKING="${{ steps.process-findings.outputs.has_blocking }}" + SKIP_INPUT="${{ inputs.skip_claude_gate }}" + + if [ "$HAS_BLOCKING" = "true" ]; then + echo "::warning::Claude review found blocking issues (CRITICAL ≥80 confidence)" + if [ "$SKIP_INPUT" = "true" ]; then + echo "::warning::Gate overridden by skip_claude_gate workflow input" + echo "passed=true" >> "$GITHUB_OUTPUT" + else + echo "::error::Blocking promotion due to CRITICAL findings (≥80 confidence)" + echo "::error::PR #${PR_NUMBER} left open with review comments" + echo "passed=false" >> "$GITHUB_OUTPUT" + exit 1 + fi + else + echo "No blocking findings. Gate passed." + echo "passed=true" >> "$GITHUB_OUTPUT" + fi + + # Merge the promotion PR + if [ -n "$PR_NUMBER" ]; then + echo "Merging promotion PR #${PR_NUMBER}" + gh pr merge "$PR_NUMBER" --merge --delete-branch || echo "::warning::Auto-merge failed for PR #${PR_NUMBER}" + fi + + # ── Update tested tag (always, so next batch covers only new commits) ── + update-tag: + name: Update staging-tested tag + needs: [check-changes, tests, e2e, create-promotion-pr, gate] + if: > + always() && + needs.check-changes.outputs.has_changes == 'true' && + needs.tests.result == 'success' && + needs.e2e.result == 'success' && + needs.create-promotion-pr.result == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: staging + fetch-depth: 1 + + - name: Update staging-tested tag + run: | + git tag -f staging-tested "${{ needs.check-changes.outputs.current_head }}" + git push origin staging-tested --force + echo "Updated staging-tested tag to ${{ needs.check-changes.outputs.current_head }}" + + # ── Report ─────────────────────────────────────────────────────── + report: + name: Staging CI Summary + needs: [check-changes, tests, e2e, create-promotion-pr, gate, update-tag] + if: always() && needs.check-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + steps: + - name: Summary + run: | + echo "## Staging CI Batch Results" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Check | Result |" >> "$GITHUB_STEP_SUMMARY" + echo "|-------|--------|" >> "$GITHUB_STEP_SUMMARY" + echo "| Tests | ${{ needs.tests.result }} |" >> "$GITHUB_STEP_SUMMARY" + echo "| E2E | ${{ needs.e2e.result }} |" >> "$GITHUB_STEP_SUMMARY" + echo "| Promotion PR | ${{ needs.create-promotion-pr.result }} |" >> "$GITHUB_STEP_SUMMARY" + echo "| Gate | ${{ needs.gate.result }} |" >> "$GITHUB_STEP_SUMMARY" + echo "| Tag Updated | ${{ needs.update-tag.result }} |" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Range: ${{ needs.check-changes.outputs.diff_range }}" >> "$GITHUB_STEP_SUMMARY" + PR_NUM="${{ needs.create-promotion-pr.outputs.pr_number }}" + if [ -n "$PR_NUM" ]; then + echo "Promotion PR: #${PR_NUM}" >> "$GITHUB_STEP_SUMMARY" + fi