From 8700df35be154070c0d292af9f1dc03b307807ef Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 May 2026 10:22:22 -0400 Subject: [PATCH 1/3] ci(ui-preview-smoke): wire MCP via .mcp.json and post via workflow steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first PR run hit two real bugs: 1. claude-code-action@v1 has no `mcp_servers` input — the action warned and ignored the block, so the Playwright MCP server was never registered. Most of the 26 permission_denials in that run were the agent attempting to call mcp__playwright__* tools that didn't exist. 2. The action does NOT auto-post structured output as a PR comment. claude-code-review.yml does it via three follow-up steps (peter-evans/find-comment → jq extract → create-or-update-comment). Skipping that wiring is why no comment appeared on the PR even though the agent returned a summary. Fixes: - Drop the broken `mcp_servers` input. Add a step that writes `.mcp.json` at the working-directory root before the agent runs. The action auto-loads it via `enableAllProjectMcpServers: true` (visible in last run's log). - Add `id: agent` to the action step so subsequent steps can read its `structured_output`. - Add the same find-comment / jq-extract / create-or-update-comment trio used by claude-code-review.yml, including the defensive double-unwrap for cases where the model nests its own JSON. - Update prompt: explicitly forbid the agent from posting comments itself, require the `` marker on the first line of `summary` so the comment is sticky across runs, and put the skip text in `summary` instead of telling the agent to post it directly. --- .github/workflows/ui-preview-smoke.yml | 130 +++++++++++++++++++------ 1 file changed, 98 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ui-preview-smoke.yml b/.github/workflows/ui-preview-smoke.yml index a2e4b13e1c..ee3a016584 100644 --- a/.github/workflows/ui-preview-smoke.yml +++ b/.github/workflows/ui-preview-smoke.yml @@ -71,8 +71,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} max_timeout: 600 check_interval: 10 - # For workflow_dispatch we need to point at the PR head commit. - # For pull_request_target the action picks up the PR sha automatically. - name: Setup Node uses: actions/setup-node@v4 @@ -84,13 +82,15 @@ jobs: npm install -g playwright playwright install --with-deps chromium - - name: Run agent against preview - uses: anthropics/claude-code-action@v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ secrets.GITHUB_TOKEN }} - mcp_servers: | - { + # claude-code-action@v1 has no `mcp_servers` input (the action ignores + # it and warns at runtime). The supported mechanism is a `.mcp.json` + # at the working-directory root, which the action picks up because it + # auto-sets `enableAllProjectMcpServers: true` in Claude's settings. + - name: Write MCP config (Playwright) + run: | + cat > .mcp.json <<'EOF' + { + "mcpServers": { "playwright": { "command": "npx", "args": [ @@ -101,19 +101,43 @@ jobs: ] } } + } + EOF + + - name: Run agent against preview + id: agent + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} prompt: | - Execute the UI test plan for PR #${{ steps.pr.outputs.number }} - on its Vercel preview deploy. + Smoke-test PR #${{ steps.pr.outputs.number }} on its Vercel + preview deploy. Preview URL: ${{ steps.vercel.outputs.url }} Repo: ${{ github.repository }} - PR body: read /tmp/pr-body.md (use the Bash cat tool). + + Read the PR body with: cat /tmp/pr-body.md This preview is built in LOCAL_MODE with a pre-configured demo ClickHouse connection and otel_logs / otel_traces sources. No registration or source setup is needed — open the URL and go. - Workflow: + CRITICAL OUTPUT REQUIREMENTS: + 1. Return a JSON object with a single "summary" field whose + VALUE is a plain markdown STRING. Do NOT put another JSON + envelope inside the string — that posts raw JSON in the + comment. + 2. The summary markdown MUST start with EXACTLY these two + lines (the comment marker is required for the workflow to + update the same comment on subsequent runs): + + ## UI Preview Smoke + 3. Do NOT post comments yourself with `gh` or any other tool. + The workflow posts (or updates) the PR comment using your + `summary` field. + + Procedure: 1. Read /tmp/pr-body.md. 2. Find the section headed exactly @@ -123,41 +147,83 @@ jobs: - "**Steps:**" — a numbered list of imperative actions. 3. If the section is missing, empty, contains only the HTML comment template placeholder, or is marked "N/A" or - "non-UI change": post a single PR comment containing exactly - the text below, then exit with status 0. + "non-UI change", return summary: + + + ## UI Preview Smoke - > - > ## UI Preview Smoke - > - > Skipped: this PR has no `How to test on Vercel preview` - > plan. Add `**Preview routes:**` and a numbered `**Steps:**` - > list to enable automated smoke testing. + Skipped: this PR has no `How to test on Vercel preview` + plan. Add `**Preview routes:**` and a numbered `**Steps:**` + list to enable automated smoke testing. 4. Otherwise, for each Preview route in order: - a. Open `` in the Playwright browser. + a. Open `` via the Playwright MCP + browser tools (mcp__playwright__*). b. Execute the numbered steps verbatim, in order. c. Treat any step beginning with "Verify", "Confirm", "Assert", "Check", or "Ensure" as an assertion. If an assertion fails, record the failure and continue to the next route. - d. After each route capture: full-page screenshot, any - console errors at level "error", any 4xx/5xx network - responses, any uncaught exception dialogs. - 5. Post a single PR comment via the JSON schema below. Use ✅ - for passed routes, ❌ for any route with at least one failed - assertion or runtime error. For every failure, include the - step text, what was asserted, and what you observed instead. + d. After each route capture: any console errors at level + "error", any 4xx/5xx network responses, any uncaught + exception dialogs. + 5. Build the summary markdown with one section per route. + Use ✅ for routes that passed every assertion, ❌ for any + route with a failed assertion, console error, or 5xx + response. For each failure include the step text, what was + asserted, and what you observed instead. Constraints: - Do not invent steps the author didn't write. - Do not exercise routes outside the "Preview routes:" list. - - If a step is ambiguous, note the ambiguity in your comment + - If a step is ambiguous, note the ambiguity in the summary and proceed with your best interpretation. Never fabricate an assertion that wasn't requested. - - Cap total runtime at 8 minutes. If a single step hangs - more than 30s, mark it failed and continue. + - Cap total runtime at 8 minutes. If a single step hangs more + than 30s, mark it failed and continue. claude_args: | --setting-sources user --allowedTools "Bash(cat /tmp/pr-body.md),Bash(gh pr view:*),mcp__playwright__*" --json-schema '{"type":"object","properties":{"summary":{"type":"string","description":"Complete markdown summary starting with on the first line and ## UI Preview Smoke on the second line"}},"required":["summary"]}' + + # The agent's structured_output is a JSON string. Pull the `summary` + # field via jq. Defensive double-unwrap mirrors the workaround in + # claude-code-review.yml: the model has been observed to nest its + # output as `{"summary":"{\"summary\":\"\"}"}`, which would + # post raw JSON instead of markdown. + - name: Extract summary from structured output + if: steps.agent.outputs.structured_output != '' + id: extract + env: + STRUCTURED_OUTPUT: ${{ steps.agent.outputs.structured_output }} + run: | + SUMMARY="$(printf '%s' "$STRUCTURED_OUTPUT" | jq -r '.summary')" + if printf '%s' "$SUMMARY" | jq -e 'type == "object" and has("summary")' >/dev/null 2>&1; then + SUMMARY="$(printf '%s' "$SUMMARY" | jq -r '.summary')" + fi + { + echo 'summary<> "$GITHUB_OUTPUT" + + - name: Find existing smoke comment + if: steps.extract.outputs.summary != '' + id: find-comment + uses: peter-evans/find-comment@v4 + with: + issue-number: ${{ steps.pr.outputs.number }} + comment-author: github-actions[bot] + body-includes: '' + direction: last + + - name: Post or update smoke comment + if: steps.extract.outputs.summary != '' + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ steps.pr.outputs.number }} + body: ${{ steps.extract.outputs.summary }} + edit-mode: replace From e054791cf269fd6009f9fdd2e17543e3a91c832e Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 May 2026 10:37:05 -0400 Subject: [PATCH 2/3] ci(ui-preview-smoke): early-skip when PR has no test plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today the agent itself decides whether to skip — but only after the workflow waits up to 10 min for Vercel, installs Node + Playwright + Chromium, and spins up the agent (~2 min, ~$0.85). For PRs where the author didn't fill in the "How to test on Vercel preview" template, this is ~7 min and ~$1 to land a one-line skip comment. Add a github-script pre-flight that parses /tmp/pr-body.md (already fetched by the previous step) and gates the expensive setup behind `has_plan == 'true'`. Required for `has_plan == 'true'`: - "### How to test on Vercel preview" heading present - Section non-empty after HTML comments are stripped - Section not marked "N/A" or "non-UI change" - "**Preview routes:**" line has non-empty content after the colon - "**Steps:**" block has at least one numbered item with content (catches the empty `1.\n2.\n3.` template placeholder) When the gate fails, post the skip comment immediately via peter-evans/create-or-update-comment using the same `` sticky marker. Move the find-comment step above both branches so it's shared. Total runtime for no-plan PRs drops to ~30s and ~$0. --- .github/workflows/ui-preview-smoke.yml | 155 +++++++++++++++++++------ 1 file changed, 122 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ui-preview-smoke.yml b/.github/workflows/ui-preview-smoke.yml index ee3a016584..9456121bbb 100644 --- a/.github/workflows/ui-preview-smoke.yml +++ b/.github/workflows/ui-preview-smoke.yml @@ -64,7 +64,106 @@ jobs: core.setOutput('number', String(pr.number)); core.setOutput('head_sha', pr.head.sha); + # Cheap pre-flight: parse the PR body for a UI test plan before we + # spend ~5 min waiting for Vercel + ~$1 of agent runtime. If the + # author didn't fill in `### How to test on Vercel preview`, we post + # a skip comment and exit immediately. Without this gate, no-plan + # PRs cost the same as full smoke runs. + - name: Check for UI test plan + id: plan + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('/tmp/pr-body.md', 'utf8'); + + const headingMatch = body.match( + /^###\s+How to test on Vercel preview\s*$/im, + ); + if (!headingMatch) { + core.setOutput('has_plan', 'false'); + core.notice('No "### How to test on Vercel preview" section.'); + return; + } + + const start = headingMatch.index + headingMatch[0].length; + const remainder = body.slice(start); + const nextHeading = remainder.match(/^###\s/m); + const section = nextHeading + ? remainder.slice(0, nextHeading.index) + : remainder; + + // Strip HTML comments — both the template explainer and any + // placeholder hints like "" inline. + const cleaned = section.replace(//g, ''); + const trimmed = cleaned.trim(); + + if (!trimmed) { + core.setOutput('has_plan', 'false'); + core.notice('Section is empty after stripping comments.'); + return; + } + if (/^(n\/?a\b|non[-\s]?ui|no[-\s]+ui)/i.test(trimmed)) { + core.setOutput('has_plan', 'false'); + core.notice('Section is marked N/A.'); + return; + } + + const routesMatch = cleaned.match( + /\*\*Preview routes:\*\*\s*([^\n]*)/i, + ); + const routes = routesMatch ? routesMatch[1].trim() : ''; + if (!routes) { + core.setOutput('has_plan', 'false'); + core.notice('"**Preview routes:**" line is empty.'); + return; + } + + const stepsMatch = cleaned.match(/\*\*Steps:\*\*([\s\S]*)/i); + const stepsBlock = stepsMatch ? stepsMatch[1] : ''; + // Need at least one numbered list item with non-whitespace + // content after the "1. " marker. + const hasStep = /^\s*\d+\.\s+\S/m.test(stepsBlock); + if (!hasStep) { + core.setOutput('has_plan', 'false'); + core.notice('No numbered **Steps:** with content.'); + return; + } + + core.setOutput('has_plan', 'true'); + core.setOutput('routes', routes); + core.notice(`UI test plan found. Routes: ${routes}`); + + # Run unconditionally — both the skip path and the full smoke path + # need it to keep the comment sticky across runs. + - name: Find existing smoke comment + id: find-comment + uses: peter-evans/find-comment@v4 + with: + issue-number: ${{ steps.pr.outputs.number }} + comment-author: github-actions[bot] + body-includes: '' + direction: last + + # ─── Skip path ──────────────────────────────────────────────────── + - name: Post skip comment + if: steps.plan.outputs.has_plan == 'false' + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ steps.pr.outputs.number }} + edit-mode: replace + body: | + + ## UI Preview Smoke + + Skipped: this PR has no `How to test on Vercel preview` plan. + Add `**Preview routes:**` and a numbered `**Steps:**` list to + enable automated smoke testing. + + # ─── Full smoke path ────────────────────────────────────────────── - name: Wait for Vercel preview + if: steps.plan.outputs.has_plan == 'true' id: vercel uses: patrickedqvist/wait-for-vercel-preview@v1.3.1 with: @@ -73,11 +172,13 @@ jobs: check_interval: 10 - name: Setup Node + if: steps.plan.outputs.has_plan == 'true' uses: actions/setup-node@v4 with: node-version: 22 - name: Install Playwright + Chromium + if: steps.plan.outputs.has_plan == 'true' run: | npm install -g playwright playwright install --with-deps chromium @@ -87,6 +188,7 @@ jobs: # at the working-directory root, which the action picks up because it # auto-sets `enableAllProjectMcpServers: true` in Claude's settings. - name: Write MCP config (Playwright) + if: steps.plan.outputs.has_plan == 'true' run: | cat > .mcp.json <<'EOF' { @@ -105,6 +207,7 @@ jobs: EOF - name: Run agent against preview + if: steps.plan.outputs.has_plan == 'true' id: agent uses: anthropics/claude-code-action@v1 with: @@ -119,6 +222,11 @@ jobs: Read the PR body with: cat /tmp/pr-body.md + The PR body is guaranteed to contain a + "### How to test on Vercel preview" section with a non-empty + `**Preview routes:**` line and at least one numbered step + (the workflow gates on this before invoking you). + This preview is built in LOCAL_MODE with a pre-configured demo ClickHouse connection and otel_logs / otel_traces sources. No registration or source setup is needed — open the URL and go. @@ -139,35 +247,22 @@ jobs: Procedure: - 1. Read /tmp/pr-body.md. - 2. Find the section headed exactly - "### How to test on Vercel preview". Within it, parse: - - "**Preview routes:**" line — comma-separated list of paths - (e.g. "/chart, /dashboards/"). Strip whitespace. - - "**Steps:**" — a numbered list of imperative actions. - 3. If the section is missing, empty, contains only the HTML - comment template placeholder, or is marked "N/A" or - "non-UI change", return summary: - - - ## UI Preview Smoke - - Skipped: this PR has no `How to test on Vercel preview` - plan. Add `**Preview routes:**` and a numbered `**Steps:**` - list to enable automated smoke testing. - - 4. Otherwise, for each Preview route in order: + 1. Read /tmp/pr-body.md and parse the "### How to test on + Vercel preview" section: + - "**Preview routes:**" line — comma-separated paths. + - "**Steps:**" — numbered list of imperative actions. + 2. For each Preview route in order: a. Open `` via the Playwright MCP browser tools (mcp__playwright__*). b. Execute the numbered steps verbatim, in order. c. Treat any step beginning with "Verify", "Confirm", "Assert", "Check", or "Ensure" as an assertion. If an - assertion fails, record the failure and continue to the - next route. + assertion fails, record the failure and continue to + the next route. d. After each route capture: any console errors at level "error", any 4xx/5xx network responses, any uncaught exception dialogs. - 5. Build the summary markdown with one section per route. + 3. Build the summary markdown with one section per route. Use ✅ for routes that passed every assertion, ❌ for any route with a failed assertion, console error, or 5xx response. For each failure include the step text, what was @@ -193,7 +288,9 @@ jobs: # output as `{"summary":"{\"summary\":\"\"}"}`, which would # post raw JSON instead of markdown. - name: Extract summary from structured output - if: steps.agent.outputs.structured_output != '' + if: + steps.plan.outputs.has_plan == 'true' && + steps.agent.outputs.structured_output != '' id: extract env: STRUCTURED_OUTPUT: ${{ steps.agent.outputs.structured_output }} @@ -209,18 +306,10 @@ jobs: echo 'UI_SMOKE_EOF' } >> "$GITHUB_OUTPUT" - - name: Find existing smoke comment - if: steps.extract.outputs.summary != '' - id: find-comment - uses: peter-evans/find-comment@v4 - with: - issue-number: ${{ steps.pr.outputs.number }} - comment-author: github-actions[bot] - body-includes: '' - direction: last - - name: Post or update smoke comment - if: steps.extract.outputs.summary != '' + if: + steps.plan.outputs.has_plan == 'true' && steps.extract.outputs.summary + != '' uses: peter-evans/create-or-update-comment@v5 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} From 3e03b8fd8468abb1b0027703fd5cd132c03e2091 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 8 May 2026 13:09:09 -0400 Subject: [PATCH 3/3] ci(ui-preview-smoke): resolve review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves all six findings on the ui-preview-smoke workflow with a disciplined +79-line diff (file lands at 397 lines). F1 — heredoc injection. Replaced static UI_SMOKE_EOF delimiter with per-run random `EOF_$(openssl rand -hex 16)`. F2 — broad MCP / gh access. Pinned @playwright/mcp@0.0.75 (was @latest). Added `--allowed-origins=${ORIGIN}` from the validated Vercel URL. Dropped `Bash(gh pr view:*)` from the agent's allowlist entirely — the prompt routes the agent to /tmp/pr-body.md, so gh was never used. Documented inline that the package's own README says --allowed-origins "is not a security boundary"; that's a residual we accept. F3 — silent sed pass-through. Replaced sed with bash regex match plus explicit abort: [[ ! "$VERCEL_URL" =~ ^(https?://[^/]+) ]] || ORIGIN=... which produces a `::error::` workflow annotation on bad input rather than silently degrading to an empty allowlist. F4 — empty summary silently skipped. Added `set -euo pipefail` and `|| SUMMARY=''` to the extract step so any malformed structured_output deterministically yields an empty summary. Then handled by F5/F6's fallback poster. F5 — plan-check throw, no signal. Added one consolidated "Post infrastructure-failure comment" step that fires `if: always() &&` when plan threw OR Vercel didn't produce a usable preview OR the agent extract produced no summary. Single sticky comment (``), links to the workflow run for the actual reason. Avoids the five-different-fallback-posters trap from the prior attempt. F6 — Vercel timeout no fallback. `continue-on-error: true` on the Vercel step. Downstream steps (Setup Node, Install Playwright, Write MCP, Run agent) gated on `steps.vercel.outcome == 'success'`. The consolidated poster covers the failure surface. What I deliberately did NOT do: - Preprocess the PR body into a strict route/step JSON before the agent reads it. The agent's output is already constrained by the JSON schema; preprocessing would require a contract change between workflow and prompt and was the path that ballooned the previous attempt to +543 lines. - Add per-failure-mode posters. One generic poster + a workflow-run link is sufficient — the run page is the source of truth for what specifically broke. --- .github/workflows/ui-preview-smoke.yml | 111 +++++++++++++++++++++---- 1 file changed, 95 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ui-preview-smoke.yml b/.github/workflows/ui-preview-smoke.yml index 9456121bbb..3571b2a56e 100644 --- a/.github/workflows/ui-preview-smoke.yml +++ b/.github/workflows/ui-preview-smoke.yml @@ -134,10 +134,12 @@ jobs: core.setOutput('routes', routes); core.notice(`UI test plan found. Routes: ${routes}`); - # Run unconditionally — both the skip path and the full smoke path - # need it to keep the comment sticky across runs. + # Run unconditionally (including when plan-check threw) so the + # consolidated infrastructure-failure poster below can update the + # sticky comment instead of creating a fresh one each broken run. - name: Find existing smoke comment id: find-comment + if: always() && steps.pr.outcome == 'success' uses: peter-evans/find-comment@v4 with: issue-number: ${{ steps.pr.outputs.number }} @@ -162,9 +164,14 @@ jobs: enable automated smoke testing. # ─── Full smoke path ────────────────────────────────────────────── + # `continue-on-error: true` so a Vercel timeout/deploy-error doesn't + # short-circuit the job — downstream steps gate on + # `vercel.outcome == 'success'` and the consolidated fallback + # poster reports the failure to the PR. - name: Wait for Vercel preview if: steps.plan.outputs.has_plan == 'true' id: vercel + continue-on-error: true uses: patrickedqvist/wait-for-vercel-preview@v1.3.1 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -172,13 +179,17 @@ jobs: check_interval: 10 - name: Setup Node - if: steps.plan.outputs.has_plan == 'true' + if: + steps.plan.outputs.has_plan == 'true' && steps.vercel.outcome == + 'success' uses: actions/setup-node@v4 with: node-version: 22 - name: Install Playwright + Chromium - if: steps.plan.outputs.has_plan == 'true' + if: + steps.plan.outputs.has_plan == 'true' && steps.vercel.outcome == + 'success' run: | npm install -g playwright playwright install --with-deps chromium @@ -187,19 +198,45 @@ jobs: # it and warns at runtime). The supported mechanism is a `.mcp.json` # at the working-directory root, which the action picks up because it # auto-sets `enableAllProjectMcpServers: true` in Claude's settings. + # + # Parse the Vercel preview URL into a `scheme://host` origin and + # pass it to `@playwright/mcp` as `--allowed-origins`. The version + # is pinned (not `@latest`) so a future MCP release can't silently + # change browser/tool behavior under us. We abort the step if the + # URL doesn't match `^https?://[^/]+`, rather than the prior `sed` + # which silently passed bogus input through and produced a + # malformed `--allowed-origins=` arg. + # + # Residual: per the package's own README, `--allowed-origins` is + # "not a security boundary" — it's a navigation hint, not a + # process-level egress control. A determined attacker who lands + # arbitrary JS in the preview origin can still issue cross-origin + # `fetch`. We accept that as residual; the upstream fix is at the + # MCP/browser layer. - name: Write MCP config (Playwright) - if: steps.plan.outputs.has_plan == 'true' + if: + steps.plan.outputs.has_plan == 'true' && steps.vercel.outcome == + 'success' + env: + VERCEL_URL: ${{ steps.vercel.outputs.url }} run: | - cat > .mcp.json <<'EOF' + set -euo pipefail + if [[ ! "$VERCEL_URL" =~ ^(https?://[^/]+) ]]; then + echo "::error::Vercel URL '$VERCEL_URL' is not a valid origin" >&2 + exit 1 + fi + ORIGIN="${BASH_REMATCH[1]}" + cat > .mcp.json < on the first line and ## UI Preview Smoke on the second line"}},"required":["summary"]}' # The agent's structured_output is a JSON string. Pull the `summary` @@ -287,23 +327,32 @@ jobs: # claude-code-review.yml: the model has been observed to nest its # output as `{"summary":"{\"summary\":\"\"}"}`, which would # post raw JSON instead of markdown. + # Per-run random heredoc delimiter so attacker-influenced summary + # content (the agent's output reflects the PR body, which on a fork + # PR is fully attacker-controlled) can't land the literal delimiter + # on its own line and inject `name=value` pairs into $GITHUB_OUTPUT. + # `set -euo pipefail` plus `|| SUMMARY=''` on the jq parse means + # any malformed structured_output yields an empty SUMMARY; the + # consolidated fallback poster below picks that up. - name: Extract summary from structured output if: steps.plan.outputs.has_plan == 'true' && steps.agent.outputs.structured_output != '' id: extract + continue-on-error: true env: STRUCTURED_OUTPUT: ${{ steps.agent.outputs.structured_output }} run: | - SUMMARY="$(printf '%s' "$STRUCTURED_OUTPUT" | jq -r '.summary')" + set -euo pipefail + SUMMARY="$(printf '%s' "$STRUCTURED_OUTPUT" | jq -r '.summary')" || SUMMARY='' if printf '%s' "$SUMMARY" | jq -e 'type == "object" and has("summary")' >/dev/null 2>&1; then - SUMMARY="$(printf '%s' "$SUMMARY" | jq -r '.summary')" + SUMMARY="$(printf '%s' "$SUMMARY" | jq -r '.summary')" || SUMMARY='' fi + DELIM="EOF_$(openssl rand -hex 16)" { - echo 'summary<> "$GITHUB_OUTPUT" - name: Post or update smoke comment @@ -316,3 +365,33 @@ jobs: issue-number: ${{ steps.pr.outputs.number }} body: ${{ steps.extract.outputs.summary }} edit-mode: replace + + # Consolidated infrastructure-failure poster. Fires when the PR + # would otherwise be left with no contextual comment, in any of: + # - plan step itself errored (regex bug, runner exception) + # - has_plan == 'true' but Vercel never produced a usable preview + # - has_plan == 'true' and Vercel succeeded but the agent / + # extract step produced no summary (timeout, malformed JSON, + # missing `.summary` field) + # Without this, F4/F5/F6 leave the PR with a red check and either + # no comment or a stale prior-run comment. + - name: Post infrastructure-failure comment + if: | + always() && steps.pr.outcome == 'success' && ( + steps.plan.outcome == 'failure' || + (steps.plan.outputs.has_plan == 'true' && + steps.vercel.outcome != 'success') || + (steps.plan.outputs.has_plan == 'true' && + steps.vercel.outcome == 'success' && + steps.extract.outputs.summary == '') + ) + uses: peter-evans/create-or-update-comment@v5 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ steps.pr.outputs.number }} + edit-mode: replace + body: | + + ## UI Preview Smoke + + Smoke run did not complete. See [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.