Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions actions/setup/js/copilot_harness.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,15 @@ function computeStartupRetryEligible(eventName) {
return eventName === "schedule" || eventName === "push";
}

/**
* Returns true when a failed attempt qualifies for the startup no-output retry budget.
* @param {{exitCode: number, hasOutput: boolean}} result
* @returns {boolean}
*/
function isStartupNoOutputRetryCandidate(result) {
return !result.hasOutput && result.exitCode === 2;
}

/**
* Read AWF config written by the compiler before the agent runs.
* @returns {any|null}
Expand Down Expand Up @@ -1258,7 +1267,11 @@ async function main() {
// (watchdogFired=true), as well as any other partial_execution failure that occurs
// after the primary task output was already produced. Retrying would reproduce the
// same pattern and exhaust the retry budget without ever posting a final safe-output.
if ((failureClass === "partial_execution" || failureClass === "long_run_exit") && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) {
// The no_output + watchdogFired case is also handled here: the post-result watchdog is
// only armed after hasTerminalSafeOutput is true, so watchdogFired on a no-stdio-output
// run means the agent completed its task (wrote safe-output) but produced no console
// output before the watchdog terminated the idle process.
if ((failureClass === "partial_execution" || failureClass === "long_run_exit" || (failureClass === "no_output" && result.watchdogFired)) && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) {
const reason = result.watchdogFired ? "post-result watchdog fired after terminal safe-output was emitted" : "partial execution after terminal safe-output was already produced";
log(`attempt ${attempt + 1}: ${reason} — treating as success (late-activity exit suppressed)`);
lastExitCode = 0;
Expand Down Expand Up @@ -1379,15 +1392,15 @@ async function main() {
// Scheduled and push-triggered runs: retry once on exit code 2 even when no output was
// produced. This specifically targets transient Copilot API outages at startup where
// there is no partial session state to continue from (Turns=0 driver-handoff failure).
if (isStartupRetryEligible && result.exitCode === 2 && !result.hasOutput && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt < maxRetries) {
if (isStartupRetryEligible && isStartupNoOutputRetryCandidate(result) && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt < maxRetries) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[/diagnosing-bugs] The watchdogFired === true && !hasOutput condition is contradictory: the post-result watchdog is armed only after hasTerminalSafeOutput(safeOutputsPath) returns true (line 1151), which requires safe-output to exist — implying hasOutput is true when the watchdog fires. This means the new watchdog-based retry branch can never trigger in practice.

💡 Root cause analysis

The watchdog's shouldArm predicate is () => hasTerminalSafeOutput(safeOutputsPath). A process that exits code 1 via watchdog will have produced terminal safe-output, so hasOutput will be true. The retry candidate predicate !result.hasOutput && exitCode === 1 && watchdogFired therefore requires a state combination the current wiring cannot produce.

Before merging: pull the actual agent-stdio.log from the failing Daily Issues Report Generator run and confirm whether hasOutput=false watchdogFired=true appears. If it does not, this branch is dead code.

@copilot please address this.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The existing review comment raises a concern about watchdogFired being true in a startup context, and there is a related correctness gap worth addressing.

If a prior attempt already wrote terminal safe-output to disk (e.g. from a partial-execution run that produced output), the watchdog on the next attempt arms on the first poll interval, fires before the process produces any console output, and yields hasOutput=false, watchdogFired=true, failureClass=no_output.

The existing terminal-safe-output guard at ~line 1273 only fires for failureClass === "partial_execution" or "long_run_exit". When hasOutput=false, failureClass is "no_output" so this guard is bypassed — and the startup no-output retry can fire even though a terminal safe-output was already written by the prior attempt.

Suggested fix: add a hasTerminalSafeOutput guard to the startup retry condition:

if (
  isStartupRetryEligible &&
  isStartupNoOutputRetryCandidate(result) &&
  !(safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) &&
  scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES &&
  attempt < maxRetries
) {

This prevents a spurious fresh retry when watchdogFired=true originates from stale safe-outputs arming the watchdog, rather than a true startup-only failure.

@copilot please address this.

scheduledExit2Retries += 1;
scheduledExit2RetryAttempted = true;
useContinueOnRetry = false;
const triggerLabel = isScheduledRun ? "scheduled" : "push";
log(`attempt ${attempt + 1}: ${triggerLabel} startup interruption (exit code 2, no output — driver-handoff Turns=0)` + ` — retrying once as fresh run (startupRetry=${scheduledExit2Retries}/${MAX_SCHEDULED_EXIT2_RETRIES})`);
continue;
}
if (isStartupRetryEligible && result.exitCode === 2 && !result.hasOutput && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt >= maxRetries) {
if (isStartupRetryEligible && isStartupNoOutputRetryCandidate(result) && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt >= maxRetries) {
log(`attempt ${attempt + 1}: startup interruption detected (driver-handoff Turns=0) but retry budget exhausted — no attempts remain`);
}

Expand Down Expand Up @@ -1416,7 +1429,7 @@ async function main() {
break;
}

if (isStartupRetryEligible && lastExitCode === 2 && scheduledExit2RetryAttempted && !lastHasOutput) {
if (isStartupRetryEligible && scheduledExit2RetryAttempted && isStartupNoOutputRetryCandidate({ exitCode: lastExitCode, hasOutput: lastHasOutput })) {
const triggerLabel = isScheduledRun ? "scheduled" : "push";
emitInfrastructureIncomplete(
`Copilot API interruption (exit code 2) persisted after automatic retry in ${triggerLabel} workflow run. ` + "This is the Turns=0 driver-handoff failure signature. Check the agent-stdio.log for startup diagnostics."
Expand Down
58 changes: 56 additions & 2 deletions actions/setup/js/copilot_harness.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -404,8 +404,9 @@ describe("copilot_harness.cjs", () => {
function shouldRetry(result, attempt, isStartupRetryEligible, scheduledExit2Retries) {
if (result.exitCode === 0) return false;

const isStartupNoOutputRetryCandidate = !result.hasOutput && result.exitCode === 2;
// Scheduled or push startup outage: retry once even when no output was produced.
if (isStartupRetryEligible && result.exitCode === 2 && !result.hasOutput && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt < MAX_RETRIES) {
if (isStartupRetryEligible && isStartupNoOutputRetryCandidate && scheduledExit2Retries < MAX_SCHEDULED_EXIT2_RETRIES && attempt < MAX_RETRIES) {
return true;
}

Expand All @@ -428,6 +429,11 @@ describe("copilot_harness.cjs", () => {
expect(shouldRetry(result, 1, isEligible, 1)).toBe(false);
});

it("does not retry exit code 1 with no output (watchdog-fired exits are suppressed as late-activity, not retried)", () => {
const result = { exitCode: 1, hasOutput: false };
expect(shouldRetry(result, 0, true, 0)).toBe(false);
});

it("does not retry on exit code 2 with no output for non-eligible triggers", () => {
const result = { exitCode: 2, hasOutput: false };
// pull_request and other event types are not startup-retry-eligible
Expand Down Expand Up @@ -465,7 +471,8 @@ describe("copilot_harness.cjs", () => {
* Returns true when the incomplete diagnostic should be emitted.
*/
function shouldEmitIncomplete({ isStartupRetryEligible, lastExitCode, retryAttempted, lastHasOutput }) {
return isStartupRetryEligible && lastExitCode === 2 && retryAttempted && !lastHasOutput;
const isStartupNoOutputRetryCandidate = !lastHasOutput && lastExitCode === 2;
return isStartupRetryEligible && isStartupNoOutputRetryCandidate && retryAttempted;
}

it("emits diagnostic when terminal attempt had no output", () => {
Expand All @@ -481,6 +488,10 @@ describe("copilot_harness.cjs", () => {
expect(shouldEmitIncomplete({ isStartupRetryEligible: true, lastExitCode: 2, retryAttempted: false, lastHasOutput: false })).toBe(false);
});

it("does not emit diagnostic for exit code 1 (watchdog-fired exits are suppressed as late-activity, not emitted as incomplete)", () => {
expect(shouldEmitIncomplete({ isStartupRetryEligible: true, lastExitCode: 1, retryAttempted: true, lastHasOutput: false })).toBe(false);
});

it("does not emit diagnostic for non-eligible event", () => {
expect(shouldEmitIncomplete({ isStartupRetryEligible: false, lastExitCode: 2, retryAttempted: true, lastHasOutput: false })).toBe(false);
});
Expand Down Expand Up @@ -2566,6 +2577,49 @@ process.exit(1);`,
expect(result.status).toBe(1);
expect(result.stderr).not.toContain("late-activity exit suppressed");
});

it("exits 0 without retrying when watchdog fires after terminal safe-output was produced (no stdio output)", () => {
const tempDir = makeHarnessTempDir("copilot-watchdog-no-stdio-suppression-");
const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl");
const stubPath = path.join(tempDir, "stub.cjs");
const promptPath = path.join(tempDir, "prompt.txt");
const callsPath = path.join(tempDir, "calls.jsonl");
// Stub writes a terminal safe-output but produces NO stdio output.
// The watchdog arms (because hasTerminalSafeOutput is true) and fires after inactivity.
// This exercises the no_output + watchdogFired + hasTerminalSafeOutput suppression path.
fs.writeFileSync(
stubPath,
`const fs = require("fs");
const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS;
const safeOutputsPath = process.env.GH_AW_SAFE_OUTPUTS;
fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n");
fs.appendFileSync(safeOutputsPath, JSON.stringify({type:"add_comment",body:"Report posted"}) + "\\n");
// No stdout output — hasOutput remains false
process.on("SIGTERM", () => process.exit(1));
setInterval(() => {}, 1000);`,
"utf8"
);
fs.writeFileSync(promptPath, "generate the report", "utf8");

const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], {
cwd: path.dirname(require.resolve("./copilot_harness.cjs")),
env: {
...process.env,
COPILOT_HARNESS_STUB_CALLS: callsPath,
GH_AW_SAFE_OUTPUTS: safeOutputsPath,
GH_AW_HARNESS_WATCHDOG_TIMEOUT_MS: "100",
},
encoding: "utf8",
timeout: 15000,
});
const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length;
// Only one attempt — no retries when watchdog suppression applies
expect(callCount).toBe(1);
// Harness exits 0 because the terminal safe-output was already produced
expect(result.status).toBe(0);
expect(result.stderr).toContain("post-result watchdog fired after terminal safe-output was emitted");
expect(result.stderr).toContain("late-activity exit suppressed");
});
});

describe("applyCopilotWireAPI", () => {
Expand Down