From 4a0b183116441e117f3a4f443e4cc4d4e65f2b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 8 Aug 2026 08:07:07 +0200 Subject: [PATCH 1/3] docs(cli): advertise open --foreground and snapshot --actions in the workflow card open --foreground (#1670/#1671) and snapshot -i --actions (#1665) shipped with no mention in the compact `help workflow` card, so a planning model never discovers either. Add one terse line each: the foreground fast-path in Bootstrap, and the merged-element custom-action guidance in Validation and evidence. Stays under the 9,000-byte compact-card budget (8493 -> 8908 bytes). Adds two help-conformance bench cases per the repo's changed-guidance rule: foreground-attach-single-sim (correct plan starts with `open --foreground` in an unambiguous single-sim scenario, fail-closed alternative forbidden) and merged-card-actions-not-directly-invokable (a merged Bluesky-style feed card's actions list is evidence, not a selector). Both use a real pinned sample rebuilt through the production snapshot renderer. --- .../help-conformance-sample-producers.ts | 49 ++++++++++++++++++ scripts/help-conformance-cases.mjs | 50 +++++++++++++++++++ scripts/help-conformance-sample-outputs.mjs | 16 ++++++ src/__tests__/cli-help.test.ts | 14 ++++++ .../parser/__tests__/cli-help-topics.test.ts | 8 +++ src/cli/parser/cli-help.ts | 2 + 6 files changed, 139 insertions(+) diff --git a/scripts/__tests__/help-conformance-sample-producers.ts b/scripts/__tests__/help-conformance-sample-producers.ts index 8e9a86a8c..b93d49bd9 100644 --- a/scripts/__tests__/help-conformance-sample-producers.ts +++ b/scripts/__tests__/help-conformance-sample-producers.ts @@ -4,6 +4,7 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, PRIVATE_AX_RECOVERY_SAMPLE, @@ -267,6 +268,54 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ ).trimEnd(); }, }, + { + name: 'MERGED_CARD_ACTIONS_SAMPLE', + producer: "the snapshot renderer with --actions naming a merged element's custom actions", + sample: MERGED_CARD_ACTIONS_SAMPLE, + render: () => { + // A Bluesky-style feed item merged into one Link node: its Reply/Repost/ + // menu controls are AX custom actions, not child nodes, so they only + // surface when --actions is passed through to the renderer. + const nodes = [ + { + index: 0, + ref: 'e1', + type: 'Application', + label: 'Bluesky', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + parentIndex: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 2, + ref: 'e3', + parentIndex: 1, + type: 'CollectionView', + interactive: true, + rect: { x: 0, y: 60, width: 390, height: 700 }, + }, + { + index: 3, + ref: 'e72', + parentIndex: 2, + type: 'Link', + label: 'feedItem-by-whiskers.test', + interactive: true, + rect: { x: 0, y: 60, width: 390, height: 140 }, + actions: ['Reply', 'Repost', 'Open post options menu'], + }, + ]; + return formatSnapshotText( + { nodes, backend: 'xctest', truncated: false }, + { interactiveOnly: true }, + ).trimEnd(); + }, + }, { name: 'DEVICE_IN_USE_SAMPLE', producer: 'the real session-open by-session conflict producer', diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 931bb2262..07c22e0f6 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -3,6 +3,7 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, SETTLE_DIFF_SAMPLE, @@ -642,4 +643,53 @@ Use the output already shown to determine whether the feed-search UI is present, { id: 'noRedundantInstall', pattern: /(?:^|\n)agent-device\s+install\b/i }, ], }, + { + id: 'foreground-attach-single-sim', + docs: ['--help:first30', 'workflow'], + task: 'You are starting fresh with no active session. The environment guarantees exactly one booted iOS simulator with exactly one app running on it -- the app you want to keep testing. Plan the command to attach to it and get its initial interactive snapshot in a single call (this only resolves unambiguously because of that guarantee, and it rejects an explicit app or device selector), then press the visible Continue control and close the session.', + expectations: ['validPlanCommands', 'fullPrefix', 'usesSettleOnMutations', 'opensAndCloses'], + matchers: [ + { + id: 'startsWithForegroundOpen', + pattern: /^agent-device\s+open\s+--foreground\b/i, + }, + { + id: 'pressesContinueAfterAttach', + pattern: /agent-device\s+press\s+[^\n]*continue[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { + id: 'noAppPositionalWithForeground', + pattern: /open[ \t]+(?!--)\S+[^\n]*--foreground\b|open[ \t]+--foreground[ \t]+(?!--)\S+/i, + }, + { + id: 'noDeviceSelectorWithForeground', + pattern: /--foreground\b[^\n]*--(?:udid|device)\b|--(?:udid|device)\b[^\n]*--foreground\b/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'merged-card-actions-not-directly-invokable', + docs: ['--help:first30', 'workflow'], + task: quiz( + MERGED_CARD_ACTIONS_SAMPLE, + 'The goal is to reply to this post. The actions list names "Reply" as a hidden affordance on @e72, but that name is not a pressable selector. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'opensCardToReachReply', + pattern: /(?:^|\n)agent-device\s+(?:press|click)\s+@e72\b[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { + id: 'noPressingActionNameAsSelector', + pattern: /(?:^|\n)agent-device\s+(?:press|click|find)\b[^\n]*(?:label|text)="?reply"?/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, ]; diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index e226ee70d..ababbfd6e 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -147,3 +147,19 @@ Next: Use the installed package or bundle identifier in open, not the app artifact name. After close, run agent-device artifacts --json --session adc-browserstack for provider video and logs.`, }; + +// Merged feed-item card on iOS (#1665): the row itself is the only ref — its +// Reply/Repost/menu controls have no separate child nodes in the tree, so +// snapshot -i alone would show a plain link with no way to act on it. +// snapshot -i --actions names the hidden affordances instead of hiding them +// silently; the names are evidence only, never directly invokable (help +// workflow: "reach via its detail screen, labeled children elsewhere, or +// coordinates"). +export const MERGED_CARD_ACTIONS_SAMPLE = { + command: 'agent-device snapshot -i --actions', + output: `Snapshot: 4 nodes +@e1 [application] "Bluesky" +@e2 [window] +@e3 [collection] +@e72 [link] "feedItem-by-whiskers.test" actions: ["Reply", "Repost", "Open post options menu"]`, +}; diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index ffbb1b645..a605d0bc1 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -135,6 +135,20 @@ test('help workflow documents open/close/relaunch runner guarantees as lifecycle assert.match(result.stdout, /Env vars: help physical-device/); }); +test('help workflow advertises open --foreground and snapshot -i --actions', async () => { + const result = await runCliCapture(['help', 'workflow']); + assert.equal(result.code, 0); + assert.equal(result.calls.length, 0); + assert.match( + result.stdout, + /No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed \(AMBIGUOUS_MATCH\) otherwise; rejects an explicit app\/--udid\/--device/, + ); + assert.match( + result.stdout, + /snapshot -i --actions names custom-action affordances hidden inside a merged element \(iOS sim only\); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates/, + ); +}); + test('help physical-device documents the runner/daemon lifecycle detail moved out of workflow (#1051)', async () => { const result = await runCliCapture(['help', 'physical-device']); assert.equal(result.code, 0); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index cfa94192b..56b3fe675 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -253,6 +253,10 @@ test('usageForCommand resolves workflow help topic', async () => { ); assert.match(help, /Known flow: batch \.\/steps\.json \(help scripting\)/); assert.match(help, /Shapes and platform quirks: help gestures/); + assert.match( + help, + /No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed \(AMBIGUOUS_MATCH\) otherwise; rejects an explicit app\/--udid\/--device/, + ); assert.match(help, /Never open artifact paths or invent package ids/); assert.match( help, @@ -302,6 +306,10 @@ test('usageForCommand resolves workflow help topic', async () => { /confirm the requested end state is actually visible on the current screen, scrolling it into view if needed/, ); assert.match(help, /get text alone, or stopping one screen early, is not enough/); + assert.match( + help, + /snapshot -i --actions names custom-action affordances hidden inside a merged element \(iOS sim only\); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates/, + ); assert.match(help, /Perf\/memory\/log\/network\/trace\/crash: help debugging/); assert.match(help, /Recording, save-script, batch, replay repair: help scripting/); assert.match(help, /help react-native for Metro\/Re\.Pack reload/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index aa8f7784f..0137e9ee2 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -226,6 +226,7 @@ Command shape: Bootstrap: agent-device devices --platform ios agent-device open MyApp --platform ios --device "iPhone 17 Pro" + No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed (AMBIGUOUS_MATCH) otherwise; rejects an explicit app/--udid/--device. Install arguments are app/package id then artifact path: agent-device install com.example.app ./dist/app.apk --platform android, then open --relaunch for fresh state. Use reinstall only when explicitly requested. Unknown app id: devices, then apps, then open . Never open artifact paths or invent package ids; ask if lookup misses the target. Apple CI: prepare ios-runner after boot/install, before replay/test (help prepare). Remote/cloud: connect -> open -> commands -> close -> disconnect (help remote). Reusable scripts, secret-safe fills, replay repair: help scripting. @@ -264,6 +265,7 @@ Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. + snapshot -i --actions names custom-action affordances hidden inside a merged element (iOS sim only); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset. From 326a1d9d50ee3f7cf59d95d089544ebc273ef116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 8 Aug 2026 08:16:54 +0200 Subject: [PATCH 2/3] fix(scripts): accept flag order in the foreground-attach conformance matcher Flag order after `open` isn't semantically meaningful (`open --platform ios --foreground` is exactly as correct as `open --foreground --platform ios`), but startsWithForegroundOpen required --foreground to be the literal next token after `open`. Rescoring the completed repeat=3 bench report shows this docked codex:gpt-5.4-mini on all 3 trials even though its plan was config-order noise, not a real deviation -- the no-positional/no-device guarantee already comes from the forbidden checks. Loosened to require --foreground anywhere on the open line; foreground-attach-single-sim now scores 54/54 across both runners. --- scripts/help-conformance-cases.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 07c22e0f6..552f32a0d 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -651,7 +651,12 @@ Use the output already shown to determine whether the feed-search UI is present, matchers: [ { id: 'startsWithForegroundOpen', - pattern: /^agent-device\s+open\s+--foreground\b/i, + // Flag order is not semantically meaningful (`open --platform ios + // --foreground` is exactly as correct as `open --foreground + // --platform ios`); this only checks that --foreground is on the + // first command and there is no app positional before it. The + // no-positional guarantee comes from the forbidden checks below. + pattern: /^agent-device\s+open\b[^\n]*--foreground\b/i, }, { id: 'pressesContinueAfterAttach', From adc836669a30e24fe598d0be14d8efadbcdbe88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 8 Aug 2026 09:23:01 +0200 Subject: [PATCH 3/3] fix: close workflow help conformance gaps --- .../__tests__/help-conformance-bench.test.ts | 29 +++++++++++++++ .../help-conformance-sample-producers.ts | 35 +++++++++++++++++-- scripts/help-conformance-case-checks.mjs | 7 ++++ scripts/help-conformance-cases.mjs | 32 ++++++++++++++--- scripts/help-conformance-command-validator.ts | 27 +++++++++++--- scripts/help-conformance-plan-validator.mjs | 6 +++- scripts/help-conformance-sample-outputs.mjs | 8 +++++ src/__tests__/cli-help.test.ts | 4 +-- .../parser/__tests__/cli-help-topics.test.ts | 4 +-- src/cli/parser/cli-help.ts | 4 +-- 10 files changed, 137 insertions(+), 19 deletions(-) diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts index aba7588f4..20d761dc2 100644 --- a/scripts/__tests__/help-conformance-bench.test.ts +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -496,6 +496,35 @@ test('case matchers score parsed tokens so shell quoting does not change results }); }); +test('foreground attach scoring rejects an explicit app positional after flags', async () => { + const validCommand = 'agent-device open --foreground --platform ios'; + const explicitTargetCommand = 'agent-device open --foreground --platform ios com.example.app'; + const [validAttach, explicitTarget] = await validatePlanCommands([ + validCommand, + explicitTargetCommand, + ]); + + assert.deepEqual(explicitTarget.agentCommand, { + command: 'open', + positionals: ['com.example.app'], + }); + assert.equal( + scoreExpectations({ expectations: ['noExplicitForegroundTarget'] }, [validCommand], '', [ + validAttach, + ]).noExplicitForegroundTarget, + true, + ); + assert.equal( + scoreExpectations( + { expectations: ['noExplicitForegroundTarget'] }, + [explicitTargetCommand], + '', + [explicitTarget], + ).noExplicitForegroundTarget, + false, + ); +}); + test('plan validator applies narrow grammar to permitted external commands', async () => { const results = await validatePlanCommands( [ diff --git a/scripts/__tests__/help-conformance-sample-producers.ts b/scripts/__tests__/help-conformance-sample-producers.ts index b93d49bd9..a0e979c5c 100644 --- a/scripts/__tests__/help-conformance-sample-producers.ts +++ b/scripts/__tests__/help-conformance-sample-producers.ts @@ -4,6 +4,7 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, @@ -14,6 +15,8 @@ import { STALE_REF_SAMPLE, } from '../help-conformance-sample-outputs.mjs'; import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts'; +import { snapshotCliOutput } from '../../src/commands/capture/output.ts'; +import { openCliOutput } from '../../src/commands/management/output.ts'; import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts'; import { buildAmbiguousMatchError } from '../../src/daemon/handlers/find.ts'; import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts'; @@ -268,6 +271,26 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ ).trimEnd(); }, }, + { + name: 'FOREGROUND_SNAPSHOT_FAILURE_SAMPLE', + producer: 'the open success renderer with a failed composed snapshot', + sample: FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + render: () => { + const warning = + 'The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i'; + return ( + openCliOutput({ + session: 'default', + warnings: [warning], + initialSnapshotError: { + code: 'COMMAND_FAILED', + message: 'capture failed', + }, + identifiers: { session: 'default' }, + }).text ?? '' + ); + }, + }, { name: 'MERGED_CARD_ACTIONS_SAMPLE', producer: "the snapshot renderer with --actions naming a merged element's custom actions", @@ -280,6 +303,7 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ { index: 0, ref: 'e1', + depth: 0, type: 'Application', label: 'Bluesky', rect: { x: 0, y: 0, width: 390, height: 844 }, @@ -288,6 +312,7 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ index: 1, ref: 'e2', parentIndex: 0, + depth: 1, type: 'Window', rect: { x: 0, y: 0, width: 390, height: 844 }, }, @@ -295,6 +320,7 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ index: 2, ref: 'e3', parentIndex: 1, + depth: 2, type: 'CollectionView', interactive: true, rect: { x: 0, y: 60, width: 390, height: 700 }, @@ -303,6 +329,7 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ index: 3, ref: 'e72', parentIndex: 2, + depth: 3, type: 'Link', label: 'feedItem-by-whiskers.test', interactive: true, @@ -310,9 +337,11 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ actions: ['Reply', 'Repost', 'Open post options menu'], }, ]; - return formatSnapshotText( - { nodes, backend: 'xctest', truncated: false }, - { interactiveOnly: true }, + return ( + snapshotCliOutput({ + result: { nodes, backend: 'xctest', truncated: false }, + interactiveOnly: true, + }).text ?? '' ).trimEnd(); }, }, diff --git a/scripts/help-conformance-case-checks.mjs b/scripts/help-conformance-case-checks.mjs index 3f7e8b4c1..227651603 100644 --- a/scripts/help-conformance-case-checks.mjs +++ b/scripts/help-conformance-case-checks.mjs @@ -16,6 +16,13 @@ const EXPECTATION_SCORERS = { // validator splits and validates each segment), so it is the only place // that can tell whether the model actually chained. usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)), + noExplicitForegroundTarget: ({ commandValidation }) => + commandValidation + .filter( + ({ tokens, agentCommand }) => + agentCommand?.command === 'open' && tokens.includes('--foreground'), + ) + .every(({ agentCommand }) => agentCommand.positionals.length === 0), noWaitStable: ({ joined }) => !joined.includes('wait stable'), verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined), usesDogfoodEvidence: ({ joined }) => diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 552f32a0d..c4bafc96c 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -3,6 +3,7 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, @@ -647,7 +648,13 @@ Use the output already shown to determine whether the feed-search UI is present, id: 'foreground-attach-single-sim', docs: ['--help:first30', 'workflow'], task: 'You are starting fresh with no active session. The environment guarantees exactly one booted iOS simulator with exactly one app running on it -- the app you want to keep testing. Plan the command to attach to it and get its initial interactive snapshot in a single call (this only resolves unambiguously because of that guarantee, and it rejects an explicit app or device selector), then press the visible Continue control and close the session.', - expectations: ['validPlanCommands', 'fullPrefix', 'usesSettleOnMutations', 'opensAndCloses'], + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSettleOnMutations', + 'opensAndCloses', + 'noExplicitForegroundTarget', + ], matchers: [ { id: 'startsWithForegroundOpen', @@ -664,10 +671,6 @@ Use the output already shown to determine whether the feed-search UI is present, }, ], forbidden: [ - { - id: 'noAppPositionalWithForeground', - pattern: /open[ \t]+(?!--)\S+[^\n]*--foreground\b|open[ \t]+--foreground[ \t]+(?!--)\S+/i, - }, { id: 'noDeviceSelectorWithForeground', pattern: /--foreground\b[^\n]*--(?:udid|device)\b|--(?:udid|device)\b[^\n]*--foreground\b/i, @@ -697,4 +700,23 @@ Use the output already shown to determine whether the feed-search UI is present, { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, ], }, + { + id: 'foreground-attach-snapshot-recovery', + docs: ['--help:first30', 'workflow'], + task: quiz( + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + 'The foreground attach succeeded and the session is still open, but its initial snapshot failed. What command should run next to get interactive refs?', + ), + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI'], + matchers: [ + { + id: 'retriesSnapshotInOpenSession', + pattern: /(?:^|\n)agent-device\s+snapshot\s+-i\b/i, + }, + ], + forbidden: [ + { id: 'noSecondOpen', pattern: /(?:^|\n)agent-device\s+open\b/i }, + { id: 'noPrematureClose', pattern: /(?:^|\n)agent-device\s+close\b/i }, + ], + }, ]; diff --git a/scripts/help-conformance-command-validator.ts b/scripts/help-conformance-command-validator.ts index 56bd92344..4abf21d65 100644 --- a/scripts/help-conformance-command-validator.ts +++ b/scripts/help-conformance-command-validator.ts @@ -5,7 +5,13 @@ import { isCommandName } from '../src/commands/command-metadata.ts'; type ValidationKind = 'agent-device-grammar' | 'pseudo-ref'; +type ParsedAgentCommand = { + command: string; + positionals: string[]; +}; + type ValidationResult = { valid: true } | { valid: false; kind: ValidationKind; error: string }; +type PlanValidationResult = ValidationResult & { agentCommand?: ParsedAgentCommand }; const TARGET_POSITION_BY_COMMAND = new Map([ ['click', 0], @@ -71,10 +77,23 @@ function isStringArray(value: unknown): value is string[] { return Array.isArray(value) && value.every((entry) => typeof entry === 'string'); } -function validateInput(value: unknown): ValidationResult | ValidationResult[] { - if (Array.isArray(value) && value.every(isStringArray)) - return value.map(validateAgentDeviceCommand); - return validateAgentDeviceCommand(value); +function validateInput(value: unknown): PlanValidationResult | PlanValidationResult[] { + if (Array.isArray(value) && value.every(isStringArray)) return value.map(validatePlanCommand); + return validatePlanCommand(value); +} + +function validatePlanCommand(value: unknown): PlanValidationResult { + const validation = validateAgentDeviceCommand(value); + if (!validation.valid || !isStringArray(value)) return validation; + const parsed = parseArgs(value, { strictFlags: true }); + if (!parsed.command) return validation; + return { + ...validation, + agentCommand: { + command: parsed.command, + positionals: [...parsed.positionals], + }, + }; } function runCli(): void { diff --git a/scripts/help-conformance-plan-validator.mjs b/scripts/help-conformance-plan-validator.mjs index 327ee3d2a..a388dada3 100644 --- a/scripts/help-conformance-plan-validator.mjs +++ b/scripts/help-conformance-plan-validator.mjs @@ -30,7 +30,7 @@ function applyCommandPolicy(parsed, agentResultState, allowedExternalCommands) { } const result = agentResultState.results[agentResultState.index++]; return result?.valid - ? parsed + ? attachAgentCommand(parsed, result.agentCommand) : withIssue( parsed, result?.kind ?? 'agent-device-grammar', @@ -38,6 +38,10 @@ function applyCommandPolicy(parsed, agentResultState, allowedExternalCommands) { ); } +function attachAgentCommand(parsed, agentCommand) { + return agentCommand ? { ...parsed, agentCommand } : parsed; +} + // The compact workflow card teaches chaining confident consecutive steps with // an unquoted `&&` (`press ... --settle && fill ... --settle`). Split on it // before tokenizing a line so each chained segment is validated as its own diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index ababbfd6e..f03e1ba22 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -148,6 +148,14 @@ Use the installed package or bundle identifier in open, not the app artifact nam After close, run agent-device artifacts --json --session adc-browserstack for provider video and logs.`, }; +// open --foreground succeeded, but its composed snapshot failed. The session +// remains usable, so recovery is snapshot -i rather than a second open. +export const FOREGROUND_SNAPSHOT_FAILURE_SAMPLE = { + command: 'agent-device open --foreground', + output: `Opened: default +Warning: The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i`, +}; + // Merged feed-item card on iOS (#1665): the row itself is the only ref — its // Reply/Repost/menu controls have no separate child nodes in the tree, so // snapshot -i alone would show a plain link with no way to act on it. diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index a605d0bc1..6fcd0f5f9 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -141,11 +141,11 @@ test('help workflow advertises open --foreground and snapshot -i --actions', asy assert.equal(result.calls.length, 0); assert.match( result.stdout, - /No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed \(AMBIGUOUS_MATCH\) otherwise; rejects an explicit app\/--udid\/--device/, + /One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/, ); assert.match( result.stdout, - /snapshot -i --actions names custom-action affordances hidden inside a merged element \(iOS sim only\); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates/, + /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, ); }); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 56b3fe675..16aeb51c4 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -255,7 +255,7 @@ test('usageForCommand resolves workflow help topic', async () => { assert.match(help, /Shapes and platform quirks: help gestures/); assert.match( help, - /No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed \(AMBIGUOUS_MATCH\) otherwise; rejects an explicit app\/--udid\/--device/, + /One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/, ); assert.match(help, /Never open artifact paths or invent package ids/); assert.match( @@ -308,7 +308,7 @@ test('usageForCommand resolves workflow help topic', async () => { assert.match(help, /get text alone, or stopping one screen early, is not enough/); assert.match( help, - /snapshot -i --actions names custom-action affordances hidden inside a merged element \(iOS sim only\); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates/, + /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, ); assert.match(help, /Perf\/memory\/log\/network\/trace\/crash: help debugging/); assert.match(help, /Recording, save-script, batch, replay repair: help scripting/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 0137e9ee2..f798f64ac 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -226,7 +226,7 @@ Command shape: Bootstrap: agent-device devices --platform ios agent-device open MyApp --platform ios --device "iPhone 17 Pro" - No session, one booted iOS simulator, one running app: open --foreground attaches and returns the initial snapshot in one call; fails closed (AMBIGUOUS_MATCH) otherwise; rejects an explicit app/--udid/--device. + One iOS sim, app running, no session: open --foreground: attach + snapshot. Capture fails; stays open: snapshot -i. App/device/ambiguity fail. Install arguments are app/package id then artifact path: agent-device install com.example.app ./dist/app.apk --platform android, then open --relaunch for fresh state. Use reinstall only when explicitly requested. Unknown app id: devices, then apps, then open . Never open artifact paths or invent package ids; ask if lookup misses the target. Apple CI: prepare ios-runner after boot/install, before replay/test (help prepare). Remote/cloud: connect -> open -> commands -> close -> disconnect (help remote). Reusable scripts, secret-safe fills, replay repair: help scripting. @@ -265,7 +265,7 @@ Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. - snapshot -i --actions names custom-action affordances hidden inside a merged element (iOS sim only); not directly invokable -- reach via its detail screen, labeled children elsewhere, or coordinates. + iOS sim: snapshot -i --actions shows merged actions; use detail/coords, not names. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset.