diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts index aba7588f4..20d761dc2 100644 --- a/scripts/__tests__/help-conformance-bench.test.ts +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -496,6 +496,35 @@ test('case matchers score parsed tokens so shell quoting does not change results }); }); +test('foreground attach scoring rejects an explicit app positional after flags', async () => { + const validCommand = 'agent-device open --foreground --platform ios'; + const explicitTargetCommand = 'agent-device open --foreground --platform ios com.example.app'; + const [validAttach, explicitTarget] = await validatePlanCommands([ + validCommand, + explicitTargetCommand, + ]); + + assert.deepEqual(explicitTarget.agentCommand, { + command: 'open', + positionals: ['com.example.app'], + }); + assert.equal( + scoreExpectations({ expectations: ['noExplicitForegroundTarget'] }, [validCommand], '', [ + validAttach, + ]).noExplicitForegroundTarget, + true, + ); + assert.equal( + scoreExpectations( + { expectations: ['noExplicitForegroundTarget'] }, + [explicitTargetCommand], + '', + [explicitTarget], + ).noExplicitForegroundTarget, + false, + ); +}); + test('plan validator applies narrow grammar to permitted external commands', async () => { const results = await validatePlanCommands( [ diff --git a/scripts/__tests__/help-conformance-sample-producers.ts b/scripts/__tests__/help-conformance-sample-producers.ts index 8e9a86a8c..a0e979c5c 100644 --- a/scripts/__tests__/help-conformance-sample-producers.ts +++ b/scripts/__tests__/help-conformance-sample-producers.ts @@ -4,6 +4,8 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, PRIVATE_AX_RECOVERY_SAMPLE, @@ -13,6 +15,8 @@ import { STALE_REF_SAMPLE, } from '../help-conformance-sample-outputs.mjs'; import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts'; +import { snapshotCliOutput } from '../../src/commands/capture/output.ts'; +import { openCliOutput } from '../../src/commands/management/output.ts'; import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts'; import { buildAmbiguousMatchError } from '../../src/daemon/handlers/find.ts'; import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts'; @@ -267,6 +271,80 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [ ).trimEnd(); }, }, + { + name: 'FOREGROUND_SNAPSHOT_FAILURE_SAMPLE', + producer: 'the open success renderer with a failed composed snapshot', + sample: FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + render: () => { + const warning = + 'The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i'; + return ( + openCliOutput({ + session: 'default', + warnings: [warning], + initialSnapshotError: { + code: 'COMMAND_FAILED', + message: 'capture failed', + }, + identifiers: { session: 'default' }, + }).text ?? '' + ); + }, + }, + { + name: 'MERGED_CARD_ACTIONS_SAMPLE', + producer: "the snapshot renderer with --actions naming a merged element's custom actions", + sample: MERGED_CARD_ACTIONS_SAMPLE, + render: () => { + // A Bluesky-style feed item merged into one Link node: its Reply/Repost/ + // menu controls are AX custom actions, not child nodes, so they only + // surface when --actions is passed through to the renderer. + const nodes = [ + { + index: 0, + ref: 'e1', + depth: 0, + type: 'Application', + label: 'Bluesky', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + parentIndex: 0, + depth: 1, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 2, + ref: 'e3', + parentIndex: 1, + depth: 2, + type: 'CollectionView', + interactive: true, + rect: { x: 0, y: 60, width: 390, height: 700 }, + }, + { + index: 3, + ref: 'e72', + parentIndex: 2, + depth: 3, + type: 'Link', + label: 'feedItem-by-whiskers.test', + interactive: true, + rect: { x: 0, y: 60, width: 390, height: 140 }, + actions: ['Reply', 'Repost', 'Open post options menu'], + }, + ]; + return ( + snapshotCliOutput({ + result: { nodes, backend: 'xctest', truncated: false }, + interactiveOnly: true, + }).text ?? '' + ).trimEnd(); + }, + }, { name: 'DEVICE_IN_USE_SAMPLE', producer: 'the real session-open by-session conflict producer', diff --git a/scripts/help-conformance-case-checks.mjs b/scripts/help-conformance-case-checks.mjs index 3f7e8b4c1..227651603 100644 --- a/scripts/help-conformance-case-checks.mjs +++ b/scripts/help-conformance-case-checks.mjs @@ -16,6 +16,13 @@ const EXPECTATION_SCORERS = { // validator splits and validates each segment), so it is the only place // that can tell whether the model actually chained. usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)), + noExplicitForegroundTarget: ({ commandValidation }) => + commandValidation + .filter( + ({ tokens, agentCommand }) => + agentCommand?.command === 'open' && tokens.includes('--foreground'), + ) + .every(({ agentCommand }) => agentCommand.positionals.length === 0), noWaitStable: ({ joined }) => !joined.includes('wait stable'), verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined), usesDogfoodEvidence: ({ joined }) => diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 931bb2262..c4bafc96c 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -3,6 +3,8 @@ import { APP_NOT_INSTALLED_SAMPLE, BROWSERSTACK_CONNECT_SAMPLE, DEVICE_IN_USE_SAMPLE, + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + MERGED_CARD_ACTIONS_SAMPLE, NOT_SETTLED_SAMPLE, OFFSCREEN_TARGET_SNAPSHOT_SAMPLE, SETTLE_DIFF_SAMPLE, @@ -642,4 +644,79 @@ Use the output already shown to determine whether the feed-search UI is present, { id: 'noRedundantInstall', pattern: /(?:^|\n)agent-device\s+install\b/i }, ], }, + { + id: 'foreground-attach-single-sim', + docs: ['--help:first30', 'workflow'], + task: 'You are starting fresh with no active session. The environment guarantees exactly one booted iOS simulator with exactly one app running on it -- the app you want to keep testing. Plan the command to attach to it and get its initial interactive snapshot in a single call (this only resolves unambiguously because of that guarantee, and it rejects an explicit app or device selector), then press the visible Continue control and close the session.', + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSettleOnMutations', + 'opensAndCloses', + 'noExplicitForegroundTarget', + ], + matchers: [ + { + id: 'startsWithForegroundOpen', + // Flag order is not semantically meaningful (`open --platform ios + // --foreground` is exactly as correct as `open --foreground + // --platform ios`); this only checks that --foreground is on the + // first command and there is no app positional before it. The + // no-positional guarantee comes from the forbidden checks below. + pattern: /^agent-device\s+open\b[^\n]*--foreground\b/i, + }, + { + id: 'pressesContinueAfterAttach', + pattern: /agent-device\s+press\s+[^\n]*continue[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { + id: 'noDeviceSelectorWithForeground', + pattern: /--foreground\b[^\n]*--(?:udid|device)\b|--(?:udid|device)\b[^\n]*--foreground\b/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'merged-card-actions-not-directly-invokable', + docs: ['--help:first30', 'workflow'], + task: quiz( + MERGED_CARD_ACTIONS_SAMPLE, + 'The goal is to reply to this post. The actions list names "Reply" as a hidden affordance on @e72, but that name is not a pressable selector. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'opensCardToReachReply', + pattern: /(?:^|\n)agent-device\s+(?:press|click)\s+@e72\b[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { + id: 'noPressingActionNameAsSelector', + pattern: /(?:^|\n)agent-device\s+(?:press|click|find)\b[^\n]*(?:label|text)="?reply"?/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'foreground-attach-snapshot-recovery', + docs: ['--help:first30', 'workflow'], + task: quiz( + FOREGROUND_SNAPSHOT_FAILURE_SAMPLE, + 'The foreground attach succeeded and the session is still open, but its initial snapshot failed. What command should run next to get interactive refs?', + ), + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI'], + matchers: [ + { + id: 'retriesSnapshotInOpenSession', + pattern: /(?:^|\n)agent-device\s+snapshot\s+-i\b/i, + }, + ], + forbidden: [ + { id: 'noSecondOpen', pattern: /(?:^|\n)agent-device\s+open\b/i }, + { id: 'noPrematureClose', pattern: /(?:^|\n)agent-device\s+close\b/i }, + ], + }, ]; diff --git a/scripts/help-conformance-command-validator.ts b/scripts/help-conformance-command-validator.ts index 56bd92344..4abf21d65 100644 --- a/scripts/help-conformance-command-validator.ts +++ b/scripts/help-conformance-command-validator.ts @@ -5,7 +5,13 @@ import { isCommandName } from '../src/commands/command-metadata.ts'; type ValidationKind = 'agent-device-grammar' | 'pseudo-ref'; +type ParsedAgentCommand = { + command: string; + positionals: string[]; +}; + type ValidationResult = { valid: true } | { valid: false; kind: ValidationKind; error: string }; +type PlanValidationResult = ValidationResult & { agentCommand?: ParsedAgentCommand }; const TARGET_POSITION_BY_COMMAND = new Map([ ['click', 0], @@ -71,10 +77,23 @@ function isStringArray(value: unknown): value is string[] { return Array.isArray(value) && value.every((entry) => typeof entry === 'string'); } -function validateInput(value: unknown): ValidationResult | ValidationResult[] { - if (Array.isArray(value) && value.every(isStringArray)) - return value.map(validateAgentDeviceCommand); - return validateAgentDeviceCommand(value); +function validateInput(value: unknown): PlanValidationResult | PlanValidationResult[] { + if (Array.isArray(value) && value.every(isStringArray)) return value.map(validatePlanCommand); + return validatePlanCommand(value); +} + +function validatePlanCommand(value: unknown): PlanValidationResult { + const validation = validateAgentDeviceCommand(value); + if (!validation.valid || !isStringArray(value)) return validation; + const parsed = parseArgs(value, { strictFlags: true }); + if (!parsed.command) return validation; + return { + ...validation, + agentCommand: { + command: parsed.command, + positionals: [...parsed.positionals], + }, + }; } function runCli(): void { diff --git a/scripts/help-conformance-plan-validator.mjs b/scripts/help-conformance-plan-validator.mjs index 327ee3d2a..a388dada3 100644 --- a/scripts/help-conformance-plan-validator.mjs +++ b/scripts/help-conformance-plan-validator.mjs @@ -30,7 +30,7 @@ function applyCommandPolicy(parsed, agentResultState, allowedExternalCommands) { } const result = agentResultState.results[agentResultState.index++]; return result?.valid - ? parsed + ? attachAgentCommand(parsed, result.agentCommand) : withIssue( parsed, result?.kind ?? 'agent-device-grammar', @@ -38,6 +38,10 @@ function applyCommandPolicy(parsed, agentResultState, allowedExternalCommands) { ); } +function attachAgentCommand(parsed, agentCommand) { + return agentCommand ? { ...parsed, agentCommand } : parsed; +} + // The compact workflow card teaches chaining confident consecutive steps with // an unquoted `&&` (`press ... --settle && fill ... --settle`). Split on it // before tokenizing a line so each chained segment is validated as its own diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index e226ee70d..f03e1ba22 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -147,3 +147,27 @@ Next: Use the installed package or bundle identifier in open, not the app artifact name. After close, run agent-device artifacts --json --session adc-browserstack for provider video and logs.`, }; + +// open --foreground succeeded, but its composed snapshot failed. The session +// remains usable, so recovery is snapshot -i rather than a second open. +export const FOREGROUND_SNAPSHOT_FAILURE_SAMPLE = { + command: 'agent-device open --foreground', + output: `Opened: default +Warning: The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i`, +}; + +// Merged feed-item card on iOS (#1665): the row itself is the only ref — its +// Reply/Repost/menu controls have no separate child nodes in the tree, so +// snapshot -i alone would show a plain link with no way to act on it. +// snapshot -i --actions names the hidden affordances instead of hiding them +// silently; the names are evidence only, never directly invokable (help +// workflow: "reach via its detail screen, labeled children elsewhere, or +// coordinates"). +export const MERGED_CARD_ACTIONS_SAMPLE = { + command: 'agent-device snapshot -i --actions', + output: `Snapshot: 4 nodes +@e1 [application] "Bluesky" +@e2 [window] +@e3 [collection] +@e72 [link] "feedItem-by-whiskers.test" actions: ["Reply", "Repost", "Open post options menu"]`, +}; diff --git a/src/__tests__/cli-help.test.ts b/src/__tests__/cli-help.test.ts index ffbb1b645..6fcd0f5f9 100644 --- a/src/__tests__/cli-help.test.ts +++ b/src/__tests__/cli-help.test.ts @@ -135,6 +135,20 @@ test('help workflow documents open/close/relaunch runner guarantees as lifecycle assert.match(result.stdout, /Env vars: help physical-device/); }); +test('help workflow advertises open --foreground and snapshot -i --actions', async () => { + const result = await runCliCapture(['help', 'workflow']); + assert.equal(result.code, 0); + assert.equal(result.calls.length, 0); + assert.match( + result.stdout, + /One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/, + ); + assert.match( + result.stdout, + /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, + ); +}); + test('help physical-device documents the runner/daemon lifecycle detail moved out of workflow (#1051)', async () => { const result = await runCliCapture(['help', 'physical-device']); assert.equal(result.code, 0); diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index cfa94192b..16aeb51c4 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -253,6 +253,10 @@ test('usageForCommand resolves workflow help topic', async () => { ); assert.match(help, /Known flow: batch \.\/steps\.json \(help scripting\)/); assert.match(help, /Shapes and platform quirks: help gestures/); + assert.match( + help, + /One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/, + ); assert.match(help, /Never open artifact paths or invent package ids/); assert.match( help, @@ -302,6 +306,10 @@ test('usageForCommand resolves workflow help topic', async () => { /confirm the requested end state is actually visible on the current screen, scrolling it into view if needed/, ); assert.match(help, /get text alone, or stopping one screen early, is not enough/); + assert.match( + help, + /iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/, + ); assert.match(help, /Perf\/memory\/log\/network\/trace\/crash: help debugging/); assert.match(help, /Recording, save-script, batch, replay repair: help scripting/); assert.match(help, /help react-native for Metro\/Re\.Pack reload/); diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index aa8f7784f..f798f64ac 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -226,6 +226,7 @@ Command shape: Bootstrap: agent-device devices --platform ios agent-device open MyApp --platform ios --device "iPhone 17 Pro" + One iOS sim, app running, no session: open --foreground: attach + snapshot. Capture fails; stays open: snapshot -i. App/device/ambiguity fail. Install arguments are app/package id then artifact path: agent-device install com.example.app ./dist/app.apk --platform android, then open --relaunch for fresh state. Use reinstall only when explicitly requested. Unknown app id: devices, then apps, then open . Never open artifact paths or invent package ids; ask if lookup misses the target. Apple CI: prepare ios-runner after boot/install, before replay/test (help prepare). Remote/cloud: connect -> open -> commands -> close -> disconnect (help remote). Reusable scripts, secret-safe fills, replay repair: help scripting. @@ -264,6 +265,7 @@ Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. + iOS sim: snapshot -i --actions shows merged actions; use detail/coords, not names. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset.