Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions scripts/__tests__/help-conformance-bench.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,35 @@ test('case matchers score parsed tokens so shell quoting does not change results
});
});

test('foreground attach scoring rejects an explicit app positional after flags', async () => {
const validCommand = 'agent-device open --foreground --platform ios';
const explicitTargetCommand = 'agent-device open --foreground --platform ios com.example.app';
const [validAttach, explicitTarget] = await validatePlanCommands([
validCommand,
explicitTargetCommand,
]);

assert.deepEqual(explicitTarget.agentCommand, {
command: 'open',
positionals: ['com.example.app'],
});
assert.equal(
scoreExpectations({ expectations: ['noExplicitForegroundTarget'] }, [validCommand], '', [
validAttach,
]).noExplicitForegroundTarget,
true,
);
assert.equal(
scoreExpectations(
{ expectations: ['noExplicitForegroundTarget'] },
[explicitTargetCommand],
'',
[explicitTarget],
).noExplicitForegroundTarget,
false,
);
});

test('plan validator applies narrow grammar to permitted external commands', async () => {
const results = await validatePlanCommands(
[
Expand Down
78 changes: 78 additions & 0 deletions scripts/__tests__/help-conformance-sample-producers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {
APP_NOT_INSTALLED_SAMPLE,
BROWSERSTACK_CONNECT_SAMPLE,
DEVICE_IN_USE_SAMPLE,
FOREGROUND_SNAPSHOT_FAILURE_SAMPLE,
MERGED_CARD_ACTIONS_SAMPLE,
NOT_SETTLED_SAMPLE,
OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
PRIVATE_AX_RECOVERY_SAMPLE,
Expand All @@ -13,6 +15,8 @@ import {
STALE_REF_SAMPLE,
} from '../help-conformance-sample-outputs.mjs';
import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts';
import { snapshotCliOutput } from '../../src/commands/capture/output.ts';
import { openCliOutput } from '../../src/commands/management/output.ts';
import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts';
import { buildAmbiguousMatchError } from '../../src/daemon/handlers/find.ts';
import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts';
Expand Down Expand Up @@ -267,6 +271,80 @@ export const SAMPLE_PRODUCERS: SampleProducer[] = [
).trimEnd();
},
},
{
name: 'FOREGROUND_SNAPSHOT_FAILURE_SAMPLE',
producer: 'the open success renderer with a failed composed snapshot',
sample: FOREGROUND_SNAPSHOT_FAILURE_SAMPLE,
render: () => {
const warning =
'The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i';
return (
openCliOutput({
session: 'default',
warnings: [warning],
initialSnapshotError: {
code: 'COMMAND_FAILED',
message: 'capture failed',
},
identifiers: { session: 'default' },
}).text ?? ''
);
},
},
{
name: 'MERGED_CARD_ACTIONS_SAMPLE',
producer: "the snapshot renderer with --actions naming a merged element's custom actions",
sample: MERGED_CARD_ACTIONS_SAMPLE,
render: () => {
// A Bluesky-style feed item merged into one Link node: its Reply/Repost/
// menu controls are AX custom actions, not child nodes, so they only
// surface when --actions is passed through to the renderer.
const nodes = [
{
index: 0,
ref: 'e1',
depth: 0,
type: 'Application',
label: 'Bluesky',
rect: { x: 0, y: 0, width: 390, height: 844 },
},
{
index: 1,
ref: 'e2',
parentIndex: 0,
depth: 1,
type: 'Window',
rect: { x: 0, y: 0, width: 390, height: 844 },
},
{
index: 2,
ref: 'e3',
parentIndex: 1,
depth: 2,
type: 'CollectionView',
interactive: true,
rect: { x: 0, y: 60, width: 390, height: 700 },
},
{
index: 3,
ref: 'e72',
parentIndex: 2,
depth: 3,
type: 'Link',
label: 'feedItem-by-whiskers.test',
interactive: true,
rect: { x: 0, y: 60, width: 390, height: 140 },
actions: ['Reply', 'Repost', 'Open post options menu'],
},
];
return (
snapshotCliOutput({
result: { nodes, backend: 'xctest', truncated: false },
interactiveOnly: true,
}).text ?? ''
).trimEnd();
},
},
{
name: 'DEVICE_IN_USE_SAMPLE',
producer: 'the real session-open by-session conflict producer',
Expand Down
7 changes: 7 additions & 0 deletions scripts/help-conformance-case-checks.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@ const EXPECTATION_SCORERS = {
// validator splits and validates each segment), so it is the only place
// that can tell whether the model actually chained.
usesConfidentChaining: ({ commands }) => commands.some((command) => /&&/.test(command)),
noExplicitForegroundTarget: ({ commandValidation }) =>
commandValidation
.filter(
({ tokens, agentCommand }) =>
agentCommand?.command === 'open' && tokens.includes('--foreground'),
)
.every(({ agentCommand }) => agentCommand.positionals.length === 0),
noWaitStable: ({ joined }) => !joined.includes('wait stable'),
verifiesNamedExpectation: ({ joined }) => /\b(wait|is|get|find)\b/.test(joined),
usesDogfoodEvidence: ({ joined }) =>
Expand Down
77 changes: 77 additions & 0 deletions scripts/help-conformance-cases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import {
APP_NOT_INSTALLED_SAMPLE,
BROWSERSTACK_CONNECT_SAMPLE,
DEVICE_IN_USE_SAMPLE,
FOREGROUND_SNAPSHOT_FAILURE_SAMPLE,
MERGED_CARD_ACTIONS_SAMPLE,
NOT_SETTLED_SAMPLE,
OFFSCREEN_TARGET_SNAPSHOT_SAMPLE,
SETTLE_DIFF_SAMPLE,
Expand Down Expand Up @@ -642,4 +644,79 @@ Use the output already shown to determine whether the feed-search UI is present,
{ id: 'noRedundantInstall', pattern: /(?:^|\n)agent-device\s+install\b/i },
],
},
{
id: 'foreground-attach-single-sim',
docs: ['--help:first30', 'workflow'],
task: 'You are starting fresh with no active session. The environment guarantees exactly one booted iOS simulator with exactly one app running on it -- the app you want to keep testing. Plan the command to attach to it and get its initial interactive snapshot in a single call (this only resolves unambiguously because of that guarantee, and it rejects an explicit app or device selector), then press the visible Continue control and close the session.',
expectations: [
'validPlanCommands',
'fullPrefix',
'usesSettleOnMutations',
'opensAndCloses',
'noExplicitForegroundTarget',
],
matchers: [
{
id: 'startsWithForegroundOpen',
// Flag order is not semantically meaningful (`open --platform ios
// --foreground` is exactly as correct as `open --foreground
// --platform ios`); this only checks that --foreground is on the
// first command and there is no app positional before it. The
// no-positional guarantee comes from the forbidden checks below.
pattern: /^agent-device\s+open\b[^\n]*--foreground\b/i,
},
{
id: 'pressesContinueAfterAttach',
pattern: /agent-device\s+press\s+[^\n]*continue[^\n]*--settle\b/i,
},
],
forbidden: [
{
id: 'noDeviceSelectorWithForeground',
pattern: /--foreground\b[^\n]*--(?:udid|device)\b|--(?:udid|device)\b[^\n]*--foreground\b/i,
},
{ id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET },
],
},
{
id: 'merged-card-actions-not-directly-invokable',
docs: ['--help:first30', 'workflow'],
task: quiz(
MERGED_CARD_ACTIONS_SAMPLE,
'The goal is to reply to this post. The actions list names "Reply" as a hidden affordance on @e72, but that name is not a pressable selector. What command should run next?',
),
expectations: ['validPlanCommands', 'fullPrefix'],
matchers: [
{
id: 'opensCardToReachReply',
pattern: /(?:^|\n)agent-device\s+(?:press|click)\s+@e72\b[^\n]*--settle\b/i,
},
],
forbidden: [
{
id: 'noPressingActionNameAsSelector',
pattern: /(?:^|\n)agent-device\s+(?:press|click|find)\b[^\n]*(?:label|text)="?reply"?/i,
},
{ id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET },
],
},
{
id: 'foreground-attach-snapshot-recovery',
docs: ['--help:first30', 'workflow'],
task: quiz(
FOREGROUND_SNAPSHOT_FAILURE_SAMPLE,
'The foreground attach succeeded and the session is still open, but its initial snapshot failed. What command should run next to get interactive refs?',
),
expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI'],
matchers: [
{
id: 'retriesSnapshotInOpenSession',
pattern: /(?:^|\n)agent-device\s+snapshot\s+-i\b/i,
},
],
forbidden: [
{ id: 'noSecondOpen', pattern: /(?:^|\n)agent-device\s+open\b/i },
{ id: 'noPrematureClose', pattern: /(?:^|\n)agent-device\s+close\b/i },
],
},
];
27 changes: 23 additions & 4 deletions scripts/help-conformance-command-validator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@ import { isCommandName } from '../src/commands/command-metadata.ts';

type ValidationKind = 'agent-device-grammar' | 'pseudo-ref';

type ParsedAgentCommand = {
command: string;
positionals: string[];
};

type ValidationResult = { valid: true } | { valid: false; kind: ValidationKind; error: string };
type PlanValidationResult = ValidationResult & { agentCommand?: ParsedAgentCommand };

const TARGET_POSITION_BY_COMMAND = new Map<string, number>([
['click', 0],
Expand Down Expand Up @@ -71,10 +77,23 @@ function isStringArray(value: unknown): value is string[] {
return Array.isArray(value) && value.every((entry) => typeof entry === 'string');
}

function validateInput(value: unknown): ValidationResult | ValidationResult[] {
if (Array.isArray(value) && value.every(isStringArray))
return value.map(validateAgentDeviceCommand);
return validateAgentDeviceCommand(value);
function validateInput(value: unknown): PlanValidationResult | PlanValidationResult[] {
if (Array.isArray(value) && value.every(isStringArray)) return value.map(validatePlanCommand);
return validatePlanCommand(value);
}

function validatePlanCommand(value: unknown): PlanValidationResult {
const validation = validateAgentDeviceCommand(value);
if (!validation.valid || !isStringArray(value)) return validation;
const parsed = parseArgs(value, { strictFlags: true });
if (!parsed.command) return validation;
return {
...validation,
agentCommand: {
command: parsed.command,
positionals: [...parsed.positionals],
},
};
}

function runCli(): void {
Expand Down
6 changes: 5 additions & 1 deletion scripts/help-conformance-plan-validator.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,18 @@ function applyCommandPolicy(parsed, agentResultState, allowedExternalCommands) {
}
const result = agentResultState.results[agentResultState.index++];
return result?.valid
? parsed
? attachAgentCommand(parsed, result.agentCommand)
: withIssue(
parsed,
result?.kind ?? 'agent-device-grammar',
result?.error ?? 'Command validation returned no result.',
);
}

function attachAgentCommand(parsed, agentCommand) {
return agentCommand ? { ...parsed, agentCommand } : parsed;
}

// The compact workflow card teaches chaining confident consecutive steps with
// an unquoted `&&` (`press ... --settle && fill ... --settle`). Split on it
// before tokenizing a line so each chained segment is validated as its own
Expand Down
24 changes: 24 additions & 0 deletions scripts/help-conformance-sample-outputs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,27 @@ Next:
Use the installed package or bundle identifier in open, not the app artifact name.
After close, run agent-device artifacts --json --session adc-browserstack for provider video and logs.`,
};

// open --foreground succeeded, but its composed snapshot failed. The session
// remains usable, so recovery is snapshot -i rather than a second open.
export const FOREGROUND_SNAPSHOT_FAILURE_SAMPLE = {
command: 'agent-device open --foreground',
output: `Opened: default
Warning: The session is open, but the initial interactive snapshot failed (COMMAND_FAILED: capture failed). Run: agent-device snapshot -i`,
};

// Merged feed-item card on iOS (#1665): the row itself is the only ref — its
// Reply/Repost/menu controls have no separate child nodes in the tree, so
// snapshot -i alone would show a plain link with no way to act on it.
// snapshot -i --actions names the hidden affordances instead of hiding them
// silently; the names are evidence only, never directly invokable (help
// workflow: "reach via its detail screen, labeled children elsewhere, or
// coordinates").
export const MERGED_CARD_ACTIONS_SAMPLE = {
command: 'agent-device snapshot -i --actions',
output: `Snapshot: 4 nodes
@e1 [application] "Bluesky"
@e2 [window]
@e3 [collection]
@e72 [link] "feedItem-by-whiskers.test" actions: ["Reply", "Repost", "Open post options menu"]`,
};
14 changes: 14 additions & 0 deletions src/__tests__/cli-help.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,20 @@ test('help workflow documents open/close/relaunch runner guarantees as lifecycle
assert.match(result.stdout, /Env vars: help physical-device/);
});

test('help workflow advertises open --foreground and snapshot -i --actions', async () => {
const result = await runCliCapture(['help', 'workflow']);
assert.equal(result.code, 0);
assert.equal(result.calls.length, 0);
assert.match(
result.stdout,
/One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/,
);
assert.match(
result.stdout,
/iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/,
);
});

test('help physical-device documents the runner/daemon lifecycle detail moved out of workflow (#1051)', async () => {
const result = await runCliCapture(['help', 'physical-device']);
assert.equal(result.code, 0);
Expand Down
8 changes: 8 additions & 0 deletions src/cli/parser/__tests__/cli-help-topics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ test('usageForCommand resolves workflow help topic', async () => {
);
assert.match(help, /Known flow: batch \.\/steps\.json \(help scripting\)/);
assert.match(help, /Shapes and platform quirks: help gestures/);
assert.match(
help,
/One iOS sim, app running, no session: open --foreground: attach \+ snapshot\. Capture fails; stays open: snapshot -i\. App\/device\/ambiguity fail/,
);
assert.match(help, /Never open artifact paths or invent package ids/);
assert.match(
help,
Expand Down Expand Up @@ -302,6 +306,10 @@ test('usageForCommand resolves workflow help topic', async () => {
/confirm the requested end state is actually visible on the current screen, scrolling it into view if needed/,
);
assert.match(help, /get text alone, or stopping one screen early, is not enough/);
assert.match(
help,
/iOS sim: snapshot -i --actions shows merged actions; use detail\/coords, not names/,
);
assert.match(help, /Perf\/memory\/log\/network\/trace\/crash: help debugging/);
assert.match(help, /Recording, save-script, batch, replay repair: help scripting/);
assert.match(help, /help react-native for Metro\/Re\.Pack reload/);
Expand Down
2 changes: 2 additions & 0 deletions src/cli/parser/cli-help.ts
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ Command shape:
Bootstrap:
agent-device devices --platform ios
agent-device open MyApp --platform ios --device "iPhone 17 Pro"
One iOS sim, app running, no session: open --foreground: attach + snapshot. Capture fails; stays open: snapshot -i. App/device/ambiguity fail.
Install arguments are app/package id then artifact path: agent-device install com.example.app ./dist/app.apk --platform android, then open <id> --relaunch for fresh state. Use reinstall only when explicitly requested.
Unknown app id: devices, then apps, then open <discovered-app-id>. Never open artifact paths or invent package ids; ask if lookup misses the target.
Apple CI: prepare ios-runner after boot/install, before replay/test (help prepare). Remote/cloud: connect -> open -> commands -> close -> disconnect (help remote). Reusable scripts, secret-safe fills, replay repair: help scripting.
Expand Down Expand Up @@ -264,6 +265,7 @@ Validation and evidence:
Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing.
Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough.
When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen.
iOS sim: snapshot -i --actions shows merged actions; use detail/coords, not names.
Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting.

React Native: help react-native for Metro/Re.Pack reload, DevTools, RN overlays. JS-only change: metro reload, find "Home"; open --relaunch for native reset.
Expand Down
Loading