diff --git a/CLAUDE.md b/CLAUDE.md index ed556a57ea537..9b85a3d23fa0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,7 +115,7 @@ gh pr create --repo microsoft/playwright --head username:fix-39562 \ --title "fix(proxy): handle SOCKS proxy authentication" \ --body "$(cat <<'EOF' ## Summary -- +- Fixes https://github.com/microsoft/playwright/issues/39562 EOF diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.md index a94cdbe4a0ba3..179bc2f22879d 100644 --- a/docs/src/api/class-locator.md +++ b/docs/src/api/class-locator.md @@ -2485,6 +2485,19 @@ This method expects [Locator] to point to an ### option: Locator.setInputFiles.timeout = %%-input-timeout-js-%% * since: v1.14 +## async method: Locator.snapshotForAI +* since: v1.59 +- returns: <[Object]> + - `full` <[string]> Accessibility snapshot of the element matching this locator. + +Returns an accessibility snapshot of the element's subtree optimized for AI consumption. + +### option: Locator.snapshotForAI.timeout = %%-input-timeout-%% +* since: v1.59 + +### option: Locator.snapshotForAI.timeout = %%-input-timeout-js-%% +* since: v1.59 + ## async method: Locator.tap * since: v1.14 diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index f1e38b9a74a4b..42c4714c487e5 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -4223,6 +4223,9 @@ Returns an accessibility snapshot of the page optimized for AI consumption. ### option: Page.snapshotForAI.timeout = %%-input-timeout-%% * since: v1.59 +### option: Page.snapshotForAI.timeout = %%-input-timeout-js-%% +* since: v1.59 + ### option: Page.snapshotForAI.track * since: v1.59 - `track` <[string]> diff --git a/package.json b/package.json index 43053cd703fe0..df5fe7fe4af65 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,11 @@ "lint": "npm run eslint && npm run tsc && npm run doc && npm run check-deps && node utils/generate_channels.js && node utils/generate_types/ && npm run lint-tests && npm run test-types && npm run lint-packages", "lint-packages": "node utils/workspace.js --ensure-consistent", "lint-tests": "node utils/lint_tests.js", - "flint": "concurrently \"npm run eslint\" \"npm run tsc\" \"npm run doc\" \"npm run check-deps\" \"node utils/generate_channels.js\" \"node utils/generate_types/\" \"npm run lint-tests\" \"npm run test-types\" \"npm run lint-packages\" \"node utils/doclint/linting-code-snippets/cli.js --js-only\"", + "flint": "concurrently \"npm run eslint\" \"npm run tsc\" \"npm run doc\" \"npm run check-deps\" \"node utils/generate_channels.js\" \"npm run lint-tests\" \"npm run test-types\" \"npm run lint-packages\" \"node utils/doclint/linting-code-snippets/cli.js --js-only\"", "clean": "node utils/build/clean.js", "build": "node utils/build/build.js", "watch": "node utils/build/build.js --watch --lint", - "test-types": "node utils/generate_types/ && tsc -p utils/generate_types/test/tsconfig.json && tsc -p ./tests/", + "test-types": "node utils/generate_types/ && concurrently \"tsc -p utils/generate_types/test/tsconfig.json\" \"tsc -p ./tests/\"", "roll": "node utils/roll_browser.js", "check-deps": "node utils/check_deps.js", "build-android-driver": "./utils/build_android_driver.sh", diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 0f3a236cf35c9..8c1d25aa20a21 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -4538,6 +4538,14 @@ export interface Page { * @param options */ snapshotForAI(options?: { + /** + * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout` + * option in the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + */ + timeout?: number; + /** * When specified, enables incremental snapshots. Subsequent calls with the same track name will return an incremental * snapshot containing only changes since the last call. @@ -14666,6 +14674,25 @@ export interface Locator { timeout?: number; }): Promise; + /** + * Returns an accessibility snapshot of the element's subtree optimized for AI consumption. + * @param options + */ + snapshotForAI(options?: { + /** + * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout` + * option in the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + */ + timeout?: number; + }): Promise<{ + /** + * Accessibility snapshot of the element matching this locator. + */ + full: string; + }>; + /** * Perform a tap gesture on the element matching the locator. For examples of emulating other gestures by manually * dispatching touch events, see the [emulating legacy touch events](https://playwright.dev/docs/touch-events) page. diff --git a/packages/playwright-core/src/cli/DEPS.list b/packages/playwright-core/src/cli/DEPS.list index fb73f81dd5381..4d2f05e6dbd7c 100644 --- a/packages/playwright-core/src/cli/DEPS.list +++ b/packages/playwright-core/src/cli/DEPS.list @@ -7,3 +7,4 @@ ../utilsBundle.ts ../client/ ../server/trace/viewer/ +../tools/cli-client/program.ts diff --git a/packages/playwright-core/src/cli/program.ts b/packages/playwright-core/src/cli/program.ts index cbd18e242e371..159a8ae4cc7ce 100644 --- a/packages/playwright-core/src/cli/program.ts +++ b/packages/playwright-core/src/cli/program.ts @@ -28,6 +28,7 @@ import { runTraceInBrowser, runTraceViewerApp } from '../server/trace/viewer/tra import { assert, getPackageManagerExecCommand } from '../utils'; import { wrapInASCIIBox } from '../server/utils/ascii'; import { dotenv, program } from '../utilsBundle'; +import { program as cliProgram } from '../tools/cli-client/program'; import type { Browser } from '../client/browser'; import type { BrowserContext } from '../client/browserContext'; @@ -353,6 +354,15 @@ Examples: $ show-trace $ show-trace https://example.com/trace.zip`); +program + .command('cli', { hidden: true }) + .allowExcessArguments(true) + .allowUnknownOption(true) + .action(async options => { + process.argv.splice(process.argv.indexOf('cli'), 1); + cliProgram(); + }); + type Options = { browser: string; channel?: string; diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts index 88af0b2eeac67..fdf1cb8e081a8 100644 --- a/packages/playwright-core/src/client/locator.ts +++ b/packages/playwright-core/src/client/locator.ts @@ -378,6 +378,10 @@ export class Locator implements api.Locator { await this._frame._channel.waitForSelector({ selector: this._selector, strict: true, omitReturnValue: true, ...options, timeout: this._frame._timeout(options) }); } + async snapshotForAI(options: TimeoutOptions = {}): Promise<{ full: string }> { + return await this._frame._page!._channel.snapshotForAI({ timeout: this._frame._timeout(options), selector: this._selector }); + } + async _expect(expression: string, options: FrameExpectParams): Promise<{ matches: boolean, received?: any, log?: string[], timedOut?: boolean, errorMessage?: string }> { return this._frame._expect(expression, { ...options, diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index eff128b98f921..65b8d843f452d 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1493,6 +1493,7 @@ scheme.PageRequestsResult = tObject({ }); scheme.PageSnapshotForAIParams = tObject({ track: tOptional(tString), + selector: tOptional(tString), timeout: tFloat, }); scheme.PageSnapshotForAIResult = tObject({ diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index e30e25808a990..894f1a49dce45 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -30,12 +30,13 @@ import { LongStandingScope, assert, renderTitleForCall, trimStringWithEllipsis } import { asLocator } from '../utils'; import { getComparator } from './utils/comparators'; import { debugLogger } from './utils/debugLogger'; -import { isInvalidSelectorError } from '../utils/isomorphic/selectorParser'; +import { isInvalidSelectorError, stringifySelector } from '../utils/isomorphic/selectorParser'; import { ManualPromise } from '../utils/isomorphic/manualPromise'; import { parseEvaluationResultValue } from '../utils/isomorphic/utilityScriptSerializers'; import { compressCallLog } from './callLog'; import * as rawBindingsControllerSource from '../generated/bindingsControllerSource'; import { Screencast } from './screencast'; +import { NonRecoverableDOMError } from './dom'; import type { Artifact } from './artifact'; import type { BrowserContextEventMap } from './browserContext'; @@ -48,6 +49,7 @@ import type * as types from './types'; import type { ImageComparatorOptions } from './utils/comparators'; import type * as channels from '@protocol/channels'; import type { BindingPayload } from '@injected/bindingsController'; +import type { SelectorInfo } from './frameSelectors'; export interface PageDelegate { readonly rawMouse: input.RawMouse; @@ -884,8 +886,23 @@ export class Page extends SdkObject { await Promise.all(this.frames().map(frame => frame.hideHighlight().catch(() => {}))); } - async snapshotForAI(progress: Progress, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string, incremental?: string }> { - const snapshot = await snapshotFrameForAI(progress, this.mainFrame(), options); + async snapshotForAI(progress: Progress, options: { track?: string, doNotRenderActive?: boolean, selector?: string } = {}): Promise<{ full: string, incremental?: string }> { + if (options.selector && options.track) + throw new Error('Cannot specify both selector and track options'); + + let frame: frames.Frame; + let info: SelectorInfo | undefined; + if (options.selector) { + const resolved = await this.mainFrame().selectors.resolveInjectedForSelector(options.selector, { strict: true }); + if (!resolved) + throw new Error(`Selector "${options.selector}" did not resolve to any element`); + frame = resolved.frame; + info = resolved.info; + } else { + frame = this.mainFrame(); + } + + const snapshot = await snapshotFrameForAI(progress, frame, { ...options, info }); return { full: snapshot.full.join('\n'), incremental: snapshot.incremental?.join('\n') }; } @@ -1030,20 +1047,33 @@ export class InitScript extends DisposableObject { } } -async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string, doNotRenderActive?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> { +async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, options: { track?: string, doNotRenderActive?: boolean, info?: SelectorInfo } = {}): Promise<{ full: string[], incremental?: string[] }> { // Only await the topmost navigations, inner frames will be empty when racing. const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async continuePolling => { try { const context = await progress.race(frame._utilityContext()); const injectedScript = await progress.race(context.injectedScript()); const snapshotOrRetry = await progress.race(injectedScript.evaluate((injected, options) => { + if (options.info) { + const element = injected.querySelector(options.info.parsed, injected.document, options.info.strict); + if (!element) + return false; + return injected.incrementalAriaSnapshot(element, { mode: 'ai', ...options }); + } const node = injected.document.body; if (!node) return true; return injected.incrementalAriaSnapshot(node, { mode: 'ai', ...options }); - }, { refPrefix: frame.seq ? 'f' + frame.seq : '', track: options.track, doNotRenderActive: options.doNotRenderActive })); + }, { + refPrefix: frame.seq ? 'f' + frame.seq : '', + track: options.track, + doNotRenderActive: options.doNotRenderActive, + info: options.info, + })); if (snapshotOrRetry === true) return continuePolling; + if (snapshotOrRetry === false) + throw new NonRecoverableDOMError(`Selector "${stringifySelector(options.info!.parsed)}" does not match any element`); return snapshotOrRetry; } catch (e) { if (frame.isNonRetriableError(e)) @@ -1093,7 +1123,7 @@ async function snapshotFrameRefForAI(progress: Progress, parentFrame: frames.Fra if (!child) return { full: [] }; try { - return await snapshotFrameForAI(progress, child.frame, options); + return await snapshotFrameForAI(progress, child.frame, { ...options, info: undefined }); } catch { return { full: [] }; } diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.ts index bcb7c23024f51..81b959af61236 100644 --- a/packages/playwright-core/src/tools/backend/response.ts +++ b/packages/playwright-core/src/tools/backend/response.ts @@ -47,6 +47,7 @@ export class Response { private _context: Context; private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none'; private _includeSnapshotFileName: string | undefined; + private _includeSnapshotSelector: string | undefined; private _isClose: boolean = false; readonly toolName: string; @@ -126,9 +127,10 @@ export class Response { this._includeSnapshot = this._context.config.snapshot?.mode || 'incremental'; } - setIncludeFullSnapshot(includeSnapshotFileName?: string) { + setIncludeFullSnapshot(includeSnapshotFileName?: string, selector?: string) { this._includeSnapshot = 'full'; this._includeSnapshotFileName = includeSnapshotFileName; + this._includeSnapshotSelector = selector; } async serialize(): Promise { @@ -193,7 +195,7 @@ export class Response { addSection('Ran Playwright code', this._code, 'js'); // Render tab titles upon changes or when more than one tab. - const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._clientWorkspace) : undefined; + const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotSelector, this._clientWorkspace) : undefined; const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot())); if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) { if (tabHeaders.length !== 1) diff --git a/packages/playwright-core/src/tools/backend/snapshot.ts b/packages/playwright-core/src/tools/backend/snapshot.ts index 2988dff0bbf1c..0506017564c14 100644 --- a/packages/playwright-core/src/tools/backend/snapshot.ts +++ b/packages/playwright-core/src/tools/backend/snapshot.ts @@ -27,13 +27,14 @@ const snapshot = defineTool({ description: 'Capture accessibility snapshot of the current page, this is better than screenshot', inputSchema: z.object({ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), + selector: z.string().optional().describe('Element selector of the root element to capture a partial snapshot instead of the whole page'), }), type: 'readOnly', }, handle: async (context, params, response) => { await context.ensureTab(); - response.setIncludeFullSnapshot(params.filename); + response.setIncludeFullSnapshot(params.filename, params.selector); }, }); diff --git a/packages/playwright-core/src/tools/backend/tab.ts b/packages/playwright-core/src/tools/backend/tab.ts index c6eebd34138f1..be8b823e983ff 100644 --- a/packages/playwright-core/src/tools/backend/tab.ts +++ b/packages/playwright-core/src/tools/backend/tab.ts @@ -372,11 +372,11 @@ export class Tab extends EventEmitter { this._requests.length = 0; } - async captureSnapshot(relativeTo: string | undefined): Promise { + async captureSnapshot(selector: string | undefined, relativeTo: string | undefined): Promise { await this._initializedPromise; let tabSnapshot: TabSnapshot | undefined; const modalStates = await this._raceAgainstModalStates(async () => { - const snapshot = await this.page.snapshotForAI({ track: 'response' }); + const snapshot: { full: string, incremental?: string } = selector ? await this.page.locator(selector).snapshotForAI() : await this.page.snapshotForAI({ track: 'response' }); tabSnapshot = { ariaSnapshot: snapshot.full, ariaSnapshotDiff: this._needsFullSnapshot ? undefined : snapshot.incremental, diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index 98e691f02f579..06b28a0cd1ce2 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -328,12 +328,14 @@ const snapshot = declareCommand({ name: 'snapshot', description: 'Capture page snapshot to obtain element ref', category: 'core', - args: z.object({}), + args: z.object({ + element: z.string().optional().describe('Element selector of the root element to capture a partial snapshot instead of the whole page'), + }), options: z.object({ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), }), toolName: 'browser_snapshot', - toolParams: ({ filename }) => ({ filename }), + toolParams: ({ filename, element }) => ({ filename, selector: element }), }); const evaluate = declareCommand({ diff --git a/packages/playwright-core/src/tools/mcp/DEPS.list b/packages/playwright-core/src/tools/mcp/DEPS.list index 398a7e9fd92ee..5488c6bb277ac 100644 --- a/packages/playwright-core/src/tools/mcp/DEPS.list +++ b/packages/playwright-core/src/tools/mcp/DEPS.list @@ -15,3 +15,6 @@ [browserFactory.ts] ../../client/connect.ts ../utils/connect.ts + +[program.ts] +../../cli/program.ts diff --git a/packages/playwright-core/src/tools/mcp/program.ts b/packages/playwright-core/src/tools/mcp/program.ts index 64a4b01b2339e..e0026dc891f46 100644 --- a/packages/playwright-core/src/tools/mcp/program.ts +++ b/packages/playwright-core/src/tools/mcp/program.ts @@ -153,8 +153,8 @@ export function decorateMCPInstallBrowserCommand(command: Command) { .option('--only-shell', 'only install headless shell when installing chromium') .option('--no-shell', 'do not install chromium headless shell') .action(async options => { - const { program } = require('../program'); const argv = process.argv.map(arg => arg === 'install-browser' ? 'install' : arg); - program.parse(argv); + const { program: mainProgram } = await import('../../cli/program'); + mainProgram.parse(argv); }); } diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 0f3a236cf35c9..8c1d25aa20a21 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -4538,6 +4538,14 @@ export interface Page { * @param options */ snapshotForAI(options?: { + /** + * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout` + * option in the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + */ + timeout?: number; + /** * When specified, enables incremental snapshots. Subsequent calls with the same track name will return an incremental * snapshot containing only changes since the last call. @@ -14666,6 +14674,25 @@ export interface Locator { timeout?: number; }): Promise; + /** + * Returns an accessibility snapshot of the element's subtree optimized for AI consumption. + * @param options + */ + snapshotForAI(options?: { + /** + * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout` + * option in the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + */ + timeout?: number; + }): Promise<{ + /** + * Accessibility snapshot of the element matching this locator. + */ + full: string; + }>; + /** * Perform a tap gesture on the element matching the locator. For examples of emulating other gestures by manually * dispatching touch events, see the [emulating legacy touch events](https://playwright.dev/docs/touch-events) page. diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index f384e99aedea4..c0a9f5c6c2544 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2608,10 +2608,12 @@ export type PageRequestsResult = { }; export type PageSnapshotForAIParams = { track?: string, + selector?: string, timeout: number, }; export type PageSnapshotForAIOptions = { track?: string, + selector?: string, }; export type PageSnapshotForAIResult = { full: string, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 824adafc32a40..e53c57c5f83ad 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2012,6 +2012,7 @@ Page: parameters: # When track is present, an incremental snapshot is returned when possible. track: string? + selector: string? timeout: float returns: full: string diff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.ts index af56680d82475..f5d1674cda788 100644 --- a/tests/mcp/cli-core.spec.ts +++ b/tests/mcp/cli-core.spec.ts @@ -263,3 +263,19 @@ test('click button with wrong css selector', async ({ cli, server }) => { const { output } = await cli('click', '#target'); expect(output).toContain(`Error: Selector #target does not match any elements.`); }); + +test('partial snapshot', async ({ cli, server }) => { + server.setContent('/', ``, 'text/html'); + const { snapshot } = await cli('open', server.PREFIX); + expect(snapshot).toContain('- button "Submit" [ref=e2]'); + expect(snapshot).toContain('- button "Cancel" [ref=e3]'); + + const { snapshot: partialSnapshot } = await cli('snapshot', '#two'); + expect(partialSnapshot).toBe(`- button "Cancel" [ref=e3]`); + + const { output: strictError } = await cli('snapshot', 'button'); + expect(strictError).toContain(`strict mode violation`); + + const { output: noMatchError } = await cli('snapshot', '#target'); + expect(noMatchError).toContain(`Selector "#target" does not match any element`); +});