feat(verifier): record agent trajectories

.changeset/verifier-trajectory-events.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Capture verifier trajectory evidence from agent evidence callbacks for offline scoring.

packages/core/lib/v3/agent/utils/captureAriaTreeProbe.ts

@@ -0,0 +1,73 @@
+/**
+ * captureAriaTreeProbe — capture a truncated accessibility tree of the active
+ * page for use as tier-2 evidence in the trajectory recorder.
+ *
+ * Shared by v3AgentHandler and v3CuaAgentHandler. Listener-gated by the
+ * callers so ordinary agent runs (no TrajectoryRecorder attached) don't pay
+ * the cost.
+ *
+ * The a11y tree is the same payload the agent's `ariaTree` tool sees, but
+ * captured by the harness (not the agent) so the verifier has independent
+ * textual ground truth for grounding non-visual claims — prices, names,
+ * dates, list contents — without OCR'ing screenshots.
+ *
+ * Budget: defaults to ~8000 tokens (32k chars). Per-step a11y captures
+ * across a ~30-step trajectory at that cap sum to ~240k tokens total,
+ * which the verifier handles via per-criterion top-K selection. The cap
+ * is configurable via VERIFIER_ARIATREE_TOKEN_BUDGET so consumers can
+ * trade RAM/disk for fidelity. Truncated content is marked explicitly so
+ * the verifier knows it was clipped.
+ */
+import type { V3 } from "../../v3.js";
+
+const APPROX_CHARS_PER_TOKEN = 4;
+const DEFAULT_TOKEN_BUDGET = 8_000;
+const DEFAULT_TIMEOUT_MS = 5_000;
+
+interface CaptureAriaTreeOptions {
+  /** Soft cap on token count (chars/4 approximation). Default 8000. */
+  tokenBudget?: number;
+  /** Hard timeout on the capture. Default 5s. */
+  timeoutMs?: number;
+}
+
+/**
+ * Returns the truncated a11y tree as a plain string, or undefined when
+ * capture fails. Never throws — a11y capture is best-effort tier-2 evidence,
+ * not a hard requirement, so failures are silently absorbed (the verifier
+ * surfaces this via evidence_insufficient).
+ */
+export async function captureAriaTreeProbe(
+  v3: V3,
+  opts: CaptureAriaTreeOptions = {},
+): Promise<string | undefined> {
+  const envBudget = parseInt(
+    process.env.VERIFIER_ARIATREE_TOKEN_BUDGET ?? "",
+    10,
+  );
+  const tokenBudget =
+    opts.tokenBudget ??
+    (Number.isFinite(envBudget) && envBudget > 0
+      ? envBudget
+      : DEFAULT_TOKEN_BUDGET);
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxChars = tokenBudget * APPROX_CHARS_PER_TOKEN;
+
+  try {
+    // v3.extract() without a schema returns { pageText } where pageText is the
+    // rendered accessibility tree — same path the agent's ariaTree tool uses.
+    const result = await v3.extract({ timeout: timeoutMs });
+    const pageText = result?.pageText;
+    if (typeof pageText !== "string" || pageText.length === 0) return undefined;
+
+    if (pageText.length > maxChars) {
+      return (
+        pageText.slice(0, maxChars) +
+        `\n\n[CONTENT TRUNCATED at ~${tokenBudget} tokens — set VERIFIER_ARIATREE_TOKEN_BUDGET to raise]`
+      );
+    }
+    return pageText;
+  } catch {
+    return undefined;
+  }
+}

packages/core/lib/v3/agent/utils/postStepProbeEvidence.ts

@@ -0,0 +1,81 @@
+import type { AgentEvidenceCallback } from "../../types/public/agentEvidenceEvents.js";
+import type { LogLine } from "../../types/public/logs.js";
+import type { V3 } from "../../v3.js";
+import { captureAriaTreeProbe } from "./captureAriaTreeProbe.js";
+
+interface CaptureProbeEvidenceOptions {
+  v3: V3;
+  url: string;
+  logger: (message: LogLine) => void;
+  warningMessage: string;
+}
+
+interface EmitPostStepProbeEvidenceOptions extends CaptureProbeEvidenceOptions {
+  evidenceCallback?: AgentEvidenceCallback;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export async function captureProbeEvidence({
+  v3,
+  url,
+  logger,
+  warningMessage,
+}: CaptureProbeEvidenceOptions): Promise<{
+  url: string;
+  screenshot?: Buffer;
+  ariaTree?: string;
+}> {
+  let probeUrl = url;
+  let screenshot: Buffer | undefined;
+  try {
+    const page = await v3.context.awaitActivePage();
+    probeUrl = page.url();
+    screenshot = await page.screenshot({ fullPage: false });
+  } catch (e) {
+    logger({
+      category: "agent",
+      message: `${warningMessage}: ${errorMessage(e)}`,
+      level: 1,
+    });
+  }
+
+  const ariaTree = await captureAriaTreeProbe(v3);
+  return {
+    url: probeUrl,
+    ...(screenshot ? { screenshot } : {}),
+    ...(ariaTree !== undefined ? { ariaTree } : {}),
+  };
+}
+
+export async function emitPostStepProbeEvidence({
+  v3,
+  url,
+  evidenceCallback,
+  logger,
+  warningMessage,
+}: EmitPostStepProbeEvidenceOptions): Promise<void> {
+  if (!evidenceCallback) return;
+
+  const probe = await captureProbeEvidence({
+    v3,
+    url,
+    logger,
+    warningMessage,
+  });
+  if (probe.screenshot) {
+    await evidenceCallback({
+      type: "screenshot",
+      screenshot: probe.screenshot,
+      url: probe.url,
+      evidenceRole: "probe",
+    });
+  }
+  await evidenceCallback({
+    type: "step_observed",
+    url: probe.url,
+    ariaTree: probe.ariaTree,
+  });
+}

packages/core/lib/v3/agent/utils/toolOutputEvidence.ts

@@ -0,0 +1,76 @@
+import type { AgentStepFinishedEvent } from "../../types/public/agentEvidenceEvents.js";
+
+const ERROR_STRING_LIMIT = 1000;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function hasOwn(value: Record<string, unknown>, key: string): boolean {
+  return Object.prototype.hasOwnProperty.call(value, key);
+}
+
+function normalizeError(value: unknown): string | undefined {
+  if (value === undefined || value === null || value === false) {
+    return undefined;
+  }
+  if (value instanceof Error) {
+    return value.message;
+  }
+  if (typeof value === "string") {
+    return value;
+  }
+  if (
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint"
+  ) {
+    return String(value);
+  }
+
+  let serialized: string;
+  try {
+    serialized = JSON.stringify(value) ?? String(value);
+  } catch {
+    serialized = String(value);
+  }
+  if (serialized.length <= ERROR_STRING_LIMIT) {
+    return serialized;
+  }
+  return `${serialized.slice(0, ERROR_STRING_LIMIT)}... [truncated]`;
+}
+
+function statusCandidates(toolResult: unknown): Record<string, unknown>[] {
+  if (!isRecord(toolResult)) {
+    return [];
+  }
+
+  const candidates = [toolResult];
+  const output = toolResult.output;
+  if (isRecord(output)) {
+    candidates.push(output);
+  }
+  return candidates;
+}
+
+export function inferToolOutput(
+  toolResult: unknown,
+): AgentStepFinishedEvent["toolOutput"] {
+  const candidates = statusCandidates(toolResult);
+  const error = candidates
+    .map((candidate) =>
+      hasOwn(candidate, "error") ? normalizeError(candidate.error) : undefined,
+    )
+    .find((message): message is string => message !== undefined);
+
+  const successFalse = candidates.some(
+    (candidate) => candidate.success === false,
+  );
+  const isError = candidates.some((candidate) => Boolean(candidate.isError));
+
+  return {
+    ok: error === undefined && !isError && !successFalse,
+    result: toolResult,
+    error,
+  };
+}

packages/core/lib/v3/agent/utils/wrapEvidenceCallback.ts

@@ -0,0 +1,27 @@
+import type { AgentEvidenceCallback } from "../../types/public/agentEvidenceEvents.js";
+import type { LogLine } from "../../types/public/logs.js";
+
+// onEvidence is a user-supplied observability hook (trajectory recording,
+// verifier capture, etc.). Wrap it once at the boundary where the handler
+// receives it so a throwing user callback can never abort the agent loop —
+// internal emit sites can then call the wrapped callback directly without
+// per-site try/catch.
+export function wrapEvidenceCallback(
+  callback: AgentEvidenceCallback | undefined,
+  logger: (message: LogLine) => void,
+): AgentEvidenceCallback | undefined {
+  if (!callback) return undefined;
+  return async (event) => {
+    try {
+      await callback(event);
+    } catch (e) {
+      logger({
+        category: "agent",
+        message: `Warning: onEvidence callback failed for ${event.type}: ${
+          e instanceof Error ? e.message : String(e)
+        }`,
+        level: 1,
+      });
+    }
+  };
+}