From d1d2ff69028922e92d2dbe61675bedb2d3dc829c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 00:01:29 -0600 Subject: [PATCH] feat(examples): P1 loop-vs-graph parity harness for the same coding cell (#694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replay one identical CellSpec { task, coderProfile, reviewerProfile, shots, budget } through both loop forms and return one comparable ParityRecord per arm: - arms.ts — runLoopArm (agent-eval multishot loop: reviewer = driver leg, coder = agent leg, maxTurns = shot budget) and runGraphArm (runGraph over the two-node reviewer->coder topology, shot budget on the delegates edge, completion oracle on the deliverable). The record documents the real asymmetries instead of hiding them: only the graph arm has an edge ledger (the loop's is always undefined, never synthesized), the loop cannot stop early (no deliverable gate), and the conserved budget reaches only the graph. - offline.ts — scripted seams generated from one per-shot script for both arms (reuses examples/graphs/shared.ts leafSeam/scriptedBrain); zero network, zero env, $0; captures each arm's seam-level inputs for equivalence proofs. - run-parity.ts — CLI: --backend offline|cli-bridge --cells N --shots S. offline is the CI-safe proof path; cli-bridge is the one-command live entry wiring VB_CLI_BRIDGE_URL / VB_CLI_BRIDGE_BEARER / VB_PARITY_MODEL (not executed by any gate). - parity.test.ts — asserts input equivalence on CAPTURED inputs (task text, profiles, and shot budget reach both execution seams), well-formed records with the measured asymmetries (graph settles at the passing shot, loop burns the full budget), and the honest cap-refusal non-convergence path. tsconfig.examples.json gains the missing ./durable path mapping — the pre-existing typecheck:examples failure in examples/chat-handler (present on origin/main) resolved by mapping the subpath the example already imports. --- examples/p1-parity/arms.ts | 347 ++++++++++++++++++++++++++++++ examples/p1-parity/offline.ts | 148 +++++++++++++ examples/p1-parity/parity.test.ts | 152 +++++++++++++ examples/p1-parity/run-parity.ts | 217 +++++++++++++++++++ tsconfig.examples.json | 1 + 5 files changed, 865 insertions(+) create mode 100644 examples/p1-parity/arms.ts create mode 100644 examples/p1-parity/offline.ts create mode 100644 examples/p1-parity/parity.test.ts create mode 100644 examples/p1-parity/run-parity.ts diff --git a/examples/p1-parity/arms.ts b/examples/p1-parity/arms.ts new file mode 100644 index 00000000..850b30b0 --- /dev/null +++ b/examples/p1-parity/arms.ts @@ -0,0 +1,347 @@ +/** + * P1 parity arms — the SAME coding cell replayed through the two loop forms, under measurement. + * + * A. `runLoopArm` — the LEGACY loop: agent-eval's multishot loop (`runMultishot`), with the + * reviewer profile as the simulated-user driver leg and the coder profile as the agent leg. + * Each driver→agent turn is one SHOT; `maxTurns` is the shot budget. + * B. `runGraphArm` — the graph form: `runGraph` over the two-node reviewer→coder topology + * (examples/graphs/shot-loop.ts as data), the shot budget on the delegates edge's + * `maxTraversals`, the completion oracle on the mandatory deliverable. + * + * Both arms take one identical {@link CellSpec} and return one {@link ParityRecord}, so a run of + * N cells yields N paired rows — the measurement harness for the loop→graph migration (#694 P1). + * The record maps each arm's OWN instrumentation onto shared field names; where the two forms + * genuinely differ (edge ledger, conserved pool, early stop) the difference is documented on the + * field and left visible in the data, never papered over. + */ + +import type { MultishotMessage, MultishotTransport } from '@tangle-network/agent-eval/multishot' +import { runMultishot } from '@tangle-network/agent-eval/multishot' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + type AgentGraph, + type AnalystRegistry, + type Budget, + type EdgeTraversal, + GraphEdgeCapError, + type MakeWorkerAgent, + promptHandle, + type RouterConfig, + type RunGraphOptions, + runGraph, + type Spend, + type ToolLoopChat, +} from '@tangle-network/agent-runtime/kernel' + +// ── The shared cell ──────────────────────────────────────────────────────────── + +/** One coding cell, fed VERBATIM to both arms — the input-equivalence contract of the harness. */ +export interface CellSpec { + /** The coding task text. Shot 1's brief in both arms: the multishot opener (loop) and the + * first spawn's task payload + the root task (graph). */ + readonly task: string + /** The coder under test. Loop arm: the agent leg's profile. Graph arm: the pinned worker node + * (`profile.name` is the node id, so it must be non-empty and differ from the reviewer's). */ + readonly coderProfile: AgentProfile + /** The reviewer driving the shots. Loop arm: the driver leg (its `prompt.systemPrompt` is the + * driver system prompt). Graph arm: the root node. */ + readonly reviewerProfile: AgentProfile + /** The shot budget. Loop arm: `maxTurns`. Graph arm: the delegates edge's `maxTraversals`. */ + readonly shots: number + /** The conserved resource pool. ENFORCED BY THE GRAPH ARM ONLY: `runGraph` reserves against it + * for every spawn. The legacy loop has no conserved-pool concept — its only enforceable limit + * is `shots` — so this field cannot reach it. That gap is a P1 finding, not a harness bug. */ + readonly budget: Budget +} + +// ── The shared record ────────────────────────────────────────────────────────── + +/** + * One arm's measured outcome for one cell. Same field names, each populated from that arm's OWN + * instrumentation — the asymmetries below are real differences between the two forms and are the + * exact thing P1 exists to measure: + * + * - `ledger` exists ONLY for the graph arm (the edge ledger is what the graph form adds). The + * loop arm's instrumentation is a transcript, not an edge ledger; its `ledger` is ALWAYS + * `undefined` and must never be synthesized from the transcript. + * - The loop arm cannot stop early: `runMultishot` has no deliverable gate, so it burns the full + * shot budget even when an early shot converges. The graph arm settles at the first shot whose + * output passes the deliverable. Expect `shotsUsed` to differ on early-convergence cells. + * - Loop `spend.tokens` is metered at the transport seam (the sum of `usage` on every agent and + * driver completion); graph `spend` is the run's reconciled `spentTotal` from the conserved + * pool's journal. Both are that form's honest total, measured by different machinery. + */ +export interface ParityRecord { + /** Did any shot satisfy the completion check? Graph arm: the run settled a winner (the + * deliverable passed). Loop arm: some turn-initial coder reply passed `shotPassed`. */ + converged: boolean + /** Coder shots actually executed. Graph arm: distinct live coder workers spawned (from the + * ledger). Loop arm: turn-initial agent replies in the transcript. */ + shotsUsed: number + /** Total measured resource spend for the arm's whole run (driver + coder legs). */ + spend: { tokens: { input: number; output: number }; usd: number } + /** Wall-clock duration of the arm call, measured identically around both arms. */ + wallMs: number + /** Corrective direction DELIVERED to the coder after the initial brief. Loop arm: driver (user) + * messages after the opener, counted with their UTF-8 bytes. Graph arm: delegates-edge + * traversals beyond the first with outcome `delivered` — re-brief spawns AND mid-run steers, + * with the bytes that actually crossed the edge. Graph bytes include the versioned edge + * directive text; loop bytes are the raw message only (the loop has no directive layer). */ + steeringDelivered: { count: number; bytes: number } + /** The graph arm's edge ledger — every traversal, outcome, and byte count. ABSENT for the loop + * arm, honestly: the legacy loop has no observable edges (that is the migration's point). */ + ledger?: ReadonlyArray +} + +// ── Backends ─────────────────────────────────────────────────────────────────── + +/** Execution seams for the loop arm. Offline: scripted transports (see ./offline.ts). + * Live: transports posting to a cli-bridge OpenAI-compatible endpoint (see ./run-parity.ts). */ +export interface LoopArmBackend { + readonly agentTransport: MultishotTransport + readonly driverTransport: MultishotTransport + /** The shared completion check, applied to each turn-initial coder reply. MUST be the same + * predicate the paired graph arm's deliverable uses, or the comparison is invalid. */ + readonly shotPassed: (assistantText: string) => boolean + /** `runMultishot` resolves apiKey/baseUrl eagerly even with both transports injected; the + * offline path passes inert placeholders so no env is required. */ + readonly apiKey?: string + readonly baseUrl?: string +} + +/** Execution seams for the graph arm: fully-scripted (offline/CI) or the live cli-bridge. */ +export type GraphArmBackend = + | { + readonly kind: 'seam' + readonly makeWorkerAgent: MakeWorkerAgent + readonly brain: ToolLoopChat + readonly analysts?: AnalystRegistry + /** Same predicate as the paired loop arm — becomes the graph's deliverable check. */ + readonly shotPassed: (workerOutText: string) => boolean + } + | { + readonly kind: 'bridge' + readonly bridgeUrl: string + readonly bridgeBearer: string + /** Fallback bridge wire id (e.g. `pi/deepseek`); the spawned profile may select its own. */ + readonly model?: string + readonly cwd?: string + /** Router substrate for the reviewer (driver) brain. */ + readonly router?: RouterConfig + readonly shotPassed: (workerOutText: string) => boolean + } + +// ── The graph topology (exported so tests can assert on the exact inputs) ────── + +export const PARITY_VERIFY_ANALYST = 'verify' + +/** The verify lens is ENVIRONMENT: it reads the coder's settle trace, never sits in the graph. */ +export function parityAnalysts(): AnalystRegistry { + return { + kinds: [ + { + id: PARITY_VERIFY_ANALYST, + description: 'read the coder trace, report shot outcome to the reviewer', + area: 'qa', + }, + ], + run: async () => [{ check: 'shot-completion', observed: 'see the settled output' }], + } +} + +/** The two-node reviewer→coder topology for one cell — plain data, the shot budget on the edge. + * The cell's profiles are used AS-IS (node id = `profile.name`), the task is the root task + * (`deliverable.describe`) and each spawn's payload, and `shotPassed` is the deliverable. */ +export function buildParityGraph( + cell: CellSpec, + shotPassed: (workerOutText: string) => boolean, +): AgentGraph { + const reviewer = requireProfileName(cell.reviewerProfile, 'reviewerProfile') + const coder = requireProfileName(cell.coderProfile, 'coderProfile') + return { + nodes: [ + { id: reviewer, profile: cell.reviewerProfile }, + { id: coder, profile: cell.coderProfile }, + ], + edges: [ + { + kind: 'delegates', + from: reviewer, + to: coder, + directive: promptHandle('delegates/worker-brief/v1'), + maxTraversals: cell.shots, + }, + { + kind: 'analyzes', + analyst: PARITY_VERIFY_ANALYST, + over: [coder], + to: reviewer, + directive: promptHandle('analyzes/findings-report/v1'), + }, + ], + deliverable: { + describe: cell.task, + check: (out) => typeof out === 'string' && shotPassed(out), + }, + budget: cell.budget, + } +} + +// ── Arm A: the legacy multishot loop ─────────────────────────────────────────── + +export async function runLoopArm(cell: CellSpec, backend: LoopArmBackend): Promise { + validateCell(cell) + const tokens = { input: 0, output: 0 } + // Meter tokens at the transport seam — the only usage channel the legacy loop exposes. + const metered = + (transport: MultishotTransport): MultishotTransport => + async (req) => { + const res = await transport(req) + tokens.input += res.usage?.prompt_tokens ?? 0 + tokens.output += res.usage?.completion_tokens ?? 0 + return res + } + const startedAt = Date.now() + const sim = await runMultishot({ + profile: cell.coderProfile, + persona: { id: 'parity-cell' }, + shape: { + buildOpener: () => cell.task, + buildDriverSystemPrompt: () => cell.reviewerProfile.prompt?.systemPrompt ?? '', + }, + tools: [], + toolExecutors: {}, + maxTurns: cell.shots, + agentModel: cell.coderProfile.model?.default ?? 'parity/unspecified', + driverModel: cell.reviewerProfile.model?.default ?? 'parity/unspecified', + agentTransport: metered(backend.agentTransport), + driverTransport: metered(backend.driverTransport), + apiKey: backend.apiKey ?? 'unused', + baseUrl: backend.baseUrl ?? 'http://unused.invalid', + }) + const wallMs = Date.now() - startedAt + const shotReplies = turnInitialAssistantReplies(sim.transcript) + const steering = sim.transcript.slice(1).filter((msg) => msg.role === 'user') + return { + converged: shotReplies.some((text) => backend.shotPassed(text)), + shotsUsed: shotReplies.length, + spend: { tokens: { ...tokens }, usd: sim.costUsd }, + wallMs, + steeringDelivered: { + count: steering.length, + bytes: steering.reduce((sum, msg) => sum + Buffer.byteLength(msg.content, 'utf8'), 0), + }, + // No `ledger`: the legacy loop has no edge instrumentation, and the harness never fakes one. + } +} + +/** The coder's per-shot replies: assistant messages that directly answer a user (driver) message. + * Tool-followup assistant messages (which follow tool results) are the same shot continuing. */ +function turnInitialAssistantReplies(transcript: ReadonlyArray): string[] { + const replies: string[] = [] + for (let i = 1; i < transcript.length; i += 1) { + const msg = transcript[i] + if (msg !== undefined && msg.role === 'assistant' && transcript[i - 1]?.role === 'user') { + replies.push(msg.content) + } + } + return replies +} + +// ── Arm B: the runGraph two-node form ────────────────────────────────────────── + +export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Promise { + validateCell(cell) + const graph = buildParityGraph(cell, backend.shotPassed) + const opts: RunGraphOptions = + backend.kind === 'seam' + ? { + makeWorkerAgent: backend.makeWorkerAgent, + brain: backend.brain, + analysts: backend.analysts ?? parityAnalysts(), + } + : { + backend: { + backend: 'bridge', + bridgeUrl: backend.bridgeUrl, + bridgeBearer: backend.bridgeBearer, + ...(backend.model !== undefined ? { model: backend.model } : {}), + ...(backend.cwd !== undefined ? { cwd: backend.cwd } : {}), + }, + ...(backend.router !== undefined ? { router: backend.router } : {}), + analysts: parityAnalysts(), + } + const startedAt = Date.now() + try { + const res = await runGraph(graph, opts) + return graphRecord( + res.result.kind === 'winner', + res.result.spentTotal, + res.ledger, + Date.now() - startedAt, + ) + } catch (err) { + if (err instanceof GraphEdgeCapError) { + // The cap (the cyclic-graph backstop), not the task, ended the run: an honest + // non-convergence row, with the full evidence the error carries. + return graphRecord(false, err.result.spentTotal, err.ledger, Date.now() - startedAt) + } + throw err + } +} + +function graphRecord( + converged: boolean, + spentTotal: Spend, + ledger: ReadonlyArray, + wallMs: number, +): ParityRecord { + const delegates = ledger.filter((row) => row.kind === 'delegates') + // Each live coder worker is one shot; steers re-use an existing worker id, refused rows have + // none — so distinct bound worker ids count executed shots exactly. + const shotsUsed = new Set( + delegates.filter((row) => row.workerId !== undefined).map((row) => row.workerId), + ).size + const steering = delegates.filter((row) => row.outcome === 'delivered' && row.traversal > 1) + return { + converged, + shotsUsed, + spend: { + tokens: { input: spentTotal.tokens.input, output: spentTotal.tokens.output }, + usd: spentTotal.usd, + }, + wallMs, + steeringDelivered: { + count: steering.length, + bytes: steering.reduce((sum, row) => sum + row.bytes, 0), + }, + ledger, + } +} + +// ── Shared validation ────────────────────────────────────────────────────────── + +function requireProfileName(profile: AgentProfile, field: string): string { + const name = profile.name + if (typeof name !== 'string' || name.length === 0) { + throw new Error( + `p1-parity: ${field}.name must be a non-empty string — it is the graph node id, and both ` + + 'arms report against it', + ) + } + return name +} + +function validateCell(cell: CellSpec): void { + const reviewer = requireProfileName(cell.reviewerProfile, 'reviewerProfile') + const coder = requireProfileName(cell.coderProfile, 'coderProfile') + if (reviewer === coder) { + throw new Error('p1-parity: reviewerProfile.name and coderProfile.name must differ') + } + if (!Number.isInteger(cell.shots) || cell.shots < 1) { + throw new Error(`p1-parity: shots must be a positive integer, got ${cell.shots}`) + } + if (typeof cell.task !== 'string' || cell.task.length === 0) { + throw new Error('p1-parity: task must be a non-empty string') + } +} diff --git a/examples/p1-parity/offline.ts b/examples/p1-parity/offline.ts new file mode 100644 index 00000000..1f6fab40 --- /dev/null +++ b/examples/p1-parity/offline.ts @@ -0,0 +1,148 @@ +/** + * OFFLINE scripted seams for the P1 parity arms — zero network, zero env, $0. Mirrors + * examples/graphs/shared.ts and reuses its `leafSeam` / `scriptedBrain` directly for the graph + * arm, so only the loop arm's transports are new scripting. Both arms' seams are generated from + * the same {@link ShotScript}, so the SAME cell produces the SAME shot outcomes in both forms — + * the CI-runnable proof path, and the capture point for the input-equivalence test. + * + * Synthetic accounting: every scripted completion reports `usage {5,5}` and `$0`, mirroring the + * leaf seam's per-shot spend, so the two arms' metering pipelines carry comparable numbers + * offline. The loop arm additionally meters its scripted DRIVER completions (the legacy loop's + * driver is an inference leg); the graph arm's scripted brain meters nothing (a live graph + * driver would meter through `spentBreakdown.driverInference`). Real numbers arrive only with + * the live backend. + */ + +import type { + MultishotTransport, + MultishotTransportRequest, +} from '@tangle-network/agent-eval/multishot' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { MakeWorkerAgent } from '@tangle-network/agent-runtime/kernel' +import { type LeafShot, leafSeam, type ScriptedTurn, scriptedBrain } from '../graphs/shared' +import type { CellSpec, GraphArmBackend, LoopArmBackend } from './arms' + +/** Per-shot scripted outcome; the last entry repeats for later shots. */ +export type ShotScript = ReadonlyArray<'pass' | 'fail'> + +export const SHOT_PASS_TEXT = 'TESTS: pass' +export const SHOT_FAIL_TEXT = 'TESTS: fail' + +/** The one completion check BOTH offline arms share (loop reply text = graph settle output). */ +export const offlineShotPassed = (text: string): boolean => text.includes(SHOT_PASS_TEXT) + +/** The reviewer's re-brief for shot N — identical wording in both arms, so steering payloads + * differ only by what each form adds (the graph's versioned edge directive). */ +export const rebriefText = (shot: number): string => + `shot ${shot}: revise — the verifier reported failing tests; make them pass` + +const scriptedUsage = () => ({ prompt_tokens: 5, completion_tokens: 5 }) + +const shotAt = (script: ShotScript, index: number): 'pass' | 'fail' => + script[Math.min(index, script.length - 1)] ?? 'fail' + +// ── Loop arm: scripted transports ────────────────────────────────────────────── + +export interface LoopCapture { + /** Every request the coder (agent) leg received, in order — the input-equivalence evidence. */ + readonly agentRequests: MultishotTransportRequest[] + /** Every request the reviewer (driver) leg received, in order. */ + readonly driverRequests: MultishotTransportRequest[] +} + +/** Scripted loop backend: coder replies follow the shot script; the reviewer re-briefs between + * shots with {@link rebriefText}. All requests are captured for assertion. */ +export function offlineLoopBackend(script: ShotScript): { + backend: LoopArmBackend + capture: LoopCapture +} { + const agentRequests: MultishotTransportRequest[] = [] + const driverRequests: MultishotTransportRequest[] = [] + const agentTransport: MultishotTransport = async (req) => { + agentRequests.push(req) + const outcome = shotAt(script, agentRequests.length - 1) + return { + message: { content: outcome === 'pass' ? SHOT_PASS_TEXT : SHOT_FAIL_TEXT }, + usage: scriptedUsage(), + costUsd: 0, + } + } + const driverTransport: MultishotTransport = async (req) => { + driverRequests.push(req) + // Driver call k re-briefs shot k+1 (the loop drives one driver turn between shots). + return { + message: { content: rebriefText(driverRequests.length + 1) }, + usage: scriptedUsage(), + costUsd: 0, + } + } + return { + backend: { agentTransport, driverTransport, shotPassed: offlineShotPassed }, + capture: { agentRequests, driverRequests }, + } +} + +// ── Graph arm: scripted brain + leaf seam ────────────────────────────────────── + +export interface GraphCapture { + /** Every profile the leaf factory received (the graph-pinned coder profile, with the + * delegates directive appended to its instructions), in spawn order. */ + readonly spawnedProfiles: AgentProfile[] + /** Every spawn's task payload, in spawn order — shot 1 must be the cell task verbatim. */ + readonly spawnedTasks: unknown[] +} + +/** Scripted graph backend for one cell: the leaf settles each shot per the script, and the + * reviewer brain spawns shot-by-shot until the first scripted pass. A script with no pass + * inside the shot budget drives one extra spawn INTO the delegates cap, so the backstop — + * not silence — ends the run (`runGraphArm` maps that to an honest non-convergence row). */ +export function offlineGraphBackend( + cell: CellSpec, + script: ShotScript, +): { backend: GraphArmBackend; capture: GraphCapture } { + const coder = cell.coderProfile.name ?? 'coder' + const spawnedProfiles: AgentProfile[] = [] + const spawnedTasks: unknown[] = [] + const shots: LeafShot[] = script.map((outcome) => ({ + out: outcome === 'pass' ? SHOT_PASS_TEXT : SHOT_FAIL_TEXT, + valid: outcome === 'pass', + })) + const seam = leafSeam(spawnedProfiles, { [coder]: { withTrace: true, shots } }) + const makeWorkerAgent: MakeWorkerAgent = (profile, context) => { + spawnedTasks.push(context?.task) + return seam(profile, context) + } + const firstPass = script.indexOf('pass') + const converges = firstPass >= 0 && firstPass < cell.shots + const spawnCount = converges ? firstPass + 1 : cell.shots + 1 + const turns: ScriptedTurn[] = [] + for (let shot = 1; shot <= spawnCount; shot += 1) { + turns.push({ + toolCalls: [ + { + name: 'spawn_agent', + arguments: { + profile: { name: coder }, + task: shot === 1 ? cell.task : rebriefText(shot), + }, + }, + ], + }) + // A delivered shot produces two bus events (settle, then its verify report). The final + // over-cap spawn of a non-converging script is REFUSED, so it awaits nothing. + if (shot <= cell.shots) { + turns.push({ toolCalls: [{ name: 'await_event', arguments: {} }] }) + turns.push({ toolCalls: [{ name: 'await_event', arguments: {} }] }) + } + } + turns.push({ content: 'done' }) + return { + backend: { + kind: 'seam', + makeWorkerAgent, + brain: scriptedBrain(turns), + shotPassed: offlineShotPassed, + }, + capture: { spawnedProfiles, spawnedTasks }, + } +} diff --git a/examples/p1-parity/parity.test.ts b/examples/p1-parity/parity.test.ts new file mode 100644 index 00000000..85b86815 --- /dev/null +++ b/examples/p1-parity/parity.test.ts @@ -0,0 +1,152 @@ +/** + * The P1 parity harness's own validity proof, fully offline: + * + * 1. INPUT EQUIVALENCE (the core property) — the same cell's task text, coder/reviewer + * profiles, and shot budget demonstrably reach BOTH arms, asserted on inputs CAPTURED at + * each arm's execution seam (the loop's transport requests, the graph's leaf factory), + * never on what the harness intended to send. + * 2. Both arms return well-formed {@link ParityRecord}s whose asymmetries are the REAL ones: + * the graph settles at the first passing shot while the legacy loop burns its whole budget + * (no deliverable gate), and only the graph arm carries an edge ledger. + * 3. The non-convergence path stays honest: a script with no passing shot drives the graph + * into its delegates cap (`GraphEdgeCapError`), which maps to `converged: false` with the + * refusal visible in the ledger — and the loop reports the same verdict from its transcript. + */ + +import { describe, expect, it } from 'vitest' +import { buildParityGraph, type CellSpec, type ParityRecord, runGraphArm, runLoopArm } from './arms' +import { offlineGraphBackend, offlineLoopBackend, offlineShotPassed } from './offline' + +const parityCell = (shots: number): CellSpec => ({ + task: 'make the failing test suite pass', + coderProfile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } }, + reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + shots, + budget: { maxIterations: 30, maxTokens: 100_000 }, +}) + +function expectWellFormed(record: ParityRecord): void { + expect(typeof record.converged).toBe('boolean') + expect(Number.isInteger(record.shotsUsed)).toBe(true) + expect(record.shotsUsed).toBeGreaterThanOrEqual(0) + for (const n of [ + record.spend.tokens.input, + record.spend.tokens.output, + record.spend.usd, + record.wallMs, + record.steeringDelivered.count, + record.steeringDelivered.bytes, + ]) { + expect(Number.isFinite(n)).toBe(true) + expect(n).toBeGreaterThanOrEqual(0) + } +} + +describe('p1-parity — the same cell reaches both arms and both report honestly', () => { + it('input equivalence: task, profiles, and shot budget arrive at both execution seams', async () => { + const cell = parityCell(2) + const script = ['fail', 'pass'] as const + + const loop = offlineLoopBackend(script) + const graph = offlineGraphBackend(cell, script) + const loopRecord = await runLoopArm(cell, loop.backend) + const graphRecord = await runGraphArm(cell, graph.backend) + + // ── Loop arm, captured at the transport seam ── + const firstAgentReq = loop.capture.agentRequests[0] + expect(firstAgentReq?.messages[0]).toEqual({ + role: 'system', + content: cell.coderProfile.prompt?.systemPrompt, + }) + expect(firstAgentReq?.messages[1]).toEqual({ role: 'user', content: cell.task }) + expect(loop.capture.driverRequests[0]?.messages[0]).toEqual({ + role: 'system', + content: cell.reviewerProfile.prompt?.systemPrompt, + }) + // The shot budget reached the loop: exactly `shots` coder completions were requested. + expect(loop.capture.agentRequests).toHaveLength(cell.shots) + + // ── Graph arm, captured at the leaf factory ── + const firstSpawn = graph.capture.spawnedProfiles[0] + expect(firstSpawn?.name).toBe(cell.coderProfile.name) + expect(firstSpawn?.prompt?.systemPrompt).toBe(cell.coderProfile.prompt?.systemPrompt) + expect(graph.capture.spawnedTasks[0]).toBe(cell.task) + // The shot budget reached the graph as the delegates edge's traversal cap, and the graph's + // nodes ARE the cell's profiles (pinned by name, not copied into role builders). + const topology = buildParityGraph(cell, offlineShotPassed) + const delegates = topology.edges.find((edge) => edge.kind === 'delegates') + expect(delegates?.kind === 'delegates' && delegates.maxTraversals).toBe(cell.shots) + expect(topology.nodes.map((node) => node.profile)).toEqual([ + cell.reviewerProfile, + cell.coderProfile, + ]) + expect(topology.deliverable.describe).toBe(cell.task) + + // Both arms converged on the same scripted cell. + expect(loopRecord.converged).toBe(true) + expect(graphRecord.converged).toBe(true) + }) + + it('both arms return well-formed ParityRecords with the REAL asymmetries visible', async () => { + // Pass on shot 2 of a 3-shot budget: the early-convergence probe. + const cell = parityCell(3) + const script = ['fail', 'pass'] as const + + const loopRecord = await runLoopArm(cell, offlineLoopBackend(script).backend) + const graphRecord = await runGraphArm(cell, offlineGraphBackend(cell, script).backend) + expectWellFormed(loopRecord) + expectWellFormed(graphRecord) + + expect(loopRecord.converged).toBe(true) + expect(graphRecord.converged).toBe(true) + + // The measured difference P1 exists to surface: the graph settles at the passing shot; the + // legacy loop has no deliverable gate and burns the full budget. + expect(graphRecord.shotsUsed).toBe(2) + expect(loopRecord.shotsUsed).toBe(3) + + // Steering: corrective deliveries after the initial brief. Graph re-briefed once (shot 2); + // the loop's driver kept steering after convergence too (turns 2 and 3). + expect(graphRecord.steeringDelivered.count).toBe(1) + expect(loopRecord.steeringDelivered.count).toBe(2) + expect(graphRecord.steeringDelivered.bytes).toBeGreaterThan(0) + expect(loopRecord.steeringDelivered.bytes).toBeGreaterThan(0) + + // Spend flows through each arm's own metering: graph = 2 leaf shots × {5,5} from the + // conserved pool's journal; loop = (3 agent + 2 driver) × {5,5} at the transport seam. + expect(graphRecord.spend).toEqual({ tokens: { input: 10, output: 10 }, usd: 0 }) + expect(loopRecord.spend).toEqual({ tokens: { input: 25, output: 25 }, usd: 0 }) + + // The honest ledger asymmetry: the graph's edge ledger is present and complete (2 delivered + // shot traversals + 2 delivered verify reports); the legacy loop HAS no edge ledger and the + // record must not fake one. + expect(loopRecord.ledger).toBeUndefined() + expect(graphRecord.ledger?.map((row) => [row.edge, row.traversal, row.outcome])).toEqual([ + ['delegates:reviewer->coder', 1, 'delivered'], + ['analyzes:verify:coder->reviewer', 1, 'delivered'], + ['delegates:reviewer->coder', 2, 'delivered'], + ['analyzes:verify:coder->reviewer', 2, 'delivered'], + ]) + }) + + it('non-convergence stays honest: the delegates cap refusal is a false verdict, not a crash', async () => { + const cell = parityCell(2) + const script = ['fail'] as const // every shot fails; the last repeats + + const loopRecord = await runLoopArm(cell, offlineLoopBackend(script).backend) + const graphRecord = await runGraphArm(cell, offlineGraphBackend(cell, script).backend) + expectWellFormed(loopRecord) + expectWellFormed(graphRecord) + + expect(loopRecord.converged).toBe(false) + expect(graphRecord.converged).toBe(false) + expect(loopRecord.shotsUsed).toBe(2) + expect(graphRecord.shotsUsed).toBe(2) + + // The graph's evidence: two delivered shots, then the cap REFUSED the third spawn — the + // refusal is a ledger row, not a swallowed error. + const last = graphRecord.ledger?.at(-1) + expect(last?.outcome).toBe('unpropagated') + expect(last?.reason).toContain('traversal-cap-exhausted') + }) +}) diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts new file mode 100644 index 00000000..f61c6a72 --- /dev/null +++ b/examples/p1-parity/run-parity.ts @@ -0,0 +1,217 @@ +/** + * P1 parity CLI — replay N identical coding cells through BOTH loop forms and print paired + * records (issue #694 P1: the loop→graph measurement harness). + * + * pnpm tsx examples/p1-parity/run-parity.ts --backend offline --cells 2 --shots 3 + * pnpm tsx examples/p1-parity/run-parity.ts --backend cli-bridge --cells 1 --shots 3 + * + * offline — scripted seams (mirrors examples/graphs/shared.ts): zero network, zero env, $0. + * Shot script per cell: fail × (shots−1), then pass — so the graph arm settles on + * the final shot while the loop arm burns its whole budget, and the paired records + * show exactly that. This mode is CI-safe and is what the vitest suite exercises. + * cli-bridge — the LIVE one-command entry for later: wires the real cli-bridge backend from + * VB_CLI_BRIDGE_URL / VB_CLI_BRIDGE_BEARER (+ VB_PARITY_MODEL for the coder's + * bridge wire id). Nothing in this repo's gates ever executes it. + */ + +import { parseArgs } from 'node:util' +import type { MultishotTransport } from '@tangle-network/agent-eval/multishot' +import type { CellSpec, GraphArmBackend, LoopArmBackend, ParityRecord } from './arms' +import { runGraphArm, runLoopArm } from './arms' +import { offlineGraphBackend, offlineLoopBackend } from './offline' + +interface CliOptions { + backend: 'offline' | 'cli-bridge' + cells: number + shots: number +} + +function parseCli(argv: string[]): CliOptions { + const { values } = parseArgs({ + args: argv, + options: { + backend: { type: 'string', default: 'offline' }, + cells: { type: 'string', default: '1' }, + shots: { type: 'string', default: '3' }, + }, + }) + const backend = values.backend + if (backend !== 'offline' && backend !== 'cli-bridge') { + throw new Error(`--backend must be offline|cli-bridge, got '${backend}'`) + } + const cells = Number.parseInt(values.cells ?? '1', 10) + const shots = Number.parseInt(values.shots ?? '3', 10) + if (!Number.isInteger(cells) || cells < 1) throw new Error(`--cells must be >= 1, got ${cells}`) + if (!Number.isInteger(shots) || shots < 1) throw new Error(`--shots must be >= 1, got ${shots}`) + return { backend, cells, shots } +} + +// ── Cells ────────────────────────────────────────────────────────────────────── + +function parityCell(index: number, shots: number): CellSpec { + return { + task: `parity cell ${index + 1}: make the failing test suite pass`, + coderProfile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } }, + reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + shots, + budget: { maxIterations: 30, maxTokens: 100_000 }, + } +} + +/** The live cell: same shape, plus the coder's bridge wire id and the marker contract the live + * completion check reads (see {@link LIVE_PASS_MARKER}). */ +function liveParityCell(index: number, shots: number, model: string): CellSpec { + const base = parityCell(index, shots) + return { + ...base, + coderProfile: { + ...base.coderProfile, + model: { default: model }, + prompt: { + systemPrompt: + 'Make tests pass. Print the exact line ' + + `'${LIVE_PASS_MARKER}' when and ONLY when the full suite genuinely passes.`, + }, + }, + } +} + +// ── Live cli-bridge wiring (NOT executed by any gate — the later live entry) ─── + +interface BridgeEnv { + url: string + bearer: string + model: string +} + +function requireBridgeEnv(): BridgeEnv { + const url = process.env.VB_CLI_BRIDGE_URL + const bearer = process.env.VB_CLI_BRIDGE_BEARER + const model = process.env.VB_PARITY_MODEL + if (!url || !bearer || !model) { + throw new Error( + 'cli-bridge backend needs VB_CLI_BRIDGE_URL, VB_CLI_BRIDGE_BEARER and VB_PARITY_MODEL ' + + '(the coder bridge wire id, e.g. pi/deepseek) in the environment', + ) + } + return { url, bearer, model } +} + +/** A multishot transport over cli-bridge's OpenAI-compatible chat-completions surface. */ +function bridgeTransport(env: BridgeEnv): MultishotTransport { + return async (req) => { + const res = await fetch(`${env.url.replace(/\/$/, '')}/v1/chat/completions`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${env.bearer}`, + }, + body: JSON.stringify({ + model: req.model, + messages: req.messages, + ...(req.tools !== undefined && req.tools.length > 0 ? { tools: req.tools } : {}), + ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), + ...(req.maxTokens !== undefined ? { max_tokens: req.maxTokens } : {}), + }), + ...(req.signal !== undefined ? { signal: req.signal } : {}), + }) + if (!res.ok) { + throw new Error(`cli-bridge completion failed: ${res.status} ${await res.text()}`) + } + const body = (await res.json()) as { + choices?: Array<{ message?: { content?: string | null; tool_calls?: never[] } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const message = body.choices?.[0]?.message + if (message === undefined) throw new Error('cli-bridge completion returned no message') + return { message, ...(body.usage !== undefined ? { usage: body.usage } : {}) } + } +} + +/** LIVE completion check: a marker oracle, pending a real verifier lens. The coder profile must + * instruct the harness to print this marker only when the suite genuinely passes. */ +const LIVE_PASS_MARKER = 'ALL TESTS PASS' +const livePassed = (text: string): boolean => text.includes(LIVE_PASS_MARKER) + +function liveBackends(env: BridgeEnv): { loop: LoopArmBackend; graph: GraphArmBackend } { + const transport = bridgeTransport(env) + return { + loop: { + agentTransport: transport, + driverTransport: transport, + shotPassed: livePassed, + apiKey: env.bearer, + baseUrl: env.url, + }, + graph: { + kind: 'bridge', + bridgeUrl: env.url, + bridgeBearer: env.bearer, + model: env.model, + shotPassed: livePassed, + }, + } +} + +// ── The run ──────────────────────────────────────────────────────────────────── + +interface PairedRow { + cell: number + arm: 'loop' | 'graph' + record: ParityRecord +} + +function printRecord(row: PairedRow): void { + const r = row.record + console.log( + `cell ${row.cell} ${row.arm.padEnd(5)} converged=${r.converged} shotsUsed=${r.shotsUsed} ` + + `tokens=${r.spend.tokens.input}/${r.spend.tokens.output} usd=${r.spend.usd} ` + + `wallMs=${r.wallMs} steering=${r.steeringDelivered.count}×/${r.steeringDelivered.bytes}B ` + + `ledger=${r.ledger === undefined ? 'none (legacy loop has no edge ledger)' : `${r.ledger.length} rows`}`, + ) + if (r.ledger !== undefined) { + for (const t of r.ledger) { + const worker = t.workerId !== undefined ? ` -> ${t.workerId}` : '' + const reason = t.reason !== undefined ? ` (${t.reason})` : '' + console.log(` #${t.traversal} ${t.edge} [${t.outcome}] ${t.bytes}B${worker}${reason}`) + } + } +} + +export async function main(): Promise { + const cli = parseCli(process.argv.slice(2)) + const rows: PairedRow[] = [] + for (let i = 0; i < cli.cells; i += 1) { + let loopBackend: LoopArmBackend + let graphBackend: GraphArmBackend + let cell: CellSpec + if (cli.backend === 'offline') { + // fail × (shots−1) then pass: converges exactly on the final shot. + const script = [...Array<'fail'>(cli.shots - 1).fill('fail'), 'pass' as const] + cell = parityCell(i, cli.shots) + loopBackend = offlineLoopBackend(script).backend + graphBackend = offlineGraphBackend(cell, script).backend + } else { + const env = requireBridgeEnv() + const backends = liveBackends(env) + cell = liveParityCell(i, cli.shots, env.model) + loopBackend = backends.loop + graphBackend = backends.graph + } + const loop = await runLoopArm(cell, loopBackend) + const graph = await runGraphArm(cell, graphBackend) + rows.push({ cell: i + 1, arm: 'loop', record: loop }) + rows.push({ cell: i + 1, arm: 'graph', record: graph }) + } + console.log(`p1-parity — backend=${cli.backend} cells=${cli.cells} shots=${cli.shots}\n`) + for (const row of rows) printRecord(row) + console.log('\nfull records (JSON):') + console.log(JSON.stringify(rows, null, 2)) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((err) => { + console.error(err) + process.exit(1) + }) +} diff --git a/tsconfig.examples.json b/tsconfig.examples.json index dfd3c252..f211c0fd 100644 --- a/tsconfig.examples.json +++ b/tsconfig.examples.json @@ -6,6 +6,7 @@ "paths": { "@tangle-network/agent-runtime": ["./src/index.ts"], "@tangle-network/agent-runtime/agent": ["./src/agent/index.ts"], + "@tangle-network/agent-runtime/durable": ["./src/durable/index.ts"], "@tangle-network/agent-runtime/intelligence": ["./src/intelligence/index.ts"], "@tangle-network/agent-runtime/kernel": ["./src/runtime/index.ts"], "@tangle-network/agent-runtime/environment-provider": [