diff --git a/examples/p1-parity/arms.ts b/examples/p1-parity/arms.ts new file mode 100644 index 00000000..850b30b0 --- /dev/null +++ b/examples/p1-parity/arms.ts @@ -0,0 +1,347 @@ +/** + * P1 parity arms — the SAME coding cell replayed through the two loop forms, under measurement. + * + * A. `runLoopArm` — the LEGACY loop: agent-eval's multishot loop (`runMultishot`), with the + * reviewer profile as the simulated-user driver leg and the coder profile as the agent leg. + * Each driver→agent turn is one SHOT; `maxTurns` is the shot budget. + * B. `runGraphArm` — the graph form: `runGraph` over the two-node reviewer→coder topology + * (examples/graphs/shot-loop.ts as data), the shot budget on the delegates edge's + * `maxTraversals`, the completion oracle on the mandatory deliverable. + * + * Both arms take one identical {@link CellSpec} and return one {@link ParityRecord}, so a run of + * N cells yields N paired rows — the measurement harness for the loop→graph migration (#694 P1). + * The record maps each arm's OWN instrumentation onto shared field names; where the two forms + * genuinely differ (edge ledger, conserved pool, early stop) the difference is documented on the + * field and left visible in the data, never papered over. + */ + +import type { MultishotMessage, MultishotTransport } from '@tangle-network/agent-eval/multishot' +import { runMultishot } from '@tangle-network/agent-eval/multishot' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + type AgentGraph, + type AnalystRegistry, + type Budget, + type EdgeTraversal, + GraphEdgeCapError, + type MakeWorkerAgent, + promptHandle, + type RouterConfig, + type RunGraphOptions, + runGraph, + type Spend, + type ToolLoopChat, +} from '@tangle-network/agent-runtime/kernel' + +// ── The shared cell ──────────────────────────────────────────────────────────── + +/** One coding cell, fed VERBATIM to both arms — the input-equivalence contract of the harness. */ +export interface CellSpec { + /** The coding task text. Shot 1's brief in both arms: the multishot opener (loop) and the + * first spawn's task payload + the root task (graph). */ + readonly task: string + /** The coder under test. Loop arm: the agent leg's profile. Graph arm: the pinned worker node + * (`profile.name` is the node id, so it must be non-empty and differ from the reviewer's). */ + readonly coderProfile: AgentProfile + /** The reviewer driving the shots. Loop arm: the driver leg (its `prompt.systemPrompt` is the + * driver system prompt). Graph arm: the root node. */ + readonly reviewerProfile: AgentProfile + /** The shot budget. Loop arm: `maxTurns`. Graph arm: the delegates edge's `maxTraversals`. */ + readonly shots: number + /** The conserved resource pool. ENFORCED BY THE GRAPH ARM ONLY: `runGraph` reserves against it + * for every spawn. The legacy loop has no conserved-pool concept — its only enforceable limit + * is `shots` — so this field cannot reach it. That gap is a P1 finding, not a harness bug. */ + readonly budget: Budget +} + +// ── The shared record ────────────────────────────────────────────────────────── + +/** + * One arm's measured outcome for one cell. Same field names, each populated from that arm's OWN + * instrumentation — the asymmetries below are real differences between the two forms and are the + * exact thing P1 exists to measure: + * + * - `ledger` exists ONLY for the graph arm (the edge ledger is what the graph form adds). The + * loop arm's instrumentation is a transcript, not an edge ledger; its `ledger` is ALWAYS + * `undefined` and must never be synthesized from the transcript. + * - The loop arm cannot stop early: `runMultishot` has no deliverable gate, so it burns the full + * shot budget even when an early shot converges. The graph arm settles at the first shot whose + * output passes the deliverable. Expect `shotsUsed` to differ on early-convergence cells. + * - Loop `spend.tokens` is metered at the transport seam (the sum of `usage` on every agent and + * driver completion); graph `spend` is the run's reconciled `spentTotal` from the conserved + * pool's journal. Both are that form's honest total, measured by different machinery. + */ +export interface ParityRecord { + /** Did any shot satisfy the completion check? Graph arm: the run settled a winner (the + * deliverable passed). Loop arm: some turn-initial coder reply passed `shotPassed`. */ + converged: boolean + /** Coder shots actually executed. Graph arm: distinct live coder workers spawned (from the + * ledger). Loop arm: turn-initial agent replies in the transcript. */ + shotsUsed: number + /** Total measured resource spend for the arm's whole run (driver + coder legs). */ + spend: { tokens: { input: number; output: number }; usd: number } + /** Wall-clock duration of the arm call, measured identically around both arms. */ + wallMs: number + /** Corrective direction DELIVERED to the coder after the initial brief. Loop arm: driver (user) + * messages after the opener, counted with their UTF-8 bytes. Graph arm: delegates-edge + * traversals beyond the first with outcome `delivered` — re-brief spawns AND mid-run steers, + * with the bytes that actually crossed the edge. Graph bytes include the versioned edge + * directive text; loop bytes are the raw message only (the loop has no directive layer). */ + steeringDelivered: { count: number; bytes: number } + /** The graph arm's edge ledger — every traversal, outcome, and byte count. ABSENT for the loop + * arm, honestly: the legacy loop has no observable edges (that is the migration's point). */ + ledger?: ReadonlyArray +} + +// ── Backends ─────────────────────────────────────────────────────────────────── + +/** Execution seams for the loop arm. Offline: scripted transports (see ./offline.ts). + * Live: transports posting to a cli-bridge OpenAI-compatible endpoint (see ./run-parity.ts). */ +export interface LoopArmBackend { + readonly agentTransport: MultishotTransport + readonly driverTransport: MultishotTransport + /** The shared completion check, applied to each turn-initial coder reply. MUST be the same + * predicate the paired graph arm's deliverable uses, or the comparison is invalid. */ + readonly shotPassed: (assistantText: string) => boolean + /** `runMultishot` resolves apiKey/baseUrl eagerly even with both transports injected; the + * offline path passes inert placeholders so no env is required. */ + readonly apiKey?: string + readonly baseUrl?: string +} + +/** Execution seams for the graph arm: fully-scripted (offline/CI) or the live cli-bridge. */ +export type GraphArmBackend = + | { + readonly kind: 'seam' + readonly makeWorkerAgent: MakeWorkerAgent + readonly brain: ToolLoopChat + readonly analysts?: AnalystRegistry + /** Same predicate as the paired loop arm — becomes the graph's deliverable check. */ + readonly shotPassed: (workerOutText: string) => boolean + } + | { + readonly kind: 'bridge' + readonly bridgeUrl: string + readonly bridgeBearer: string + /** Fallback bridge wire id (e.g. `pi/deepseek`); the spawned profile may select its own. */ + readonly model?: string + readonly cwd?: string + /** Router substrate for the reviewer (driver) brain. */ + readonly router?: RouterConfig + readonly shotPassed: (workerOutText: string) => boolean + } + +// ── The graph topology (exported so tests can assert on the exact inputs) ────── + +export const PARITY_VERIFY_ANALYST = 'verify' + +/** The verify lens is ENVIRONMENT: it reads the coder's settle trace, never sits in the graph. */ +export function parityAnalysts(): AnalystRegistry { + return { + kinds: [ + { + id: PARITY_VERIFY_ANALYST, + description: 'read the coder trace, report shot outcome to the reviewer', + area: 'qa', + }, + ], + run: async () => [{ check: 'shot-completion', observed: 'see the settled output' }], + } +} + +/** The two-node reviewer→coder topology for one cell — plain data, the shot budget on the edge. + * The cell's profiles are used AS-IS (node id = `profile.name`), the task is the root task + * (`deliverable.describe`) and each spawn's payload, and `shotPassed` is the deliverable. */ +export function buildParityGraph( + cell: CellSpec, + shotPassed: (workerOutText: string) => boolean, +): AgentGraph { + const reviewer = requireProfileName(cell.reviewerProfile, 'reviewerProfile') + const coder = requireProfileName(cell.coderProfile, 'coderProfile') + return { + nodes: [ + { id: reviewer, profile: cell.reviewerProfile }, + { id: coder, profile: cell.coderProfile }, + ], + edges: [ + { + kind: 'delegates', + from: reviewer, + to: coder, + directive: promptHandle('delegates/worker-brief/v1'), + maxTraversals: cell.shots, + }, + { + kind: 'analyzes', + analyst: PARITY_VERIFY_ANALYST, + over: [coder], + to: reviewer, + directive: promptHandle('analyzes/findings-report/v1'), + }, + ], + deliverable: { + describe: cell.task, + check: (out) => typeof out === 'string' && shotPassed(out), + }, + budget: cell.budget, + } +} + +// ── Arm A: the legacy multishot loop ─────────────────────────────────────────── + +export async function runLoopArm(cell: CellSpec, backend: LoopArmBackend): Promise { + validateCell(cell) + const tokens = { input: 0, output: 0 } + // Meter tokens at the transport seam — the only usage channel the legacy loop exposes. + const metered = + (transport: MultishotTransport): MultishotTransport => + async (req) => { + const res = await transport(req) + tokens.input += res.usage?.prompt_tokens ?? 0 + tokens.output += res.usage?.completion_tokens ?? 0 + return res + } + const startedAt = Date.now() + const sim = await runMultishot({ + profile: cell.coderProfile, + persona: { id: 'parity-cell' }, + shape: { + buildOpener: () => cell.task, + buildDriverSystemPrompt: () => cell.reviewerProfile.prompt?.systemPrompt ?? '', + }, + tools: [], + toolExecutors: {}, + maxTurns: cell.shots, + agentModel: cell.coderProfile.model?.default ?? 'parity/unspecified', + driverModel: cell.reviewerProfile.model?.default ?? 'parity/unspecified', + agentTransport: metered(backend.agentTransport), + driverTransport: metered(backend.driverTransport), + apiKey: backend.apiKey ?? 'unused', + baseUrl: backend.baseUrl ?? 'http://unused.invalid', + }) + const wallMs = Date.now() - startedAt + const shotReplies = turnInitialAssistantReplies(sim.transcript) + const steering = sim.transcript.slice(1).filter((msg) => msg.role === 'user') + return { + converged: shotReplies.some((text) => backend.shotPassed(text)), + shotsUsed: shotReplies.length, + spend: { tokens: { ...tokens }, usd: sim.costUsd }, + wallMs, + steeringDelivered: { + count: steering.length, + bytes: steering.reduce((sum, msg) => sum + Buffer.byteLength(msg.content, 'utf8'), 0), + }, + // No `ledger`: the legacy loop has no edge instrumentation, and the harness never fakes one. + } +} + +/** The coder's per-shot replies: assistant messages that directly answer a user (driver) message. + * Tool-followup assistant messages (which follow tool results) are the same shot continuing. */ +function turnInitialAssistantReplies(transcript: ReadonlyArray): string[] { + const replies: string[] = [] + for (let i = 1; i < transcript.length; i += 1) { + const msg = transcript[i] + if (msg !== undefined && msg.role === 'assistant' && transcript[i - 1]?.role === 'user') { + replies.push(msg.content) + } + } + return replies +} + +// ── Arm B: the runGraph two-node form ────────────────────────────────────────── + +export async function runGraphArm(cell: CellSpec, backend: GraphArmBackend): Promise { + validateCell(cell) + const graph = buildParityGraph(cell, backend.shotPassed) + const opts: RunGraphOptions = + backend.kind === 'seam' + ? { + makeWorkerAgent: backend.makeWorkerAgent, + brain: backend.brain, + analysts: backend.analysts ?? parityAnalysts(), + } + : { + backend: { + backend: 'bridge', + bridgeUrl: backend.bridgeUrl, + bridgeBearer: backend.bridgeBearer, + ...(backend.model !== undefined ? { model: backend.model } : {}), + ...(backend.cwd !== undefined ? { cwd: backend.cwd } : {}), + }, + ...(backend.router !== undefined ? { router: backend.router } : {}), + analysts: parityAnalysts(), + } + const startedAt = Date.now() + try { + const res = await runGraph(graph, opts) + return graphRecord( + res.result.kind === 'winner', + res.result.spentTotal, + res.ledger, + Date.now() - startedAt, + ) + } catch (err) { + if (err instanceof GraphEdgeCapError) { + // The cap (the cyclic-graph backstop), not the task, ended the run: an honest + // non-convergence row, with the full evidence the error carries. + return graphRecord(false, err.result.spentTotal, err.ledger, Date.now() - startedAt) + } + throw err + } +} + +function graphRecord( + converged: boolean, + spentTotal: Spend, + ledger: ReadonlyArray, + wallMs: number, +): ParityRecord { + const delegates = ledger.filter((row) => row.kind === 'delegates') + // Each live coder worker is one shot; steers re-use an existing worker id, refused rows have + // none — so distinct bound worker ids count executed shots exactly. + const shotsUsed = new Set( + delegates.filter((row) => row.workerId !== undefined).map((row) => row.workerId), + ).size + const steering = delegates.filter((row) => row.outcome === 'delivered' && row.traversal > 1) + return { + converged, + shotsUsed, + spend: { + tokens: { input: spentTotal.tokens.input, output: spentTotal.tokens.output }, + usd: spentTotal.usd, + }, + wallMs, + steeringDelivered: { + count: steering.length, + bytes: steering.reduce((sum, row) => sum + row.bytes, 0), + }, + ledger, + } +} + +// ── Shared validation ────────────────────────────────────────────────────────── + +function requireProfileName(profile: AgentProfile, field: string): string { + const name = profile.name + if (typeof name !== 'string' || name.length === 0) { + throw new Error( + `p1-parity: ${field}.name must be a non-empty string — it is the graph node id, and both ` + + 'arms report against it', + ) + } + return name +} + +function validateCell(cell: CellSpec): void { + const reviewer = requireProfileName(cell.reviewerProfile, 'reviewerProfile') + const coder = requireProfileName(cell.coderProfile, 'coderProfile') + if (reviewer === coder) { + throw new Error('p1-parity: reviewerProfile.name and coderProfile.name must differ') + } + if (!Number.isInteger(cell.shots) || cell.shots < 1) { + throw new Error(`p1-parity: shots must be a positive integer, got ${cell.shots}`) + } + if (typeof cell.task !== 'string' || cell.task.length === 0) { + throw new Error('p1-parity: task must be a non-empty string') + } +} diff --git a/examples/p1-parity/offline.ts b/examples/p1-parity/offline.ts new file mode 100644 index 00000000..1f6fab40 --- /dev/null +++ b/examples/p1-parity/offline.ts @@ -0,0 +1,148 @@ +/** + * OFFLINE scripted seams for the P1 parity arms — zero network, zero env, $0. Mirrors + * examples/graphs/shared.ts and reuses its `leafSeam` / `scriptedBrain` directly for the graph + * arm, so only the loop arm's transports are new scripting. Both arms' seams are generated from + * the same {@link ShotScript}, so the SAME cell produces the SAME shot outcomes in both forms — + * the CI-runnable proof path, and the capture point for the input-equivalence test. + * + * Synthetic accounting: every scripted completion reports `usage {5,5}` and `$0`, mirroring the + * leaf seam's per-shot spend, so the two arms' metering pipelines carry comparable numbers + * offline. The loop arm additionally meters its scripted DRIVER completions (the legacy loop's + * driver is an inference leg); the graph arm's scripted brain meters nothing (a live graph + * driver would meter through `spentBreakdown.driverInference`). Real numbers arrive only with + * the live backend. + */ + +import type { + MultishotTransport, + MultishotTransportRequest, +} from '@tangle-network/agent-eval/multishot' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { MakeWorkerAgent } from '@tangle-network/agent-runtime/kernel' +import { type LeafShot, leafSeam, type ScriptedTurn, scriptedBrain } from '../graphs/shared' +import type { CellSpec, GraphArmBackend, LoopArmBackend } from './arms' + +/** Per-shot scripted outcome; the last entry repeats for later shots. */ +export type ShotScript = ReadonlyArray<'pass' | 'fail'> + +export const SHOT_PASS_TEXT = 'TESTS: pass' +export const SHOT_FAIL_TEXT = 'TESTS: fail' + +/** The one completion check BOTH offline arms share (loop reply text = graph settle output). */ +export const offlineShotPassed = (text: string): boolean => text.includes(SHOT_PASS_TEXT) + +/** The reviewer's re-brief for shot N — identical wording in both arms, so steering payloads + * differ only by what each form adds (the graph's versioned edge directive). */ +export const rebriefText = (shot: number): string => + `shot ${shot}: revise — the verifier reported failing tests; make them pass` + +const scriptedUsage = () => ({ prompt_tokens: 5, completion_tokens: 5 }) + +const shotAt = (script: ShotScript, index: number): 'pass' | 'fail' => + script[Math.min(index, script.length - 1)] ?? 'fail' + +// ── Loop arm: scripted transports ────────────────────────────────────────────── + +export interface LoopCapture { + /** Every request the coder (agent) leg received, in order — the input-equivalence evidence. */ + readonly agentRequests: MultishotTransportRequest[] + /** Every request the reviewer (driver) leg received, in order. */ + readonly driverRequests: MultishotTransportRequest[] +} + +/** Scripted loop backend: coder replies follow the shot script; the reviewer re-briefs between + * shots with {@link rebriefText}. All requests are captured for assertion. */ +export function offlineLoopBackend(script: ShotScript): { + backend: LoopArmBackend + capture: LoopCapture +} { + const agentRequests: MultishotTransportRequest[] = [] + const driverRequests: MultishotTransportRequest[] = [] + const agentTransport: MultishotTransport = async (req) => { + agentRequests.push(req) + const outcome = shotAt(script, agentRequests.length - 1) + return { + message: { content: outcome === 'pass' ? SHOT_PASS_TEXT : SHOT_FAIL_TEXT }, + usage: scriptedUsage(), + costUsd: 0, + } + } + const driverTransport: MultishotTransport = async (req) => { + driverRequests.push(req) + // Driver call k re-briefs shot k+1 (the loop drives one driver turn between shots). + return { + message: { content: rebriefText(driverRequests.length + 1) }, + usage: scriptedUsage(), + costUsd: 0, + } + } + return { + backend: { agentTransport, driverTransport, shotPassed: offlineShotPassed }, + capture: { agentRequests, driverRequests }, + } +} + +// ── Graph arm: scripted brain + leaf seam ────────────────────────────────────── + +export interface GraphCapture { + /** Every profile the leaf factory received (the graph-pinned coder profile, with the + * delegates directive appended to its instructions), in spawn order. */ + readonly spawnedProfiles: AgentProfile[] + /** Every spawn's task payload, in spawn order — shot 1 must be the cell task verbatim. */ + readonly spawnedTasks: unknown[] +} + +/** Scripted graph backend for one cell: the leaf settles each shot per the script, and the + * reviewer brain spawns shot-by-shot until the first scripted pass. A script with no pass + * inside the shot budget drives one extra spawn INTO the delegates cap, so the backstop — + * not silence — ends the run (`runGraphArm` maps that to an honest non-convergence row). */ +export function offlineGraphBackend( + cell: CellSpec, + script: ShotScript, +): { backend: GraphArmBackend; capture: GraphCapture } { + const coder = cell.coderProfile.name ?? 'coder' + const spawnedProfiles: AgentProfile[] = [] + const spawnedTasks: unknown[] = [] + const shots: LeafShot[] = script.map((outcome) => ({ + out: outcome === 'pass' ? SHOT_PASS_TEXT : SHOT_FAIL_TEXT, + valid: outcome === 'pass', + })) + const seam = leafSeam(spawnedProfiles, { [coder]: { withTrace: true, shots } }) + const makeWorkerAgent: MakeWorkerAgent = (profile, context) => { + spawnedTasks.push(context?.task) + return seam(profile, context) + } + const firstPass = script.indexOf('pass') + const converges = firstPass >= 0 && firstPass < cell.shots + const spawnCount = converges ? firstPass + 1 : cell.shots + 1 + const turns: ScriptedTurn[] = [] + for (let shot = 1; shot <= spawnCount; shot += 1) { + turns.push({ + toolCalls: [ + { + name: 'spawn_agent', + arguments: { + profile: { name: coder }, + task: shot === 1 ? cell.task : rebriefText(shot), + }, + }, + ], + }) + // A delivered shot produces two bus events (settle, then its verify report). The final + // over-cap spawn of a non-converging script is REFUSED, so it awaits nothing. + if (shot <= cell.shots) { + turns.push({ toolCalls: [{ name: 'await_event', arguments: {} }] }) + turns.push({ toolCalls: [{ name: 'await_event', arguments: {} }] }) + } + } + turns.push({ content: 'done' }) + return { + backend: { + kind: 'seam', + makeWorkerAgent, + brain: scriptedBrain(turns), + shotPassed: offlineShotPassed, + }, + capture: { spawnedProfiles, spawnedTasks }, + } +} diff --git a/examples/p1-parity/parity.test.ts b/examples/p1-parity/parity.test.ts new file mode 100644 index 00000000..85b86815 --- /dev/null +++ b/examples/p1-parity/parity.test.ts @@ -0,0 +1,152 @@ +/** + * The P1 parity harness's own validity proof, fully offline: + * + * 1. INPUT EQUIVALENCE (the core property) — the same cell's task text, coder/reviewer + * profiles, and shot budget demonstrably reach BOTH arms, asserted on inputs CAPTURED at + * each arm's execution seam (the loop's transport requests, the graph's leaf factory), + * never on what the harness intended to send. + * 2. Both arms return well-formed {@link ParityRecord}s whose asymmetries are the REAL ones: + * the graph settles at the first passing shot while the legacy loop burns its whole budget + * (no deliverable gate), and only the graph arm carries an edge ledger. + * 3. The non-convergence path stays honest: a script with no passing shot drives the graph + * into its delegates cap (`GraphEdgeCapError`), which maps to `converged: false` with the + * refusal visible in the ledger — and the loop reports the same verdict from its transcript. + */ + +import { describe, expect, it } from 'vitest' +import { buildParityGraph, type CellSpec, type ParityRecord, runGraphArm, runLoopArm } from './arms' +import { offlineGraphBackend, offlineLoopBackend, offlineShotPassed } from './offline' + +const parityCell = (shots: number): CellSpec => ({ + task: 'make the failing test suite pass', + coderProfile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } }, + reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + shots, + budget: { maxIterations: 30, maxTokens: 100_000 }, +}) + +function expectWellFormed(record: ParityRecord): void { + expect(typeof record.converged).toBe('boolean') + expect(Number.isInteger(record.shotsUsed)).toBe(true) + expect(record.shotsUsed).toBeGreaterThanOrEqual(0) + for (const n of [ + record.spend.tokens.input, + record.spend.tokens.output, + record.spend.usd, + record.wallMs, + record.steeringDelivered.count, + record.steeringDelivered.bytes, + ]) { + expect(Number.isFinite(n)).toBe(true) + expect(n).toBeGreaterThanOrEqual(0) + } +} + +describe('p1-parity — the same cell reaches both arms and both report honestly', () => { + it('input equivalence: task, profiles, and shot budget arrive at both execution seams', async () => { + const cell = parityCell(2) + const script = ['fail', 'pass'] as const + + const loop = offlineLoopBackend(script) + const graph = offlineGraphBackend(cell, script) + const loopRecord = await runLoopArm(cell, loop.backend) + const graphRecord = await runGraphArm(cell, graph.backend) + + // ── Loop arm, captured at the transport seam ── + const firstAgentReq = loop.capture.agentRequests[0] + expect(firstAgentReq?.messages[0]).toEqual({ + role: 'system', + content: cell.coderProfile.prompt?.systemPrompt, + }) + expect(firstAgentReq?.messages[1]).toEqual({ role: 'user', content: cell.task }) + expect(loop.capture.driverRequests[0]?.messages[0]).toEqual({ + role: 'system', + content: cell.reviewerProfile.prompt?.systemPrompt, + }) + // The shot budget reached the loop: exactly `shots` coder completions were requested. + expect(loop.capture.agentRequests).toHaveLength(cell.shots) + + // ── Graph arm, captured at the leaf factory ── + const firstSpawn = graph.capture.spawnedProfiles[0] + expect(firstSpawn?.name).toBe(cell.coderProfile.name) + expect(firstSpawn?.prompt?.systemPrompt).toBe(cell.coderProfile.prompt?.systemPrompt) + expect(graph.capture.spawnedTasks[0]).toBe(cell.task) + // The shot budget reached the graph as the delegates edge's traversal cap, and the graph's + // nodes ARE the cell's profiles (pinned by name, not copied into role builders). + const topology = buildParityGraph(cell, offlineShotPassed) + const delegates = topology.edges.find((edge) => edge.kind === 'delegates') + expect(delegates?.kind === 'delegates' && delegates.maxTraversals).toBe(cell.shots) + expect(topology.nodes.map((node) => node.profile)).toEqual([ + cell.reviewerProfile, + cell.coderProfile, + ]) + expect(topology.deliverable.describe).toBe(cell.task) + + // Both arms converged on the same scripted cell. + expect(loopRecord.converged).toBe(true) + expect(graphRecord.converged).toBe(true) + }) + + it('both arms return well-formed ParityRecords with the REAL asymmetries visible', async () => { + // Pass on shot 2 of a 3-shot budget: the early-convergence probe. + const cell = parityCell(3) + const script = ['fail', 'pass'] as const + + const loopRecord = await runLoopArm(cell, offlineLoopBackend(script).backend) + const graphRecord = await runGraphArm(cell, offlineGraphBackend(cell, script).backend) + expectWellFormed(loopRecord) + expectWellFormed(graphRecord) + + expect(loopRecord.converged).toBe(true) + expect(graphRecord.converged).toBe(true) + + // The measured difference P1 exists to surface: the graph settles at the passing shot; the + // legacy loop has no deliverable gate and burns the full budget. + expect(graphRecord.shotsUsed).toBe(2) + expect(loopRecord.shotsUsed).toBe(3) + + // Steering: corrective deliveries after the initial brief. Graph re-briefed once (shot 2); + // the loop's driver kept steering after convergence too (turns 2 and 3). + expect(graphRecord.steeringDelivered.count).toBe(1) + expect(loopRecord.steeringDelivered.count).toBe(2) + expect(graphRecord.steeringDelivered.bytes).toBeGreaterThan(0) + expect(loopRecord.steeringDelivered.bytes).toBeGreaterThan(0) + + // Spend flows through each arm's own metering: graph = 2 leaf shots × {5,5} from the + // conserved pool's journal; loop = (3 agent + 2 driver) × {5,5} at the transport seam. + expect(graphRecord.spend).toEqual({ tokens: { input: 10, output: 10 }, usd: 0 }) + expect(loopRecord.spend).toEqual({ tokens: { input: 25, output: 25 }, usd: 0 }) + + // The honest ledger asymmetry: the graph's edge ledger is present and complete (2 delivered + // shot traversals + 2 delivered verify reports); the legacy loop HAS no edge ledger and the + // record must not fake one. + expect(loopRecord.ledger).toBeUndefined() + expect(graphRecord.ledger?.map((row) => [row.edge, row.traversal, row.outcome])).toEqual([ + ['delegates:reviewer->coder', 1, 'delivered'], + ['analyzes:verify:coder->reviewer', 1, 'delivered'], + ['delegates:reviewer->coder', 2, 'delivered'], + ['analyzes:verify:coder->reviewer', 2, 'delivered'], + ]) + }) + + it('non-convergence stays honest: the delegates cap refusal is a false verdict, not a crash', async () => { + const cell = parityCell(2) + const script = ['fail'] as const // every shot fails; the last repeats + + const loopRecord = await runLoopArm(cell, offlineLoopBackend(script).backend) + const graphRecord = await runGraphArm(cell, offlineGraphBackend(cell, script).backend) + expectWellFormed(loopRecord) + expectWellFormed(graphRecord) + + expect(loopRecord.converged).toBe(false) + expect(graphRecord.converged).toBe(false) + expect(loopRecord.shotsUsed).toBe(2) + expect(graphRecord.shotsUsed).toBe(2) + + // The graph's evidence: two delivered shots, then the cap REFUSED the third spawn — the + // refusal is a ledger row, not a swallowed error. + const last = graphRecord.ledger?.at(-1) + expect(last?.outcome).toBe('unpropagated') + expect(last?.reason).toContain('traversal-cap-exhausted') + }) +}) diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts new file mode 100644 index 00000000..f61c6a72 --- /dev/null +++ b/examples/p1-parity/run-parity.ts @@ -0,0 +1,217 @@ +/** + * P1 parity CLI — replay N identical coding cells through BOTH loop forms and print paired + * records (issue #694 P1: the loop→graph measurement harness). + * + * pnpm tsx examples/p1-parity/run-parity.ts --backend offline --cells 2 --shots 3 + * pnpm tsx examples/p1-parity/run-parity.ts --backend cli-bridge --cells 1 --shots 3 + * + * offline — scripted seams (mirrors examples/graphs/shared.ts): zero network, zero env, $0. + * Shot script per cell: fail × (shots−1), then pass — so the graph arm settles on + * the final shot while the loop arm burns its whole budget, and the paired records + * show exactly that. This mode is CI-safe and is what the vitest suite exercises. + * cli-bridge — the LIVE one-command entry for later: wires the real cli-bridge backend from + * VB_CLI_BRIDGE_URL / VB_CLI_BRIDGE_BEARER (+ VB_PARITY_MODEL for the coder's + * bridge wire id). Nothing in this repo's gates ever executes it. + */ + +import { parseArgs } from 'node:util' +import type { MultishotTransport } from '@tangle-network/agent-eval/multishot' +import type { CellSpec, GraphArmBackend, LoopArmBackend, ParityRecord } from './arms' +import { runGraphArm, runLoopArm } from './arms' +import { offlineGraphBackend, offlineLoopBackend } from './offline' + +interface CliOptions { + backend: 'offline' | 'cli-bridge' + cells: number + shots: number +} + +function parseCli(argv: string[]): CliOptions { + const { values } = parseArgs({ + args: argv, + options: { + backend: { type: 'string', default: 'offline' }, + cells: { type: 'string', default: '1' }, + shots: { type: 'string', default: '3' }, + }, + }) + const backend = values.backend + if (backend !== 'offline' && backend !== 'cli-bridge') { + throw new Error(`--backend must be offline|cli-bridge, got '${backend}'`) + } + const cells = Number.parseInt(values.cells ?? '1', 10) + const shots = Number.parseInt(values.shots ?? '3', 10) + if (!Number.isInteger(cells) || cells < 1) throw new Error(`--cells must be >= 1, got ${cells}`) + if (!Number.isInteger(shots) || shots < 1) throw new Error(`--shots must be >= 1, got ${shots}`) + return { backend, cells, shots } +} + +// ── Cells ────────────────────────────────────────────────────────────────────── + +function parityCell(index: number, shots: number): CellSpec { + return { + task: `parity cell ${index + 1}: make the failing test suite pass`, + coderProfile: { name: 'coder', prompt: { systemPrompt: 'Make tests pass.' } }, + reviewerProfile: { name: 'reviewer', prompt: { systemPrompt: 'Verify.' } }, + shots, + budget: { maxIterations: 30, maxTokens: 100_000 }, + } +} + +/** The live cell: same shape, plus the coder's bridge wire id and the marker contract the live + * completion check reads (see {@link LIVE_PASS_MARKER}). */ +function liveParityCell(index: number, shots: number, model: string): CellSpec { + const base = parityCell(index, shots) + return { + ...base, + coderProfile: { + ...base.coderProfile, + model: { default: model }, + prompt: { + systemPrompt: + 'Make tests pass. Print the exact line ' + + `'${LIVE_PASS_MARKER}' when and ONLY when the full suite genuinely passes.`, + }, + }, + } +} + +// ── Live cli-bridge wiring (NOT executed by any gate — the later live entry) ─── + +interface BridgeEnv { + url: string + bearer: string + model: string +} + +function requireBridgeEnv(): BridgeEnv { + const url = process.env.VB_CLI_BRIDGE_URL + const bearer = process.env.VB_CLI_BRIDGE_BEARER + const model = process.env.VB_PARITY_MODEL + if (!url || !bearer || !model) { + throw new Error( + 'cli-bridge backend needs VB_CLI_BRIDGE_URL, VB_CLI_BRIDGE_BEARER and VB_PARITY_MODEL ' + + '(the coder bridge wire id, e.g. pi/deepseek) in the environment', + ) + } + return { url, bearer, model } +} + +/** A multishot transport over cli-bridge's OpenAI-compatible chat-completions surface. */ +function bridgeTransport(env: BridgeEnv): MultishotTransport { + return async (req) => { + const res = await fetch(`${env.url.replace(/\/$/, '')}/v1/chat/completions`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${env.bearer}`, + }, + body: JSON.stringify({ + model: req.model, + messages: req.messages, + ...(req.tools !== undefined && req.tools.length > 0 ? { tools: req.tools } : {}), + ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), + ...(req.maxTokens !== undefined ? { max_tokens: req.maxTokens } : {}), + }), + ...(req.signal !== undefined ? { signal: req.signal } : {}), + }) + if (!res.ok) { + throw new Error(`cli-bridge completion failed: ${res.status} ${await res.text()}`) + } + const body = (await res.json()) as { + choices?: Array<{ message?: { content?: string | null; tool_calls?: never[] } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } + } + const message = body.choices?.[0]?.message + if (message === undefined) throw new Error('cli-bridge completion returned no message') + return { message, ...(body.usage !== undefined ? { usage: body.usage } : {}) } + } +} + +/** LIVE completion check: a marker oracle, pending a real verifier lens. The coder profile must + * instruct the harness to print this marker only when the suite genuinely passes. */ +const LIVE_PASS_MARKER = 'ALL TESTS PASS' +const livePassed = (text: string): boolean => text.includes(LIVE_PASS_MARKER) + +function liveBackends(env: BridgeEnv): { loop: LoopArmBackend; graph: GraphArmBackend } { + const transport = bridgeTransport(env) + return { + loop: { + agentTransport: transport, + driverTransport: transport, + shotPassed: livePassed, + apiKey: env.bearer, + baseUrl: env.url, + }, + graph: { + kind: 'bridge', + bridgeUrl: env.url, + bridgeBearer: env.bearer, + model: env.model, + shotPassed: livePassed, + }, + } +} + +// ── The run ──────────────────────────────────────────────────────────────────── + +interface PairedRow { + cell: number + arm: 'loop' | 'graph' + record: ParityRecord +} + +function printRecord(row: PairedRow): void { + const r = row.record + console.log( + `cell ${row.cell} ${row.arm.padEnd(5)} converged=${r.converged} shotsUsed=${r.shotsUsed} ` + + `tokens=${r.spend.tokens.input}/${r.spend.tokens.output} usd=${r.spend.usd} ` + + `wallMs=${r.wallMs} steering=${r.steeringDelivered.count}×/${r.steeringDelivered.bytes}B ` + + `ledger=${r.ledger === undefined ? 'none (legacy loop has no edge ledger)' : `${r.ledger.length} rows`}`, + ) + if (r.ledger !== undefined) { + for (const t of r.ledger) { + const worker = t.workerId !== undefined ? ` -> ${t.workerId}` : '' + const reason = t.reason !== undefined ? ` (${t.reason})` : '' + console.log(` #${t.traversal} ${t.edge} [${t.outcome}] ${t.bytes}B${worker}${reason}`) + } + } +} + +export async function main(): Promise { + const cli = parseCli(process.argv.slice(2)) + const rows: PairedRow[] = [] + for (let i = 0; i < cli.cells; i += 1) { + let loopBackend: LoopArmBackend + let graphBackend: GraphArmBackend + let cell: CellSpec + if (cli.backend === 'offline') { + // fail × (shots−1) then pass: converges exactly on the final shot. + const script = [...Array<'fail'>(cli.shots - 1).fill('fail'), 'pass' as const] + cell = parityCell(i, cli.shots) + loopBackend = offlineLoopBackend(script).backend + graphBackend = offlineGraphBackend(cell, script).backend + } else { + const env = requireBridgeEnv() + const backends = liveBackends(env) + cell = liveParityCell(i, cli.shots, env.model) + loopBackend = backends.loop + graphBackend = backends.graph + } + const loop = await runLoopArm(cell, loopBackend) + const graph = await runGraphArm(cell, graphBackend) + rows.push({ cell: i + 1, arm: 'loop', record: loop }) + rows.push({ cell: i + 1, arm: 'graph', record: graph }) + } + console.log(`p1-parity — backend=${cli.backend} cells=${cli.cells} shots=${cli.shots}\n`) + for (const row of rows) printRecord(row) + console.log('\nfull records (JSON):') + console.log(JSON.stringify(rows, null, 2)) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((err) => { + console.error(err) + process.exit(1) + }) +} diff --git a/tsconfig.examples.json b/tsconfig.examples.json index dfd3c252..f211c0fd 100644 --- a/tsconfig.examples.json +++ b/tsconfig.examples.json @@ -6,6 +6,7 @@ "paths": { "@tangle-network/agent-runtime": ["./src/index.ts"], "@tangle-network/agent-runtime/agent": ["./src/agent/index.ts"], + "@tangle-network/agent-runtime/durable": ["./src/durable/index.ts"], "@tangle-network/agent-runtime/intelligence": ["./src/intelligence/index.ts"], "@tangle-network/agent-runtime/kernel": ["./src/runtime/index.ts"], "@tangle-network/agent-runtime/environment-provider": [