diff --git a/benchmarks/trace-analysis/tools/build-verified-dataset.mjs b/benchmarks/trace-analysis/tools/build-verified-dataset.mjs new file mode 100644 index 00000000..1cffd129 --- /dev/null +++ b/benchmarks/trace-analysis/tools/build-verified-dataset.mjs @@ -0,0 +1,94 @@ +#!/usr/bin/env node +// Build a verified-findings dataset from a replay-verify batch run. +// +// Joins batch-report.json (executed verdicts + fix arms) with the gold label +// corpora and the normalized trajectories, then writes: +// /rows.jsonl one VerifiedFindingRow per replayable case +// /manifest.json summary + full source provenance + emitted-file shas +// +// Imports the package's own join from dist/ (build first: pnpm build). +// +// Usage: +// build-verified-dataset.mjs --report PATH --run-id ID --out DIR \ +// --corpus NAME=LABELS_PATH::PREPARED_DIR [--corpus ...] \ +// [--run-dir DIR] [--max-obs N] +// +// --run-dir points at the batch run directory holding per-case +// `--/replay-verdict.json`; when given, every case must have +// one (divergence detail + arm A command + run ids join into the rows). +// Every join failure is fatal — a partially joined dataset is never written. + +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const dist = join(fileURLToPath(new URL('.', import.meta.url)), '..', '..', '..', 'dist', 'rl.js') +const { loadVerifiedFindingsDataset, verifiedFindingsToJsonl, VERIFIED_FINDING_SCHEMA } = await import(dist) + +function parseArgs(argv) { + const args = { corpora: {}, maxObservationChars: undefined, runDir: undefined } + for (let i = 0; i < argv.length; i++) { + const flag = argv[i] + const value = () => { + const v = argv[++i] + if (v === undefined) throw new Error(`missing value for ${flag}`) + return v + } + if (flag === '--report') args.batchReportPath = value() + else if (flag === '--run-id') args.runId = value() + else if (flag === '--out') args.out = value() + else if (flag === '--run-dir') args.runDir = value() + else if (flag === '--max-obs') args.maxObservationChars = Number(value()) + else if (flag === '--corpus') { + const spec = value() + const eq = spec.indexOf('=') + const sep = spec.indexOf('::') + if (eq < 1 || sep < eq) throw new Error(`--corpus expects NAME=LABELS_PATH::PREPARED_DIR, got: ${spec}`) + args.corpora[spec.slice(0, eq)] = { + labelsPath: spec.slice(eq + 1, sep), + preparedDir: spec.slice(sep + 2), + } + } else throw new Error(`unknown flag: ${flag}`) + } + for (const required of ['batchReportPath', 'runId', 'out']) { + if (!args[required]) throw new Error(`--${required.replace(/[A-Z]/g, (c) => `-${c.toLowerCase()}`)} is required`) + } + if (Object.keys(args.corpora).length === 0) throw new Error('at least one --corpus is required') + return args +} + +const args = parseArgs(process.argv.slice(2)) +const dataset = loadVerifiedFindingsDataset({ + batchReportPath: args.batchReportPath, + runId: args.runId, + corpora: args.corpora, + runDir: args.runDir, + maxObservationChars: args.maxObservationChars, +}) + +mkdirSync(args.out, { recursive: true }) +const jsonl = verifiedFindingsToJsonl(dataset.rows) +const rowsPath = join(args.out, 'rows.jsonl') +writeFileSync(rowsPath, jsonl) +const rowsSha256 = createHash('sha256').update(readFileSync(rowsPath)).digest('hex') + +const manifest = { + schema: VERIFIED_FINDING_SCHEMA, + generatedAt: new Date().toISOString(), + summary: dataset.summary, + provenance: dataset.provenance, + files: { 'rows.jsonl': { sha256: rowsSha256, bytes: Buffer.byteLength(jsonl) } }, +} +writeFileSync(join(args.out, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`) + +const s = dataset.summary +console.log(`verified-findings dataset → ${args.out}`) +console.log(`rows: ${s.rows} | reproduced: ${s.reproduced} | signature-strict: ${s.signatureStrict}`) +console.log( + `fix: flipped ${s.fix.flipped}, not-flipped ${s.fix['not-flipped']}, generation-failed ${s.fix['generation-failed']}, not-attempted ${s.fix['not-attempted']}`, +) +for (const [corpus, c] of Object.entries(s.byCorpus)) { + console.log(` ${corpus}: rows ${c.rows}, reproduced ${c.reproduced}, fix-flipped ${c.fixFlipped}`) +} +console.log(`rows.jsonl sha256 ${rowsSha256}`) diff --git a/benchmarks/trace-analysis/verified-dataset-v0/README.md b/benchmarks/trace-analysis/verified-dataset-v0/README.md new file mode 100644 index 00000000..f6063a48 --- /dev/null +++ b/benchmarks/trace-analysis/verified-dataset-v0/README.md @@ -0,0 +1,65 @@ +# verified-dataset-v0 — execution-verified gold labels as RL rows + +The first artifact of the verified-labels flywheel: gold "incorrect step" annotations that a replay-verify batch re-executed inside the original docker image, joined with their trajectories into trainer-ready rows. +The label on every row was decided by execution (returncode/signature comparison at the gold step, plus a fix arm), not by a rater. +The pipeline is the deliverable; rows scale with every future replay batch. +Flywheel context and the phase-2 capture spec live in [`docs/verified-labels-flywheel.md`](../../../docs/verified-labels-flywheel.md). + +## Artifact (out of git — 1.4 MB, sha-pinned) + +Location: `~/bench-cache/ctb-20260801/verified-dataset-v0/` + +| file | sha256 | contents | +| --- | --- | --- | +| `rows.jsonl` | `afcbfb7b21c14868f8654bb27d113f42ba66fd951bcdc64b5f32b1e8b33641e3` | 22 `VerifiedFindingRow` lines (1,436,306 bytes) | +| `manifest.json` | committed by the builder run | summary + source provenance + emitted-file shas | + +Source pins (embedded per row and in the manifest): + +| source | sha256 | +| --- | --- | +| `replay-batch/run2-20260802/batch-report.json` | `be6236dce691ea9df7c850040c995dd1f2274aaf559c45836c726bc480288fb3` | +| `ctb-holdout-labels.json` (holdout-1) | `53af5ffe3962f3378f2d65419b92b8a56fe7d6c8efc619a0bc2b8f0872bc4f83` | +| `ctb-holdout2-labels.json` (holdout-2) | `2db46579b7993edc376acbbcacf67a1d0ddfcdb94e28930c2bb8dfcf1dc32fb2` | +| `split3/ctb-split3-labels.json` (split3) | `d0347ec7a5ec9a07bd3fcd16aa06b07bcb33ffabca39cb0b0f7a564fb500ae08` | + +Run2 composition (n=22): 16 reproduced, 13 signature-strict, fix arms 9 flipped / 2 not-flipped / 5 generation-failed / 6 not-attempted. +Per corpus — holdout-1: 4 rows (2 reproduced, 1 flipped); holdout-2: 9 (9, 7); split3: 9 (5, 1). + +## Row schema — `agent-eval/verified-finding@0` + +One row per replayable case (JSONL). Full types: `src/rl/verified-findings-dataset.ts` (exported from `@tangle-network/agent-eval/rl`). + +| field | meaning | +| --- | --- | +| `caseId` | `//` — unique across batches | +| `task` | agent, model, task name, difficulty, solved, step count (from the gold label entry) | +| `gold.stepK` | the verified gold step — earliest replayable incorrect step | +| `gold.actionAtK` | exact command the agent ran at k (never truncated — it is the labeled object) | +| `gold.goldIncorrectSteps` / `gold.labelIncorrectSteps` | replay targets vs every labeled incorrect step | +| `gold.recordedReturncodeAtK` | returncode the original trajectory recorded at k | +| `trajectory` | prefix steps 1..k (action + observation, observations truncated at 4000 chars with original length kept); post-k steps are excluded so a trainer never sees the future | +| `verification.reproduced` | batch verdict: prefix divergence ≤ tolerance AND arm A reproduced the recorded returncode at k | +| `verification.signatureStrict` | arm A also matched the failure signature (raw evidence — can be true on a non-reproduced case) | +| `verification.prefixDivergenceDetail` | per-step `{step, expectedReturncode, actualExit}` from the per-case verdict | +| `verification.armAExit` / `armACommand` | executed evidence at k | +| `fix.outcome` | `flipped` \| `not-flipped` \| `generation-failed` \| `not-attempted` (batch report is authoritative; arm B exit carried) | +| `provenance` | run id, batch/labels/steps sha256s, docker image + derived replay image, cwd, original/arm-A run ids | + +Join discipline: any missing or inconsistent join (label absent, step-count mismatch, k outside the gold set, arm B verdict missing on a fix command) throws — a partially joined dataset is never written. + +## Regenerate + +```bash +pnpm build +node benchmarks/trace-analysis/tools/build-verified-dataset.mjs \ + --report ~/bench-cache/ctb-20260801/replay-batch/run2-20260802/batch-report.json \ + --run-id run2-20260802 \ + --run-dir ~/bench-cache/ctb-20260801/replay-batch/run2-20260802 \ + --out ~/bench-cache/ctb-20260801/verified-dataset-v0 \ + --corpus 'holdout-1=/home/drew/bench-cache/ctb-20260801/ctb-holdout-labels.json::/home/drew/bench-cache/ctb-20260801/ctb-holdout-prepared' \ + --corpus 'holdout-2=/home/drew/bench-cache/ctb-20260801/ctb-holdout2-labels.json::/home/drew/bench-cache/ctb-20260801/ctb-holdout2-prepared' \ + --corpus 'split3=/home/drew/bench-cache/ctb-20260801/split3/ctb-split3-labels.json::/home/drew/bench-cache/ctb-20260801/split3/ctb-split3-prepared' +``` + +The build is deterministic given the same inputs: `rows.jsonl` reproduces byte-identical (manifest `generatedAt` varies). diff --git a/docs/verified-labels-flywheel.md b/docs/verified-labels-flywheel.md new file mode 100644 index 00000000..62186025 --- /dev/null +++ b/docs/verified-labels-flywheel.md @@ -0,0 +1,46 @@ +# Verified-labels flywheel — own-traffic replay eligibility (phase-2 spec) + +Phase 1 shipped the dataset pipeline: `src/rl/verified-findings-dataset.ts` joins replay-verify batch verdicts with gold labels and trajectories into execution-verified RL rows (`agent-eval/verified-finding@0`, see `benchmarks/trace-analysis/verified-dataset-v0/README.md`). +Those rows came from public benchmark trajectories (mini-SWE / CodeTraceBench). +The flywheel's real fuel is our own traffic: fleet sessions run inside sandboxes where the image is known. +This document maps which local session classes are replay-eligible today, which are not and why, and the concrete capture changes that make future sessions eligible. + +## What replay eligibility requires + +Derived from what the run2 replay batch actually consumed (its enumeration excluded 111/133 label entries): + +1. **Pinned environment** — a docker image (or digest) the trajectory ran in; `no-docker-image` alone excluded 21 entries. +2. **Working directory** — the cwd commands were executed from. +3. **Ordered step commands** — the exact action string per step; `no-swe-raw-trajectory` excluded 65 entries. +4. **Per-step recorded returncodes** — needed for prefix-divergence checking (the replay batch aborts when >10% of prefix steps diverge from recorded returncodes). +5. **A verifiable target step** — a finding/label on a *command* step (submit-only golds excluded 21 entries; findings on prose are not executable). + +## Local session stores surveyed (2026-08-03, this host) + +| store | volume | environment (req 1–2) | steps (req 3) | returncodes (req 4) | eligible today | +| --- | --- | --- | --- | --- | --- | +| Claude Code transcripts `~/.claude/projects` | 474 projects, 7,579 session JSONLs, 4.6 GB | no image; `cwd` + `gitBranch` + harness `version` per message | yes — full tool calls + results | **no** — Bash `toolUseResult` records `stdout`/`stderr`/`interrupted` only, no exit code field | no (host env unpinned, no rc) | +| Codex sessions `~/.codex/sessions` | 4,332 rollout files, 113 GB | no image; `session_meta` has `cwd` + git `commit_hash`/`branch`/`repository_url` + `cli_version` | yes | shell events carry exit codes in payloads (format varies by version) | no (host env unpinned) | +| traces CLI normalized envelope (`~/code/traces`, 18 harness adapters: claude, codex, amp, gemini, opencode, pi, copilot, factory, forge, qwen, …) | imports the two stores above | `cwd` filter exists; **no image/sandbox field in the envelope** | yes | adapter-dependent | no — the schema itself cannot express environment identity | +| Sandbox sessions (agent-dev-container `PersistedSession`) | remote fleet; none stored on this host | runtime knows the image — `runtime.ready` event carries `image?` + `sandboxId` + backend — but `PersistedSession` persists only `workspaceRoot` + free-form `metadata`; `image?` is optional even on the event | yes (message store) | via tool parts, not normalized | **almost** — the image is in hand at runtime and dropped at persistence time | +| mini-SWE / CodeTraceBench benchmark trajectories (`~/bench-cache/ctb-20260801`) | 133 labeled, 22 replayable | yes — `mswebench/*` images + cwd | yes | yes — `N` in every observation | **yes — the only eligible class; run2 proved 16/22 reproduce** | + +Conclusion: today only benchmark-imported trajectories are replay-eligible. +Our own sessions fail on environment pinning (all classes) and returncode capture (Claude Code). +The sandbox class is one persistence field away from eligibility — the runtime already knows the image. + +## Phase-2 capture changes (ranked by unlock per line of code) + +1. **Persist the sandbox image at session start** (agent-dev-container): copy `runtime.ready`'s `image` (as a digest, not a tag) + `sandboxId` into `PersistedSession` as first-class fields, and make `image` required on the event. + This single change makes every future fleet sandbox session satisfy requirements 1–2 — the highest-leverage line in the flywheel. +2. **Add environment identity to the traces envelope** (traces repo): an optional `environment: { image?, imageDigest?, cwd, gitCommit? }` block on the normalized session, populated by adapters where known. + Without it, eligible sandbox sessions lose their eligibility at import time. +3. **Record exit codes in Claude Code tool results**: the harness owns `toolUseResult`; until it carries `exitCode`, replay divergence checking cannot run on Claude transcripts even inside a pinned sandbox. + Workaround for sandboxed Claude sessions: derive returncodes from the sandbox's own command journal instead of the transcript. +4. **Emit a replay descriptor per session** (the join target this package consumes): `{ image, cwd, steps: [{action, returncode}], findings: [{stepId, claim}] }` — exactly the shape `loadVerifiedFindingsDataset` joins today, so phase-3 needs no new pipeline code. + +## Why this matters + +Run2 measured: 72.7% of eligible trajectories reproduce their recorded failure at the gold step, and 81.8% of generated fixes flip it. +Execution-verified labels at fleet scale are training data that cannot be bought — AgenTracer-8B showed +18pp from a specialist localizer trained on *unverified* labels; ours carry executed proof per row. +Every capture change above turns a session class from "readable" into "verifiable", and the phase-1 pipeline converts verifiable sessions into dataset rows with zero new code. diff --git a/src/rl/index.ts b/src/rl/index.ts index 59126121..32fec556 100644 --- a/src/rl/index.ts +++ b/src/rl/index.ts @@ -43,6 +43,8 @@ export * from './sim-fidelity' export * from './tournament' /** @stable Verifiable reward extraction (compile / test / schema) with judge-noise filtering. */ export * from './verifiable-reward' +/** Execution-verified gold labels joined into RL-ready rows (replay-verify batches → trainer input). */ +export * from './verified-findings-dataset' // ── Deployment-outcome store (predictive-validity calibration) ────── // Promoted to public so external consumers don't have to inline the diff --git a/src/rl/verified-findings-dataset.test.ts b/src/rl/verified-findings-dataset.test.ts new file mode 100644 index 00000000..38ee6a1a --- /dev/null +++ b/src/rl/verified-findings-dataset.test.ts @@ -0,0 +1,353 @@ +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { beforeEach, describe, expect, it } from 'vitest' +import { + buildVerifiedFindingRow, + type GoldLabelEntry, + loadVerifiedFindingsDataset, + type NormalizedStep, + type ReplayBatchCase, + summarizeVerifiedFindings, + VERIFIED_FINDING_SCHEMA, + type VerifiedFindingRow, + verifiedFindingsToJsonl, +} from './verified-findings-dataset' + +// Pins the verified-labels join: a replay batch's executed verdicts must land +// on the exact gold step and trajectory they verified. A regression here means +// training rows carry the wrong label, the wrong context, or silently drop a +// case — the three failure modes that make an execution-verified dataset +// worthless. + +const DIR = join(tmpdir(), 'agent-eval-verified-findings-test') + +function makeCase(overrides: Partial = {}): ReplayBatchCase { + return { + corpus: 'holdout-x', + trajId: 'traj-1', + image: 'bench/img:pr-1', + cwd: '/home', + cwdSource: 'run-config', + k: 2, + stepCount: 3, + goldIncorrectSteps: [2, 3], + recordedReturncodeAtK: 127, + derivedImage: 'ctb-replay:abc-uid1000', + signature: 'command not found', + status: 'ok', + error: null, + prefixExecuted: 1, + prefixDivergences: 0, + prefixDivergencePct: 0, + armAExit: 127, + armAReturncodeMatch: true, + armASignatureMatch: true, + replayed: true, + fix: { + attempted: true, + sampledOut: false, + command: 'apt-get install -y foo', + llmError: null, + armBExit: 0, + failureVanished: true, + }, + wallMs: 1200, + ...overrides, + } +} + +function makeLabel(overrides: Partial = {}): GoldLabelEntry { + return { + traj_id: 'traj-1', + solved: false, + step_count: 3, + agent: 'mini-SWE-agent', + model: 'OpenAI/GPT-5', + task_name: 'fix-the-thing', + difficulty: 'medium', + incorrect_stages: [ + { stage_id: 1, incorrect_step_ids: [2] }, + { stage_id: 2, incorrect_step_ids: [3] }, + ], + ...overrides, + } +} + +function makeSteps(): NormalizedStep[] { + return [ + { step_id: 1, action: 'ls', observation: '0' }, + { + step_id: 2, + action: 'foo --run', + observation: '127\nfoo: command not found', + }, + { step_id: 3, action: 'echo done', observation: '0' }, + ] +} + +function baseArgs() { + return { + batchCase: makeCase(), + label: makeLabel(), + steps: makeSteps(), + runId: 'run-test', + batchGeneratedAt: '2026-08-02T00:00:00.000Z', + batchReportSha256: 'r'.repeat(64), + labelsPath: '/labels.json', + labelsSha256: 'l'.repeat(64), + stepsPath: '/steps.json', + stepsSha256: 's'.repeat(64), + } +} + +describe('buildVerifiedFindingRow', () => { + it('joins case, label, and trajectory into a fully provenanced row', () => { + const row = buildVerifiedFindingRow(baseArgs()) + expect(row.schema).toBe(VERIFIED_FINDING_SCHEMA) + expect(row.caseId).toBe('run-test/holdout-x/traj-1') + expect(row.gold.stepK).toBe(2) + expect(row.gold.actionAtK).toBe('foo --run') + expect(row.gold.labelIncorrectSteps).toEqual([2, 3]) + expect(row.trajectory.window).toEqual({ start: 1, end: 2 }) + expect(row.trajectory.steps.map((s) => s.stepId)).toEqual([1, 2]) + expect(row.verification.reproduced).toBe(true) + expect(row.fix.outcome).toBe('flipped') + expect(row.fix.armBExit).toBe(0) + expect(row.provenance.labelsSha256).toBe('l'.repeat(64)) + expect(row.provenance.image).toBe('bench/img:pr-1') + }) + + it('excludes post-k steps so the trainer never sees the future', () => { + const row = buildVerifiedFindingRow(baseArgs()) + expect(row.trajectory.steps.some((s) => s.stepId > 2)).toBe(false) + }) + + it('truncates long observations and records the original length', () => { + const args = baseArgs() + args.steps[0]!.observation = 'x'.repeat(500) + const row = buildVerifiedFindingRow({ ...args, maxObservationChars: 100 }) + const step = row.trajectory.steps[0]! + expect(step.observation).toHaveLength(100) + expect(step.observationTruncated).toBe(true) + expect(step.observationChars).toBe(500) + expect(row.gold.actionAtK).toBe('foo --run') + }) + + it('maps fix records to outcomes: generation failure', () => { + const args = baseArgs() + args.batchCase = makeCase({ + fix: { + attempted: true, + sampledOut: false, + command: null, + llmError: 'aborted', + armBExit: null, + failureVanished: null, + }, + }) + expect(buildVerifiedFindingRow(args).fix.outcome).toBe('generation-failed') + }) + + it('maps fix records to outcomes: not flipped', () => { + const args = baseArgs() + args.batchCase = makeCase({ + fix: { + attempted: true, + sampledOut: false, + command: 'try', + llmError: null, + armBExit: 127, + failureVanished: false, + }, + }) + expect(buildVerifiedFindingRow(args).fix.outcome).toBe('not-flipped') + }) + + it('marks non-replayed cases without a fix record as not-attempted', () => { + const args = baseArgs() + args.batchCase = makeCase({ + replayed: false, + armAReturncodeMatch: false, + armASignatureMatch: false, + fix: null, + }) + const row = buildVerifiedFindingRow(args) + expect(row.verification.reproduced).toBe(false) + expect(row.fix.outcome).toBe('not-attempted') + }) + + it('throws when a replayed case is missing its fix record', () => { + const args = baseArgs() + args.batchCase = makeCase({ fix: null }) + expect(() => buildVerifiedFindingRow(args)).toThrow(/replayed case has no fix record/) + }) + + it('throws when a fix command has no arm B verdict', () => { + const args = baseArgs() + args.batchCase = makeCase({ + fix: { + attempted: true, + sampledOut: false, + command: 'try', + llmError: null, + armBExit: null, + failureVanished: null, + }, + }) + expect(() => buildVerifiedFindingRow(args)).toThrow(/failureVanished missing/) + }) + + it('throws when the label step count disagrees with the case', () => { + const args = baseArgs() + args.label = makeLabel({ step_count: 5 }) + expect(() => buildVerifiedFindingRow(args)).toThrow(/step_count 5 != case stepCount 3/) + }) + + it('throws when the trajectory is missing steps', () => { + const args = baseArgs() + args.steps = args.steps.slice(0, 2) + expect(() => buildVerifiedFindingRow(args)).toThrow(/has 2 steps, case expects 3/) + }) + + it('throws when the gold step is not in the label incorrect steps', () => { + const args = baseArgs() + args.label = makeLabel({ incorrect_stages: [{ stage_id: 1, incorrect_step_ids: [3] }] }) + expect(() => buildVerifiedFindingRow(args)).toThrow(/absent from the label's incorrect steps/) + }) + + it('throws when k is not among the case gold steps', () => { + const args = baseArgs() + args.batchCase = makeCase({ k: 1, goldIncorrectSteps: [2, 3] }) + expect(() => buildVerifiedFindingRow(args)).toThrow(/k=1 is not in goldIncorrectSteps/) + }) + + it('cross-checks the per-case verdict detail against the report', () => { + const args = baseArgs() + const detail = { + k: 3, + prefixExecuted: 1, + recordedReturncode: 127, + signatureBasis: 'returncode-only', + prefixDivergences: [], + armACommand: 'foo --run', + runIds: { original: 'o-1', armA: 'a-1' }, + } + expect(() => buildVerifiedFindingRow({ ...args, detail })).toThrow(/verdict k=3 != report k=2/) + const row = buildVerifiedFindingRow({ ...args, detail: { ...detail, k: 2 } }) + expect(row.provenance.originalRunId).toBe('o-1') + expect(row.verification.armACommand).toBe('foo --run') + }) +}) + +describe('summarizeVerifiedFindings + jsonl', () => { + it('counts verdicts by corpus and serializes one row per line', () => { + const flipped = buildVerifiedFindingRow(baseArgs()) + const args = baseArgs() + args.batchCase = makeCase({ + trajId: 'traj-1', + corpus: 'holdout-x', + replayed: false, + fix: null, + armASignatureMatch: false, + }) + const notReplayed = buildVerifiedFindingRow(args) + const summary = summarizeVerifiedFindings([flipped, notReplayed]) + expect(summary.rows).toBe(2) + expect(summary.reproduced).toBe(1) + expect(summary.signatureStrict).toBe(1) + expect(summary.fix.flipped).toBe(1) + expect(summary.fix['not-attempted']).toBe(1) + expect(summary.byCorpus['holdout-x']).toEqual({ rows: 2, reproduced: 1, fixFlipped: 1 }) + + const jsonl = verifiedFindingsToJsonl([flipped, notReplayed]) + const lines = jsonl.trimEnd().split('\n') + expect(lines).toHaveLength(2) + expect((JSON.parse(lines[0]!) as VerifiedFindingRow).caseId).toBe(flipped.caseId) + }) +}) + +describe('loadVerifiedFindingsDataset', () => { + beforeEach(() => { + rmSync(DIR, { recursive: true, force: true }) + mkdirSync(DIR, { recursive: true }) + }) + + function writeFixture(options: { withLabelFor?: string[] } = {}) { + const report = { + generatedAt: '2026-08-02T00:00:00.000Z', + cases: [ + makeCase(), + makeCase({ + trajId: 'traj-2', + k: 3, + goldIncorrectSteps: [3], + replayed: false, + fix: null, + armAReturncodeMatch: false, + armASignatureMatch: false, + }), + ], + } + writeFileSync(join(DIR, 'batch-report.json'), JSON.stringify(report)) + const labelIds = options.withLabelFor ?? ['traj-1', 'traj-2'] + const labels = labelIds.map((id) => + makeLabel({ traj_id: id, incorrect_stages: [{ stage_id: 1, incorrect_step_ids: [2, 3] }] }), + ) + writeFileSync(join(DIR, 'labels.json'), JSON.stringify(labels)) + for (const id of ['traj-1', 'traj-2']) { + const dir = join(DIR, 'prepared', 'normalized', id) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, 'steps.json'), JSON.stringify(makeSteps())) + } + } + + function source() { + return { + batchReportPath: join(DIR, 'batch-report.json'), + runId: 'run-fixture', + corpora: { + 'holdout-x': { labelsPath: join(DIR, 'labels.json'), preparedDir: join(DIR, 'prepared') }, + }, + } + } + + it('loads, joins, and orders rows deterministically with file provenance', () => { + writeFixture() + const dataset = loadVerifiedFindingsDataset(source()) + expect(dataset.rows.map((r) => r.trajId)).toEqual(['traj-1', 'traj-2']) + expect(dataset.summary.rows).toBe(2) + expect(dataset.summary.reproduced).toBe(1) + expect(dataset.provenance.batchReportSha256).toMatch(/^[0-9a-f]{64}$/) + const corpus = dataset.provenance.corpora['holdout-x']! + expect(corpus.labelsSha256).toMatch(/^[0-9a-f]{64}$/) + for (const row of dataset.rows) { + expect(row.provenance.labelsSha256).toBe(corpus.labelsSha256) + expect(row.provenance.stepsSha256).toMatch(/^[0-9a-f]{64}$/) + } + }) + + it('throws when a case has no label entry', () => { + writeFixture({ withLabelFor: ['traj-1'] }) + expect(() => loadVerifiedFindingsDataset(source())).toThrow(/traj-2: no label entry/) + }) + + it('throws when the corpus is not configured', () => { + writeFixture() + const bad = { ...source(), corpora: {} } + expect(() => loadVerifiedFindingsDataset(bad)).toThrow(/no labels\/preparedDir was configured/) + }) + + it('throws when a trajectory steps file is missing', () => { + writeFixture() + rmSync(join(DIR, 'prepared', 'normalized', 'traj-2'), { recursive: true }) + expect(() => loadVerifiedFindingsDataset(source())).toThrow( + /cannot read trajectory steps for traj-2/, + ) + }) + + it('throws on duplicate label traj_ids', () => { + writeFixture({ withLabelFor: ['traj-1', 'traj-1', 'traj-2'] }) + expect(() => loadVerifiedFindingsDataset(source())).toThrow(/duplicate traj_id 'traj-1'/) + }) +}) diff --git a/src/rl/verified-findings-dataset.ts b/src/rl/verified-findings-dataset.ts new file mode 100644 index 00000000..59a9c7b2 --- /dev/null +++ b/src/rl/verified-findings-dataset.ts @@ -0,0 +1,633 @@ +/** + * Verified-findings dataset — execution-verified gold labels as RL-ready rows. + * + * A replay-verify batch re-executes a labeled trajectory prefix inside the + * original docker image and checks, at the gold "incorrect" step k, whether + * the recorded failure reproduces (arm A) and whether a generated fix makes + * it vanish (arm B). That turns an annotation into an *executed* label: the + * verdict is a returncode/signature comparison, not a rater's opinion. + * + * This module joins three artifact families into one row per replayed case: + * + * 1. the batch report (`batch-report.json` — per-case verdicts, fix arms), + * 2. the gold label corpus (`*-labels.json` — incorrect step annotations), + * 3. the normalized trajectory (`normalized//steps.json` — the + * action/observation sequence the agent actually took). + * + * The emitted `VerifiedFindingRow` carries the trajectory prefix up to k, + * the gold label, the execution verdict with its evidence (exit codes, + * failure signature, prefix divergences), the fix arm when present, and + * per-row provenance (label/steps/report sha256s, docker images, run ids). + * Rows are trainer input for step-level localizer/critic models; the reward + * is deterministic because execution decided it. + * + * Join discipline: every missing or inconsistent join throws — a dataset + * built from partially joined artifacts would silently train on wrong + * labels. The batch report is authoritative for fix outcomes (per-case + * `replay-verdict.json` files are written before the fix arm completes); + * per-case files contribute prefix-divergence detail and run ids only, and + * are cross-checked against the report where they overlap. + */ + +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +export const VERIFIED_FINDING_SCHEMA = 'agent-eval/verified-finding@0' + +// ── Input shapes (parsed artifacts) ───────────────────────────────── + +/** One case row from a replay-verify `batch-report.json`. */ +export interface ReplayBatchCase { + corpus: string + trajId: string + image: string + cwd: string + cwdSource: string + k: number + stepCount: number + goldIncorrectSteps: number[] + recordedReturncodeAtK: number + derivedImage: string | null + signature: string | null + status: string + error: string | null + prefixExecuted: number + prefixDivergences: number + prefixDivergencePct: number + armAExit: number | null + armAReturncodeMatch: boolean + armASignatureMatch: boolean + /** Batch verdict: prefix divergence within tolerance AND arm A reproduced the recorded returncode at k. */ + replayed: boolean + fix: ReplayBatchFix | null + wallMs: number +} + +export interface ReplayBatchFix { + attempted: boolean + sampledOut: boolean + command: string | null + llmError: string | null + armBExit: number | null + failureVanished: boolean | null +} + +export interface ReplayBatchReport { + generatedAt: string + cases: ReplayBatchCase[] +} + +/** Gold label entry for one trajectory (CodeTraceBench annotation format). */ +export interface GoldLabelEntry { + traj_id: string + solved: boolean + step_count: number + agent?: string + model?: string + task_name?: string + difficulty?: string + incorrect_stages: Array<{ stage_id: number; incorrect_step_ids: number[] }> +} + +/** One step from a normalized trajectory `steps.json`. */ +export interface NormalizedStep { + step_id: number + action: string + observation?: string | null +} + +export interface PrefixDivergence { + step: number + expectedReturncode: number + actualExit: number +} + +/** Optional extract from a per-case `replay-verdict.json` (arm A detail only). */ +export interface CaseVerdictDetail { + k: number + prefixExecuted: number + recordedReturncode: number + signatureBasis: string | null + prefixDivergences: PrefixDivergence[] + armACommand: string | null + runIds: { original: string | null; armA: string | null } +} + +// ── Row schema ────────────────────────────────────────────────────── + +export interface TrajectoryStep { + stepId: number + action: string + observation: string | null + /** True when the observation was cut at `maxObservationChars`; `observationChars` keeps the original length. */ + observationTruncated: boolean + observationChars: number +} + +export type FixOutcome = 'flipped' | 'not-flipped' | 'generation-failed' | 'not-attempted' + +export interface VerifiedFindingFix { + outcome: FixOutcome + command: string | null + llmError: string | null + armBExit: number | null + failureVanished: boolean | null +} + +export interface VerifiedFindingRow { + schema: typeof VERIFIED_FINDING_SCHEMA + /** `//` — unique across batches. */ + caseId: string + corpus: string + trajId: string + task: { + agent: string | null + model: string | null + taskName: string | null + difficulty: string | null + solved: boolean + stepCount: number + } + gold: { + /** The verified gold step — the earliest replayable incorrect step. */ + stepK: number + /** The exact command the agent ran at step k (never truncated — it is the labeled object). */ + actionAtK: string + /** Incorrect steps the batch considered replay targets (submit-step golds excluded). */ + goldIncorrectSteps: number[] + /** Every incorrect step in the label entry, across stages. */ + labelIncorrectSteps: number[] + recordedReturncodeAtK: number + } + /** Prefix context 1..k — post-k steps are excluded so a trainer never sees the future. */ + trajectory: { + window: { start: number; end: number } + steps: TrajectoryStep[] + } + verification: { + reproduced: boolean + /** Arm A output also contained the recorded error substring (or returncode-only basis matched). */ + signatureStrict: boolean + signatureBasis: string | null + signature: string | null + prefixExecuted: number + prefixDivergences: number + prefixDivergencePct: number + prefixDivergenceDetail: PrefixDivergence[] | null + armAExit: number | null + armAReturncodeMatch: boolean + armACommand: string | null + wallMs: number + } + fix: VerifiedFindingFix + provenance: { + runId: string + batchGeneratedAt: string + batchReportSha256: string + labelsPath: string + labelsSha256: string + stepsPath: string + stepsSha256: string + image: string + derivedImage: string | null + cwd: string + cwdSource: string + originalRunId: string | null + armARunId: string | null + } +} + +export interface VerifiedFindingsSummary { + rows: number + reproduced: number + /** Reproduced AND arm A matched the failure signature — the batch report's headline strict rate. + * Row-level `verification.signatureStrict` is raw arm A evidence and can be true on a + * non-reproduced case (signature matched but the prefix diverged past tolerance). */ + signatureStrict: number + fix: Record + byCorpus: Record +} + +// ── Pure join ─────────────────────────────────────────────────────── + +const DEFAULT_MAX_OBSERVATION_CHARS = 4000 + +export interface BuildVerifiedFindingRowArgs { + batchCase: ReplayBatchCase + label: GoldLabelEntry + steps: NormalizedStep[] + runId: string + batchGeneratedAt: string + batchReportSha256: string + labelsPath: string + labelsSha256: string + stepsPath: string + stepsSha256: string + detail?: CaseVerdictDetail + maxObservationChars?: number +} + +function fail(caseId: string, message: string): never { + throw new Error(`verified-findings: ${caseId}: ${message}`) +} + +function deriveFixOutcome(caseId: string, batchCase: ReplayBatchCase): VerifiedFindingFix { + const fix = batchCase.fix + if (fix === null) { + if (batchCase.replayed) { + fail( + caseId, + 'replayed case has no fix record — the batch always records the fix arm for replayed cases', + ) + } + return { + outcome: 'not-attempted', + command: null, + llmError: null, + armBExit: null, + failureVanished: null, + } + } + const base = { + command: fix.command, + llmError: fix.llmError, + armBExit: fix.armBExit, + failureVanished: fix.failureVanished, + } + if (fix.command !== null) { + if (fix.failureVanished === null) { + fail( + caseId, + 'fix command present but failureVanished missing — arm B verdict was never recorded', + ) + } + return { outcome: fix.failureVanished ? 'flipped' : 'not-flipped', ...base } + } + if (fix.llmError !== null) return { outcome: 'generation-failed', ...base } + if (!fix.attempted || fix.sampledOut) return { outcome: 'not-attempted', ...base } + fail(caseId, 'unrecognized fix record state (attempted, no command, no llmError)') +} + +function truncateObservation( + observation: string | null | undefined, + maxChars: number, +): Pick { + if (observation === null || observation === undefined) { + return { observation: null, observationTruncated: false, observationChars: 0 } + } + if (observation.length <= maxChars) { + return { observation, observationTruncated: false, observationChars: observation.length } + } + return { + observation: observation.slice(0, maxChars), + observationTruncated: true, + observationChars: observation.length, + } +} + +/** + * Join one batch case with its gold label and trajectory into a row. + * Throws on any join inconsistency — never emits a partially joined row. + */ +export function buildVerifiedFindingRow(args: BuildVerifiedFindingRowArgs): VerifiedFindingRow { + const { batchCase, label, steps, detail } = args + const caseId = `${args.runId}/${batchCase.corpus}/${batchCase.trajId}` + const maxObservationChars = args.maxObservationChars ?? DEFAULT_MAX_OBSERVATION_CHARS + + if (batchCase.status !== 'ok') { + fail( + caseId, + `case status is '${batchCase.status}' (error: ${batchCase.error ?? 'none'}) — only ok cases join`, + ) + } + if (label.traj_id !== batchCase.trajId) { + fail(caseId, `label traj_id '${label.traj_id}' does not match the case`) + } + if (label.step_count !== batchCase.stepCount) { + fail(caseId, `label step_count ${label.step_count} != case stepCount ${batchCase.stepCount}`) + } + if (steps.length !== batchCase.stepCount) { + fail(caseId, `steps.json has ${steps.length} steps, case expects ${batchCase.stepCount}`) + } + for (let i = 0; i < steps.length; i++) { + const step = steps[i]! + if (step.step_id !== i + 1) { + fail(caseId, `steps.json is not contiguous 1..n: index ${i} has step_id ${step.step_id}`) + } + } + const k = batchCase.k + if (k < 1 || k > batchCase.stepCount) { + fail(caseId, `gold step k=${k} is outside 1..${batchCase.stepCount}`) + } + if (!batchCase.goldIncorrectSteps.includes(k)) { + fail( + caseId, + `gold step k=${k} is not in goldIncorrectSteps [${batchCase.goldIncorrectSteps.join(', ')}]`, + ) + } + const labelIncorrectSteps = [ + ...new Set(label.incorrect_stages.flatMap((s) => s.incorrect_step_ids)), + ].sort((a, b) => a - b) + for (const goldStep of batchCase.goldIncorrectSteps) { + if (!labelIncorrectSteps.includes(goldStep)) { + fail( + caseId, + `case gold step ${goldStep} is absent from the label's incorrect steps — label/report mismatch`, + ) + } + } + if (detail !== undefined) { + if (detail.k !== k) fail(caseId, `per-case verdict k=${detail.k} != report k=${k}`) + if (detail.prefixExecuted !== batchCase.prefixExecuted) { + fail( + caseId, + `per-case verdict prefixExecuted=${detail.prefixExecuted} != report ${batchCase.prefixExecuted}`, + ) + } + if (detail.recordedReturncode !== batchCase.recordedReturncodeAtK) { + fail( + caseId, + `per-case verdict recordedReturncode=${detail.recordedReturncode} != report ${batchCase.recordedReturncodeAtK}`, + ) + } + } + + const stepAtK = steps[k - 1]! + const trajectorySteps: TrajectoryStep[] = steps.slice(0, k).map((step) => ({ + stepId: step.step_id, + action: step.action, + ...truncateObservation(step.observation, maxObservationChars), + })) + + return { + schema: VERIFIED_FINDING_SCHEMA, + caseId, + corpus: batchCase.corpus, + trajId: batchCase.trajId, + task: { + agent: label.agent ?? null, + model: label.model ?? null, + taskName: label.task_name ?? null, + difficulty: label.difficulty ?? null, + solved: label.solved, + stepCount: batchCase.stepCount, + }, + gold: { + stepK: k, + actionAtK: stepAtK.action, + goldIncorrectSteps: [...batchCase.goldIncorrectSteps].sort((a, b) => a - b), + labelIncorrectSteps, + recordedReturncodeAtK: batchCase.recordedReturncodeAtK, + }, + trajectory: { + window: { start: 1, end: k }, + steps: trajectorySteps, + }, + verification: { + reproduced: batchCase.replayed, + signatureStrict: batchCase.armASignatureMatch, + signatureBasis: detail?.signatureBasis ?? null, + signature: batchCase.signature, + prefixExecuted: batchCase.prefixExecuted, + prefixDivergences: batchCase.prefixDivergences, + prefixDivergencePct: batchCase.prefixDivergencePct, + prefixDivergenceDetail: detail?.prefixDivergences ?? null, + armAExit: batchCase.armAExit, + armAReturncodeMatch: batchCase.armAReturncodeMatch, + armACommand: detail?.armACommand ?? null, + wallMs: batchCase.wallMs, + }, + fix: deriveFixOutcome(caseId, batchCase), + provenance: { + runId: args.runId, + batchGeneratedAt: args.batchGeneratedAt, + batchReportSha256: args.batchReportSha256, + labelsPath: args.labelsPath, + labelsSha256: args.labelsSha256, + stepsPath: args.stepsPath, + stepsSha256: args.stepsSha256, + image: batchCase.image, + derivedImage: batchCase.derivedImage, + cwd: batchCase.cwd, + cwdSource: batchCase.cwdSource, + originalRunId: detail?.runIds.original ?? null, + armARunId: detail?.runIds.armA ?? null, + }, + } +} + +export function summarizeVerifiedFindings(rows: VerifiedFindingRow[]): VerifiedFindingsSummary { + const summary: VerifiedFindingsSummary = { + rows: rows.length, + reproduced: 0, + signatureStrict: 0, + fix: { flipped: 0, 'not-flipped': 0, 'generation-failed': 0, 'not-attempted': 0 }, + byCorpus: {}, + } + for (const row of rows) { + if (row.verification.reproduced) summary.reproduced++ + if (row.verification.reproduced && row.verification.signatureStrict) summary.signatureStrict++ + summary.fix[row.fix.outcome]++ + let corpus = summary.byCorpus[row.corpus] + if (corpus === undefined) { + corpus = { rows: 0, reproduced: 0, fixFlipped: 0 } + summary.byCorpus[row.corpus] = corpus + } + corpus.rows++ + if (row.verification.reproduced) corpus.reproduced++ + if (row.fix.outcome === 'flipped') corpus.fixFlipped++ + } + return summary +} + +export function verifiedFindingsToJsonl(rows: VerifiedFindingRow[]): string { + return rows.map((row) => JSON.stringify(row)).join('\n') + (rows.length > 0 ? '\n' : '') +} + +// ── Filesystem loader ─────────────────────────────────────────────── + +export interface VerifiedFindingsCorpusSource { + labelsPath: string + /** Directory containing `normalized//steps.json`. */ + preparedDir: string +} + +export interface VerifiedFindingsSource { + batchReportPath: string + /** Batch run identifier embedded in every caseId, e.g. 'run2-20260802'. */ + runId: string + /** Corpus name (as it appears in the batch report) → label + trajectory locations. */ + corpora: Record + /** Batch run directory holding `--/replay-verdict.json`; when set, every case must have one. */ + runDir?: string + maxObservationChars?: number +} + +export interface VerifiedFindingsDataset { + rows: VerifiedFindingRow[] + summary: VerifiedFindingsSummary + provenance: { + runId: string + batchReportPath: string + batchReportSha256: string + batchGeneratedAt: string + corpora: Record + } +} + +function sha256(buffer: Buffer): string { + return createHash('sha256').update(buffer).digest('hex') +} + +function readJson(path: string, what: string): { value: unknown; sha256: string } { + let buffer: Buffer + try { + buffer = readFileSync(path) + } catch (error) { + throw new Error( + `verified-findings: cannot read ${what} at ${path}: ${(error as Error).message}`, + ) + } + try { + return { value: JSON.parse(buffer.toString('utf8')), sha256: sha256(buffer) } + } catch (error) { + throw new Error( + `verified-findings: ${what} at ${path} is not valid JSON: ${(error as Error).message}`, + ) + } +} + +interface CaseVerdictFile { + k: number + prefixExecuted: number + recordedReturncode: number + signatureBasis?: string | null + prefixDivergences?: PrefixDivergence[] + armA?: { command?: string | null } | null + runIds?: { original?: string | null; armA?: string | null } | null +} + +function loadCaseVerdictDetail(runDir: string, batchCase: ReplayBatchCase): CaseVerdictDetail { + const path = join(runDir, `${batchCase.corpus}--${batchCase.trajId}`, 'replay-verdict.json') + const parsed = readJson(path, `per-case verdict for ${batchCase.trajId}`).value as CaseVerdictFile + return { + k: parsed.k, + prefixExecuted: parsed.prefixExecuted, + recordedReturncode: parsed.recordedReturncode, + signatureBasis: parsed.signatureBasis ?? null, + prefixDivergences: parsed.prefixDivergences ?? [], + armACommand: parsed.armA?.command ?? null, + runIds: { + original: parsed.runIds?.original ?? null, + armA: parsed.runIds?.armA ?? null, + }, + } +} + +/** + * Load a replay-verify batch and join it into verified-finding rows. + * Every case in the report must join: an unresolvable corpus, a missing + * label entry, or a missing trajectory throws instead of dropping the row. + */ +export function loadVerifiedFindingsDataset( + source: VerifiedFindingsSource, +): VerifiedFindingsDataset { + const report = readJson(source.batchReportPath, 'batch report') + const parsedReport = report.value as ReplayBatchReport + if (!Array.isArray(parsedReport.cases) || parsedReport.cases.length === 0) { + throw new Error(`verified-findings: batch report at ${source.batchReportPath} has no cases`) + } + if (typeof parsedReport.generatedAt !== 'string' || parsedReport.generatedAt.length === 0) { + throw new Error( + `verified-findings: batch report at ${source.batchReportPath} has no generatedAt`, + ) + } + + const labelCache = new Map }>() + const corporaProvenance: VerifiedFindingsDataset['provenance']['corpora'] = {} + + const resolveCorpus = (corpus: string) => { + const config = source.corpora[corpus] + if (config === undefined) { + throw new Error( + `verified-findings: batch report references corpus '${corpus}' but no labels/preparedDir was configured for it`, + ) + } + let cached = labelCache.get(corpus) + if (cached === undefined) { + const labels = readJson(config.labelsPath, `labels for corpus '${corpus}'`) + const entries = labels.value as GoldLabelEntry[] + if (!Array.isArray(entries)) { + throw new Error( + `verified-findings: labels for corpus '${corpus}' at ${config.labelsPath} are not an array`, + ) + } + const byTrajId = new Map() + for (const entry of entries) { + if (byTrajId.has(entry.traj_id)) { + throw new Error( + `verified-findings: labels for corpus '${corpus}' contain duplicate traj_id '${entry.traj_id}'`, + ) + } + byTrajId.set(entry.traj_id, entry) + } + cached = { sha256: labels.sha256, byTrajId } + labelCache.set(corpus, cached) + corporaProvenance[corpus] = { + labelsPath: config.labelsPath, + labelsSha256: labels.sha256, + preparedDir: config.preparedDir, + } + } + return { config, ...cached } + } + + const rows: VerifiedFindingRow[] = [] + for (const batchCase of parsedReport.cases) { + const { config, sha256: labelsSha256, byTrajId } = resolveCorpus(batchCase.corpus) + const label = byTrajId.get(batchCase.trajId) + if (label === undefined) { + throw new Error( + `verified-findings: ${source.runId}/${batchCase.corpus}/${batchCase.trajId}: no label entry in ${config.labelsPath}`, + ) + } + const stepsPath = join(config.preparedDir, 'normalized', batchCase.trajId, 'steps.json') + const stepsFile = readJson(stepsPath, `trajectory steps for ${batchCase.trajId}`) + const steps = stepsFile.value as NormalizedStep[] + if (!Array.isArray(steps)) { + throw new Error(`verified-findings: trajectory steps at ${stepsPath} are not an array`) + } + const detail = + source.runDir === undefined ? undefined : loadCaseVerdictDetail(source.runDir, batchCase) + rows.push( + buildVerifiedFindingRow({ + batchCase, + label, + steps, + runId: source.runId, + batchGeneratedAt: parsedReport.generatedAt, + batchReportSha256: report.sha256, + labelsPath: config.labelsPath, + labelsSha256, + stepsPath, + stepsSha256: stepsFile.sha256, + detail, + maxObservationChars: source.maxObservationChars, + }), + ) + } + rows.sort((a, b) => a.caseId.localeCompare(b.caseId)) + + return { + rows, + summary: summarizeVerifiedFindings(rows), + provenance: { + runId: source.runId, + batchReportPath: source.batchReportPath, + batchReportSha256: report.sha256, + batchGeneratedAt: parsedReport.generatedAt, + corpora: corporaProvenance, + }, + } +}