feat(rl): verified-findings dataset — execution-verified labels as RL rows + the flywheel phase-2 spec

benchmarks/trace-analysis/tools/build-verified-dataset.mjs

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+// Build a verified-findings dataset from a replay-verify batch run.
+//
+// Joins batch-report.json (executed verdicts + fix arms) with the gold label
+// corpora and the normalized trajectories, then writes:
+//   <out>/rows.jsonl      one VerifiedFindingRow per replayable case
+//   <out>/manifest.json   summary + full source provenance + emitted-file shas
+//
+// Imports the package's own join from dist/ (build first: pnpm build).
+//
+// Usage:
+//   build-verified-dataset.mjs --report PATH --run-id ID --out DIR \
+//     --corpus NAME=LABELS_PATH::PREPARED_DIR [--corpus ...] \
+//     [--run-dir DIR] [--max-obs N]
+//
+// --run-dir points at the batch run directory holding per-case
+// `<corpus>--<trajId>/replay-verdict.json`; when given, every case must have
+// one (divergence detail + arm A command + run ids join into the rows).
+// Every join failure is fatal — a partially joined dataset is never written.
+
+import { createHash } from 'node:crypto'
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const dist = join(fileURLToPath(new URL('.', import.meta.url)), '..', '..', '..', 'dist', 'rl.js')
+const { loadVerifiedFindingsDataset, verifiedFindingsToJsonl, VERIFIED_FINDING_SCHEMA } = await import(dist)
+
+function parseArgs(argv) {
+  const args = { corpora: {}, maxObservationChars: undefined, runDir: undefined }
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i]
+    const value = () => {
+      const v = argv[++i]
+      if (v === undefined) throw new Error(`missing value for ${flag}`)
+      return v
+    }
+    if (flag === '--report') args.batchReportPath = value()
+    else if (flag === '--run-id') args.runId = value()
+    else if (flag === '--out') args.out = value()
+    else if (flag === '--run-dir') args.runDir = value()
+    else if (flag === '--max-obs') args.maxObservationChars = Number(value())
+    else if (flag === '--corpus') {
+      const spec = value()
+      const eq = spec.indexOf('=')
+      const sep = spec.indexOf('::')
+      if (eq < 1 || sep < eq) throw new Error(`--corpus expects NAME=LABELS_PATH::PREPARED_DIR, got: ${spec}`)
+      args.corpora[spec.slice(0, eq)] = {
+        labelsPath: spec.slice(eq + 1, sep),
+        preparedDir: spec.slice(sep + 2),
+      }
+    } else throw new Error(`unknown flag: ${flag}`)
+  }
+  for (const required of ['batchReportPath', 'runId', 'out']) {
+    if (!args[required]) throw new Error(`--${required.replace(/[A-Z]/g, (c) => `-${c.toLowerCase()}`)} is required`)
+  }
+  if (Object.keys(args.corpora).length === 0) throw new Error('at least one --corpus is required')
+  return args
+}
+
+const args = parseArgs(process.argv.slice(2))
+const dataset = loadVerifiedFindingsDataset({
+  batchReportPath: args.batchReportPath,
+  runId: args.runId,
+  corpora: args.corpora,
+  runDir: args.runDir,
+  maxObservationChars: args.maxObservationChars,
+})
+
+mkdirSync(args.out, { recursive: true })
+const jsonl = verifiedFindingsToJsonl(dataset.rows)
+const rowsPath = join(args.out, 'rows.jsonl')
+writeFileSync(rowsPath, jsonl)
+const rowsSha256 = createHash('sha256').update(readFileSync(rowsPath)).digest('hex')
+
+const manifest = {
+  schema: VERIFIED_FINDING_SCHEMA,
+  generatedAt: new Date().toISOString(),
+  summary: dataset.summary,
+  provenance: dataset.provenance,
+  files: { 'rows.jsonl': { sha256: rowsSha256, bytes: Buffer.byteLength(jsonl) } },
+}
+writeFileSync(join(args.out, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
+
+const s = dataset.summary
+console.log(`verified-findings dataset → ${args.out}`)
+console.log(`rows: ${s.rows} | reproduced: ${s.reproduced} | signature-strict: ${s.signatureStrict}`)
+console.log(
+  `fix: flipped ${s.fix.flipped}, not-flipped ${s.fix['not-flipped']}, generation-failed ${s.fix['generation-failed']}, not-attempted ${s.fix['not-attempted']}`,
+)
+for (const [corpus, c] of Object.entries(s.byCorpus)) {
+  console.log(`  ${corpus}: rows ${c.rows}, reproduced ${c.reproduced}, fix-flipped ${c.fixFlipped}`)
+}
+console.log(`rows.jsonl sha256 ${rowsSha256}`)

benchmarks/trace-analysis/verified-dataset-v0/README.md

@@ -0,0 +1,65 @@
+# verified-dataset-v0 — execution-verified gold labels as RL rows
+
+The first artifact of the verified-labels flywheel: gold "incorrect step" annotations that a replay-verify batch re-executed inside the original docker image, joined with their trajectories into trainer-ready rows.
+The label on every row was decided by execution (returncode/signature comparison at the gold step, plus a fix arm), not by a rater.
+The pipeline is the deliverable; rows scale with every future replay batch.
+Flywheel context and the phase-2 capture spec live in [`docs/verified-labels-flywheel.md`](../../../docs/verified-labels-flywheel.md).
+
+## Artifact (out of git — 1.4 MB, sha-pinned)
+
+Location: `~/bench-cache/ctb-20260801/verified-dataset-v0/`
+
+| file | sha256 | contents |
+| --- | --- | --- |
+| `rows.jsonl` | `afcbfb7b21c14868f8654bb27d113f42ba66fd951bcdc64b5f32b1e8b33641e3` | 22 `VerifiedFindingRow` lines (1,436,306 bytes) |
+| `manifest.json` | committed by the builder run | summary + source provenance + emitted-file shas |
+
+Source pins (embedded per row and in the manifest):
+
+| source | sha256 |
+| --- | --- |
+| `replay-batch/run2-20260802/batch-report.json` | `be6236dce691ea9df7c850040c995dd1f2274aaf559c45836c726bc480288fb3` |
+| `ctb-holdout-labels.json` (holdout-1) | `53af5ffe3962f3378f2d65419b92b8a56fe7d6c8efc619a0bc2b8f0872bc4f83` |
+| `ctb-holdout2-labels.json` (holdout-2) | `2db46579b7993edc376acbbcacf67a1d0ddfcdb94e28930c2bb8dfcf1dc32fb2` |
+| `split3/ctb-split3-labels.json` (split3) | `d0347ec7a5ec9a07bd3fcd16aa06b07bcb33ffabca39cb0b0f7a564fb500ae08` |
+
+Run2 composition (n=22): 16 reproduced, 13 signature-strict, fix arms 9 flipped / 2 not-flipped / 5 generation-failed / 6 not-attempted.
+Per corpus — holdout-1: 4 rows (2 reproduced, 1 flipped); holdout-2: 9 (9, 7); split3: 9 (5, 1).
+
+## Row schema — `agent-eval/verified-finding@0`
+
+One row per replayable case (JSONL). Full types: `src/rl/verified-findings-dataset.ts` (exported from `@tangle-network/agent-eval/rl`).
+
+| field | meaning |
+| --- | --- |
+| `caseId` | `<runId>/<corpus>/<trajId>` — unique across batches |
+| `task` | agent, model, task name, difficulty, solved, step count (from the gold label entry) |
+| `gold.stepK` | the verified gold step — earliest replayable incorrect step |
+| `gold.actionAtK` | exact command the agent ran at k (never truncated — it is the labeled object) |
+| `gold.goldIncorrectSteps` / `gold.labelIncorrectSteps` | replay targets vs every labeled incorrect step |
+| `gold.recordedReturncodeAtK` | returncode the original trajectory recorded at k |
+| `trajectory` | prefix steps 1..k (action + observation, observations truncated at 4000 chars with original length kept); post-k steps are excluded so a trainer never sees the future |
+| `verification.reproduced` | batch verdict: prefix divergence ≤ tolerance AND arm A reproduced the recorded returncode at k |
+| `verification.signatureStrict` | arm A also matched the failure signature (raw evidence — can be true on a non-reproduced case) |
+| `verification.prefixDivergenceDetail` | per-step `{step, expectedReturncode, actualExit}` from the per-case verdict |
+| `verification.armAExit` / `armACommand` | executed evidence at k |
+| `fix.outcome` | `flipped` \| `not-flipped` \| `generation-failed` \| `not-attempted` (batch report is authoritative; arm B exit carried) |
+| `provenance` | run id, batch/labels/steps sha256s, docker image + derived replay image, cwd, original/arm-A run ids |
+
+Join discipline: any missing or inconsistent join (label absent, step-count mismatch, k outside the gold set, arm B verdict missing on a fix command) throws — a partially joined dataset is never written.
+
+## Regenerate
+
+```bash
+pnpm build
+node benchmarks/trace-analysis/tools/build-verified-dataset.mjs \
+  --report ~/bench-cache/ctb-20260801/replay-batch/run2-20260802/batch-report.json \
+  --run-id run2-20260802 \
+  --run-dir ~/bench-cache/ctb-20260801/replay-batch/run2-20260802 \
+  --out ~/bench-cache/ctb-20260801/verified-dataset-v0 \
+  --corpus 'holdout-1=/home/drew/bench-cache/ctb-20260801/ctb-holdout-labels.json::/home/drew/bench-cache/ctb-20260801/ctb-holdout-prepared' \
+  --corpus 'holdout-2=/home/drew/bench-cache/ctb-20260801/ctb-holdout2-labels.json::/home/drew/bench-cache/ctb-20260801/ctb-holdout2-prepared' \
+  --corpus 'split3=/home/drew/bench-cache/ctb-20260801/split3/ctb-split3-labels.json::/home/drew/bench-cache/ctb-20260801/split3/ctb-split3-prepared'
+```
+
+The build is deterministic given the same inputs: `rows.jsonl` reproduces byte-identical (manifest `generatedAt` varies).

docs/verified-labels-flywheel.md

@@ -0,0 +1,46 @@
+# Verified-labels flywheel — own-traffic replay eligibility (phase-2 spec)
+
+Phase 1 shipped the dataset pipeline: `src/rl/verified-findings-dataset.ts` joins replay-verify batch verdicts with gold labels and trajectories into execution-verified RL rows (`agent-eval/verified-finding@0`, see `benchmarks/trace-analysis/verified-dataset-v0/README.md`).
+Those rows came from public benchmark trajectories (mini-SWE / CodeTraceBench).
+The flywheel's real fuel is our own traffic: fleet sessions run inside sandboxes where the image is known.
+This document maps which local session classes are replay-eligible today, which are not and why, and the concrete capture changes that make future sessions eligible.
+
+## What replay eligibility requires
+
+Derived from what the run2 replay batch actually consumed (its enumeration excluded 111/133 label entries):
+
+1. **Pinned environment** — a docker image (or digest) the trajectory ran in; `no-docker-image` alone excluded 21 entries.
+2. **Working directory** — the cwd commands were executed from.
+3. **Ordered step commands** — the exact action string per step; `no-swe-raw-trajectory` excluded 65 entries.
+4. **Per-step recorded returncodes** — needed for prefix-divergence checking (the replay batch aborts when >10% of prefix steps diverge from recorded returncodes).
+5. **A verifiable target step** — a finding/label on a *command* step (submit-only golds excluded 21 entries; findings on prose are not executable).
+
+## Local session stores surveyed (2026-08-03, this host)
+
+| store | volume | environment (req 1–2) | steps (req 3) | returncodes (req 4) | eligible today |
+| --- | --- | --- | --- | --- | --- |
+| Claude Code transcripts `~/.claude/projects` | 474 projects, 7,579 session JSONLs, 4.6 GB | no image; `cwd` + `gitBranch` + harness `version` per message | yes — full tool calls + results | **no** — Bash `toolUseResult` records `stdout`/`stderr`/`interrupted` only, no exit code field | no (host env unpinned, no rc) |
+| Codex sessions `~/.codex/sessions` | 4,332 rollout files, 113 GB | no image; `session_meta` has `cwd` + git `commit_hash`/`branch`/`repository_url` + `cli_version` | yes | shell events carry exit codes in payloads (format varies by version) | no (host env unpinned) |
+| traces CLI normalized envelope (`~/code/traces`, 18 harness adapters: claude, codex, amp, gemini, opencode, pi, copilot, factory, forge, qwen, …) | imports the two stores above | `cwd` filter exists; **no image/sandbox field in the envelope** | yes | adapter-dependent | no — the schema itself cannot express environment identity |
+| Sandbox sessions (agent-dev-container `PersistedSession`) | remote fleet; none stored on this host | runtime knows the image — `runtime.ready` event carries `image?` + `sandboxId` + backend — but `PersistedSession` persists only `workspaceRoot` + free-form `metadata`; `image?` is optional even on the event | yes (message store) | via tool parts, not normalized | **almost** — the image is in hand at runtime and dropped at persistence time |
+| mini-SWE / CodeTraceBench benchmark trajectories (`~/bench-cache/ctb-20260801`) | 133 labeled, 22 replayable | yes — `mswebench/*` images + cwd | yes | yes — `<returncode>N</returncode>` in every observation | **yes — the only eligible class; run2 proved 16/22 reproduce** |
+
+Conclusion: today only benchmark-imported trajectories are replay-eligible.
+Our own sessions fail on environment pinning (all classes) and returncode capture (Claude Code).
+The sandbox class is one persistence field away from eligibility — the runtime already knows the image.
+
+## Phase-2 capture changes (ranked by unlock per line of code)
+
+1. **Persist the sandbox image at session start** (agent-dev-container): copy `runtime.ready`'s `image` (as a digest, not a tag) + `sandboxId` into `PersistedSession` as first-class fields, and make `image` required on the event.
+   This single change makes every future fleet sandbox session satisfy requirements 1–2 — the highest-leverage line in the flywheel.
+2. **Add environment identity to the traces envelope** (traces repo): an optional `environment: { image?, imageDigest?, cwd, gitCommit? }` block on the normalized session, populated by adapters where known.
+   Without it, eligible sandbox sessions lose their eligibility at import time.
+3. **Record exit codes in Claude Code tool results**: the harness owns `toolUseResult`; until it carries `exitCode`, replay divergence checking cannot run on Claude transcripts even inside a pinned sandbox.
+   Workaround for sandboxed Claude sessions: derive returncodes from the sandbox's own command journal instead of the transcript.
+4. **Emit a replay descriptor per session** (the join target this package consumes): `{ image, cwd, steps: [{action, returncode}], findings: [{stepId, claim}] }` — exactly the shape `loadVerifiedFindingsDataset` joins today, so phase-3 needs no new pipeline code.
+
+## Why this matters
+
+Run2 measured: 72.7% of eligible trajectories reproduce their recorded failure at the gold step, and 81.8% of generated fixes flip it.
+Execution-verified labels at fleet scale are training data that cannot be bought — AgenTracer-8B showed +18pp from a specialist localizer trained on *unverified* labels; ours carry executed proof per row.
+Every capture change above turns a session class from "readable" into "verifiable", and the phase-1 pipeline converts verifiable sessions into dataset rows with zero new code.

src/rl/index.ts

@@ -43,6 +43,8 @@ export * from './sim-fidelity'
 export * from './tournament'
 /** @stable Verifiable reward extraction (compile / test / schema) with judge-noise filtering. */
 export * from './verifiable-reward'
+/** Execution-verified gold labels joined into RL-ready rows (replay-verify batches → trainer input). */
+export * from './verified-findings-dataset'
 
 // ── Deployment-outcome store (predictive-validity calibration) ──────
 // Promoted to public so external consumers don't have to inline the