From 932f8c46cf82ce088216479706a93deb0c11f241 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 16 Jul 2026 23:01:42 +0300 Subject: [PATCH] feat(savings): ship procedure foundry evidence ledger --- AGENTS.md | 8 +- CHANGELOG.md | 6 + README.md | 6 +- docs/COMMANDS.md | 6 +- docs/ci-integration.md | 2 +- llms-install.md | 4 +- skills/roam/SKILL.md | 2 +- src/roam/capability.py | 8 + src/roam/cli.py | 4 + src/roam/commands/cmd_at.py | 1 + src/roam/commands/cmd_brief.py | 1 + src/roam/commands/cmd_diff.py | 1 + src/roam/commands/cmd_grep.py | 1 + src/roam/commands/cmd_help_search.py | 1 + src/roam/commands/cmd_savings.py | 190 +++ src/roam/commands/cmd_savings_backfill.py | 137 ++ src/roam/commands/cmd_verify.py | 1 + src/roam/competitor_site_data.py | 2 +- src/roam/modes/policy.py | 5 + src/roam/procedure_mining.py | 1013 ++++++++++++ src/roam/savings.py | 1210 ++++++++++++++ src/roam/transcript_backfill.py | 1406 +++++++++++++++++ .../distribution/landing-page/changelog.html | 7 + .../landing-page/docs/agent-contract.html | 2 +- .../docs/integration-tutorials.html | 2 +- templates/distribution/landing-page/llms.txt | 2 +- .../distribution/landing-page/pricing.html | 2 +- .../distribution/landing-page/setup.html | 4 +- tests/test_procedure_mining.py | 259 +++ tests/test_savings.py | 309 ++++ tests/test_savings_aggregate.py | 86 + tests/test_transcript_backfill.py | 406 +++++ 32 files changed, 5074 insertions(+), 20 deletions(-) create mode 100644 src/roam/commands/cmd_savings.py create mode 100644 src/roam/commands/cmd_savings_backfill.py create mode 100644 src/roam/procedure_mining.py create mode 100644 src/roam/savings.py create mode 100644 src/roam/transcript_backfill.py create mode 100644 tests/test_procedure_mining.py create mode 100644 tests/test_savings.py create mode 100644 tests/test_savings_aggregate.py create mode 100644 tests/test_transcript_backfill.py diff --git a/AGENTS.md b/AGENTS.md index 49c25e82b..648b9419c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,11 +5,11 @@ roam-code is a local codebase intelligence CLI for developers and AI coding agents. It pre-indexes symbols, call graphs, dependencies, architecture, and git history into -a local SQLite DB. **279 commands · 244 MCP tools (16 in the default `core` preset) · 28 languages · 100% local · zero API keys.** +a local SQLite DB. **281 commands · 244 MCP tools (16 in the default `core` preset) · 28 languages · 100% local · zero API keys.** -Authoritative counts (AST-derived, env-independent): `command_count: 279 · canonical_count: 272 · category_count: 7 · mcp tools registered: 244 · mcp tools in core preset: 16`. The `roam surface --json` envelope additionally exposes `mcp_tool_count_by_preset` for per-preset counts. +Authoritative counts (AST-derived, env-independent): `command_count: 281 · canonical_count: 274 · category_count: 7 · mcp tools registered: 244 · mcp tools in core preset: 16`. The `roam surface --json` envelope additionally exposes `mcp_tool_count_by_preset` for per-preset counts. **Package:** `roam-code` on PyPI. Entry point: `roam.cli:cli`. @@ -151,7 +151,7 @@ roam health ``` src/roam/ - cli.py # Click CLI entry point — LazyGroup, _COMMANDS dict, _CATEGORIES. 279 command names (272 canonical + 7 aliases). + cli.py # Click CLI entry point — LazyGroup, _COMMANDS dict, _CATEGORIES. 281 command names (274 canonical + 7 aliases). mcp_server.py # FastMCP server (16 tools in core preset; 244 in `full`) + `roam mcp` CLI command mcp_extras/ # MCP-native enhancements: sampling, watcher, session, progress, completions sampling.py # Sampling-driven result compression (summarize=True) via Context.sample @@ -711,7 +711,7 @@ Index-aware text search (added on top of grep / refs): - `roam delete-check [--source working|staged|pr|head] [--ci]` — gates the diff on surviving references; exits 5 on BREAK-RISK with `--ci`. - `roam history-grep [--polarity]` — git pickaxe (-S/-G) with author/date and introduced/removed annotation. -Run `roam --help` for the 5-verb core; `roam --help-all` for all 279 command names; `roam surface --json` for the machine-readable inventory. Use `roam --json ` for structured output. +Run `roam --help` for the 5-verb core; `roam --help-all` for all 281 command names; `roam surface --json` for the machine-readable inventory. Use `roam --json ` for structured output. Use `roam --sarif health` for CI integration (SARIF 2.1.0). ## Compiler tooling (2026-06-02 wave) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a4a0c8f7..8c116312a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added +- **`roam savings`** — materializes prospective hook events, compile records, and value-redacted historical transcript episodes into a local SQLite ledger; reports coverage and sensor canaries before allowing any measurement or routing claim. Repeated patterns remain candidate-only until joined live outcomes satisfy the declared evidence gates. +- **`roam savings-backfill`** — derives replaceable historical discovery episodes from Claude and Codex transcript stores without persisting raw prompts, responses, paths, source, or shell-command values. The backfill keeps closed templates, fingerprints, counts, timing buckets, and friction signals for Procedure Foundry ranking. +- **Producer-side `roam savings --aggregate` privacy boundary** — emits only fixed summary fields, scalar coverage, closed-state counts, and aggregate opportunity counts for platform dashboards. Transcript-derived pattern text, task prefixes, commands, paths, identifiers, and per-episode rows are structurally excluded. +- **Capability displacement metadata** — `@roam_capability(displaces=(...))` declares which repeated-work family a capability is intended to replace. Procedure Foundry reads the declarations through AST without importing lazy command modules; declarations nominate prospective interventions but never claim adoption or savings. + ### Fixed - **`roam capsule --redact-paths` no longer leaks the directory tree via cluster labels** — the "zero-source" architecture capsule is meant for external review *without* sharing source, and it correctly hashes every symbol's `file` path, but `clusters[].label` (which is derived from a file/directory path) was emitted verbatim — leaking the exact tree the flag exists to anonymize (e.g. `workflows/checkout`, `tests/test_cmd_pr_replay_...`). Cluster labels are now hashed with the same per-component scheme as symbol paths under `--redact-paths`. Surfaced by real dogfooding of the redaction path. - **`roam --json postmortem` no longer corrupts its own JSON on stdout** — the `click.progressbar("Replaying detectors")` wrote its label to stdout ahead of the JSON, so `roam --json postmortem > out.json` produced an unparseable file. The progress chrome now goes to stderr (matching `pr-replay`), leaving stdout pure JSON. diff --git a/README.md b/README.md index c3b0df945..6c2772b61 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Credential-free · 100% local by default (opt-in `metrics-push` is the only outbound surface) · tamper-evident `ChangeEvidence` packets · Apache 2.0 · runs entirely on your machine -279 commands · 244 MCP tools (16 in the default `core` preset) · 28 languages +281 commands · 244 MCP tools (16 in the default `core` preset) · 28 languages ![roam terminal demo](docs/assets/roam-terminal-demo.gif) @@ -417,7 +417,7 @@ Roam's surfaces differ in how rigorously they've been validated — know which i ## Core commands -**Lead with the 5 verbs.** The [5 core commands](#core-commands) cover ~80% of agent workflows: `understand`, `context`, `retrieve`, `preflight`, `critique`. The remaining ~274 commands are detail surface for specialised workflows (taint, fleet, cga, oracle, eval, …) — they're called by agents on demand, not memorised. This is intentional design; under the hood the canonical surface is **279 commands (272 canonical + 7 aliases) organised into 7 categories** (aliases for muscle memory: `math` → `algo`, `churn` → `weather`, `digest` / `snapshot` / `trend` → `trends`, `onboard` → `understand`, `refs` → `uses`), but you don't need to know that to start. +**Lead with the 5 verbs.** The [5 core commands](#core-commands) cover ~80% of agent workflows: `understand`, `context`, `retrieve`, `preflight`, `critique`. The remaining ~276 commands are detail surface for specialised workflows (taint, fleet, cga, oracle, eval, …) — they're called by agents on demand, not memorised. This is intentional design; under the hood the canonical surface is **281 commands (274 canonical + 7 aliases) organised into 7 categories** (aliases for muscle memory: `math` → `algo`, `churn` → `weather`, `digest` / `snapshot` / `trend` → `trends`, `onboard` → `understand`, `refs` → `uses`), but you don't need to know that to start. | Verb | What it does | @@ -432,7 +432,7 @@ The full surface spans **7 categories** — Getting Started, Daily Workflow, Cod
-Full command reference — canonical command list (all 272) +Full command reference — canonical command list (all 274) The complete, always-current list with flags and examples lives in the [Command Reference](https://roam-code.com/docs/command-reference). diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index ec8e6d81e..89e17fbfb 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -2,7 +2,7 @@ > **Generated — do not hand-edit.** Regenerate with `python scripts/build_commands_doc.py`. Kept in sync by `tests/test_commands_doc_synced.py` (a command dropped from this index, or a new command left undocumented, fails CI — the reconcile-survival invariant). -**279 commands** (272 canonical + aliases) across 7 categories · 244 MCP tools · roam v13.8.0 +**281 commands** (274 canonical + aliases) across 7 categories · 244 MCP tools · roam v13.8.0 ## Getting Started (41) @@ -50,7 +50,7 @@ | `workflow` | stable | ✓ | — | | `ws` | internal | — | — | -## Daily Workflow (86) +## Daily Workflow (88) | Command | Maturity | MCP | Aliases | |---------|----------|-----|---------| @@ -127,6 +127,8 @@ | `retrieve` | stable | ✓ | — | | `rules-validate` | stable | ✓ | — | | `runs` | stable | — | — | +| `savings` | stable | — | — | +| `savings-backfill` | stable | — | — | | `semantic-diff` | stable | ✓ | — | | `service-report` | stable | — | — | | `skill-generate` | stable | — | — | diff --git a/docs/ci-integration.md b/docs/ci-integration.md index 3c49fba16..3d6a1b8d5 100644 --- a/docs/ci-integration.md +++ b/docs/ci-integration.md @@ -324,7 +324,7 @@ Any roam command can be passed via the `commands` input. Common choices: | `breaking` | Detect breaking API changes | | `conventions` | Naming convention violations | -Run `roam --help` for all 279 commands. +Run `roam --help` for all 281 commands. ## Exit Codes diff --git a/llms-install.md b/llms-install.md index 70d0e17df..cd36ae6e2 100644 --- a/llms-install.md +++ b/llms-install.md @@ -4,7 +4,7 @@ roam-code is the local CLI that runs pre-change gates before every agent edit and compiles tamper-evident, content-hashed evidence packets after every change. 100% local, no API keys, no telemetry. -279 commands, 244 MCP tools, 28 languages, 100% local, zero API keys. +281 commands, 244 MCP tools, 28 languages, 100% local, zero API keys. ## Cross-references @@ -262,5 +262,5 @@ and `roam surface --json` for the machine-readable inventory. -Run `roam --help` for all 279 commands (+ alias pairs). +Run `roam --help` for all 281 commands (+ alias pairs). diff --git a/skills/roam/SKILL.md b/skills/roam/SKILL.md index 30ba611c1..06ec7dd36 100644 --- a/skills/roam/SKILL.md +++ b/skills/roam/SKILL.md @@ -196,7 +196,7 @@ roam minimap --update # update sentinel block in CLAUDE.md ## Discovering More Commands -This skill covers the most common commands, but roam has 279 commands. +This skill covers the most common commands, but roam has 281 commands. To explore what's available: ```bash diff --git a/src/roam/capability.py b/src/roam/capability.py index a47599b46..c6e0dbe1d 100644 --- a/src/roam/capability.py +++ b/src/roam/capability.py @@ -65,6 +65,9 @@ class Capability: task_required: bool = False # required for task plans (validate_plan etc.) destructive: bool = False # rm-style operations stale_sensitive: bool = True # results invalidate when index goes stale + # Closed repeated-work hypotheses this capability is intended to replace. + # Declarations are discovery metadata, never proof of adoption or savings. + displaces: tuple[str, ...] = () @dataclass @@ -125,6 +128,7 @@ def as_registry_catalog(self) -> dict[str, Any]: "task_required": cap.task_required, "destructive": cap.destructive, "stale_sensitive": cap.stale_sensitive, + "displaces": list(cap.displaces), } for cap in self.all() ], @@ -162,6 +166,7 @@ def roam_capability( task_required: bool = False, destructive: bool = False, stale_sensitive: bool = True, + displaces: tuple[str, ...] | list[str] = (), ) -> Callable[[F], F]: """Mark a Click command as a capability for introspection. @@ -197,6 +202,7 @@ def deco(func: F) -> F: task_required=task_required, destructive=destructive, stale_sensitive=stale_sensitive, + displaces=tuple(displaces), ) REGISTRY.register(cap) # Stash on the function for later introspection (e.g. roam capabilities --explain X) @@ -262,6 +268,8 @@ def emit_yaml() -> str: lines.append(f" - {_yaml_str(ex)}") if cap["tags"]: lines.append(f" tags: [{', '.join(_yaml_str(t) for t in cap['tags'])}]") + if cap["displaces"]: + lines.append(f" displaces: [{', '.join(_yaml_str(t) for t in cap['displaces'])}]") lines.append(f" ai_safe: {str(cap['ai_safe']).lower()}") lines.append(f" requires_index: {str(cap['requires_index']).lower()}") if cap["since"]: diff --git a/src/roam/cli.py b/src/roam/cli.py index 322d48a52..a5b902ebe 100644 --- a/src/roam/cli.py +++ b/src/roam/cli.py @@ -313,6 +313,8 @@ def _deprecation_record(name: str) -> dict | None: "plan": ("roam.commands.cmd_plan", "plan"), "compile": ("roam.commands.cmd_compile", "compile_"), "compile-stats": ("roam.commands.cmd_compile_stats", "compile_stats"), + "savings": ("roam.commands.cmd_savings", "savings"), + "savings-backfill": ("roam.commands.cmd_savings_backfill", "savings_backfill"), "compile-cache": ("roam.commands.cmd_compile_cache", "compile_cache"), "compile-daemon": ("roam.commands.cmd_compile_daemon", "compile_daemon"), "envelope-diff": ("roam.commands.cmd_envelope_diff", "envelope_diff"), @@ -574,6 +576,8 @@ def _deprecation_record(name: str) -> dict | None: "plan", "compile", "compile-stats", + "savings", + "savings-backfill", "compile-cache", "envelope-diff", "dispatch-trace", diff --git a/src/roam/commands/cmd_at.py b/src/roam/commands/cmd_at.py index e335b1f73..dcae166aa 100644 --- a/src/roam/commands/cmd_at.py +++ b/src/roam/commands/cmd_at.py @@ -95,6 +95,7 @@ def _callers_of(conn, symbol_id: int, cap: int = 8) -> list[str]: outputs=("location_context_envelope",), examples=("roam at src/roam/cli.py:42", "roam at src/roam/plan/compiler.py:6730 --context 8 --callers"), tags=("exploration", "location", "context"), + displaces=("repeated_code_slicing",), ) @click.command(name="at") @click.argument("location") diff --git a/src/roam/commands/cmd_brief.py b/src/roam/commands/cmd_brief.py index 220de09a0..fe182daf7 100644 --- a/src/roam/commands/cmd_brief.py +++ b/src/roam/commands/cmd_brief.py @@ -620,6 +620,7 @@ def _render_text( task_required=False, destructive=False, stale_sensitive=True, + displaces=("orientation_tax",), ) @click.command(name="brief") @click.option( diff --git a/src/roam/commands/cmd_diff.py b/src/roam/commands/cmd_diff.py index 0d1d3b2cf..2045e3fdd 100644 --- a/src/roam/commands/cmd_diff.py +++ b/src/roam/commands/cmd_diff.py @@ -384,6 +384,7 @@ def _check_naming_rule_scoped(rule, conn, changed_fids): task_required=False, destructive=False, stale_sensitive=False, + displaces=("post_edit_context_recovery",), ) @click.command("diff") @click.argument("commit_range", required=False, default=None) diff --git a/src/roam/commands/cmd_grep.py b/src/roam/commands/cmd_grep.py index 20bf55eb7..fa6568d85 100644 --- a/src/roam/commands/cmd_grep.py +++ b/src/roam/commands/cmd_grep.py @@ -280,6 +280,7 @@ def _read_patterns_file(p: Path) -> list[str]: stale_sensitive=True, ai_safe=True, requires_index=True, + displaces=("search_inspect_thrash", "repeated_code_slicing"), ) @click.command("grep") @click.argument("positional", required=False) diff --git a/src/roam/commands/cmd_help_search.py b/src/roam/commands/cmd_help_search.py index ed786c093..222213b89 100644 --- a/src/roam/commands/cmd_help_search.py +++ b/src/roam/commands/cmd_help_search.py @@ -66,6 +66,7 @@ def _score(query: str, name: str, help_text: str) -> int: stale_sensitive=False, ai_safe=True, requires_index=False, + displaces=("command_discoverability",), ) @click.command(name="help-search") @click.argument("query", nargs=-1, required=True) diff --git a/src/roam/commands/cmd_savings.py b/src/roam/commands/cmd_savings.py new file mode 100644 index 000000000..cef8de958 --- /dev/null +++ b/src/roam/commands/cmd_savings.py @@ -0,0 +1,190 @@ +"""Measure compiler savings only when episode evidence is admissible. + +SARIF is deliberately NOT emitted because savings evidence is episode- and +experiment-scoped rather than a file-located code finding. There are no source +coordinates that could populate SARIF ``locations[]`` without inventing them. +""" + +from __future__ import annotations + +import click + +from roam.capability import roam_capability +from roam.output.formatter import json_envelope, to_json +from roam.savings import ( + EVENT_FIELD_DEFINITIONS, + aggregate_savings_payload, + analyze_ledger, +) + + +@click.command(name="savings") +@click.option( + "--root", + default=".", + show_default=True, + help="Project root containing .roam/episodes.jsonl and .roam/compile-runs.jsonl.", +) +@click.option("--schema", is_flag=True, help="Print the episode-event field contract and exit.") +@click.option( + "--aggregate", + is_flag=True, + help="Emit only browser-safe scalar coverage and counts; excludes pattern and episode values.", +) +@click.pass_context +@roam_capability( + name="savings", + category="planning", + summary="Gate compiler-savings claims on joined prompt, compile, and terminal episode evidence.", + inputs=("--root",), + outputs=("summary_envelope", "coverage", "repeat_candidates"), + examples=("roam savings", "roam --json savings --root /path/to/project"), + tags=("planning", "telemetry", "compiler", "episodes", "savings"), + ai_safe=True, + requires_index=False, + mcp_expose=False, + mcp_preset=(), + side_effect=True, + stale_sensitive=False, +) +def savings(ctx: click.Context, root: str, schema: bool, aggregate: bool) -> None: + """Materialize the local episode ledger and report admissible savings evidence.""" + json_mode = ctx.obj.get("json") if ctx.obj else False + token_budget = int(ctx.obj.get("budget", 0) or 0) if ctx.obj else 0 + if schema: + if json_mode: + click.echo( + to_json( + json_envelope( + "savings", + budget=token_budget, + event_schema={"fields": EVENT_FIELD_DEFINITIONS}, + ) + ) + ) + else: + click.echo("episodes.jsonl event schema:") + for field, meaning in EVENT_FIELD_DEFINITIONS.items(): + click.echo(f" {field:<18s} {meaning}") + return + + result = analyze_ledger(root) + if aggregate: + aggregate_payload = aggregate_savings_payload(result) + aggregate_summary = aggregate_payload.pop("summary") + click.echo( + to_json( + json_envelope( + "savings", + summary=aggregate_summary, + budget=token_budget, + **aggregate_payload, + ) + ) + ) + return + + result = dict(result) + summary = result.pop("summary") + coverage = result.get("coverage") or {} + next_commands = ["roam hooks claude --write", "roam savings"] + if summary["measurement_admissible"]: + next_commands = ["roam compile-stats --by-procedure", "roam savings"] + + if json_mode: + click.echo( + to_json( + json_envelope( + "savings", + summary=summary, + agent_contract={ + "facts": [ + f"{coverage.get('prompt_starts', 0)} prompt-start events", + f"{coverage.get('terminal_outcomes', 0)} terminal outcome records", + f"{coverage.get('fully_joined_episodes', 0)} fully joined event records", + f"{len(result.get('historical_candidates', []))} historical candidate patterns", + f"{len((result.get('procedure_atlas') or {}).get('opportunities', []))} ranked work findings", + f"{sum(row.get('declaration_state') == 'unclaimed' for row in (result.get('procedure_atlas') or {}).get('intervention_mappings', []))} unclaimed intervention capabilities", + f"{result['sensor_canaries']['passed']} sensor canaries passed", + ], + "next_commands": next_commands, + "risks": ( + [] + if summary["policy_admissible"] + else ["Savings claims remain gated by incomplete joined episodes"] + ), + "confidence": None, + }, + budget=token_budget, + **result, + ) + ) + ) + return + + click.echo(f"VERDICT: {summary['verdict']}") + click.echo(f"state: {summary['state']}") + click.echo(f"prompt starts: {coverage.get('prompt_starts', 0)}") + click.echo(f"eligible prompt starts: {coverage.get('eligible_prompt_starts', 0)}") + click.echo(f"terminal coverage: {coverage.get('terminal_coverage_pct')}%") + click.echo(f"compile + terminal join: {coverage.get('episode_join_coverage_pct')}%") + click.echo(f"compile identity coverage: {coverage.get('compile_identity_coverage_pct')}%") + click.echo(f"execution-health coverage: {coverage.get('health_context_coverage_pct')}%") + click.echo( + f"sensor canaries: {result['sensor_canaries']['passed']}/" + f"{result['sensor_canaries']['total']} {result['sensor_canaries']['state']}" + ) + if result["repeat_candidates"]: + click.echo("") + click.echo("Repeated candidates (observed, not promoted):") + for candidate in result["repeat_candidates"][:10]: + click.echo( + f" {candidate['episodes']:>3} episodes {candidate['procedure']:<22s} {candidate['task_prefix'][:70]}" + ) + if result["historical_candidates"]: + click.echo("") + click.echo("Historical patterns (candidate-only):") + for candidate in result["historical_candidates"][:10]: + click.echo(f" {candidate['episodes']:>4} episodes {candidate['kind']:<15s} {candidate['pattern'][:100]}") + opportunities = (result.get("procedure_atlas") or {}).get("opportunities") or [] + if opportunities: + click.echo("") + click.echo("Cross-project work opportunities (historical, non-causal):") + for opportunity in opportunities[:10]: + click.echo( + f" {opportunity['opportunity_score']:>5.1f} " + f"{opportunity['episodes']:>5} episodes " + f"{opportunity['projects']:>4} projects " + f"{opportunity['title']}" + ) + failure_signatures = (result.get("procedure_atlas") or {}).get("failure_signatures") or [] + if failure_signatures: + click.echo("") + click.echo("Sanitized failure signatures (historical, non-causal):") + for signature in failure_signatures[:10]: + click.echo( + f" {signature['failures']:>5}/{signature['attempts']:<5} failed " + f"{signature['projects']:>4} projects " + f"{signature['template'][:100]}" + ) + recovery_targets = (result.get("procedure_atlas") or {}).get("recovery_targets") or [] + if recovery_targets: + click.echo("") + click.echo("Closed failure-recovery targets (raw results discarded):") + for target in recovery_targets[:10]: + click.echo( + f" {target['failures']:>5} failures {target['projects']:>4} projects {target['failure_class']}" + ) + intervention_mappings = (result.get("procedure_atlas") or {}).get("intervention_mappings") or [] + if intervention_mappings: + click.echo("") + click.echo("Intervention hypotheses (all prospectively unmeasured):") + for gap in intervention_mappings[:10]: + click.echo( + f" {gap['research_priority_score']:>5.1f} " + f"{gap['declaration_state']:<18} " + f"{gap['projects']:>4} projects " + f"{gap['title']}" + ) + click.echo("") + click.echo("Run `roam hooks claude --write` to refresh the episode-aware hooks.") diff --git a/src/roam/commands/cmd_savings_backfill.py b/src/roam/commands/cmd_savings_backfill.py new file mode 100644 index 000000000..3998eb403 --- /dev/null +++ b/src/roam/commands/cmd_savings_backfill.py @@ -0,0 +1,137 @@ +"""Derive privacy-preserving discovery episodes from local agent transcripts. + +SARIF is deliberately NOT emitted because transcript backfill reports +corpus-level extraction coverage, not file-located code findings. Transcript +paths are intentionally excluded from the output privacy contract. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import click + +from roam.capability import roam_capability +from roam.output.formatter import json_envelope, to_json +from roam.transcript_backfill import backfill_transcripts + + +def _since_value(_ctx: click.Context, _param: click.Parameter, value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value).replace(tzinfo=timezone.utc) + except ValueError as exc: + raise click.BadParameter("use YYYY-MM-DD") from exc + + +@click.command(name="savings-backfill") +@click.option( + "--transcripts-dir", + required=True, + multiple=True, + type=click.Path(exists=True, file_okay=False), + help="Transcript root; repeat to merge Claude and Codex stores.", +) +@click.option("--root", default=".", show_default=True, help="Project root receiving the derived snapshot.") +@click.option( + "--source", + type=click.Choice(["auto", "claude", "codex"], case_sensitive=False), + default="auto", + show_default=True, +) +@click.option("--since", callback=_since_value, help="Include transcript files modified on/after YYYY-MM-DD.") +@click.option( + "--max-files", + type=click.IntRange(min=0), + default=0, + show_default=True, + help="Newest files to scan per --transcripts-dir; 0 scans all.", +) +@click.option( + "--all-projects", + is_flag=True, + help="Include every cwd in the source tree; default keeps only episodes under --root.", +) +@click.option("--dry-run", is_flag=True, help="Scan and report without writing transcript-episodes.jsonl.") +@click.pass_context +@roam_capability( + name="savings-backfill", + category="planning", + summary="Extract privacy-preserving historical discovery episodes from Claude or Codex transcripts.", + inputs=("--transcripts-dir", "--root", "--source", "--since", "--max-files"), + outputs=("summary_envelope", "privacy_contract", "episode_counts"), + examples=( + "roam savings-backfill --transcripts-dir ~/.claude/projects", + "roam savings-backfill --transcripts-dir ~/.codex/sessions --source codex", + ), + tags=("planning", "telemetry", "transcripts", "privacy", "savings"), + ai_safe=True, + requires_index=False, + mcp_expose=False, + mcp_preset=(), + side_effect=True, + stale_sensitive=False, +) +def savings_backfill( + ctx: click.Context, + transcripts_dir: tuple[str, ...], + root: str, + source: str, + since: datetime | None, + max_files: int, + all_projects: bool, + dry_run: bool, +) -> None: + """Write a value-redacted historical episode snapshot for repeated-pattern discovery.""" + token_budget = int(ctx.obj.get("budget", 0) or 0) if ctx.obj else 0 + result = backfill_transcripts( + root, + transcripts_dir, + source=source.lower(), + since=since, + max_files=max_files, + all_projects=all_projects, + dry_run=dry_run, + ) + verdict = ( + f"Derived {result['episodes']} historical discovery episodes from " + f"{result['files_with_episodes']} transcript files" + ) + summary = { + "verdict": verdict, + "state": result["state"], + "partial_success": bool(result["unknown_format_files"]), + "episodes": result["episodes"], + "files_with_episodes": result["files_with_episodes"], + "policy_admissible": False, + } + json_mode = ctx.obj.get("json") if ctx.obj else False + if json_mode: + click.echo( + to_json( + json_envelope( + "savings-backfill", + summary=summary, + agent_contract={ + "facts": [ + f"{result['episodes']} historical discovery events", + f"{result['files_with_episodes']} transcript files", + "0 raw prompt, response, path, or shell-command texts persisted", + ], + "next_commands": ["roam savings"], + "risks": ["Historical proxy outcomes require prospective validation before policy promotion"], + "confidence": None, + }, + budget=token_budget, + **result, + ) + ) + ) + return + click.echo(f"VERDICT: {verdict}") + click.echo(f"events: {result['events']}") + click.echo(f"unknown formats: {result['unknown_format_files']}") + click.echo(f"output: {result['output']}") + click.echo("privacy: raw values excluded; sanitized shell templates retained") + click.echo("Run `roam savings` to rank historical candidates separately from live evidence.") diff --git a/src/roam/commands/cmd_verify.py b/src/roam/commands/cmd_verify.py index 5a2dec8ba..20508ea4f 100644 --- a/src/roam/commands/cmd_verify.py +++ b/src/roam/commands/cmd_verify.py @@ -5779,6 +5779,7 @@ def _dispatch_verify_command( stale_sensitive=True, ai_safe=True, requires_index=True, + displaces=("verification_retry",), ) @click.command() @click.option( diff --git a/src/roam/competitor_site_data.py b/src/roam/competitor_site_data.py index 0448f0956..fda1221e4 100644 --- a/src/roam/competitor_site_data.py +++ b/src/roam/competitor_site_data.py @@ -1361,7 +1361,7 @@ "relationship": "self", "peer": True, "graph": "PageRank + Tarjan + Louvain + layers", - "note": "Graph algorithms (PageRank, SCC, Louvain, Fiedler) on tree-sitter ASTs fused with git history in SQLite. 244 MCP tools, 279 CLI commands. 19 Python idiom detectors (v12.7+). 54 algo detectors (12.40). Audit-trail commands: permit / postmortem / article-12-check.", + "note": "Graph algorithms (PageRank, SCC, Louvain, Fiedler) on tree-sitter ASTs fused with git history in SQLite. 244 MCP tools, 281 CLI commands. 19 Python idiom detectors (v12.7+). 54 algo detectors (12.40). Audit-trail commands: permit / postmortem / article-12-check.", "version_evaluated": "12.48", "repo_url": "https://github.com/Cranot/roam-code", }, diff --git a/src/roam/modes/policy.py b/src/roam/modes/policy.py index 402139bcd..c2b9d1e47 100644 --- a/src/roam/modes/policy.py +++ b/src/roam/modes/policy.py @@ -158,6 +158,11 @@ "bench-compile", # W40 — compile-stats reads .roam/compile-runs.jsonl; no edits. "compile-stats", + # Procedure Foundry reads local episode telemetry and writes only + # replaceable `.roam/` ledger/snapshot state. This is the same + # local-state boundary as compile-cache and compile-daemon. + "savings", + "savings-backfill", # W56 — compile-cache (group: stats/clear/build); writes to # .roam/compile-envelope-cache.sqlite, which is local-state. "compile-cache", diff --git a/src/roam/procedure_mining.py b/src/roam/procedure_mining.py new file mode 100644 index 000000000..0a1a89450 --- /dev/null +++ b/src/roam/procedure_mining.py @@ -0,0 +1,1013 @@ +"""Mine private transcript telemetry for general agent-work opportunities. + +The module ranks observed friction and repeated intent clusters. Its counters +are deliberately named ``associated_*`` or ``addressable_*``: historical +telemetry can identify work that an intervention may remove, but it cannot +establish causal savings. +""" + +from __future__ import annotations + +import ast +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from statistics import median +from typing import Any, Callable + + +def _int(value: Any) -> int: + try: + return max(0, int(value)) + except (TypeError, ValueError): + return 0 + + +def _friction(episode: dict[str, Any], key: str) -> int: + values = episode.get("friction") + return _int(values.get(key)) if isinstance(values, dict) else 0 + + +_STRUCTURED_PROJECTION_RE = re.compile( + r"(?:^|(?:&&|\|\||;)\s*)roam\b[^;]*" + r"(?:\|\s*(?:jq|python(?:3)?|head|tail|sed|awk|cut|select-object)\b|head\s+-c\b)", + re.IGNORECASE, +) + + +def _structured_projection_calls(episode: dict[str, Any]) -> int: + recorded = _friction(episode, "structured_output_postprocess_calls") + if recorded: + return recorded + templates = episode.get("shell_templates") + if not isinstance(templates, dict): + return 0 + return sum(_int(count) for template, count in templates.items() if _STRUCTURED_PROJECTION_RE.search(str(template))) + + +_NO_EDIT_EXPECTED_ARCHETYPES = frozenset({"review", "research", "document", "plan"}) + + +def _high_tool_no_edit_transition(episode: dict[str, Any]) -> int: + """Count long no-edit work only when the observed intent is not passive.""" + if _int(episode.get("edit_actions")) or _int(episode.get("tool_calls")) < 8: + return 0 + archetypes = _archetype_set(episode) + if not archetypes or archetypes.issubset(_NO_EDIT_EXPECTED_ARCHETYPES): + return 0 + return 1 + + +def normalized_episode_tokens(episode: dict[str, Any]) -> int: + """Normalize provider token fields without double-counting cached input.""" + input_tokens = _int(episode.get("input_tokens")) + output_tokens = _int(episode.get("output_tokens")) + cached_tokens = _int(episode.get("cached_input_tokens")) + cache_creation_tokens = _int(episode.get("cache_creation_tokens")) + reasoning_tokens = _int(episode.get("reasoning_output_tokens")) + source = str(episode.get("transcript_source") or "") + if source == "claude": + normalized_input = input_tokens + cached_tokens + cache_creation_tokens + elif source == "codex": + normalized_input = input_tokens + else: + normalized_input = max(input_tokens, cached_tokens + cache_creation_tokens) + return normalized_input + output_tokens + reasoning_tokens + + +def _pct(numerator: int, denominator: int) -> float | None: + if denominator <= 0: + return None + return round(100.0 * numerator / denominator, 1) + + +def _median(values: list[int]) -> int | None: + return int(median(values)) if values else None + + +@dataclass(frozen=True) +class _Opportunity: + key: str + title: str + signal_definition: str + intervention_shape: str + counter: Callable[[dict[str, Any]], int] + leverage: float + precision: str = "direct" + + +_OPPORTUNITIES = ( + _Opportunity( + "repeated_code_slicing", + "Repeated code-window reconstruction", + "Line-window inspection calls beyond the first call in one episode", + "Return one location-aware structural context envelope", + lambda ep: max(0, _friction(ep, "slice_calls") - 1), + 1.0, + ), + _Opportunity( + "search_inspect_thrash", + "Search and inspection thrash", + "Observed search→inspect→search or inspect→search→inspect cycles", + "Combine ranked search hits with bounded structural context", + lambda ep: _friction(ep, "search_inspect_cycles"), + 1.0, + ), + _Opportunity( + "exact_shell_replay", + "Repeated shell-template execution", + "Exact sanitized shell-template executions beyond the first execution", + "Cache, parameterize, or compile the repeated deterministic operation", + lambda ep: _friction(ep, "exact_shell_replays"), + 0.9, + ), + _Opportunity( + "output_postprocessing", + "Structured-output post-processing", + "Roam shell calls that pipe structured output into projection or truncation utilities", + "Expose a native bounded projection at the producing tool boundary", + _structured_projection_calls, + 1.0, + ), + _Opportunity( + "orientation_tax", + "Repeated workspace orientation", + "Workspace-orientation calls beyond the first call in one episode", + "Inject compact dirty-tree, branch, and project identity facts", + lambda ep: max(0, _friction(ep, "orientation_calls") - 1), + 0.85, + ), + _Opportunity( + "verification_retry", + "Repeated verification selection", + "Verification or build attempts beyond the first attempt in one episode", + "Select and order the smallest executable verification ladder", + lambda ep: _friction(ep, "verification_retries"), + 1.0, + ), + _Opportunity( + "failed_action_retry", + "Failed action replay", + "Failed actions retried with the same shell template within three actions", + "Diagnose the failure before retrying or repair the command contract", + lambda ep: _friction(ep, "failed_action_retries"), + 0.95, + ), + _Opportunity( + "post_edit_context_recovery", + "Context reacquisition after editing", + "Orientation, search, inspection, or intelligence calls after the first edit and before verification", + "Carry forward the edit-local context and changed-symbol closure", + lambda ep: _friction(ep, "post_edit_context_calls"), + 0.95, + ), + _Opportunity( + "command_discoverability", + "Repeated command-help lookup", + "Observed shell invocations containing --help or -h", + "Compile the selected command signature and one executable example", + lambda ep: _friction(ep, "help_calls"), + 0.8, + ), + _Opportunity( + "user_correction_loop", + "User correction after an agent episode", + "The next user turn begins with a correction marker", + "Mine the preceding trajectory as a negative procedure example", + lambda ep: int(bool(ep.get("correction_after"))), + 1.0, + "proxy", + ), + _Opportunity( + "tool_failure_recovery", + "Tool failure and recovery work", + "Tool results classified as failed in the episode", + "Route known failure signatures to deterministic recovery procedures", + lambda ep: _int(ep.get("tool_errors")), + 0.9, + ), + _Opportunity( + "high_tool_no_edit", + "Long exploration with no edit", + "No-edit episodes containing at least eight tool calls after excluding " + "unknown and passive-only intent archetypes", + "Return a stopping verdict or a compressed research brief earlier", + _high_tool_no_edit_transition, + 0.75, + "conditional", + ), + _Opportunity( + "large_result_handling", + "Large tool-result handling", + "Episodes receiving at least 64 KiB of bucketed tool-result content", + "Summarize, page, or project large results before they enter context", + lambda ep: int(_int(ep.get("tool_result_bytes_bucket")) >= 65_536), + 0.9, + "conditional", + ), +) + + +_PLATFORM_DISPLACEMENT_CLAIMS: tuple[dict[str, str], ...] = ( + { + "capability": "global --select", + "opportunity": "output_postprocessing", + "support": "native", + "contract": "Project JSON envelopes before token budgeting without jq or Python.", + }, + { + "capability": "automatic response handles", + "opportunity": "large_result_handling", + "support": "native", + "contract": "Page or persist oversized command results before they enter agent context.", + }, + { + "capability": "compile-cache", + "opportunity": "exact_shell_replay", + "support": "partial", + "contract": "Cache compiled agent procedures; arbitrary deterministic shell replay remains uncovered.", + }, +) + +_TRANSITION_ELIGIBILITY: dict[str, str] = { + "repeated_code_slicing": ( + "A live location or search hit exists and a later action reads source before the next edit or verification." + ), + "search_inspect_thrash": ("A search and inspection transition repeats before any edit or terminal verdict."), + "exact_shell_replay": ( + "The same deterministic sanitized template repeats with equivalent inputs and no intervening state mutation." + ), + "output_postprocessing": ("A Roam JSON result is immediately projected or truncated by another process."), + "orientation_tax": ("Workspace identity was already observed and no branch/cwd mutation occurred."), + "verification_retry": ("A verification attempt repeats or widens after a result that can be classified."), + "failed_action_retry": ("A failed action is replayed before a diagnosis or relevant state change."), + "post_edit_context_recovery": ( + "An edit occurred and the agent re-reads facts available from the changed-symbol closure." + ), + "command_discoverability": ("A help lookup follows an intent that can be mapped to a registered capability."), + "user_correction_loop": ("A correction follows an exposed intervention with joined intent and terminal evidence."), + "tool_failure_recovery": ("A failure class has validated precision and a deterministic recovery precondition."), + "high_tool_no_edit": ( + "A non-passive closed intent archetype is present; unknown, research-only, " + "review-only, planning-only, and documentation-only tasks are excluded." + ), + "large_result_handling": ( + "A result crosses the size threshold before projection, pagination, or file persistence." + ), +} + + +@lru_cache(maxsize=1) +def declared_displacement_claims() -> tuple[dict[str, str], ...]: + """AST-read command displacement contracts without importing commands. + + Lazy command loading is performance-critical, so the Foundry must not + import hundreds of command modules merely to learn coverage. The + ``@roam_capability(displaces=(...))`` literals are parsed directly from + source and combined with platform-wide claims such as ``--select``. + """ + claims: list[dict[str, str]] = [dict(row) for row in _PLATFORM_DISPLACEMENT_CLAIMS] + commands_dir = Path(__file__).resolve().parent / "commands" + for path in sorted(commands_dir.glob("cmd_*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError, UnicodeError): + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func_name = ( + node.func.id + if isinstance(node.func, ast.Name) + else node.func.attr + if isinstance(node.func, ast.Attribute) + else "" + ) + if func_name != "roam_capability": + continue + keywords = {kw.arg: kw.value for kw in node.keywords if kw.arg} + displaces_node = keywords.get("displaces") + if displaces_node is None: + continue + try: + displaced = ast.literal_eval(displaces_node) + except (ValueError, TypeError): + continue + if isinstance(displaced, str): + displaced = (displaced,) + if not isinstance(displaced, (tuple, list)): + continue + capability = path.stem.removeprefix("cmd_").replace("_", "-") + name_node = keywords.get("name") + if name_node is not None: + try: + explicit_name = ast.literal_eval(name_node) + except (ValueError, TypeError): + explicit_name = None + if isinstance(explicit_name, str) and explicit_name: + capability = explicit_name + for opportunity in displaced: + if isinstance(opportunity, str) and opportunity: + claims.append( + { + "capability": capability, + "opportunity": opportunity, + "support": "native", + "contract": (f"Capability `{capability}` declares this repeated-work family."), + } + ) + claims.sort( + key=lambda row: ( + row["opportunity"], + row["support"], + row["capability"], + ) + ) + return tuple(claims) + + +def _intervention_mapping_rows( + opportunities: list[dict[str, Any]], + claims: tuple[dict[str, str], ...] | None = None, +) -> list[dict[str, Any]]: + """Map declared interventions without pretending they close the gap. + + Historical support determines research priority. Declarations only say + which intervention should enter a prospective test; they do not discount + priority until live episodes establish adoption and non-inferior outcomes. + """ + coverage_claims = claims if claims is not None else declared_displacement_claims() + claims_by_opportunity: dict[str, list[dict[str, str]]] = defaultdict(list) + for claim in coverage_claims: + claims_by_opportunity[str(claim.get("opportunity") or "")].append(dict(claim)) + rows: list[dict[str, Any]] = [] + for opportunity in opportunities: + key = str(opportunity.get("opportunity") or "") + matching = claims_by_opportunity.get(key, []) + supports = {str(claim.get("support") or "") for claim in matching} + if "native" in supports: + declaration_state = "declared_native" + elif "partial" in supports: + declaration_state = "declared_partial" + else: + declaration_state = "unclaimed" + opportunity_score = float(opportunity.get("opportunity_score") or 0.0) + rows.append( + { + "opportunity": key, + "title": opportunity.get("title"), + "declaration_state": declaration_state, + "effectiveness_state": "unmeasured", + "research_priority_score": opportunity_score, + "residual_gap_score": None, + "opportunity_score": opportunity_score, + "organic_episode_estimate": _int(opportunity.get("organic_episode_estimate")), + "organic_addressable_actions": _int(opportunity.get("organic_addressable_actions")), + "projects": _int(opportunity.get("projects")), + "capability_claims": matching, + "next_architecture_move": opportunity.get("intervention_shape"), + "evidence_status": "declared_intervention_mapping_only", + "causal_savings_claimed": False, + } + ) + rows.sort( + key=lambda row: ( + -row["research_priority_score"], + -row["organic_addressable_actions"], + row["opportunity"], + ) + ) + return rows + + +def _intervention_test_rows( + opportunities: list[dict[str, Any]], + mappings: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Build prospective tests around transitions, not command counts.""" + mapping_by_key = {str(row.get("opportunity") or ""): row for row in mappings} + rows = [] + for opportunity in opportunities: + key = str(opportunity.get("opportunity") or "") + mapping = mapping_by_key.get(key, {}) + rows.append( + { + "transition": key, + "eligibility_rule": _TRANSITION_ELIGIBILITY.get( + key, + "A preconditioned repeated transition is observed in a joined live episode.", + ), + "baseline_signal": opportunity.get("signal_definition"), + "intervention_candidates": mapping.get("capability_claims") or [], + "exposure_state": "not_instrumented", + "effectiveness_state": "unmeasured", + "primary_metric": ("eligible downstream transition count per joined episode"), + "experimental_design": { + "assignment_unit": "session_id", + "analysis_unit": "eligible_episode", + "assignment_strategy": ("randomized clustered holdout or pre-registered stepped wedge"), + "analysis_population": "intent_to_treat", + "control_condition": ( + "candidate capability remains available but is not surfaced as the assigned intervention" + ), + "exposed_condition": ( + "the assigned intervention is explicitly surfaced before the eligible transition" + ), + "required_event_pair": [ + "intervention_assignment", + "intervention_observation", + ], + "stratification_fields": [ + "project_id", + "intent_archetype", + "agent_surface", + "baseline_task_burden", + ], + "contamination_guard": ("keep one assignment per session and intervention version"), + }, + "resource_metrics": [ + "tool_calls", + "normalized_tokens", + "episode_wall_ms", + "tool_result_bytes", + ], + "outcome_guards": [ + "terminal_success_rate", + "verification_pass_rate", + "user_correction_rate", + "partial_success_rate", + ], + "minimum_promotion_gate": { + "minimum_exposed_episodes": 30, + "minimum_control_episodes": 30, + "join_coverage_pct": 95.0, + "power_analysis_required": True, + "minimum_relative_transition_reduction_pct": 10.0, + "transition_effect_confidence_level_pct": 95.0, + "transition_effect_interval_must_exclude_zero": True, + "terminal_success_non_inferiority_margin_pp": 2.0, + "verification_pass_non_inferiority_margin_pp": 2.0, + "user_correction_inferiority_margin_pp": 2.0, + "partial_success_inferiority_margin_pp": 2.0, + "sequential_testing_policy": ("fixed horizon or pre-registered alpha-spending rule"), + "require_transition_reduction": True, + "require_outcome_non_inferiority": True, + }, + "causal_savings_claimed": False, + } + ) + rows.sort( + key=lambda row: ( + -float(mapping_by_key.get(row["transition"], {}).get("research_priority_score", 0.0)), + row["transition"], + ) + ) + return rows + + +def _opportunity_score(row: dict[str, Any], max_episodes: int, max_actions: int) -> float: + episode_scale = math.log1p(row["organic_episode_estimate"]) / max(1.0, math.log1p(max_episodes)) + action_scale = math.log1p(row["organic_addressable_actions"]) / max(1.0, math.log1p(max_actions)) + project_scale = min(1.0, row["projects"] / 12.0) + burden_scale = min(1.0, (row["median_tool_calls"] or 0) / 20.0) + correction_scale = min(1.0, (row["correction_pct"] or 0.0) / 20.0) + score = 100.0 * ( + 0.25 * episode_scale + + 0.20 * action_scale + + 0.20 * project_scale + + 0.10 * burden_scale + + 0.10 * correction_scale + + 0.15 * float(row["implementation_leverage"]) + ) + return round(score, 1) + + +def _likely_automated_episode_ids(episodes: list[dict[str, Any]]) -> set[str]: + """Identify repeated exact prompts that look like scripted/eval cohorts.""" + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for episode in episodes: + fingerprint = str(episode.get("prompt_hmac_sha256") or "") + if fingerprint: + grouped[fingerprint].append(episode) + automated: set[str] = set() + for rows in grouped.values(): + sessions = {str(row.get("session_id") or "") for row in rows} + sessions.discard("") + if len(rows) >= 20 and len(sessions) >= 20: + automated.update(str(row.get("episode_id") or "") for row in rows) + automated.discard("") + return automated + + +def _opportunity_rows( + episodes: list[dict[str, Any]], + likely_automated_ids: set[str], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for spec in _OPPORTUNITIES: + selected: list[tuple[dict[str, Any], int]] = [] + for episode in episodes: + count = spec.counter(episode) + if count > 0: + selected.append((episode, count)) + if not selected: + continue + members = [episode for episode, _count in selected] + organic_selected = [ + (episode, count) + for episode, count in selected + if str(episode.get("episode_id") or "") not in likely_automated_ids + ] + organic_members = [episode for episode, _count in organic_selected] + projects = {str(ep.get("project_id") or "") for ep in organic_members} + projects.discard("") + analysis_members = organic_members or members + sources = Counter(str(ep.get("transcript_source") or "unknown") for ep in analysis_members) + archetypes = Counter( + str(archetype) + for episode in analysis_members + for archetype in ( + episode.get("intent_archetypes") if isinstance(episode.get("intent_archetypes"), list) else [] + ) + ) + tokens = [normalized_episode_tokens(ep) for ep in members] + tools = [_int(ep.get("tool_calls")) for ep in members] + organic_tokens = [normalized_episode_tokens(ep) for ep in organic_members] + organic_tools = [_int(ep.get("tool_calls")) for ep in organic_members] + analysis_tokens = [normalized_episode_tokens(ep) for ep in analysis_members] + analysis_tools = [_int(ep.get("tool_calls")) for ep in analysis_members] + wall = [_int(ep.get("duration_ms")) for ep in members if ep.get("duration_ms") is not None] + rows.append( + { + "opportunity": spec.key, + "title": spec.title, + "signal_definition": spec.signal_definition, + "intervention_shape": spec.intervention_shape, + "precision": spec.precision, + "episodes": len(members), + "organic_episode_estimate": len(organic_members), + "likely_automated_episodes": len(members) - len(organic_members), + "automation_contamination_pct": _pct( + len(members) - len(organic_members), + len(members), + ), + "projects": len(projects), + "addressable_actions": sum(count for _episode, count in selected), + "organic_addressable_actions": sum(count for _episode, count in organic_selected), + "associated_tool_calls": sum(tools), + "associated_tokens": sum(tokens), + "organic_associated_tool_calls": sum(organic_tools), + "organic_associated_tokens": sum(organic_tokens), + "associated_episode_wall_ms": sum(wall), + "median_tool_calls": _median(analysis_tools), + "median_tokens": _median(analysis_tokens), + "median_episode_wall_ms": _median(wall), + "correction_pct": _pct( + sum(bool(ep.get("correction_after")) for ep in analysis_members), + len(analysis_members), + ), + "tool_error_pct": _pct( + sum(_int(ep.get("tool_errors")) > 0 for ep in analysis_members), + len(analysis_members), + ), + "verified_edit_pct": _pct( + sum(str(ep.get("outcome") or "") == "historical_acted_verified_proxy" for ep in analysis_members), + len(analysis_members), + ), + "sources": dict(sources.most_common()), + "top_intent_archetypes": dict(archetypes.most_common(6)), + "implementation_leverage": spec.leverage, + "evidence_status": "historical_opportunity_only", + } + ) + max_episodes = max((row["organic_episode_estimate"] for row in rows), default=1) + max_actions = max((row["organic_addressable_actions"] for row in rows), default=1) + for row in rows: + row["opportunity_score"] = _opportunity_score(row, max_episodes, max_actions) + rows.sort( + key=lambda row: ( + -row["opportunity_score"], + -row["projects"], + -row["organic_addressable_actions"], + row["opportunity"], + ) + ) + return rows + + +def _behavior_signature(rows: list[dict[str, Any]]) -> dict[str, Any]: + trajectories = Counter(str(row.get("phase_sequence_template") or "") for row in rows) + trajectories.pop("", None) + shell_patterns = Counter( + str(pattern) + for row in rows + for pattern in (row.get("shell_templates").keys() if isinstance(row.get("shell_templates"), dict) else []) + ) + archetypes = Counter( + str(archetype) + for row in rows + for archetype in (row.get("intent_archetypes") if isinstance(row.get("intent_archetypes"), list) else []) + ) + modal_phase, modal_count = trajectories.most_common(1)[0] if trajectories else ("", 0) + return { + "modal_phase_sequence": modal_phase, + "modal_phase_sequence_pct": _pct(modal_count, len(rows)), + "top_shell_templates": [pattern for pattern, _count in shell_patterns.most_common(3)], + "top_intent_archetypes": dict(archetypes.most_common(4)), + } + + +def _exact_intent_clusters(episodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for episode in episodes: + fingerprint = str(episode.get("prompt_hmac_sha256") or "") + if fingerprint: + grouped[fingerprint].append(episode) + clusters: list[dict[str, Any]] = [] + for fingerprint, rows in grouped.items(): + sessions = {str(row.get("session_id") or "") for row in rows} + sessions.discard("") + if len(rows) < 3 or len(sessions) < 2: + continue + projects = {str(row.get("project_id") or "") for row in rows} + projects.discard("") + outcomes = Counter(str(row.get("outcome") or "unknown") for row in rows) + tools = [_int(row.get("tool_calls")) for row in rows] + tokens = [normalized_episode_tokens(row) for row in rows] + clusters.append( + { + "intent_fingerprint": fingerprint, + "episodes": len(rows), + "sessions": len(sessions), + "projects": len(projects), + "associated_tool_calls": sum(tools), + "associated_tokens": sum(tokens), + "median_tool_calls": _median(tools), + "median_tokens": _median(tokens), + "correction_pct": _pct(sum(bool(row.get("correction_after")) for row in rows), len(rows)), + "automation_likelihood": ("high" if len(rows) >= 20 and len(sessions) >= 20 else "low"), + "outcomes": dict(outcomes.most_common()), + "evidence_status": "historical_repeated_intent_only", + **_behavior_signature(rows), + } + ) + clusters.sort( + key=lambda row: ( + -row["associated_tool_calls"], + -row["episodes"], + row["intent_fingerprint"], + ) + ) + return clusters[:50] + + +class _UnionFind: + def __init__(self, size: int) -> None: + self.parent = list(range(size)) + self.rank = [0] * size + + def find(self, value: int) -> int: + while self.parent[value] != value: + self.parent[value] = self.parent[self.parent[value]] + value = self.parent[value] + return value + + def union(self, left: int, right: int) -> None: + left_root = self.find(left) + right_root = self.find(right) + if left_root == right_root: + return + if self.rank[left_root] < self.rank[right_root]: + left_root, right_root = right_root, left_root + self.parent[right_root] = left_root + if self.rank[left_root] == self.rank[right_root]: + self.rank[left_root] += 1 + + +def _archetype_set(episode: dict[str, Any]) -> set[str]: + values = episode.get("intent_archetypes") + if not isinstance(values, list): + return set() + return {str(value) for value in values if value and value != "other"} + + +def _near_intent_clusters(episodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Cluster conservative paraphrases using keyed SimHash locality.""" + eligible = [ + episode + for episode in episodes + if str(episode.get("intent_simhash64") or "") not in {"", "0000000000000000"} + and _int(episode.get("prompt_tokens_bucket")) >= 25 + and _archetype_set(episode) + ] + if len(eligible) < 3: + return [] + hashes: list[int] = [] + filtered: list[dict[str, Any]] = [] + for episode in eligible: + try: + value = int(str(episode["intent_simhash64"]), 16) + except (TypeError, ValueError): + continue + hashes.append(value) + filtered.append(episode) + union_find = _UnionFind(len(filtered)) + buckets: dict[tuple[int, int], list[int]] = defaultdict(list) + for index, value in enumerate(hashes): + for band in range(4): + buckets[(band, (value >> (band * 16)) & 0xFFFF)].append(index) + compared: set[tuple[int, int]] = set() + for members in buckets.values(): + # Very large LSH buckets represent generic language rather than a + # useful procedure neighborhood. Exact-intent mining still covers them. + if len(members) > 500: + continue + for offset, left in enumerate(members): + left_archetypes = _archetype_set(filtered[left]) + for right in members[offset + 1 :]: + pair = (left, right) if left < right else (right, left) + if pair in compared: + continue + compared.add(pair) + if not left_archetypes.intersection(_archetype_set(filtered[right])): + continue + if (hashes[left] ^ hashes[right]).bit_count() <= 4: + union_find.union(left, right) + grouped: dict[int, list[dict[str, Any]]] = defaultdict(list) + for index, episode in enumerate(filtered): + grouped[union_find.find(index)].append(episode) + clusters: list[dict[str, Any]] = [] + for rows in grouped.values(): + exact_intents = {str(row.get("prompt_hmac_sha256") or "") for row in rows} + exact_intents.discard("") + sessions = {str(row.get("session_id") or "") for row in rows} + sessions.discard("") + if len(rows) < 3 or len(exact_intents) < 2 or len(sessions) < 2: + continue + projects = {str(row.get("project_id") or "") for row in rows} + projects.discard("") + tools = [_int(row.get("tool_calls")) for row in rows] + tokens = [normalized_episode_tokens(row) for row in rows] + outcomes = Counter(str(row.get("outcome") or "unknown") for row in rows) + hashes_in_cluster = sorted(str(row.get("intent_simhash64") or "") for row in rows) + clusters.append( + { + "intent_cluster_id": f"sim_{hashes_in_cluster[0]}", + "episodes": len(rows), + "distinct_exact_intents": len(exact_intents), + "sessions": len(sessions), + "projects": len(projects), + "associated_tool_calls": sum(tools), + "associated_tokens": sum(tokens), + "median_tool_calls": _median(tools), + "median_tokens": _median(tokens), + "correction_pct": _pct(sum(bool(row.get("correction_after")) for row in rows), len(rows)), + "automation_likelihood": ("high" if len(rows) >= 20 and len(sessions) >= 20 else "medium"), + "outcomes": dict(outcomes.most_common()), + "evidence_status": "historical_near_intent_only", + **_behavior_signature(rows), + } + ) + clusters.sort( + key=lambda row: ( + -row["associated_tool_calls"], + -row["episodes"], + row["intent_cluster_id"], + ) + ) + return clusters[:50] + + +def _template_outcome_rankings( + episodes: list[dict[str, Any]], + likely_automated_ids: set[str], +) -> dict[str, list[dict[str, Any]]]: + aggregates: dict[str, Counter[str]] = defaultdict(Counter) + episode_sets: dict[str, set[str]] = defaultdict(set) + project_sets: dict[str, set[str]] = defaultdict(set) + recovery_counts: Counter[str] = Counter() + recovery_templates: dict[str, set[str]] = defaultdict(set) + recovery_projects: dict[str, set[str]] = defaultdict(set) + for episode in episodes: + if str(episode.get("episode_id") or "") in likely_automated_ids: + continue + outcomes = episode.get("shell_template_outcomes") + if not isinstance(outcomes, dict): + continue + for template, values in outcomes.items(): + if not template or not isinstance(values, dict): + continue + aggregate = aggregates[str(template)] + for key in ( + "attempts", + "failures", + "no_results", + "retries_after_failure", + "result_bytes_bucket", + ): + aggregate[key] += _int(values.get(key)) + failure_classes = values.get("failure_classes") + if isinstance(failure_classes, dict): + for failure_class, count in failure_classes.items(): + label = str(failure_class or "") + amount = _int(count) + if not label or amount <= 0: + continue + aggregate[f"failure_class:{label}"] += amount + recovery_counts[label] += amount + recovery_templates[label].add(str(template)) + project_id = str(episode.get("project_id") or "") + if project_id: + recovery_projects[label].add(project_id) + episode_sets[str(template)].add(str(episode.get("episode_id") or "")) + project_id = str(episode.get("project_id") or "") + if project_id: + project_sets[str(template)].add(project_id) + rows: list[dict[str, Any]] = [] + for template, counts in aggregates.items(): + attempts = counts["attempts"] + if attempts < 3: + continue + failures = counts["failures"] + no_results = counts["no_results"] + result_bytes = counts["result_bytes_bucket"] + projects = len(project_sets[template]) + rows.append( + { + "template": template, + "episodes": len(episode_sets[template] - {""}), + "projects": projects, + "attempts": attempts, + "failures": failures, + "failure_rate_pct": _pct(failures, attempts), + "no_results": no_results, + "failure_rate_excluding_no_results_pct": _pct( + failures, + max(0, attempts - no_results), + ), + "retries_after_failure": counts["retries_after_failure"], + "failure_classes": { + key.removeprefix("failure_class:"): value + for key, value in sorted(counts.items()) + if key.startswith("failure_class:") and value > 0 + }, + "associated_bucketed_result_bytes": result_bytes, + "failure_priority_score": round( + math.log1p(failures) + * (1.0 + min(2.0, projects / 10.0)) + * (1.0 + min(1.0, counts["retries_after_failure"] / 20.0)), + 2, + ), + "result_volume_score": round( + math.log1p(result_bytes) * (1.0 + min(2.0, projects / 10.0)), + 2, + ), + "evidence_status": "historical_template_outcome_only", + } + ) + failure_signatures = [row for row in rows if row["failures"] > 0] + failure_signatures.sort( + key=lambda row: ( + -row["failure_priority_score"], + -row["failures"], + -row["projects"], + row["template"], + ) + ) + result_producers = [row for row in rows if row["associated_bucketed_result_bytes"] > 0] + result_producers.sort( + key=lambda row: ( + -row["result_volume_score"], + -row["associated_bucketed_result_bytes"], + -row["projects"], + row["template"], + ) + ) + recovery_targets = [ + { + "failure_class": failure_class, + "failures": failures, + "templates": len(recovery_templates[failure_class]), + "projects": len(recovery_projects[failure_class]), + "evidence_status": "closed_failure_class_only", + "raw_result_content_persisted": False, + "classification_status": "heuristic_unvalidated", + "routing_eligible": False, + } + for failure_class, failures in recovery_counts.items() + ] + recovery_targets.sort( + key=lambda row: ( + -row["failures"], + -row["projects"], + -row["templates"], + row["failure_class"], + ) + ) + return { + "failure_signatures": failure_signatures[:100], + "large_result_producers": result_producers[:100], + "recovery_targets": recovery_targets, + } + + +def build_procedure_atlas(episodes: list[dict[str, Any]]) -> dict[str, Any]: + """Return cross-project opportunity and repeated-intent evidence.""" + historical = [ + episode + for episode in episodes + if episode.get("evidence_source") == "transcript_backfill" and episode.get("terminal") + ] + projects = {str(episode.get("project_id") or "") for episode in historical} + projects.discard("") + sources = Counter(str(episode.get("transcript_source") or "unknown") for episode in historical) + likely_automated_ids = _likely_automated_episode_ids(historical) + template_outcomes = _template_outcome_rankings( + historical, + likely_automated_ids, + ) + opportunities = _opportunity_rows(historical, likely_automated_ids) + intervention_mappings = _intervention_mapping_rows(opportunities) + intervention_tests = _intervention_test_rows( + opportunities, + intervention_mappings, + ) + return { + "summary": { + "verdict": ( + f"{len(historical)} historical episodes across {len(projects)} project identities " + "rank private procedure opportunities" + ), + "episodes": len(historical), + "projects": len(projects), + "sources": dict(sources.most_common()), + "likely_automated_episodes": len(likely_automated_ids), + "organic_episode_estimate": len(historical) - len(likely_automated_ids), + "causal_savings_claimed": False, + "declared_native_interventions": sum( + row["declaration_state"] == "declared_native" for row in intervention_mappings + ), + "declared_partial_interventions": sum( + row["declaration_state"] == "declared_partial" for row in intervention_mappings + ), + "unclaimed_intervention_families": sum( + row["declaration_state"] == "unclaimed" for row in intervention_mappings + ), + }, + "opportunities": opportunities, + "intervention_mappings": intervention_mappings, + "intervention_tests": intervention_tests, + "exact_intent_clusters": _exact_intent_clusters(historical), + "near_intent_clusters": _near_intent_clusters(historical), + **template_outcomes, + "definitions": { + "addressable_actions": ( + "directly counted historical actions matching the opportunity signal; not actions proven removable" + ), + "associated_tokens": ( + "provider-normalized input plus output/reasoning tokens in episodes " + "containing the signal; not tokens attributable to the signal" + ), + "opportunity_score": ( + "non-causal prioritization score combining episode support, action support, " + "cross-project support, episode burden, correction rate, and implementation leverage; " + "support terms use the organic estimate" + ), + "likely_automated_episode": ( + "episode whose exact keyed intent repeats across at least 20 episodes " + "and 20 sessions; a contamination proxy, not proof of automation" + ), + "near_intent_cluster": ( + "keyed 64-bit intent SimHashes within Hamming distance 4, sharing " + "a closed intent archetype, spanning at least two exact intents and sessions" + ), + "failure_signature": ( + "sanitized shell-template attempts, failed results, and same-template " + "retries aggregated from transcript tool results" + ), + "recovery_target": ( + "closed failure-reason labels classified transiently from tool results; " + "raw result content is discarded; historical regex labels remain " + "routing-ineligible until validated" + ), + "intervention_test": ( + "prospective transition-level test requiring explicit exposure, joined " + "terminal outcomes, transition reduction, and outcome non-inferiority" + ), + "associated_bucketed_result_bytes": ( + "sum of 4 KiB-rounded result-size buckets; output content is not persisted" + ), + "declaration_state": ( + "declared_native or declared_partial when an intervention hypothesis " + "exists, otherwise unclaimed; declarations do not establish coverage" + ), + "research_priority_score": ("historical opportunity score, intentionally undiscounted by declarations"), + "residual_gap_score": ( + "null until prospective episodes establish intervention adoption and " + "outcome non-inferiority; only then may residual priority be estimated" + ), + }, + } diff --git a/src/roam/savings.py b/src/roam/savings.py new file mode 100644 index 000000000..10289110d --- /dev/null +++ b/src/roam/savings.py @@ -0,0 +1,1210 @@ +"""Proof-oriented episode ledger for compiler savings measurement. + +The hook boundary writes append-only JSONL because it must stay tiny, +dependency-free, and fail-open. Analysis materializes those events plus +``compile-runs.jsonl`` into a local SQLite database so joins are idempotent, +deduplicated, and queryable. + +This module deliberately separates: + +* **measurement admissibility** — are prompt, compile, and terminal records + joined with enough coverage to support analysis? +* **policy admissibility** — do we also know the execution-health context + needed to turn repeated failures into routing or quarantine policy? + +Frequency alone never promotes a savings claim. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from roam.procedure_mining import build_procedure_atlas, normalized_episode_tokens + +EPISODE_SCHEMA_VERSION = 1 +LEDGER_SCHEMA_VERSION = 3 +MIN_ADMISSIBLE_EPISODES = 30 +MIN_COVERAGE_PCT = 95.0 +TERMINAL_GRACE_SECONDS = 600 +MIN_LIVE_HOOK_VERSION = 6 + +EVENT_LOG_NAME = "episodes.jsonl" +TRANSCRIPT_EVENT_LOG_NAME = "transcript-episodes.jsonl" +COMPILE_LOG_NAME = "compile-runs.jsonl" +LEDGER_DB_NAME = "episodes.sqlite" +VALIDATED_HEALTH_STATES = frozenset({"verification_passed", "verification_failed"}) + +EVENT_FIELD_DEFINITIONS: dict[str, str] = { + "event_id": "stable unique event identifier used for idempotent ingestion", + "episode_id": "join key spanning prompt compilation and terminal outcome", + "event_type": ( + "closed event kind: prompt_submitted, stop_decision, stop_continuation, " + "intervention_assignment, or intervention_observation" + ), + "ts": "UTC event timestamp in ISO-8601 form", + "session_id": "host session identifier; local-only join metadata", + "turn_seq": "monotonic prompt sequence within the host session", + "terminal": "whether this event closes the episode", + "outcome": "closed terminal or intermediate outcome classification", + "duration_ms": "elapsed milliseconds from prompt submission to this event", + "changed_files": "count of changed tracked and untracked files at stop time", + "diff_sha256": "content hash of the local tracked diff plus untracked path identities", + "health_state": "closed execution-health state; only explicit verification pass/fail is validated", + "evidence_source": "live_hook for prospective evidence or transcript_backfill for historical discovery", + "hook_version": "live hook body version that produced the event", + "intervention_id": "closed repeated-transition identifier for prospective intervention events", + "intervention_version": "versioned intervention behavior assigned to the episode", + "eligibility_rule_version": "version of the pre-intervention eligibility rule", + "assignment": "control, exposed, or shadow assignment made before intervention delivery", + "assignment_cluster": "session-level cluster identifier used to prevent cross-arm contamination", + "eligible_transition": "whether the pre-registered transition eligibility rule passed", + "delivered": "whether the assigned intervention was actually surfaced", + "adopted": "whether the agent used the surfaced intervention", + "downstream_transition_count": "eligible repeated transitions observed after assignment", +} + +_INTERVENTION_ASSIGNMENTS = frozenset({"control", "exposed", "shadow"}) +_INTERVENTION_EVENT_TYPES = frozenset({"intervention_assignment", "intervention_observation"}) + + +def _parse_ts(value: Any) -> datetime | None: + raw = str(value or "").strip() + if not raw: + return None + try: + return datetime.fromisoformat(raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError: + return None + + +def _read_jsonl(path: Path) -> tuple[list[dict[str, Any]], int]: + if not path.exists(): + return [], 0 + rows: list[dict[str, Any]] = [] + invalid = 0 + with path.open(encoding="utf-8", errors="replace") as fh: + for line_number, line in enumerate(fh, start=1): + line = line.strip() + if not line: + continue + try: + value = json.loads(line) + except json.JSONDecodeError: + invalid += 1 + continue + if isinstance(value, dict): + value = dict(value) + value["__source_line"] = line_number + rows.append(value) + else: + invalid += 1 + return rows, invalid + + +def _canonical_hash(prefix: str, value: dict[str, Any]) -> str: + raw = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + return prefix + hashlib.sha256(raw.encode("utf-8", "replace")).hexdigest()[:24] + + +def _open_ledger(path: Path) -> sqlite3.Connection: + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + prior_version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + if prior_version and prior_version < LEDGER_SCHEMA_VERSION: + conn.executescript( + """ + DROP TABLE IF EXISTS episode_events; + DROP TABLE IF EXISTS compile_records; + DROP TABLE IF EXISTS ledger_meta; + """ + ) + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS episode_events ( + event_id TEXT PRIMARY KEY, + episode_id TEXT NOT NULL, + event_type TEXT NOT NULL, + ts TEXT NOT NULL, + session_id TEXT NOT NULL DEFAULT '', + turn_seq INTEGER, + terminal INTEGER NOT NULL DEFAULT 0, + outcome TEXT NOT NULL DEFAULT '', + duration_ms INTEGER, + changed_files INTEGER, + diff_sha256 TEXT NOT NULL DEFAULT '', + health_state TEXT NOT NULL DEFAULT 'unknown', + evidence_source TEXT NOT NULL DEFAULT 'live_hook', + hook_version INTEGER, + payload_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_episode_events_episode + ON episode_events(episode_id, ts); + CREATE INDEX IF NOT EXISTS idx_episode_events_type + ON episode_events(event_type, ts); + + CREATE TABLE IF NOT EXISTS compile_records ( + record_id TEXT PRIMARY KEY, + episode_id TEXT NOT NULL DEFAULT '', + ts TEXT NOT NULL, + task_hash TEXT NOT NULL DEFAULT '', + task_prefix TEXT NOT NULL DEFAULT '', + procedure TEXT NOT NULL DEFAULT '', + classifier_conf REAL, + art_label TEXT NOT NULL DEFAULT '', + envelope_bytes INTEGER, + compile_ms REAL, + injection_advice TEXT NOT NULL DEFAULT '', + cache_hit INTEGER NOT NULL DEFAULT 0, + agent_mode TEXT NOT NULL DEFAULT 'unknown', + payload_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_compile_records_episode + ON compile_records(episode_id, ts); + CREATE INDEX IF NOT EXISTS idx_compile_records_task + ON compile_records(task_hash, ts); + + CREATE TABLE IF NOT EXISTS ledger_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """ + ) + conn.execute(f"PRAGMA user_version={LEDGER_SCHEMA_VERSION}") + return conn + + +def _ingest_events(conn: sqlite3.Connection, rows: Iterable[dict[str, Any]]) -> tuple[int, int]: + inserted = 0 + rejected = 0 + for row in rows: + episode_id = str(row.get("episode_id") or "").strip() + event_type = str(row.get("event_type") or "").strip() + ts = str(row.get("ts") or "").strip() + if not episode_id or not event_type or _parse_ts(ts) is None: + rejected += 1 + continue + event_id = str(row.get("event_id") or "").strip() or _canonical_hash("evt_", row) + payload = {key: value for key, value in row.items() if key != "__source_line"} + cur = conn.execute( + """ + INSERT OR IGNORE INTO episode_events ( + event_id, episode_id, event_type, ts, session_id, turn_seq, + terminal, outcome, duration_ms, changed_files, diff_sha256, + health_state, evidence_source, hook_version, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + event_id, + episode_id, + event_type, + ts, + str(row.get("session_id") or ""), + _int_or_none(row.get("turn_seq")), + int(bool(row.get("terminal"))), + str(row.get("outcome") or ""), + _int_or_none(row.get("duration_ms")), + _int_or_none(row.get("changed_files")), + str(row.get("diff_sha256") or ""), + str(row.get("health_state") or "unknown"), + str(row.get("evidence_source") or "live_hook"), + _int_or_none(row.get("hook_version")), + json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str), + ), + ) + inserted += int(cur.rowcount > 0) + return inserted, rejected + + +def _ingest_compiles(conn: sqlite3.Connection, rows: Iterable[dict[str, Any]]) -> tuple[int, int]: + inserted = 0 + rejected = 0 + for row in rows: + ts = str(row.get("ts") or "").strip() + if _parse_ts(ts) is None: + rejected += 1 + continue + source_line = _int_or_none(row.get("__source_line")) + payload = {key: value for key, value in row.items() if key != "__source_line"} + record_id = _canonical_hash("cmp_", {"source_line": source_line, "payload": payload}) + cur = conn.execute( + """ + INSERT OR IGNORE INTO compile_records ( + record_id, episode_id, ts, task_hash, task_prefix, procedure, + classifier_conf, art_label, envelope_bytes, compile_ms, + injection_advice, cache_hit, agent_mode, payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + record_id, + str(row.get("episode_id") or ""), + ts, + str(row.get("task_hash") or ""), + str(row.get("task_prefix") or ""), + str(row.get("procedure") or ""), + _float_or_none(row.get("classifier_conf")), + str(row.get("art_label") or ""), + _int_or_none(row.get("envelope_bytes")), + _float_or_none(row.get("compile_ms")), + str(row.get("injection_advice") or ""), + int(row.get("cache_hit") is True), + str(row.get("agent_mode") or "unknown"), + json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str), + ), + ) + inserted += int(cur.rowcount > 0) + return inserted, rejected + + +def _int_or_none(value: Any) -> int | None: + if isinstance(value, bool): + return int(value) + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _float_or_none(value: Any) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + +def materialize_ledger(root: str | Path) -> dict[str, Any]: + root_path = Path(root).resolve() + roam_dir = root_path / ".roam" + live_event_rows, invalid_live_events = _read_jsonl(roam_dir / EVENT_LOG_NAME) + transcript_event_rows, invalid_transcript_events = _read_jsonl(roam_dir / TRANSCRIPT_EVENT_LOG_NAME) + for row in live_event_rows: + row.setdefault("evidence_source", "live_hook") + for row in transcript_event_rows: + row["evidence_source"] = "transcript_backfill" + event_rows = [*live_event_rows, *transcript_event_rows] + invalid_events = invalid_live_events + invalid_transcript_events + compile_rows, invalid_compiles = _read_jsonl(roam_dir / COMPILE_LOG_NAME) + conn = _open_ledger(roam_dir / LEDGER_DB_NAME) + try: + # transcript-episodes.jsonl is a replaceable derived snapshot, unlike + # the append-only live hook stream. Reconcile it exactly on every run. + conn.execute("DELETE FROM episode_events WHERE evidence_source='transcript_backfill'") + event_inserted, event_rejected = _ingest_events(conn, event_rows) + compile_inserted, compile_rejected = _ingest_compiles(conn, compile_rows) + conn.execute( + "INSERT OR REPLACE INTO ledger_meta(key, value) VALUES ('last_materialized_at', ?)", + (datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),), + ) + conn.commit() + totals = { + "event_records": conn.execute("SELECT COUNT(*) FROM episode_events").fetchone()[0], + "compile_records": conn.execute("SELECT COUNT(*) FROM compile_records").fetchone()[0], + } + finally: + conn.close() + return { + "database": str(roam_dir / LEDGER_DB_NAME), + "event_rows_read": len(event_rows), + "live_event_rows_read": len(live_event_rows), + "transcript_event_rows_read": len(transcript_event_rows), + "compile_rows_read": len(compile_rows), + "event_rows_inserted": event_inserted, + "compile_rows_inserted": compile_inserted, + "invalid_event_rows": invalid_events + event_rejected, + "invalid_compile_rows": invalid_compiles + compile_rejected, + **totals, + } + + +def _known_answer_canaries() -> dict[str, Any]: + """Run tiny deterministic fixtures through the classification assumptions.""" + fixtures = [ + { + "name": "clean_no_edit_terminal", + "events": [ + {"event_type": "prompt_submitted", "terminal": False}, + {"event_type": "stop_decision", "terminal": True, "outcome": "no_edit"}, + ], + "expected": ("no_edit", True), + }, + { + "name": "blocked_then_completed", + "events": [ + {"event_type": "prompt_submitted", "terminal": False}, + {"event_type": "stop_decision", "terminal": False, "outcome": "verification_blocked"}, + {"event_type": "stop_continuation", "terminal": True, "outcome": "continued_after_block"}, + ], + "expected": ("continued_after_block", True), + }, + { + "name": "degraded_verify_terminal", + "events": [ + {"event_type": "prompt_submitted", "terminal": False}, + {"event_type": "stop_decision", "terminal": True, "outcome": "verify_unavailable"}, + ], + "expected": ("verify_unavailable", True), + }, + ] + failures: list[str] = [] + for fixture in fixtures: + terminal = [event for event in fixture["events"] if event.get("terminal")] + observed = ( + str(terminal[-1].get("outcome") or "") if terminal else "", + bool(terminal), + ) + if observed != fixture["expected"]: + failures.append(fixture["name"]) + return { + "state": "passed" if not failures else "failed", + "passed": len(fixtures) - len(failures), + "total": len(fixtures), + "failures": failures, + } + + +def _pct(numerator: int, denominator: int) -> float | None: + if denominator <= 0: + return None + return round(100.0 * numerator / denominator, 1) + + +def _episode_health_state( + episode_events: list[dict[str, Any]], + start: dict[str, Any], + terminal: dict[str, Any] | None, +) -> str: + states = [str(row.get("health_state") or "unknown") for row in episode_events] + if "verification_failed" in states: + return "verification_failed" + if "verification_passed" in states: + return "verification_passed" + return str((terminal or start).get("health_state") or "unknown") + + +def _payload(row: dict[str, Any]) -> dict[str, Any]: + try: + value = json.loads(row.get("payload_json") or "{}") + except (TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def _episode_rows(conn: sqlite3.Connection) -> list[dict[str, Any]]: + events = [dict(row) for row in conn.execute("SELECT * FROM episode_events ORDER BY ts, event_id")] + compiles = [dict(row) for row in conn.execute("SELECT * FROM compile_records ORDER BY ts, record_id")] + events_by_episode: dict[str, list[dict[str, Any]]] = defaultdict(list) + compiles_by_episode: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in events: + events_by_episode[row["episode_id"]].append(row) + for row in compiles: + if row["episode_id"]: + compiles_by_episode[row["episode_id"]].append(row) + + now = datetime.now(timezone.utc) + out: list[dict[str, Any]] = [] + for episode_id, episode_events in events_by_episode.items(): + starts = [row for row in episode_events if row["event_type"] == "prompt_submitted"] + if not starts: + continue + start = starts[0] + start_payload = _payload(start) + start_ts = _parse_ts(start["ts"]) + terminals = [row for row in episode_events if row["terminal"]] + terminal = terminals[-1] if terminals else None + terminal_payload = _payload(terminal or {}) + age_s = (now - start_ts).total_seconds() if start_ts else None + eligible = age_s is not None and age_s >= TERMINAL_GRACE_SECONDS + compile_row = compiles_by_episode.get(episode_id, [None])[-1] + out.append( + { + "episode_id": episode_id, + "session_id": str(start.get("session_id") or ""), + "started_at": start["ts"], + "eligible": eligible, + "terminal": terminal is not None, + "outcome": str((terminal or {}).get("outcome") or ""), + "duration_ms": (terminal or {}).get("duration_ms"), + "changed_files": (terminal or {}).get("changed_files"), + "health_state": _episode_health_state(episode_events, start, terminal), + "evidence_source": str(start.get("evidence_source") or "live_hook"), + "transcript_source": str(start_payload.get("transcript_source") or ""), + "hook_version": start.get("hook_version"), + "compile_expected": bool(start_payload.get("compile_expected", True)), + "compile_joined": compile_row is not None, + "project_id": str(start_payload.get("project_id") or ""), + "project_identity_basis": str(start_payload.get("project_identity_basis") or ""), + "intent_archetypes": ( + start_payload.get("intent_archetypes") + if isinstance(start_payload.get("intent_archetypes"), list) + else [] + ), + "intent_simhash64": str(start_payload.get("intent_simhash64") or ""), + "prompt_hmac_sha256": str(start_payload.get("prompt_hmac_sha256") or ""), + "prompt_tokens_bucket": (_int_or_none(start_payload.get("prompt_tokens_bucket")) or 0), + "trajectory_fingerprint": str(terminal_payload.get("trajectory_fingerprint") or ""), + "trajectory_template": str(terminal_payload.get("trajectory_template") or ""), + "phase_sequence_template": str(terminal_payload.get("phase_sequence_template") or ""), + "command_sequence_fingerprint": str(terminal_payload.get("command_sequence_fingerprint") or ""), + "command_sequence_template": str(terminal_payload.get("command_sequence_template") or ""), + "shell_templates": ( + terminal_payload.get("shell_templates") + if isinstance(terminal_payload.get("shell_templates"), dict) + else {} + ), + "shell_ngrams": ( + terminal_payload.get("shell_ngrams") + if isinstance(terminal_payload.get("shell_ngrams"), dict) + else {} + ), + "tool_ngrams": ( + terminal_payload.get("tool_ngrams") if isinstance(terminal_payload.get("tool_ngrams"), dict) else {} + ), + "phase_ngrams": ( + terminal_payload.get("phase_ngrams") + if isinstance(terminal_payload.get("phase_ngrams"), dict) + else {} + ), + "friction": ( + terminal_payload.get("friction") if isinstance(terminal_payload.get("friction"), dict) else {} + ), + "shell_template_outcomes": ( + terminal_payload.get("shell_template_outcomes") + if isinstance(terminal_payload.get("shell_template_outcomes"), dict) + else {} + ), + "phase_outcomes": ( + terminal_payload.get("phase_outcomes") + if isinstance(terminal_payload.get("phase_outcomes"), dict) + else {} + ), + "command_class_outcomes": ( + terminal_payload.get("command_class_outcomes") + if isinstance(terminal_payload.get("command_class_outcomes"), dict) + else {} + ), + "tool_calls": _int_or_none(terminal_payload.get("tool_calls")) or 0, + "tool_errors": _int_or_none(terminal_payload.get("tool_errors")) or 0, + "tool_result_bytes_bucket": (_int_or_none(terminal_payload.get("tool_result_bytes_bucket")) or 0), + "assistant_messages": (_int_or_none(terminal_payload.get("assistant_messages")) or 0), + "time_to_first_tool_ms": _int_or_none(terminal_payload.get("time_to_first_tool_ms")), + "time_to_first_edit_ms": _int_or_none(terminal_payload.get("time_to_first_edit_ms")), + "edit_actions": _int_or_none(terminal_payload.get("edit_actions")) or 0, + "verification_attempts": (_int_or_none(terminal_payload.get("verification_attempts")) or 0), + "verification_failures": (_int_or_none(terminal_payload.get("verification_failures")) or 0), + "input_tokens": _int_or_none(terminal_payload.get("input_tokens")) or 0, + "output_tokens": _int_or_none(terminal_payload.get("output_tokens")) or 0, + "cached_input_tokens": (_int_or_none(terminal_payload.get("cached_input_tokens")) or 0), + "cache_creation_tokens": (_int_or_none(terminal_payload.get("cache_creation_tokens")) or 0), + "reasoning_output_tokens": (_int_or_none(terminal_payload.get("reasoning_output_tokens")) or 0), + "correction_after": bool(terminal_payload.get("correction_after")), + "task_hash": str((compile_row or {}).get("task_hash") or ""), + "task_prefix": str((compile_row or {}).get("task_prefix") or ""), + "procedure": str((compile_row or {}).get("procedure") or ""), + "art_label": str((compile_row or {}).get("art_label") or ""), + "classifier_conf": (compile_row or {}).get("classifier_conf"), + } + ) + return out + + +def _intervention_evidence(conn: sqlite3.Connection) -> dict[str, Any]: + """Summarize pre-assignment/post-observation joins without estimating effects.""" + rows = [ + dict(row) + for row in conn.execute( + """ + SELECT event_id, episode_id, event_type, ts, session_id, payload_json + FROM episode_events + WHERE event_type IN ('intervention_assignment', 'intervention_observation') + ORDER BY ts, event_id + """ + ) + ] + terminal_times = { + str(row[0]): _parse_ts(row[1]) + for row in conn.execute( + """ + SELECT episode_id, MAX(ts) + FROM episode_events + WHERE terminal=1 + GROUP BY episode_id + """ + ) + } + assignments: dict[tuple[str, str, str], dict[str, Any]] = {} + observations: dict[tuple[str, str, str], dict[str, Any]] = {} + invalid_rows = 0 + duplicate_rows = 0 + for row in rows: + payload = _payload(row) + intervention_id = str(payload.get("intervention_id") or "").strip() + version = str(payload.get("intervention_version") or "").strip() + key = (str(row.get("episode_id") or ""), intervention_id, version) + if not all(key): + invalid_rows += 1 + continue + if row["event_type"] == "intervention_assignment": + assignment = str(payload.get("assignment") or "") + rule_version = str(payload.get("eligibility_rule_version") or "") + assignment_cluster = str(payload.get("assignment_cluster") or "") + eligible_transition = payload.get("eligible_transition") + if ( + assignment not in _INTERVENTION_ASSIGNMENTS + or not rule_version + or not assignment_cluster + or assignment_cluster != str(row.get("session_id") or "") + or eligible_transition is not True + ): + invalid_rows += 1 + continue + if key in assignments: + duplicate_rows += 1 + assignments[key] = { + **payload, + "session_id": str(row.get("session_id") or ""), + "assigned_at": str(row.get("ts") or ""), + } + continue + transition_count = _int_or_none(payload.get("downstream_transition_count")) + if transition_count is None or transition_count < 0: + invalid_rows += 1 + continue + if key in observations: + duplicate_rows += 1 + observations[key] = { + **payload, + "observed_at": str(row.get("ts") or ""), + } + + grouped: dict[tuple[str, str], list[tuple[tuple[str, str, str], dict[str, Any]]]] = defaultdict(list) + for key, assignment in assignments.items(): + grouped[(key[1], key[2])].append((key, assignment)) + + experiments = [] + for (intervention_id, version), assigned_rows in grouped.items(): + arm_counts = Counter(str(assignment.get("assignment") or "") for _, assignment in assigned_rows) + session_arms: dict[str, set[str]] = defaultdict(set) + joined = 0 + measured = 0 + delivered = 0 + adopted = 0 + ordering_violations = 0 + for key, assignment in assigned_rows: + session_arms[str(assignment.get("assignment_cluster") or "")].add(str(assignment.get("assignment") or "")) + observation = observations.get(key) + assigned_at = _parse_ts(assignment.get("assigned_at")) + observed_at = _parse_ts((observation or {}).get("observed_at")) + terminal_at = terminal_times.get(key[0]) + if observation is None or terminal_at is None: + continue + if assigned_at is None or observed_at is None or assigned_at >= observed_at or observed_at > terminal_at: + ordering_violations += 1 + continue + joined += 1 + measured += int(_int_or_none(observation.get("downstream_transition_count")) is not None) + delivered += int(observation.get("delivered") is True) + adopted += int(observation.get("adopted") is True) + contaminated_sessions = sum(len(arms) > 1 for session, arms in session_arms.items() if session) + control = arm_counts["control"] + exposed = arm_counts["exposed"] + join_coverage = _pct(joined, len(assigned_rows)) + if ordering_violations: + readiness = "event_ordering_violation" + elif control < 30 or exposed < 30: + readiness = "insufficient_sample" + elif (join_coverage or 0.0) < MIN_COVERAGE_PCT: + readiness = "incomplete_observation_join" + elif contaminated_sessions: + readiness = "assignment_contamination" + else: + readiness = "ready_for_preregistered_effect_estimation" + experiments.append( + { + "intervention_id": intervention_id, + "intervention_version": version, + "assigned_episodes": len(assigned_rows), + "assignment_counts": dict(sorted(arm_counts.items())), + "terminal_observation_joins": joined, + "observation_join_coverage_pct": join_coverage, + "transition_measurement_coverage_pct": _pct(measured, len(assigned_rows)), + "delivery_rate_pct": _pct(delivered, exposed), + "adoption_rate_pct": _pct(adopted, exposed), + "contaminated_sessions": contaminated_sessions, + "event_ordering_violations": ordering_violations, + "promotion_readiness": readiness, + "effectiveness_state": "unmeasured", + "causal_savings_claimed": False, + } + ) + experiments.sort( + key=lambda row: ( + -row["assigned_episodes"], + row["intervention_id"], + row["intervention_version"], + ) + ) + joined_total = sum(row["terminal_observation_joins"] for row in experiments) + assignment_total = sum(row["assigned_episodes"] for row in experiments) + ordering_violations = sum(row["event_ordering_violations"] for row in experiments) + return { + "summary": { + "state": "instrumented" if assignment_total else "not_instrumented", + "assignment_events": assignment_total, + "terminal_observation_joins": joined_total, + "observation_join_coverage_pct": _pct(joined_total, assignment_total), + "invalid_intervention_events": invalid_rows, + "duplicate_intervention_events": duplicate_rows, + "event_ordering_violations": ordering_violations, + "causal_savings_claimed": False, + }, + "experiments": experiments, + "event_contract": { + "required_pair": [ + "intervention_assignment", + "intervention_observation", + ], + "assignment_must_precede_delivery": True, + "assignment_unit": "session_id", + "analysis_population": "intent_to_treat", + }, + } + + +def _repeat_candidates(episodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for episode in episodes: + if episode["task_hash"] and episode["terminal"]: + grouped[episode["task_hash"]].append(episode) + candidates: list[dict[str, Any]] = [] + for task_hash, rows in grouped.items(): + if len(rows) < 3: + continue + outcomes = Counter(row["outcome"] or "unknown" for row in rows) + duration_values = [ + int(row["duration_ms"]) + for row in rows + if isinstance(row.get("duration_ms"), (int, float)) and row["duration_ms"] >= 0 + ] + health_known = sum(row["health_state"] in VALIDATED_HEALTH_STATES for row in rows) + candidates.append( + { + "task_hash": task_hash, + "task_prefix": rows[-1]["task_prefix"], + "procedure": rows[-1]["procedure"], + "episodes": len(rows), + "outcomes": dict(outcomes.most_common()), + "observed_wall_ms": sum(duration_values), + "health_context_pct": _pct(health_known, len(rows)), + "evidence_status": ( + "candidate" if health_known == len(rows) else "candidate_only_health_context_missing" + ), + } + ) + candidates.sort(key=lambda row: (-row["observed_wall_ms"], -row["episodes"], row["task_hash"])) + return candidates[:20] + + +def _interpret_historical_pattern(kind: str, pattern: str) -> dict[str, str]: + lowered = pattern.lower() + if "roam " in lowered and any(token in lowered for token in (" | python", " | jq", "head -c")): + return { + "pattern_family": "roam_projection_postprocessing", + "candidate_disposition": "projection_gap", + "priority": "high", + "automation_hypothesis": ( + "Return the exact compact projection from Roam so agents stop parsing or truncating it in shell" + ), + "existing_surface": "roam fetch-handle --jq or a command-native --summary projection", + "next_experiment": "Add one native projection and measure disappearance of the post-processing shell n-gram", + } + if "roam " in lowered and "--help" in lowered: + return { + "pattern_family": "command_discoverability", + "candidate_disposition": "discoverability_gap", + "priority": "medium", + "automation_hypothesis": ( + "Compile the relevant command contract before execution so agents stop opening help repeatedly" + ), + "existing_surface": "roam explain-command ", + "next_experiment": "Inject the selected command signature and compare follow-up --help calls", + } + if "rg " in lowered and "sed -n" in lowered: + return { + "pattern_family": "search_then_slice", + "candidate_disposition": "automation_candidate", + "priority": "high", + "automation_hypothesis": ( + "Collapse search plus manual line slicing into one ranked bounded-context result" + ), + "existing_surface": "roam grep --group-by symbol or roam retrieve ", + "next_experiment": "Route a held-out search cohort through one context-enriched call", + } + if lowered.count("sed -n") >= 2 or ("nl -ba" in lowered and "sed -n" in lowered): + return { + "pattern_family": "repeated_code_slicing", + "candidate_disposition": "automation_candidate", + "priority": "high", + "automation_hypothesis": ( + "Replace repeated line-window reads with one location-aware structural context call" + ), + "existing_surface": "roam at FILE:LINE or roam context --for-file PATH", + "next_experiment": "Compare repeated slice count after compiling roam at/context into the first turn", + } + if "git diff" in lowered and any(token in lowered for token in ("sed -n", "rg ", "pytest", "roam verify")): + return { + "pattern_family": "diff_drilldown", + "candidate_disposition": "automation_candidate", + "priority": "high", + "automation_hypothesis": ( + "Bundle changed hunks, structural blast radius, and verification targets into one review envelope" + ), + "existing_surface": "roam diff --full and git diff | roam critique", + "next_experiment": "Compile one diff-review envelope and measure follow-up git/sed/rg calls", + } + if "py_compile" in lowered and "pytest" in lowered: + return { + "pattern_family": "verification_ladder", + "candidate_disposition": "automation_candidate", + "priority": "high", + "automation_hypothesis": ( + "Turn syntax checking and focused test selection into one deterministic verification recipe" + ), + "existing_surface": "roam syntax-check plus roam test-impact", + "next_experiment": "Emit one executable verification recipe and compare retries and failed checks", + } + if "git status" in lowered and kind != "shell_template": + return { + "pattern_family": "workspace_orientation", + "candidate_disposition": "compiler_context_candidate", + "priority": "medium", + "automation_hypothesis": ( + "Include the minimal dirty-tree state in compiled context so orientation does not require a shell turn" + ), + "existing_surface": "roam brief plus compile envelope facts", + "next_experiment": "Add a compact dirty-tree fact and measure follow-up git status calls", + } + if kind == "tool_ngram" and "edit" in lowered and "shell" in lowered: + return { + "pattern_family": "edit_shell_loop", + "candidate_disposition": "measurement_refinement", + "priority": "medium", + "automation_hypothesis": ( + "Distinguish verification shells from exploration shells before proposing loop automation" + ), + "existing_surface": "prospective hook verification outcomes", + "next_experiment": "Join shell command class to live edit outcomes before promoting a recipe", + } + if lowered.startswith("roam ") and all(token not in lowered for token in ("=>", "|", "&&", ";")): + return { + "pattern_family": "already_compiled_operation", + "candidate_disposition": "already_boring", + "priority": "low", + "automation_hypothesis": "Observe adoption and outcome quality; no new wrapper is justified by frequency alone", + "existing_surface": pattern, + "next_experiment": "Compare success and fallback rates before changing the command", + } + return { + "pattern_family": "unclassified_repetition", + "candidate_disposition": "candidate_only", + "priority": "medium" if kind in {"shell_ngram", "shell_sequence"} else "low", + "automation_hypothesis": "Inspect the repeated structure and identify the invariant input/output contract", + "existing_surface": "", + "next_experiment": "Label a held-out sample and test one deterministic replacement", + } + + +def _historical_candidates(episodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Rank repeated transcript patterns without promoting them to savings facts.""" + groups: dict[str, dict[str, list[tuple[dict[str, Any], int]]]] = { + "shell_ngram": defaultdict(list), + "shell_sequence": defaultdict(list), + "tool_ngram": defaultdict(list), + "shell_template": defaultdict(list), + "tool_trajectory": defaultdict(list), + } + for episode in episodes: + if not episode["terminal"]: + continue + for source_field, kind in ( + ("shell_ngrams", "shell_ngram"), + ("tool_ngrams", "tool_ngram"), + ("shell_templates", "shell_template"), + ): + for pattern, count in episode.get(source_field, {}).items(): + try: + occurrences = int(count) + except (TypeError, ValueError): + continue + if pattern and occurrences > 0: + groups[kind][str(pattern)].append((episode, occurrences)) + trajectory = str(episode.get("trajectory_template") or "") + if trajectory: + groups["tool_trajectory"][trajectory].append((episode, 1)) + command_sequence = str(episode.get("command_sequence_template") or "") + if command_sequence: + groups["shell_sequence"][command_sequence].append((episode, 1)) + + ranked_by_kind: dict[str, list[dict[str, Any]]] = {} + for kind, patterns in groups.items(): + ranked: list[dict[str, Any]] = [] + for pattern, records in patterns.items(): + if len(records) < 3: + continue + if len(pattern) > 260 or pattern.count("") > 2: + continue + if kind == "shell_sequence" and "=>" not in pattern and "×" not in pattern: + continue + if kind == "tool_ngram": + families = {part.strip() for part in pattern.split("=>")} + if len(families) <= 1: + continue + if kind == "tool_trajectory": + families = { + re.sub(r"\*\d+$", "", part.strip()) + for part in pattern.split(">") + if part.strip() and part.strip() != "" + } + if len(families) <= 1: + continue + rows = [episode for episode, _count in records] + outcomes = Counter(row["outcome"] or "unknown" for row in rows) + associated_tokens = sum(normalized_episode_tokens(row) for row in rows) + ranked.append( + { + "kind": kind, + "pattern": pattern, + "episodes": len(rows), + "occurrences": sum(count for _episode, count in records), + "associated_episode_wall_ms": sum( + int(row["duration_ms"]) + for row in rows + if isinstance(row.get("duration_ms"), (int, float)) and row["duration_ms"] >= 0 + ), + "associated_tool_calls": sum(int(row.get("tool_calls") or 0) for row in rows), + "associated_tokens": associated_tokens, + "correction_pct": _pct( + sum(bool(row.get("correction_after")) for row in rows), + len(rows), + ), + "outcomes": dict(outcomes.most_common()), + "evidence_status": "historical_candidate_only", + **_interpret_historical_pattern(kind, pattern), + } + ) + ranked.sort( + key=lambda row: ( + -row["associated_episode_wall_ms"], + -row["episodes"], + -row["occurrences"], + row["pattern"], + ) + ) + ranked_by_kind[kind] = ranked + + priority = ( + "shell_ngram", + "shell_sequence", + "tool_ngram", + "shell_template", + "tool_trajectory", + ) + interleaved: list[dict[str, Any]] = [] + for rank in range(12): + for kind in priority: + rows = ranked_by_kind.get(kind, []) + if rank < len(rows): + interleaved.append(rows[rank]) + return interleaved + + +_AGGREGATE_SUMMARY_FIELDS = ( + "verdict", + "state", + "partial_success", + "measurement_admissible", + "policy_admissible", + "integrity_clean", + "north_star", +) +_AGGREGATE_COVERAGE_FIELDS = ( + "prompt_starts", + "prospective_prompt_starts", + "historical_prompt_starts", + "eligible_prompt_starts", + "terminal_outcomes", + "compile_expected_episodes", + "compile_joined_episodes", + "fully_joined_episodes", + "health_context_episodes", + "health_expected_episodes", + "current_hook_episodes", + "terminal_coverage_pct", + "episode_join_coverage_pct", + "health_context_coverage_pct", + "hook_version_coverage_pct", + "compile_identity_coverage_pct", + "terminal_coverage_definition", + "episode_join_coverage_definition", + "compile_identity_coverage_definition", + "health_context_coverage_definition", + "hook_version_coverage_definition", +) +_SAFE_SCALAR_TYPES = (bool, int, float, str) + + +def _aggregate_scalars( + value: object, + fields: tuple[str, ...], +) -> dict[str, object]: + source = value if isinstance(value, dict) else {} + out: dict[str, object] = {} + for field in fields: + field_value = source.get(field) + if field_value is None or isinstance(field_value, _SAFE_SCALAR_TYPES): + if field in source: + out[field] = field_value[:500] if isinstance(field_value, str) else field_value + return out + + +def _closed_state_counts( + rows: object, + field: str, + allowed: frozenset[str], +) -> dict[str, int]: + counts: Counter[str] = Counter() + for row in rows if isinstance(rows, list) else []: + value = str(row.get(field) or "unknown") if isinstance(row, dict) else "unknown" + counts[value if value in allowed else "unknown"] += 1 + return dict(sorted(counts.items(), key=lambda item: (-item[1], item[0]))) + + +def aggregate_savings_payload(payload: dict[str, Any]) -> dict[str, Any]: + """Return the browser-safe savings contract without transcript-derived values. + + Only scalar coverage, fixed verdict vocabulary, and counts cross this + boundary. Pattern text, task prefixes, commands, paths, identifiers, and + per-episode rows are deliberately unreachable from the returned object. + """ + atlas = payload.get("procedure_atlas") if isinstance(payload.get("procedure_atlas"), dict) else {} + intervention = ( + payload.get("intervention_evidence") if isinstance(payload.get("intervention_evidence"), dict) else {} + ) + assignments = intervention.get("assignments") if isinstance(intervention.get("assignments"), list) else [] + experiments = intervention.get("experiments") if isinstance(intervention.get("experiments"), list) else [] + mappings = atlas.get("intervention_mappings") if isinstance(atlas.get("intervention_mappings"), list) else [] + summary = _aggregate_scalars(payload.get("summary"), _AGGREGATE_SUMMARY_FIELDS) + summary["causal_savings_claimed"] = False + return { + "summary": summary, + "coverage": _aggregate_scalars( + payload.get("coverage"), + _AGGREGATE_COVERAGE_FIELDS, + ), + "sensor_canaries": _aggregate_scalars( + payload.get("sensor_canaries"), + ("state", "passed", "total"), + ), + "opportunity_counts": { + "repeated_live_candidates": len(payload.get("repeat_candidates") or []), + "historical_pattern_candidates": len(payload.get("historical_candidates") or []), + "ranked_work_opportunities": len(atlas.get("opportunities") or []), + "failure_signatures": len(atlas.get("failure_signatures") or []), + "recovery_targets": len(atlas.get("recovery_targets") or []), + "intervention_mappings": len(mappings), + }, + "intervention_state": { + "declaration_states": _closed_state_counts( + mappings, + "declaration_state", + frozenset({"declared_native", "declared_partial", "unclaimed"}), + ), + "assignments": len(assignments), + "experiments": len(experiments), + "assignment_states": _closed_state_counts( + assignments, + "assignment", + frozenset({"control", "exposed", "shadow"}), + ), + "causal_savings_claimed": False, + }, + "privacy": { + "aggregate_only": True, + "raw_transcripts_returned": False, + "prompt_or_response_text_returned": False, + "shell_command_text_returned": False, + "source_or_path_text_returned": False, + }, + } + + +def analyze_ledger(root: str | Path) -> dict[str, Any]: + materialization = materialize_ledger(root) + db_path = Path(materialization["database"]) + conn = _open_ledger(db_path) + try: + episodes = _episode_rows(conn) + event_counts = dict( + conn.execute( + "SELECT event_type, COUNT(*) AS n FROM episode_events GROUP BY event_type ORDER BY n DESC" + ).fetchall() + ) + first_start_row = conn.execute( + """ + SELECT MIN(ts) FROM episode_events + WHERE event_type='prompt_submitted' AND evidence_source='live_hook' + """ + ).fetchone() + first_start = first_start_row[0] if first_start_row else None + compile_window = [] + if first_start: + compile_window = [ + dict(row) + for row in conn.execute( + "SELECT episode_id, agent_mode FROM compile_records WHERE ts >= ?", + (first_start,), + ) + ] + intervention_evidence = _intervention_evidence(conn) + finally: + conn.close() + + canaries = _known_answer_canaries() + prospective = [episode for episode in episodes if episode["evidence_source"] == "live_hook"] + historical = [episode for episode in episodes if episode["evidence_source"] == "transcript_backfill"] + eligible = [episode for episode in prospective if episode["eligible"]] + terminal = [episode for episode in eligible if episode["terminal"]] + compile_expected = [episode for episode in eligible if episode["compile_expected"]] + compile_joined = [episode for episode in compile_expected if episode["compile_joined"]] + fully_joined = [ + episode + for episode in eligible + if episode["terminal"] and (episode["compile_joined"] or not episode["compile_expected"]) + ] + health_expected = [ + episode + for episode in eligible + if isinstance(episode.get("changed_files"), int) and episode["changed_files"] > 0 + ] + health_known = [episode for episode in health_expected if episode["health_state"] in VALIDATED_HEALTH_STATES] + current_hook = [ + episode + for episode in eligible + if isinstance(episode.get("hook_version"), int) and episode["hook_version"] >= MIN_LIVE_HOOK_VERSION + ] + production_compiles = [ + row + for row in compile_window + if (row.get("agent_mode") or "unknown") + not in {"bench", "corpus", "trace", "envelope_diff", "compile_cache_build", "test"} + ] + identified_compiles = [row for row in production_compiles if row.get("episode_id")] + + coverage = { + "prompt_starts": len(episodes), + "prospective_prompt_starts": len(prospective), + "historical_prompt_starts": len(historical), + "eligible_prompt_starts": len(eligible), + "terminal_outcomes": len(terminal), + "compile_expected_episodes": len(compile_expected), + "compile_joined_episodes": len(compile_joined), + "fully_joined_episodes": len(fully_joined), + "health_context_episodes": len(health_known), + "health_expected_episodes": len(health_expected), + "current_hook_episodes": len(current_hook), + "terminal_coverage_pct": _pct(len(terminal), len(eligible)), + "episode_join_coverage_pct": _pct(len(fully_joined), len(eligible)), + "health_context_coverage_pct": _pct(len(health_known), len(health_expected)), + "hook_version_coverage_pct": _pct(len(current_hook), len(eligible)), + "compile_identity_coverage_pct": _pct(len(identified_compiles), len(production_compiles)), + "terminal_coverage_definition": ( + "eligible prompt starts with a terminal stop event / prompt starts older than the grace window" + ), + "episode_join_coverage_definition": ( + "eligible prospective prompt starts with a terminal event and either the expected compile " + "record or an explicit compile_expected=false skip / eligible prospective prompt starts" + ), + "compile_identity_coverage_definition": ( + "production compile rows with episode_id / production compile rows since the first prompt-start event" + ), + "health_context_coverage_definition": ( + "eligible edited episodes with explicit verification_passed or verification_failed state / " + "eligible edited episodes" + ), + "hook_version_coverage_definition": ( + f"eligible prospective prompt starts emitted by hook version >= {MIN_LIVE_HOOK_VERSION} / " + "eligible prospective prompt starts" + ), + } + + integrity_clean = materialization["invalid_event_rows"] == 0 and materialization["invalid_compile_rows"] == 0 + measurement_admissible = ( + canaries["state"] == "passed" + and integrity_clean + and len(eligible) >= MIN_ADMISSIBLE_EPISODES + and (coverage["terminal_coverage_pct"] or 0) >= MIN_COVERAGE_PCT + and (coverage["episode_join_coverage_pct"] or 0) >= MIN_COVERAGE_PCT + and (coverage["compile_identity_coverage_pct"] or 0) >= MIN_COVERAGE_PCT + and (coverage["hook_version_coverage_pct"] or 0) >= MIN_COVERAGE_PCT + ) + policy_admissible = ( + measurement_admissible + and bool(health_expected) + and (coverage["health_context_coverage_pct"] or 0) >= MIN_COVERAGE_PCT + ) + + if not episodes: + state = "not_initialized" + verdict = "Savings ledger not initialized — install refreshed hooks and complete agent episodes" + elif not measurement_admissible: + state = "insufficient_evidence" + if not integrity_clean: + verdict = "Savings claims withheld — malformed telemetry rows failed the integrity gate" + else: + verdict = ( + f"Savings claims withheld — {len(fully_joined)}/{len(eligible)} eligible prospective " + "episodes carry compile and terminal records" + ) + elif not policy_admissible: + state = "measurement_ready" + verdict = "Episode joins are measurement-ready; routing savings remain gated on execution-health context" + else: + state = "policy_ready" + verdict = "Episode ledger is admissible for outcome-conditioned savings experiments" + + candidates = _repeat_candidates(eligible) if measurement_admissible else [] + historical_candidates = _historical_candidates(historical) + procedure_atlas = build_procedure_atlas(historical) + return { + "summary": { + "verdict": verdict, + "state": state, + "partial_success": state != "policy_ready", + "measurement_admissible": measurement_admissible, + "policy_admissible": policy_admissible, + "integrity_clean": integrity_clean, + "north_star": "durable successful outcomes per unit of constrained resource", + }, + "coverage": coverage, + "sensor_canaries": canaries, + "event_distribution": event_counts, + "outcome_distribution": dict(Counter(ep["outcome"] or "open" for ep in episodes).most_common()), + "repeat_candidates": candidates, + "historical_candidates": historical_candidates, + "procedure_atlas": procedure_atlas, + "intervention_evidence": intervention_evidence, + "materialization": materialization, + "thresholds": { + "minimum_eligible_episodes": MIN_ADMISSIBLE_EPISODES, + "minimum_coverage_pct": MIN_COVERAGE_PCT, + "terminal_grace_seconds": TERMINAL_GRACE_SECONDS, + }, + } diff --git a/src/roam/transcript_backfill.py b/src/roam/transcript_backfill.py new file mode 100644 index 000000000..94a819f74 --- /dev/null +++ b/src/roam/transcript_backfill.py @@ -0,0 +1,1406 @@ +"""Privacy-preserving historical episode extraction from agent transcripts. + +The extractor reads raw Claude Code or Codex JSONL locally and emits a compact +derived snapshot. Raw prompts, responses, command values, paths, and tool +arguments never enter the snapshot. Sanitized shell templates retain the +executable, safe subcommand/flags, and control-flow shape. Historical episodes +are discovery evidence only; they cannot satisfy the prospective measurement +gate in :mod:`roam.savings`. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import re +import shlex +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Any, Iterable + +BACKFILL_VERSION = 5 +MAX_TRANSCRIPT_BYTES = 50 * 1024 * 1024 +OUTPUT_NAME = "transcript-episodes.jsonl" +SALT_NAME = "savings-backfill.key" + +_TOKEN_RE = re.compile(r"[A-Za-z0-9_./\\:-]+") +_CORRECTION_RE = re.compile( + r"^\s*(no\b|stop\b|wait\b|actually\b|don'?t\b|instead\b|not that|" + r"that'?s wrong|revert|undo|why did|you (missed|forgot|broke)|try again)", + re.IGNORECASE, +) +_SYSTEM_BLOCK_RE = re.compile( + r"<(system-reminder|local-command-caveat|command-name|command-message)>.*?", + re.IGNORECASE | re.DOTALL, +) +_VERIFY_RE = re.compile( + r"\b(pytest|unittest|jest|vitest|mocha|cargo\s+test|go\s+test|dotnet\s+test|" + r"mvn\s+test|gradle\s+test|npm\s+(run\s+)?test|pnpm\s+(run\s+)?test|" + r"yarn\s+test|ruff|eslint|mypy|pyright|tsc|typecheck|lint|check\.py|" + r"roam\s+verify|prepush|pre-commit)\b", + re.IGNORECASE, +) +_BUILD_RE = re.compile( + r"\b(npm|pnpm|yarn)\s+(run\s+)?build\b|\b(cargo|go|mvn|gradle|dotnet)\s+build\b", + re.IGNORECASE, +) +_SEARCH_RE = re.compile(r"\b(rg|grep|git\s+grep|findstr|select-string)\b", re.IGNORECASE) +_FORMAT_RE = re.compile(r"\b(black|prettier|ruff\s+format|gofmt|rustfmt)\b", re.IGNORECASE) +_INSPECT_RE = re.compile( + r"\b(sed\s+-n|nl\s+-ba|head\b|tail\b|cat\b|less\b|more\b|" + r"get-content\b|select-object\s+-(?:first|last|skip))", + re.IGNORECASE, +) +_ORIENT_RE = re.compile( + r"\b(pwd|tree|git\s+(?:status|branch|log|rev-parse|remote)|" + r"get-location|get-childitem|ls|dir)\b", + re.IGNORECASE, +) +_DIFF_RE = re.compile(r"\b(git\s+diff|roam\s+(?:diff|critique))\b", re.IGNORECASE) +_VCS_WRITE_RE = re.compile( + r"\bgit\s+(?:add|commit|push|merge|rebase|cherry-pick|tag)\b", + re.IGNORECASE, +) +_DEPLOY_RE = re.compile( + r"\b(kubectl|helm|terraform|ansible|systemctl|docker\s+(?:compose|run|build|push)|" + r"ssh|scp|rsync)\b", + re.IGNORECASE, +) +_DEPENDENCY_RE = re.compile( + r"\b(pip|pip3|uv|poetry|npm|pnpm|yarn|cargo|go)\s+" + r"(?:install|add|remove|update|get|mod|sync)\b", + re.IGNORECASE, +) +_PROJECTION_RE = re.compile( + r"(?:\|\s*(?:jq|python(?:3)?|head|tail|sed|awk|cut|select-object)\b|head\s+-c\b)", + re.IGNORECASE, +) +_STRUCTURED_PROJECTION_RE = re.compile( + r"(?:^|(?:&&|\|\||;)\s*)roam\b[^;]*" + r"(?:\|\s*(?:jq|python(?:3)?|head|tail|sed|awk|cut|select-object)\b|head\s+-c\b)", + re.IGNORECASE, +) +_SLICE_TEMPLATE_RE = re.compile( + r"\b(?:sed\s+-n|nl\s+-ba|head\b|tail\b|get-content\b|" + r"select-object\s+-(?:first|last|skip))", + re.IGNORECASE, +) +_HELP_RE = re.compile(r"(?:^|\s)--help(?:\s|$)|(?:^|\s)-h(?:\s|$)", re.IGNORECASE) +_EXIT_CODE_RE = re.compile( + r"(?:process exited with code|exit code|return code)\s*[:=]?\s*(-?\d+)", + re.I, +) +_FAILURE_CLASSIFIERS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ( + "invalid_invocation", + re.compile( + r"\b(unrecognized arguments?|unknown option|invalid option|no such option|" + r"unexpected argument|usage error)\b", + re.I, + ), + ), + ( + "command_unavailable", + re.compile( + r"\b(command not found|not recognized as an internal or external command|" + r"executable file not found|could not find executable)\b", + re.I, + ), + ), + ( + "dependency_unavailable", + re.compile( + r"\b(modulenotfounderror|no module named|cannot find module|" + r"package .* not found|missing dependency|could not import)\b", + re.I, + ), + ), + ( + "path_unavailable", + re.compile( + r"\b(no such file or directory|file not found|path not found|" + r"cannot find the (?:file|path)|does not exist)\b", + re.I, + ), + ), + ( + "permission_or_auth", + re.compile( + r"\b(permission denied|access denied|unauthorized|forbidden|" + r"authentication failed|not authorized)\b", + re.I, + ), + ), + ( + "timeout", + re.compile( + r"\b(timed? out|timeout expired|deadline exceeded)\b", + re.I, + ), + ), + ( + "network", + re.compile( + r"\b(connection refused|network is unreachable|could not resolve|" + r"name resolution|dns failure|connection reset)\b", + re.I, + ), + ), + ( + "resource_exhausted", + re.compile( + r"\b(no space left|out of memory|memoryerror|too many open files|" + r"resource temporarily unavailable)\b", + re.I, + ), + ), + ( + "state_conflict", + re.compile( + r"\b(merge conflict|would be overwritten|index\.lock|already exists|" + r"non-fast-forward|working tree.*dirty)\b", + re.I, + ), + ), + ( + "syntax_or_compile", + re.compile( + r"\b(syntaxerror|syntax error|parse error|compilation failed|" + r"compile error|typecheck failed|type error)\b", + re.I, + ), + ), + ( + "test_failure", + re.compile( + r"(?:\b(?:assertionerror|assertion failed|tests? failed|" + r"failures?:\s*[1-9])\b|\b[1-9]\d* failed(?:,|\s|$))", + re.I, + ), + ), +) +_URL_RE = re.compile(r"^(?:https?|ssh|git)://", re.IGNORECASE) +_SECRET_RE = re.compile( + r"(?:^|[_-])(token|secret|password|passwd|api[_-]?key|private[_-]?key|authorization)(?:$|[_-])", + re.IGNORECASE, +) +_SAFE_WORD_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.+-]{0,39}$") +_PATHISH_RE = re.compile( + r"(?:[/\\]|^\.+$|\.(?:py|js|ts|tsx|jsx|go|rs|java|cs|php|rb|sh|ps1|json|ya?ml|toml|md|txt|sql)$)", re.I +) +_KNOWN_SUBCOMMAND_EXECUTABLES = frozenset( + { + "git", + "roam", + "npm", + "pnpm", + "yarn", + "cargo", + "go", + "docker", + "kubectl", + "systemctl", + "journalctl", + "pip", + "pip3", + } +) +_FLAGS_WITH_VALUES = frozenset( + { + "-c", + "-e", + "-f", + "-m", + "-o", + "-name", + "-path", + "-maxdepth", + "-mindepth", + "--command", + "--eval", + "--filter", + "--format", + "--glob", + "--include", + "--name", + "--output", + "--pattern", + "--root", + "--select", + } +) + +_EDIT_TOOLS = frozenset({"Edit", "Write", "MultiEdit", "NotebookEdit", "apply_patch"}) + +_INTENT_ARCHETYPES: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("debug", re.compile(r"\b(debug|diagnos|root cause|why (?:is|does|did)|broken|failure|error)\b", re.I)), + ("implement", re.compile(r"\b(implement|build|create|add|ship|develop|make)\b", re.I)), + ("refactor", re.compile(r"\b(refactor|simplif|clean up|restructur|extract|rename)\b", re.I)), + ("review", re.compile(r"\b(review|audit|critique|inspect|check the diff|code review)\b", re.I)), + ("verify", re.compile(r"\b(test|verify|validate|lint|typecheck|prove|regression)\b", re.I)), + ("performance", re.compile(r"\b(performance|latency|speed|optimi[sz]|memory|cpu|faster)\b", re.I)), + ("security", re.compile(r"\b(security|vulnerab|secret|auth|permission|threat|exploit)\b", re.I)), + ("deploy", re.compile(r"\b(deploy|release|production|server|service|container|kubernetes)\b", re.I)), + ("research", re.compile(r"\b(research|compare|investigate|look into|find out|explore)\b", re.I)), + ("document", re.compile(r"\b(document|readme|docs|guide|explain|write up)\b", re.I)), + ("plan", re.compile(r"\b(plan|design|architecture|strategy|roadmap|approach)\b", re.I)), + ("data", re.compile(r"\b(data|database|sql|query|migration|schema|analytics)\b", re.I)), + ("ui", re.compile(r"\b(ui|ux|frontend|layout|screen|page|component|style|css)\b", re.I)), + ("git", re.compile(r"\b(git|commit|branch|merge|rebase|pull request|\bpr\b)\b", re.I)), +) + + +def _parse_ts(value: Any) -> datetime | None: + raw = str(value or "").strip() + if not raw: + return None + try: + return datetime.fromisoformat(raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError: + return None + + +def _iso(value: datetime | None) -> str: + return (value or datetime.now(timezone.utc)).isoformat().replace("+00:00", "Z") + + +def _bucket(value: int, width: int) -> int: + return max(width, ((max(0, value) + width - 1) // width) * width) + + +def _load_or_create_key(root: Path, *, create: bool = True) -> bytes: + path = root / ".roam" / SALT_NAME + try: + raw = path.read_text(encoding="ascii").strip() + key = bytes.fromhex(raw) + if len(key) >= 16: + return key + except (OSError, ValueError): + pass + if not create: + return os.urandom(32) + path.parent.mkdir(parents=True, exist_ok=True) + key = os.urandom(32) + tmp = path.with_name(path.name + f".tmp-{os.getpid()}") + tmp.write_text(key.hex() + "\n", encoding="ascii") + try: + os.chmod(tmp, 0o600) + except OSError: + pass + os.replace(tmp, path) + return key + + +def _keyed_hex(key: bytes, purpose: str, value: str, length: int = 24) -> str: + digest = hmac.new(key, f"{purpose}\0{value}".encode("utf-8", "replace"), hashlib.sha256) + return digest.hexdigest()[:length] + + +def _normalize_prompt(text: str) -> list[str]: + text = _SYSTEM_BLOCK_RE.sub(" ", text) + tokens: list[str] = [] + for raw in _TOKEN_RE.findall(text.lower())[:1024]: + if len(raw) > 80: + raw = "" + elif re.fullmatch(r"[0-9a-f]{12,}", raw): + raw = "" + elif re.fullmatch(r"\d+(?:\.\d+)*", raw): + raw = "" + elif "/" in raw or "\\" in raw: + raw = "" + tokens.append(raw) + return tokens + + +def _intent_fingerprints(text: str, key: bytes) -> tuple[str, str, int]: + tokens = _normalize_prompt(text) + normalized = " ".join(tokens) + exact = _keyed_hex(key, "intent-exact", normalized, 32) + if not tokens: + return exact, "0000000000000000", 0 + weights = [0] * 64 + features = tokens + [f"{a}\0{b}" for a, b in zip(tokens, tokens[1:])] + for feature in features: + digest = hashlib.blake2b( + feature.encode("utf-8", "replace"), + key=key, + digest_size=8, + person=b"roam-int", + ).digest() + value = int.from_bytes(digest, "big") + for bit in range(64): + weights[bit] += 1 if value & (1 << bit) else -1 + simhash = sum(1 << bit for bit, weight in enumerate(weights) if weight >= 0) + return exact, f"{simhash:016x}", len(tokens) + + +def _text_content(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + return "\n".join( + str(block.get("text") or "") for block in content if isinstance(block, dict) and block.get("type") == "text" + ).strip() + + +def _is_correction(text: str) -> bool: + cleaned = _SYSTEM_BLOCK_RE.sub(" ", text) + cleaned = re.sub(r"<[^>]+>.*?]+>", " ", cleaned, flags=re.DOTALL) + return bool(_CORRECTION_RE.match(cleaned.lstrip())) + + +def _tool_family(name: str) -> str: + if name in _EDIT_TOOLS: + return "edit" + if name in {"Bash", "exec_command", "write_stdin"}: + return "shell" + if name in {"Read"}: + return "read" + if name in {"Grep", "Glob", "ToolSearch"}: + return "search" + if name.startswith("mcp__roam-code__") or name.startswith("roam_"): + return "roam" + if name in {"WebSearch", "WebFetch"} or name.startswith("web_search"): + return "web" + if name in {"Agent", "Task", "TaskCreate", "TaskUpdate", "TaskOutput"}: + return "agent" + return "other" + + +def _command_class(command: str) -> str: + if _VERIFY_RE.search(command): + return "verify" + if _BUILD_RE.search(command): + return "build" + if _FORMAT_RE.search(command): + return "format" + if _DIFF_RE.search(command): + return "review" + if _VCS_WRITE_RE.search(command): + return "vcs_write" + if _DEPLOY_RE.search(command): + return "deploy" + if _DEPENDENCY_RE.search(command): + return "dependency" + if _SEARCH_RE.search(command): + return "search" + if _INSPECT_RE.search(command): + return "inspect" + if _ORIENT_RE.search(command): + return "orient" + if re.search(r"\bgit\b", command, re.IGNORECASE): + return "git" + return "other" + + +def _tool_phase(name: str, command: str = "") -> str: + family = _tool_family(name) + if family == "shell": + command_class = _command_class(command) + return { + "verify": "verify", + "build": "verify", + "format": "format", + "review": "review", + "vcs_write": "publish", + "deploy": "deploy", + "dependency": "setup", + "search": "search", + "inspect": "inspect", + "orient": "orient", + "git": "orient", + }.get(command_class, "shell") + return { + "edit": "edit", + "read": "inspect", + "search": "search", + "roam": "intelligence", + "web": "research", + "agent": "delegate", + }.get(family, "other") + + +def _intent_archetypes(text: str) -> list[str]: + cleaned = _SYSTEM_BLOCK_RE.sub(" ", text) + matches = [name for name, pattern in _INTENT_ARCHETYPES if pattern.search(cleaned)] + return matches[:4] or ["other"] + + +def _content_size(value: Any) -> int: + if isinstance(value, str): + return len(value.encode("utf-8", "replace")) + try: + return len(json.dumps(value, default=str).encode("utf-8", "replace")) + except (TypeError, ValueError): + return 0 + + +def _exit_code(value: Any) -> int | None: + if isinstance(value, str): + text = value + else: + try: + text = json.dumps(value, default=str) + except (TypeError, ValueError): + return None + match = _EXIT_CODE_RE.search(text) + return int(match.group(1)) if match else None + + +def _result_text(value: Any) -> str: + """Flatten transient tool-result text for classification only.""" + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(_result_text(item) for item in value) + if isinstance(value, dict): + return "\n".join( + _result_text(value.get(key)) + for key in ("content", "text", "output", "error", "message", "stderr") + if key in value + ) + return "" + + +def _failure_class(value: Any) -> str: + """Return one closed failure label without retaining result content.""" + text = _result_text(value)[:200_000] + for label, pattern in _FAILURE_CLASSIFIERS: + if pattern.search(text): + return label + return "unknown" + + +def _elapsed_ms(started_at: datetime, observed_at: datetime | None) -> int | None: + if observed_at is None: + return None + return max(0, int((observed_at - started_at).total_seconds() * 1000)) + + +@lru_cache(maxsize=8192) +def _project_scope(cwd: str) -> tuple[str, str]: + """Return the nearest live Git root, otherwise the normalized workspace.""" + if not cwd: + return "", "missing" + path = Path(cwd).expanduser() + try: + resolved = path.resolve() + except OSError: + resolved = path + for candidate in (resolved, *resolved.parents): + try: + if (candidate / ".git").exists(): + return os.path.normcase(os.path.normpath(str(candidate))), "git_root" + except OSError: + break + return os.path.normcase(os.path.normpath(str(resolved))), "workspace" + + +def _friction_metrics( + actions: list[dict[str, Any]], + *, + verification_attempts: int, +) -> dict[str, int]: + phases = [str(action.get("phase") or "other") for action in actions] + templates = [str(action.get("template") or "") for action in actions if str(action.get("template") or "")] + template_counts = Counter(templates) + exact_shell_replays = sum(max(0, count - 1) for count in template_counts.values()) + adjacent_shell_replays = sum(bool(left and left == right) for left, right in zip(templates, templates[1:])) + failed_action_retries = 0 + for index, action in enumerate(actions): + template = str(action.get("template") or "") + if not action.get("failed") or not template: + continue + if any(str(candidate.get("template") or "") == template for candidate in actions[index + 1 : index + 4]): + failed_action_retries += 1 + first_edit = next((index for index, phase in enumerate(phases) if phase == "edit"), None) + post_edit_context_calls = 0 + if first_edit is not None: + for phase in phases[first_edit + 1 :]: + if phase == "verify": + break + if phase in {"orient", "search", "inspect", "intelligence"}: + post_edit_context_calls += 1 + search_inspect_cycles = sum( + window + in { + ("search", "inspect", "search"), + ("inspect", "search", "inspect"), + } + for window in zip(phases, phases[1:], phases[2:]) + ) + phase_switches = sum(left != right for left, right in zip(phases, phases[1:])) + return { + "orientation_calls": phases.count("orient"), + "search_calls": phases.count("search"), + "inspection_calls": phases.count("inspect"), + "slice_calls": sum(bool(_SLICE_TEMPLATE_RE.search(template)) for template in templates), + "output_postprocess_calls": sum(bool(_PROJECTION_RE.search(template)) for template in templates), + "structured_output_postprocess_calls": sum( + bool(_STRUCTURED_PROJECTION_RE.search(template)) for template in templates + ), + "help_calls": sum(bool(_HELP_RE.search(template)) for template in templates), + "exact_shell_replays": exact_shell_replays, + "adjacent_shell_replays": adjacent_shell_replays, + "failed_action_retries": failed_action_retries, + "verification_retries": max(0, verification_attempts - 1), + "post_edit_context_calls": post_edit_context_calls, + "search_inspect_cycles": search_inspect_cycles, + "phase_switches": phase_switches, + } + + +def _action_outcome_tables( + actions: list[dict[str, Any]], +) -> dict[str, dict[str, dict[str, Any]]]: + shell: dict[str, Counter[str]] = defaultdict(Counter) + phases: dict[str, Counter[str]] = defaultdict(Counter) + command_classes: dict[str, Counter[str]] = defaultdict(Counter) + shell_failure_classes: dict[str, Counter[str]] = defaultdict(Counter) + phase_failure_classes: dict[str, Counter[str]] = defaultdict(Counter) + command_failure_classes: dict[str, Counter[str]] = defaultdict(Counter) + for index, action in enumerate(actions): + failed = bool(action.get("failed")) + result_size = _int_nonnegative(action.get("result_size")) + phase = str(action.get("phase") or "other") + command_class = str(action.get("command_class") or "") + template = str(action.get("template") or "") + for table, key in ( + (phases, phase), + (command_classes, command_class), + (shell, template), + ): + if not key: + continue + table[key]["attempts"] += 1 + table[key]["failures"] += int(failed) + table[key]["no_results"] += int(str(action.get("result_state") or "") == "no_results") + table[key]["result_bytes"] += result_size + failure_class = str(action.get("failure_class") or "") + if failed and failure_class: + for table, key in ( + (phase_failure_classes, phase), + (command_failure_classes, command_class), + (shell_failure_classes, template), + ): + if key: + table[key][failure_class] += 1 + if ( + failed + and template + and any(str(candidate.get("template") or "") == template for candidate in actions[index + 1 : index + 4]) + ): + shell[template]["retries_after_failure"] += 1 + + def render( + table: dict[str, Counter[str]], + failure_classes: dict[str, Counter[str]], + ) -> dict[str, dict[str, Any]]: + return { + key: { + "attempts": counts["attempts"], + "failures": counts["failures"], + "no_results": counts["no_results"], + "retries_after_failure": counts["retries_after_failure"], + "result_bytes_bucket": (_bucket(counts["result_bytes"], 4096) if counts["result_bytes"] else 0), + "failure_classes": dict(sorted(failure_classes.get(key, Counter()).items())), + } + for key, counts in sorted(table.items()) + } + + return { + "shell_template_outcomes": render(shell, shell_failure_classes), + "phase_outcomes": render(phases, phase_failure_classes), + "command_class_outcomes": render( + command_classes, + command_failure_classes, + ), + } + + +def _int_nonnegative(value: Any) -> int: + try: + return max(0, int(value)) + except (TypeError, ValueError): + return 0 + + +def _run_length_values(values: list[str]) -> list[tuple[str, int]]: + if not values: + return [] + runs: list[tuple[str, int]] = [] + current = values[0] + count = 1 + for value in values[1:]: + if value == current: + count += 1 + continue + runs.append((current, count)) + current = value + count = 1 + runs.append((current, count)) + return runs + + +def _compressed_sequence(values: list[str], *, limit: int = 20) -> str: + runs = _run_length_values(values) + rendered = [f"{value}*{count}" if count > 1 else value for value, count in runs[:limit]] + if len(runs) > limit: + rendered.append("") + return ">".join(rendered) + + +def _compressed_command_sequence(values: list[str], *, limit: int = 12) -> str: + runs = _run_length_values(values) + rendered = [f"{value} ×{count}" if count > 1 else value for value, count in runs[:limit]] + if len(runs) > limit: + rendered.append("") + return " => ".join(rendered) + + +def _sequence_ngrams( + values: list[str], + *, + minimum: int = 2, + maximum: int = 4, +) -> Counter[str]: + out: Counter[str] = Counter() + for width in range(minimum, maximum + 1): + for index in range(0, len(values) - width + 1): + out[" => ".join(values[index : index + width])] += 1 + return out + + +def _command_from_input(value: Any) -> str: + if isinstance(value, dict): + for key in ("command", "cmd"): + command = value.get(key) + if isinstance(command, str): + return command + if isinstance(command, list): + return " ".join(str(part) for part in command) + return "" + if not isinstance(value, str): + return "" + try: + decoded = json.loads(value) + except (TypeError, ValueError): + return value + return _command_from_input(decoded) + + +def _safe_executable(token: str) -> str: + token = token.strip("\"'`") + base = re.split(r"[/\\]", token)[-1].lower() + if base.endswith(".exe"): + base = base[:-4] + return base if _SAFE_WORD_RE.fullmatch(base) else "" + + +def _safe_argument(token: str, *, executable: str, positional_index: int) -> str: + raw = token.strip() + clean = raw.strip("\"'`") + if not clean: + return "" + if raw.startswith((">", "2>", "1>", "<")): + return "" + if _URL_RE.search(clean): + return "" + if _PATHISH_RE.search(clean): + return "" + if "=" in clean and not clean.startswith(("==", "!=")): + name, _value = clean.split("=", 1) + if name.startswith("-"): + flag = name if _SAFE_WORD_RE.fullmatch(name.lstrip("-")) else "--" + return f"{flag}=" + return "=" + if _SECRET_RE.search(clean) or re.fullmatch(r"[A-Za-z0-9+/=_-]{32,}", clean): + return "" + if clean.startswith("-"): + if len(clean) <= 32 and re.fullmatch(r"--?[A-Za-z0-9][A-Za-z0-9_.-]*", clean): + return clean.lower() + return "-" + if re.fullmatch(r"\d+(?:\.\d+)*", clean): + return "" + if positional_index == 0 and executable in _KNOWN_SUBCOMMAND_EXECUTABLES: + return clean.lower() if _SAFE_WORD_RE.fullmatch(clean) else "" + if ( + positional_index == 1 + and executable in {"npm", "pnpm", "yarn"} + and clean.lower() in {"build", "test", "lint", "typecheck", "format"} + ): + return clean.lower() + if executable in {"python", "python3", "py"}: + return clean.lower() if clean in {"pytest", "unittest", "pip"} else "" + return "" + + +def _split_shell_control(command: str) -> list[str]: + """Split control operators outside quotes; normalize newlines to ``;``.""" + out: list[str] = [] + buf: list[str] = [] + quote = "" + escaped = False + index = 0 + + def flush() -> None: + value = "".join(buf).strip() + if value: + out.append(value) + buf.clear() + + while index < len(command): + char = command[index] + if escaped: + buf.append(char) + escaped = False + index += 1 + continue + if char == "\\" and quote != "'": + buf.append(char) + escaped = True + index += 1 + continue + if quote: + buf.append(char) + if char == quote: + quote = "" + index += 1 + continue + if char in {"'", '"', "`"}: + quote = char + buf.append(char) + index += 1 + continue + pair = command[index : index + 2] + if pair in {"&&", "||"}: + flush() + out.append(pair) + index += 2 + continue + if char in {"|", ";", "\n"}: + flush() + operator = "|" if char == "|" else ";" + if not out or out[-1] != operator: + out.append(operator) + index += 1 + continue + buf.append(char) + index += 1 + flush() + return out + + +def sanitize_command_template(command: str) -> str: + """Return a useful shell shape while removing values, paths, and secrets.""" + command = command.replace("\r", "\n").strip() + if not command: + return "" + parts = _split_shell_control(command) + rendered: list[str] = [] + for part in parts: + part = part.strip() + if not part: + continue + if part in {"&&", "||", "|", ";"}: + rendered.append(part) + continue + if "\n" in part and any(marker in part for marker in ("<<", "@'", '@"')): + head = part.splitlines()[0] + part = head + " " + try: + # Transcript parsing must be reproducible across the host that runs + # the miner. Agent shell records are predominantly POSIX-shaped; + # using the mining host's ``os.name`` made one corpus produce + # different templates on Windows and Linux. + tokens = shlex.split(part, posix=True) + except ValueError: + tokens = re.findall(r"""(?:"[^"]*"|'[^']*'|`[^`]*`|\S+)""", part) + if not tokens: + continue + env_prefix: list[str] = [] + token_index = 0 + while token_index < len(tokens) and "=" in tokens[token_index] and not tokens[token_index].startswith("-"): + env_prefix.append("=") + token_index += 1 + if token_index >= len(tokens): + rendered.extend(env_prefix or ["="]) + continue + executable = _safe_executable(tokens[token_index]) + segment = [*env_prefix, executable] + positional_index = 0 + value_expected = "" + for token in tokens[token_index + 1 : token_index + 20]: + if value_expected: + if ( + value_expected == "-m" + and executable in {"python", "python3", "py"} + and token.strip("\"'`").lower() + in {"pytest", "unittest", "pip", "py_compile", "compileall", "ruff", "mypy"} + ): + safe = token.strip("\"'`").lower() + else: + safe = "" if value_expected in {"-c", "--eval", "--command"} else "" + value_expected = "" + else: + safe = _safe_argument( + token, + executable=executable, + positional_index=positional_index, + ) + segment.append(safe) + if safe in _FLAGS_WITH_VALUES: + value_expected = safe + if not token.startswith("-"): + positional_index += 1 + if len(tokens) - token_index > 20: + segment.append("") + rendered.append(" ".join(segment)) + template = " ".join(rendered) + return template[:320] + + +def _cwd_matches(cwd: str, root: Path, all_projects: bool) -> bool: + if all_projects: + return True + if not cwd: + return False + left = os.path.normcase(os.path.normpath(cwd)) + right = os.path.normcase(os.path.normpath(str(root))) + return left == right or left.startswith(right + os.sep) + + +@dataclass +class _Episode: + source: str + session_key: str + turn_seq: int + started_at: datetime + prompt: str + cwd: str + key: bytes + last_at: datetime | None = None + explicit_duration_ms: int | None = None + tool_sequence: list[str] = field(default_factory=list) + tool_counts: Counter[str] = field(default_factory=Counter) + command_counts: Counter[str] = field(default_factory=Counter) + command_templates: Counter[str] = field(default_factory=Counter) + command_sequence: list[str] = field(default_factory=list) + actions: list[dict[str, Any]] = field(default_factory=list) + pending: dict[str, int] = field(default_factory=dict) + tool_errors: int = 0 + edit_actions: int = 0 + failed_edit_actions: int = 0 + verification_attempts: int = 0 + verification_failures: int = 0 + input_tokens: int = 0 + output_tokens: int = 0 + cached_input_tokens: int = 0 + cache_creation_tokens: int = 0 + reasoning_output_tokens: int = 0 + tool_result_bytes: int = 0 + assistant_messages: int = 0 + first_tool_at: datetime | None = None + first_edit_at: datetime | None = None + correction_after: bool = False + explicit_complete: bool = False + + def add_tool( + self, + name: str, + call_id: str, + tool_input: Any, + observed_at: datetime | None = None, + ) -> None: + family = _tool_family(name) + self.tool_sequence.append(family) + self.tool_counts[family] += 1 + command_class = "" + raw_command = "" + template = "" + if family == "shell": + raw_command = _command_from_input(tool_input) + command_class = _command_class(raw_command) + self.command_counts[command_class] += 1 + template = sanitize_command_template(raw_command) + if template: + self.command_templates[template] += 1 + self.command_sequence.append(template) + if command_class in {"verify", "build"}: + self.verification_attempts += 1 + if name in _EDIT_TOOLS: + self.edit_actions += 1 + if self.first_tool_at is None: + self.first_tool_at = observed_at + if family == "edit" and self.first_edit_at is None: + self.first_edit_at = observed_at + self.actions.append( + { + "family": family, + "phase": _tool_phase(name, raw_command), + "command_class": command_class, + "template": template, + "failed": False, + "result_state": "unknown", + "result_size": 0, + } + ) + if call_id: + self.pending[call_id] = len(self.actions) - 1 + + def add_result( + self, + call_id: str, + failed: bool, + result_size: int = 0, + exit_code: int | None = None, + result_value: Any = "", + ) -> None: + self.tool_result_bytes += max(0, result_size) + action_index = self.pending.pop(call_id, -1) + action = self.actions[action_index] if 0 <= action_index < len(self.actions) else {} + family = str(action.get("family") or "") + command_class = str(action.get("command_class") or "") + no_results = bool(failed and command_class == "search" and exit_code == 1) + effective_failure = bool(failed and not no_results) + if action: + action["failed"] = effective_failure + action["result_state"] = "no_results" if no_results else "failure" if effective_failure else "success" + action["result_size"] = max(0, result_size) + action["failure_class"] = _failure_class(result_value) if effective_failure else "" + if effective_failure: + self.tool_errors += 1 + if family == "edit": + self.failed_edit_actions += 1 + if command_class in {"verify", "build"}: + self.verification_failures += 1 + + def finish(self, terminal_at: datetime | None) -> tuple[dict[str, Any], dict[str, Any]]: + terminal_at = terminal_at or self.last_at or self.started_at + duration_ms = self.explicit_duration_ms + if duration_ms is None: + duration_ms = max(0, int((terminal_at - self.started_at).total_seconds() * 1000)) + exact, simhash, token_count = _intent_fingerprints(self.prompt, self.key) + episode_id = "hist_" + _keyed_hex( + self.key, + "episode", + f"{self.source}\0{self.session_key}\0{self.turn_seq}", + 24, + ) + session_id = "hist_" + _keyed_hex(self.key, "session", self.session_key, 16) + successful_edits = max(0, self.edit_actions - self.failed_edit_actions) + if successful_edits and self.verification_attempts and not self.verification_failures: + outcome = "historical_acted_verified_proxy" + health_state = "proxy_verification_passed" + elif successful_edits and self.verification_failures: + outcome = "historical_acted_verification_failed_proxy" + health_state = "proxy_verification_failed" + elif successful_edits: + outcome = "historical_acted_unverified" + health_state = "proxy_unverified" + elif self.tool_errors: + outcome = "historical_no_edit_tool_error" + health_state = "proxy_tool_error" + else: + outcome = "historical_no_edit" + health_state = "not_applicable" + trajectory = ",".join(self.tool_sequence) + trajectory_fingerprint = _keyed_hex(self.key, "trajectory", trajectory, 24) + command_sequence = "\n".join(self.command_sequence) + command_sequence_fingerprint = _keyed_hex(self.key, "command-sequence", command_sequence, 24) + event_base = { + "schema_version": 1, + "backfill_version": BACKFILL_VERSION, + "evidence_source": "transcript_backfill", + "transcript_source": self.source, + "episode_id": episode_id, + "session_id": session_id, + "turn_seq": self.turn_seq, + } + project_scope, project_identity_basis = _project_scope(self.cwd) + project_id = "proj_" + _keyed_hex(self.key, "project", project_scope, 20) if project_scope else "" + phases = [str(action.get("phase") or "other") for action in self.actions] + friction = _friction_metrics( + self.actions, + verification_attempts=self.verification_attempts, + ) + outcome_tables = _action_outcome_tables(self.actions) + start = { + **event_base, + "event_id": f"evt_{episode_id[5:]}_start", + "event_type": "prompt_submitted", + "ts": _iso(self.started_at), + "terminal": False, + "outcome": "pending", + "compile_expected": False, + "prompt_hmac_sha256": exact, + "intent_simhash64": simhash, + "prompt_chars_bucket": _bucket(len(self.prompt), 100), + "prompt_tokens_bucket": _bucket(token_count, 25), + "project_id": project_id, + "project_identity_basis": project_identity_basis, + "cwd_hmac_sha256": _keyed_hex(self.key, "cwd", self.cwd, 24), + "intent_archetypes": _intent_archetypes(self.prompt), + "health_state": "unknown", + } + terminal = { + **event_base, + "event_id": f"evt_{episode_id[5:]}_terminal", + "event_type": "transcript_terminal", + "ts": _iso(terminal_at), + "terminal": True, + "outcome": outcome, + "duration_ms": duration_ms, + "changed_files": None, + "health_state": health_state, + "trajectory_fingerprint": trajectory_fingerprint, + "trajectory_template": _compressed_sequence(self.tool_sequence), + "phase_sequence_template": _compressed_sequence(phases), + "phase_ngrams": dict(_sequence_ngrams(phases, minimum=2, maximum=5).most_common(80)), + "command_sequence_fingerprint": command_sequence_fingerprint, + "command_sequence_template": _compressed_command_sequence(self.command_sequence), + "shell_templates": dict(self.command_templates.most_common(30)), + "shell_ngrams": dict(_sequence_ngrams(self.command_sequence).most_common(80)), + "tool_ngrams": dict(_sequence_ngrams(self.tool_sequence, minimum=2, maximum=5).most_common(80)), + "tool_calls": sum(self.tool_counts.values()), + "tool_errors": self.tool_errors, + "failure_classes": dict( + sorted( + Counter( + str(action.get("failure_class") or "") + for action in self.actions + if action.get("failed") and action.get("failure_class") + ).items() + ) + ), + "tool_result_bytes_bucket": (_bucket(self.tool_result_bytes, 4096) if self.tool_result_bytes else 0), + "assistant_messages": self.assistant_messages, + "time_to_first_tool_ms": _elapsed_ms(self.started_at, self.first_tool_at), + "time_to_first_edit_ms": _elapsed_ms(self.started_at, self.first_edit_at), + "tool_families": dict(sorted(self.tool_counts.items())), + "command_classes": dict(sorted(self.command_counts.items())), + "friction": friction, + **outcome_tables, + "edit_actions": successful_edits, + "verification_attempts": self.verification_attempts, + "verification_failures": self.verification_failures, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "cached_input_tokens": self.cached_input_tokens, + "cache_creation_tokens": self.cache_creation_tokens, + "reasoning_output_tokens": self.reasoning_output_tokens, + "correction_after": self.correction_after, + "explicit_complete": self.explicit_complete, + } + return start, terminal + + +def _iter_jsonl(path: Path) -> Iterable[dict[str, Any]]: + try: + with path.open(encoding="utf-8", errors="replace") as fh: + for line in fh: + try: + value = json.loads(line) + except (TypeError, ValueError): + continue + if isinstance(value, dict): + yield value + except OSError: + return + + +def _scan_claude(path: Path, root: Path, key: bytes, all_projects: bool) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + current: _Episode | None = None + turn_seq = 0 + session_key = path.stem + session_cwd = "" + last_ts: datetime | None = None + for row in _iter_jsonl(path): + ts = _parse_ts(row.get("timestamp")) or last_ts + if ts: + last_ts = ts + cwd = str(row.get("cwd") or "") + if cwd: + session_cwd = cwd + message = row.get("message") + if not isinstance(message, dict): + continue + content = message.get("content") + if row.get("type") == "assistant" and current: + current.assistant_messages += 1 + usage = message.get("usage") + if isinstance(usage, dict): + current.input_tokens += int(usage.get("input_tokens") or 0) + current.output_tokens += int(usage.get("output_tokens") or 0) + current.cached_input_tokens += int(usage.get("cache_read_input_tokens") or 0) + current.cache_creation_tokens += int(usage.get("cache_creation_input_tokens") or 0) + if isinstance(content, list): + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + current.add_tool( + str(block.get("name") or ""), + str(block.get("id") or ""), + block.get("input"), + ts, + ) + current.last_at = ts or current.last_at + continue + if row.get("type") != "user": + continue + if isinstance(content, list) and current: + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + current.add_result( + str(block.get("tool_use_id") or ""), + block.get("is_error") is True, + _content_size(block.get("content")), + _exit_code(block.get("content")), + block.get("content"), + ) + current.last_at = ts or current.last_at + prompt = _text_content(content) + if not prompt: + continue + if current: + current.correction_after = _is_correction(prompt) + events.extend(current.finish(ts)) + turn_seq += 1 + effective_cwd = cwd or session_cwd + if not _cwd_matches(effective_cwd, root, all_projects): + current = None + continue + current = _Episode( + source="claude", + session_key=session_key, + turn_seq=turn_seq, + started_at=ts or datetime.now(timezone.utc), + prompt=prompt, + cwd=effective_cwd, + key=key, + ) + if current: + events.extend(current.finish(last_ts)) + return events + + +def _scan_codex(path: Path, root: Path, key: bytes, all_projects: bool) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + current: _Episode | None = None + turn_seq = 0 + session_key = path.stem + cwd = "" + last_ts: datetime | None = None + for row in _iter_jsonl(path): + ts = _parse_ts(row.get("timestamp")) or last_ts + if ts: + last_ts = ts + payload = row.get("payload") + if not isinstance(payload, dict): + continue + if not cwd and isinstance(payload.get("cwd"), str): + cwd = payload["cwd"] + payload_type = str(payload.get("type") or "") + if row.get("type") == "event_msg" and payload_type == "user_message": + prompt = str(payload.get("message") or "") + if not prompt: + continue + if current: + current.correction_after = _is_correction(prompt) + events.extend(current.finish(ts)) + turn_seq += 1 + current = _Episode( + source="codex", + session_key=session_key, + turn_seq=turn_seq, + started_at=ts or datetime.now(timezone.utc), + prompt=prompt, + cwd=cwd, + key=key, + ) + continue + if not current: + continue + if row.get("type") == "response_item" and payload_type in { + "function_call", + "custom_tool_call", + }: + current.add_tool( + str(payload.get("name") or ""), + str(payload.get("call_id") or ""), + payload.get("arguments") if "arguments" in payload else payload.get("input"), + ts, + ) + elif row.get("type") == "response_item" and payload_type in { + "function_call_output", + "custom_tool_call_output", + }: + output = str(payload.get("output") or "") + exit_code = _exit_code(output) + current.add_result( + str(payload.get("call_id") or ""), + exit_code not in {None, 0}, + _content_size(output), + exit_code, + output, + ) + elif row.get("type") == "event_msg" and payload_type == "patch_apply_end": + if payload.get("success") is not True: + current.tool_errors += 1 + current.failed_edit_actions += 1 + elif row.get("type") == "event_msg" and payload_type == "token_count": + info = payload.get("info") + last_usage = info.get("last_token_usage") if isinstance(info, dict) else {} + if isinstance(last_usage, dict): + current.input_tokens = int(last_usage.get("input_tokens") or 0) + current.output_tokens = int(last_usage.get("output_tokens") or 0) + current.cached_input_tokens = int(last_usage.get("cached_input_tokens") or 0) + current.reasoning_output_tokens = int(last_usage.get("reasoning_output_tokens") or 0) + elif row.get("type") == "event_msg" and payload_type == "task_complete": + current.explicit_complete = True + try: + current.explicit_duration_ms = int(payload.get("duration_ms")) + except (TypeError, ValueError): + pass + current.last_at = ts or current.last_at + events.extend(current.finish(ts)) + current = None + continue + current.last_at = ts or current.last_at + if current: + events.extend(current.finish(last_ts)) + if all_projects or _cwd_matches(cwd, root, False): + return events + return [] + + +def _detect_source(path: Path) -> str: + for row in _iter_jsonl(path): + if set(row) == {"timestamp", "type", "payload"}: + return "codex" + if "message" in row or "sessionId" in row: + return "claude" + break + return "unknown" + + +def _candidate_files( + transcripts_dir: Path, + *, + since: datetime | None, + max_files: int, +) -> list[Path]: + paths: list[Path] = [] + for path in transcripts_dir.rglob("*.jsonl"): + try: + stat = path.stat() + except OSError: + continue + if not path.is_file() or stat.st_size > MAX_TRANSCRIPT_BYTES: + continue + if since and datetime.fromtimestamp(stat.st_mtime, timezone.utc) < since: + continue + paths.append(path) + paths.sort(key=lambda path: (path.stat().st_mtime, str(path))) + if max_files > 0: + paths = paths[-max_files:] + return paths + + +def backfill_transcripts( + root: str | Path, + transcripts_dir: str | Path | Iterable[str | Path], + *, + source: str = "auto", + since: datetime | None = None, + max_files: int = 0, + all_projects: bool = False, + dry_run: bool = False, +) -> dict[str, Any]: + root_path = Path(root).resolve() + if isinstance(transcripts_dir, (str, Path)): + source_values = [transcripts_dir] + else: + source_values = list(transcripts_dir) + source_paths: list[Path] = [] + seen_sources: set[str] = set() + for value in source_values: + path = Path(value).expanduser().resolve() + identity = os.path.normcase(str(path)) + if identity not in seen_sources: + seen_sources.add(identity) + source_paths.append(path) + key = _load_or_create_key(root_path, create=not dry_run) + files: list[Path] = [] + seen_files: set[str] = set() + for source_path in source_paths: + for path in _candidate_files(source_path, since=since, max_files=max_files): + identity = os.path.normcase(str(path.resolve())) + if identity not in seen_files: + seen_files.add(identity) + files.append(path) + files.sort(key=lambda path: (path.stat().st_mtime, str(path))) + events: list[dict[str, Any]] = [] + source_counts: Counter[str] = Counter() + skipped_unknown = 0 + for path in files: + detected = source if source != "auto" else _detect_source(path) + if detected == "claude": + extracted = _scan_claude(path, root_path, key, all_projects) + elif detected == "codex": + extracted = _scan_codex(path, root_path, key, all_projects) + else: + skipped_unknown += 1 + continue + if extracted: + source_counts[detected] += 1 + events.extend(extracted) + events.sort(key=lambda row: (str(row.get("ts") or ""), str(row.get("event_id") or ""))) + output = root_path / ".roam" / OUTPUT_NAME + if not dry_run: + output.parent.mkdir(parents=True, exist_ok=True) + tmp = output.with_name(output.name + f".tmp-{os.getpid()}") + with tmp.open("w", encoding="utf-8", newline="\n") as fh: + for event in events: + fh.write(json.dumps(event, sort_keys=True, separators=(",", ":")) + "\n") + fh.flush() + os.fsync(fh.fileno()) + try: + os.chmod(tmp, 0o600) + except OSError: + pass + os.replace(tmp, output) + return { + "state": "dry_run" if dry_run else "written", + "output": str(output), + "files_considered": len(files), + "source_directories": [str(path) for path in source_paths], + "files_with_episodes": sum(source_counts.values()), + "files_by_source": dict(sorted(source_counts.items())), + "unknown_format_files": skipped_unknown, + "episodes": len(events) // 2, + "events": len(events), + "privacy_contract": { + "prompt_text_persisted": False, + "assistant_text_persisted": False, + "raw_commands_persisted": False, + "sanitized_command_templates_persisted": True, + "paths_persisted": False, + "tool_arguments_persisted": False, + "keyed_fingerprints": True, + "closed_intent_archetypes_persisted": True, + "phase_and_friction_counts_persisted": True, + "sanitized_template_outcomes_persisted": True, + "search_no_results_separated_from_failures": True, + "closed_failure_classes_persisted": True, + "tool_result_content_persisted": False, + }, + } diff --git a/templates/distribution/landing-page/changelog.html b/templates/distribution/landing-page/changelog.html index 4a582d09f..c5daca517 100644 --- a/templates/distribution/landing-page/changelog.html +++ b/templates/distribution/landing-page/changelog.html @@ -76,6 +76,13 @@

Changelog

All notable changes to roam-code will be documented in this file.

The format is based on Keep a Changelog, and this project adheres to Semantic Versioning.

[Unreleased]

+

Added

+
    +
  • roam savings — materializes prospective hook events, compile records, and value-redacted historical transcript episodes into a local SQLite ledger; reports coverage and sensor canaries before allowing any measurement or routing claim. Repeated patterns remain candidate-only until joined live outcomes satisfy the declared evidence gates.
    • +
  • roam savings-backfill — derives replaceable historical discovery episodes from Claude and Codex transcript stores without persisting raw prompts, responses, paths, source, or shell-command values. The backfill keeps closed templates, fingerprints, counts, timing buckets, and friction signals for Procedure Foundry ranking.
    • +
  • Producer-side roam savings --aggregate privacy boundary — emits only fixed summary fields, scalar coverage, closed-state counts, and aggregate opportunity counts for platform dashboards. Transcript-derived pattern text, task prefixes, commands, paths, identifiers, and per-episode rows are structurally excluded.
    • +
  • Capability displacement metadata — @roam_capability(displaces=(...)) declares which repeated-work family a capability is intended to replace. Procedure Foundry reads the declarations through AST without importing lazy command modules; declarations nominate prospective interventions but never claim adoption or savings.
    • +

Fixed

  • roam capsule --redact-paths no longer leaks the directory tree via cluster labels — the "zero-source" architecture capsule is meant for external review without sharing source, and it correctly hashes every symbol's file path, but clusters[].label (which is derived from a file/directory path) was emitted verbatim — leaking the exact tree the flag exists to anonymize (e.g. workflows/checkout, tests/test_cmd_pr_replay_...). Cluster labels are now hashed with the same per-component scheme as symbol paths under --redact-paths. Surfaced by real dogfooding of the redaction path.
    • diff --git a/templates/distribution/landing-page/docs/agent-contract.html b/templates/distribution/landing-page/docs/agent-contract.html index c3a2e1ab7..b001b3599 100644 --- a/templates/distribution/landing-page/docs/agent-contract.html +++ b/templates/distribution/landing-page/docs/agent-contract.html @@ -98,7 +98,7 @@

    Surface at a glance

    - + diff --git a/templates/distribution/landing-page/docs/integration-tutorials.html b/templates/distribution/landing-page/docs/integration-tutorials.html index 876d0fec2..320ae5462 100644 --- a/templates/distribution/landing-page/docs/integration-tutorials.html +++ b/templates/distribution/landing-page/docs/integration-tutorials.html @@ -79,7 +79,7 @@

    Integration Tutorials

    Scope: MCP host setup (5 platforms), CI integration (5 platforms via roam ci-setup), gateway integration via the McpDecisionReceipt schema and HMAC-chained run ledger, and the OSCAL v1.2 control-mapping projection. Non-goal: hosted Roam deployment; private-deployment pilots are scoped separately by SOW.

    - Surface scale on v13.9.0: 279 commands, 244 MCP tools (16 in the + Surface scale on v13.9.0: 281 commands, 244 MCP tools (16 in the default core preset, opt into full via ROAM_MCP_PRESET), 28 indexed languages. All local; no API keys. diff --git a/templates/distribution/landing-page/llms.txt b/templates/distribution/landing-page/llms.txt index cad8d5eea..44d27851f 100644 --- a/templates/distribution/landing-page/llms.txt +++ b/templates/distribution/landing-page/llms.txt @@ -5,7 +5,7 @@ roam-code (`pip install roam-code`, Python 3.10+) is a local code-graph engine. It parses your repo once, stores structural facts in a local SQLite graph (symbols, dependencies, call graphs, architecture layers, git history, -runtime traces), and exposes the graph through 279 CLI commands and 244 MCP +runtime traces), and exposes the graph through 281 CLI commands and 244 MCP tools (16 in the default `core` preset) across 28 languages. AI agents (Claude Code, Cursor, Codex, Aider, your own) call it before, during, and after every change. diff --git a/templates/distribution/landing-page/pricing.html b/templates/distribution/landing-page/pricing.html index f6de7eabb..375a3890e 100644 --- a/templates/distribution/landing-page/pricing.html +++ b/templates/distribution/landing-page/pricing.html @@ -168,7 +168,7 @@

    The buying path

    - + diff --git a/templates/distribution/landing-page/setup.html b/templates/distribution/landing-page/setup.html index 3b8bb28fd..a732fa025 100644 --- a/templates/distribution/landing-page/setup.html +++ b/templates/distribution/landing-page/setup.html @@ -94,7 +94,7 @@

    1. Install the CLI

    (incremental, near-instant — only changed files re-parse).

    - Discover everything: roam --help (279 commands + Discover everything: roam --help (281 commands grouped by what an agent asks) or roam tour (guided walk through the five core verbs). Full reference at /docs/command-reference. @@ -154,7 +154,7 @@

    2. Wire it into your editor

    Browse the source - 279 commands live in src/roam/commands/, registered in cli.py. Apache 2.0. + 281 commands live in src/roam/commands/, registered in cli.py. Apache 2.0.
    Need help? diff --git a/tests/test_procedure_mining.py b/tests/test_procedure_mining.py new file mode 100644 index 000000000..386ce9d4a --- /dev/null +++ b/tests/test_procedure_mining.py @@ -0,0 +1,259 @@ +"""Known-answer tests for private transcript opportunity mining.""" + +from __future__ import annotations + +from roam.procedure_mining import ( + build_procedure_atlas, + declared_displacement_claims, + normalized_episode_tokens, +) + + +def _episode(index: int, **overrides): + row = { + "episode_id": f"hist_{index}", + "session_id": f"session_{index // 2}", + "project_id": f"project_{index % 3}", + "evidence_source": "transcript_backfill", + "transcript_source": "codex" if index % 2 else "claude", + "terminal": True, + "outcome": "historical_no_edit", + "duration_ms": 10_000, + "tool_calls": 10, + "tool_errors": 0, + "tool_result_bytes_bucket": 0, + "edit_actions": 0, + "input_tokens": 1000, + "output_tokens": 100, + "cached_input_tokens": 500, + "cache_creation_tokens": 0, + "reasoning_output_tokens": 50, + "correction_after": False, + "prompt_hmac_sha256": "same-intent", + "intent_simhash64": "1234567890abcdef", + "prompt_tokens_bucket": 50, + "intent_archetypes": ["debug"], + "phase_sequence_template": "orient>search>inspect*2", + "shell_templates": { + "git status --short": 1, + "rg -n ": 1, + "sed -n ": 2, + }, + "shell_template_outcomes": { + "python -m pytest -q": { + "attempts": 2, + "failures": 1, + "retries_after_failure": 1, + "result_bytes_bucket": 8192, + "failure_classes": {"test_failure": 1}, + } + }, + "friction": { + "slice_calls": 3, + "search_inspect_cycles": 1, + "exact_shell_replays": 2, + "output_postprocess_calls": 0, + "orientation_calls": 1, + "verification_retries": 0, + "failed_action_retries": 0, + "post_edit_context_calls": 0, + "help_calls": 0, + }, + } + row.update(overrides) + return row + + +def test_atlas_ranks_direct_actions_without_calling_them_savings() -> None: + atlas = build_procedure_atlas([_episode(index) for index in range(6)]) + assert atlas["summary"]["causal_savings_claimed"] is False + slicing = next(row for row in atlas["opportunities"] if row["opportunity"] == "repeated_code_slicing") + assert slicing["episodes"] == 6 + assert slicing["projects"] == 3 + assert slicing["addressable_actions"] == 12 + assert slicing["evidence_status"] == "historical_opportunity_only" + assert "saved" not in " ".join(slicing).lower() + + +def test_exact_intent_cluster_requires_multiple_sessions() -> None: + atlas = build_procedure_atlas([_episode(index) for index in range(6)]) + clusters = atlas["exact_intent_clusters"] + assert len(clusters) == 1 + assert clusters[0]["episodes"] == 6 + assert clusters[0]["sessions"] == 3 + assert clusters[0]["projects"] == 3 + assert clusters[0]["modal_phase_sequence"] == "orient>search>inspect*2" + + +def test_live_episodes_never_enter_historical_atlas() -> None: + atlas = build_procedure_atlas([_episode(index, evidence_source="live_hook") for index in range(6)]) + assert atlas["summary"]["episodes"] == 0 + assert atlas["opportunities"] == [] + assert atlas["exact_intent_clusters"] == [] + assert atlas["near_intent_clusters"] == [] + + +def test_token_normalization_does_not_double_count_codex_cache() -> None: + codex = _episode( + 1, + transcript_source="codex", + input_tokens=1000, + cached_input_tokens=800, + output_tokens=100, + reasoning_output_tokens=50, + ) + claude = _episode( + 2, + transcript_source="claude", + input_tokens=200, + cached_input_tokens=800, + cache_creation_tokens=100, + output_tokens=100, + reasoning_output_tokens=0, + ) + assert normalized_episode_tokens(codex) == 1150 + assert normalized_episode_tokens(claude) == 1200 + + +def test_near_intent_cluster_requires_distinct_exact_intents() -> None: + base = int("1234567890abcdef", 16) + episodes = [ + _episode( + index, + prompt_hmac_sha256=f"intent-{index}", + intent_simhash64=f"{base ^ (1 << index):016x}", + ) + for index in range(3) + ] + atlas = build_procedure_atlas(episodes) + assert len(atlas["near_intent_clusters"]) == 1 + cluster = atlas["near_intent_clusters"][0] + assert cluster["episodes"] == 3 + assert cluster["distinct_exact_intents"] == 3 + assert cluster["sessions"] == 2 + + +def test_scripted_exact_intent_is_disclosed_and_removed_from_organic_support() -> None: + episodes = [_episode(index, session_id=f"script-session-{index}") for index in range(20)] + atlas = build_procedure_atlas(episodes) + assert atlas["summary"]["likely_automated_episodes"] == 20 + slicing = next(row for row in atlas["opportunities"] if row["opportunity"] == "repeated_code_slicing") + assert slicing["episodes"] == 20 + assert slicing["organic_episode_estimate"] == 0 + assert slicing["likely_automated_episodes"] == 20 + assert slicing["organic_addressable_actions"] == 0 + + +def test_structured_projection_does_not_count_code_slicing_pipeline() -> None: + episodes = [ + _episode( + 1, + shell_templates={ + "nl -ba | sed -n ": 8, + "roam complexity --json | python3 -c ": 2, + }, + friction={"output_postprocess_calls": 10}, + ) + ] + atlas = build_procedure_atlas(episodes) + projection = next(row for row in atlas["opportunities"] if row["opportunity"] == "output_postprocessing") + assert projection["addressable_actions"] == 2 + + +def test_template_outcome_rankings_attribute_failures_and_result_volume() -> None: + atlas = build_procedure_atlas([_episode(index) for index in range(6)]) + failure = atlas["failure_signatures"][0] + assert failure["template"] == "python -m pytest -q" + assert failure["attempts"] == 12 + assert failure["failures"] == 6 + assert failure["retries_after_failure"] == 6 + assert failure["projects"] == 3 + assert failure["failure_classes"] == {"test_failure": 6} + result = atlas["large_result_producers"][0] + assert result["associated_bucketed_result_bytes"] == 49152 + assert atlas["recovery_targets"] == [ + { + "failure_class": "test_failure", + "failures": 6, + "templates": 1, + "projects": 3, + "evidence_status": "closed_failure_class_only", + "raw_result_content_persisted": False, + "classification_status": "heuristic_unvalidated", + "routing_eligible": False, + } + ] + + +def test_declared_displacements_are_ast_read_without_importing_commands() -> None: + claims = declared_displacement_claims() + pairs = {(claim["capability"], claim["opportunity"]) for claim in claims} + + assert ("at", "repeated_code_slicing") in pairs + assert ("grep", "search_inspect_thrash") in pairs + assert ("verify", "verification_retry") in pairs + assert ("global --select", "output_postprocessing") in pairs + + +def test_intervention_mappings_keep_declarations_prospectively_unmeasured() -> None: + atlas = build_procedure_atlas( + [ + _episode( + index, + tool_errors=1, + friction={ + "slice_calls": 3, + "search_inspect_cycles": 1, + "exact_shell_replays": 2, + "output_postprocess_calls": 0, + "orientation_calls": 1, + "verification_retries": 0, + "failed_action_retries": 0, + "post_edit_context_calls": 0, + "help_calls": 0, + }, + ) + for index in range(6) + ] + ) + gaps = {row["opportunity"]: row for row in atlas["intervention_mappings"]} + + assert gaps["repeated_code_slicing"]["declaration_state"] == "declared_native" + assert gaps["search_inspect_thrash"]["declaration_state"] == "declared_native" + assert gaps["exact_shell_replay"]["declaration_state"] == "declared_partial" + assert gaps["tool_failure_recovery"]["declaration_state"] == "unclaimed" + assert all(row["effectiveness_state"] == "unmeasured" for row in gaps.values()) + assert all(row["residual_gap_score"] is None for row in gaps.values()) + assert ( + gaps["repeated_code_slicing"]["research_priority_score"] == gaps["repeated_code_slicing"]["opportunity_score"] + ) + assert all(row["causal_savings_claimed"] is False for row in gaps.values()) + + tests = {row["transition"]: row for row in atlas["intervention_tests"]} + slicing = tests["repeated_code_slicing"] + assert slicing["exposure_state"] == "not_instrumented" + assert slicing["effectiveness_state"] == "unmeasured" + assert slicing["minimum_promotion_gate"]["require_transition_reduction"] is True + assert slicing["minimum_promotion_gate"]["require_outcome_non_inferiority"] is True + assert slicing["minimum_promotion_gate"]["power_analysis_required"] is True + assert slicing["minimum_promotion_gate"]["transition_effect_interval_must_exclude_zero"] is True + assert slicing["experimental_design"]["assignment_unit"] == "session_id" + assert slicing["experimental_design"]["analysis_population"] == "intent_to_treat" + assert slicing["experimental_design"]["required_event_pair"] == [ + "intervention_assignment", + "intervention_observation", + ] + + +def test_high_tool_no_edit_excludes_passive_and_unknown_intents() -> None: + episodes = [ + _episode(1, intent_archetypes=["research"]), + _episode(2, intent_archetypes=["review", "plan"]), + _episode(3, intent_archetypes=[]), + _episode(4, intent_archetypes=["implement"]), + _episode(5, intent_archetypes=["research", "implement"]), + ] + atlas = build_procedure_atlas(episodes) + opportunity = next(row for row in atlas["opportunities"] if row["opportunity"] == "high_tool_no_edit") + assert opportunity["episodes"] == 2 + assert opportunity["addressable_actions"] == 2 diff --git a/tests/test_savings.py b/tests/test_savings.py new file mode 100644 index 000000000..e7763e492 --- /dev/null +++ b/tests/test_savings.py @@ -0,0 +1,309 @@ +"""Known-answer and admissibility tests for the episode savings ledger.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from click.testing import CliRunner + +from roam.commands.cmd_savings import savings +from roam.savings import ( + _episode_health_state, + _interpret_historical_pattern, + analyze_ledger, +) + + +def _write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def _fixture_rows( + *, + count: int = 30, + terminal: bool = True, + health_state: str = "verification_passed", +): + events: list[dict] = [] + compiles: list[dict] = [] + for i in range(count): + episode_id = f"ep_{i:03d}" + session_id = f"s_{i // 3}" + events.append( + { + "schema_version": 1, + "hook_version": 6, + "evidence_source": "live_hook", + "event_id": f"start_{i}", + "episode_id": episode_id, + "event_type": "prompt_submitted", + "ts": "2026-01-01T00:00:00Z", + "session_id": session_id, + "turn_seq": i + 1, + "terminal": False, + "outcome": "pending", + "compile_expected": True, + "health_state": health_state, + } + ) + if terminal: + events.append( + { + "schema_version": 1, + "hook_version": 6, + "evidence_source": "live_hook", + "event_id": f"stop_{i}", + "episode_id": episode_id, + "event_type": "stop_decision", + "ts": "2026-01-01T00:00:10Z", + "session_id": session_id, + "turn_seq": i + 1, + "terminal": True, + "outcome": "verified_clean" if i % 3 else "no_edit", + "duration_ms": 10_000 + i, + "changed_files": 1, + "diff_sha256": f"{i:064x}", + "health_state": health_state, + } + ) + compiles.append( + { + "ts": "2026-01-01T00:00:01Z", + "task_hash": "repeated-task", + "task_prefix": "investigate repeated login latency", + "procedure": "freeform_explore", + "classifier_conf": 0.35, + "art_label": "facts", + "prefetched_keys": [], + "envelope_bytes": 247, + "compile_ms": 4.0, + "agent_mode": "hook", + "session_id": session_id, + "turn_seq": str(i + 1), + "episode_id": episode_id, + "compiler_fp": "fixture", + "injection_advice": "inject", + "cache_hit": False, + } + ) + return events, compiles + + +def _seed(tmp_path: Path, **kwargs) -> None: + events, compiles = _fixture_rows(**kwargs) + _write_jsonl(tmp_path / ".roam" / "episodes.jsonl", events) + _write_jsonl(tmp_path / ".roam" / "compile-runs.jsonl", compiles) + + +def test_policy_ready_requires_complete_join_and_health_context(tmp_path: Path) -> None: + _seed(tmp_path) + result = analyze_ledger(tmp_path) + assert result["summary"]["state"] == "policy_ready" + assert result["summary"]["measurement_admissible"] is True + assert result["summary"]["policy_admissible"] is True + assert result["coverage"]["terminal_coverage_pct"] == 100.0 + assert result["coverage"]["episode_join_coverage_pct"] == 100.0 + assert result["coverage"]["compile_identity_coverage_pct"] == 100.0 + assert result["coverage"]["health_context_coverage_pct"] == 100.0 + assert result["repeat_candidates"][0]["episodes"] == 30 + assert result["repeat_candidates"][0]["evidence_status"] == "candidate" + + +def test_health_unknown_allows_measurement_but_blocks_policy(tmp_path: Path) -> None: + _seed(tmp_path, health_state="unknown") + result = analyze_ledger(tmp_path) + assert result["summary"]["state"] == "measurement_ready" + assert result["summary"]["measurement_admissible"] is True + assert result["summary"]["policy_admissible"] is False + assert result["repeat_candidates"][0]["evidence_status"] == "candidate_only_health_context_missing" + + +def test_missing_terminal_outcomes_withholds_savings_claims(tmp_path: Path) -> None: + _seed(tmp_path, terminal=False) + result = analyze_ledger(tmp_path) + assert result["summary"]["state"] == "insufficient_evidence" + assert result["summary"]["partial_success"] is True + assert result["coverage"]["terminal_coverage_pct"] == 0.0 + assert result["repeat_candidates"] == [] + + +def test_materialization_is_idempotent_and_preserves_identical_compile_calls(tmp_path: Path) -> None: + _seed(tmp_path) + first = analyze_ledger(tmp_path) + second = analyze_ledger(tmp_path) + assert first["materialization"]["event_records"] == 60 + assert first["materialization"]["compile_records"] == 30 + assert second["materialization"]["event_rows_inserted"] == 0 + assert second["materialization"]["compile_rows_inserted"] == 0 + assert second["materialization"]["compile_records"] == 30 + + +def test_invalid_jsonl_is_disclosed_without_crashing(tmp_path: Path) -> None: + events, compiles = _fixture_rows() + roam_dir = tmp_path / ".roam" + _write_jsonl(roam_dir / "episodes.jsonl", events) + with (roam_dir / "episodes.jsonl").open("a", encoding="utf-8") as fh: + fh.write("not-json\n") + _write_jsonl(roam_dir / "compile-runs.jsonl", compiles) + result = analyze_ledger(tmp_path) + assert result["materialization"]["invalid_event_rows"] == 1 + assert result["summary"]["state"] == "insufficient_evidence" + assert result["summary"]["integrity_clean"] is False + assert result["repeat_candidates"] == [] + + +def test_cli_not_initialized_is_honest_and_structured(tmp_path: Path) -> None: + (tmp_path / ".roam").mkdir() + result = CliRunner().invoke(savings, ["--root", str(tmp_path)], obj={"json": True}) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["summary"]["state"] == "not_initialized" + assert payload["summary"]["partial_success"] is True + assert payload["sensor_canaries"]["state"] == "passed" + assert payload["repeat_candidates"] == [] + + +def test_schema_works_without_telemetry(tmp_path: Path) -> None: + result = CliRunner().invoke( + savings, + ["--root", str(tmp_path), "--schema"], + obj={"json": True}, + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + fields = payload["event_schema"]["fields"] + assert {"event_id", "episode_id", "event_type", "terminal", "outcome"} <= set(fields) + assert { + "intervention_id", + "intervention_version", + "eligibility_rule_version", + "assignment", + "downstream_transition_count", + } <= set(fields) + + +def test_intervention_evidence_requires_assignment_observation_join( + tmp_path: Path, +) -> None: + events, compiles = _fixture_rows(count=2) + for index, assignment in enumerate(("control", "exposed")): + episode_id = f"ep_{index:03d}" + common = { + "schema_version": 1, + "hook_version": 6, + "evidence_source": "live_hook", + "episode_id": episode_id, + "session_id": f"cluster_{index}", + "terminal": False, + "outcome": "intervention_measurement", + "health_state": "unknown", + "intervention_id": "repeated_code_slicing", + "intervention_version": "grep-packets-v1", + } + events.append( + { + **common, + "event_id": f"assignment_{index}", + "event_type": "intervention_assignment", + "ts": "2026-01-01T00:00:02Z", + "eligibility_rule_version": "slice-transition-v1", + "eligible_transition": True, + "assignment": assignment, + "assignment_cluster": f"cluster_{index}", + } + ) + events.append( + { + **common, + "event_id": f"observation_{index}", + "event_type": "intervention_observation", + "ts": "2026-01-01T00:00:09Z", + "delivered": assignment == "exposed", + "adopted": assignment == "exposed", + "downstream_transition_count": index, + } + ) + _write_jsonl(tmp_path / ".roam" / "episodes.jsonl", events) + _write_jsonl(tmp_path / ".roam" / "compile-runs.jsonl", compiles) + + evidence = analyze_ledger(tmp_path)["intervention_evidence"] + assert evidence["summary"]["state"] == "instrumented" + assert evidence["summary"]["assignment_events"] == 2 + assert evidence["summary"]["terminal_observation_joins"] == 2 + experiment = evidence["experiments"][0] + assert experiment["assignment_counts"] == {"control": 1, "exposed": 1} + assert experiment["observation_join_coverage_pct"] == 100.0 + assert experiment["event_ordering_violations"] == 0 + assert experiment["promotion_readiness"] == "insufficient_sample" + assert experiment["effectiveness_state"] == "unmeasured" + assert experiment["causal_savings_claimed"] is False + + +def test_intervention_evidence_rejects_post_terminal_observation( + tmp_path: Path, +) -> None: + events, compiles = _fixture_rows(count=1) + events.extend( + [ + { + "event_id": "assignment", + "episode_id": "ep_000", + "event_type": "intervention_assignment", + "ts": "2026-01-01T00:00:02Z", + "session_id": "cluster", + "terminal": False, + "intervention_id": "repeated_code_slicing", + "intervention_version": "grep-packets-v1", + "eligibility_rule_version": "slice-transition-v1", + "eligible_transition": True, + "assignment": "exposed", + "assignment_cluster": "cluster", + }, + { + "event_id": "observation", + "episode_id": "ep_000", + "event_type": "intervention_observation", + "ts": "2026-01-01T00:00:11Z", + "session_id": "cluster", + "terminal": False, + "intervention_id": "repeated_code_slicing", + "intervention_version": "grep-packets-v1", + "delivered": True, + "adopted": True, + "downstream_transition_count": 0, + }, + ] + ) + _write_jsonl(tmp_path / ".roam" / "episodes.jsonl", events) + _write_jsonl(tmp_path / ".roam" / "compile-runs.jsonl", compiles) + + evidence = analyze_ledger(tmp_path)["intervention_evidence"] + assert evidence["summary"]["terminal_observation_joins"] == 0 + assert evidence["summary"]["event_ordering_violations"] == 1 + experiment = evidence["experiments"][0] + assert experiment["promotion_readiness"] == "event_ordering_violation" + + +def test_historical_pattern_interpretation_maps_repetition_to_existing_surfaces() -> None: + slice_hint = _interpret_historical_pattern( + "shell_ngram", + "rg -n => sed -n ", + ) + assert slice_hint["pattern_family"] == "search_then_slice" + assert slice_hint["priority"] == "high" + assert "roam retrieve" in slice_hint["existing_surface"] + + projection_hint = _interpret_historical_pattern( + "shell_sequence", + "roam complexity --json | python3 -c ", + ) + assert projection_hint["candidate_disposition"] == "projection_gap" + + +def test_blocked_continuation_preserves_validated_failure_health() -> None: + start = {"health_state": "unknown"} + blocked = {"health_state": "verification_failed", "terminal": 0} + continuation = {"health_state": "continuation_unverified", "terminal": 1} + assert _episode_health_state([start, blocked, continuation], start, continuation) == "verification_failed" diff --git a/tests/test_savings_aggregate.py b/tests/test_savings_aggregate.py new file mode 100644 index 000000000..92c71d349 --- /dev/null +++ b/tests/test_savings_aggregate.py @@ -0,0 +1,86 @@ +"""Producer-side privacy contract for Prakteon's savings dashboard.""" + +from __future__ import annotations + +import json + +from click.testing import CliRunner + +import roam.commands.cmd_savings as cmd_savings +from roam.savings import aggregate_savings_payload + + +def _rich_payload() -> dict: + return { + "summary": { + "verdict": "Savings claims withheld", + "state": "insufficient_evidence", + "measurement_admissible": False, + "north_star": {"private": "nested prompt"}, + }, + "coverage": { + "prompt_starts": 100, + "historical_prompt_starts": 90, + }, + "sensor_canaries": {"state": "passed", "passed": 3, "total": 3}, + "historical_candidates": [{"pattern": "secret shell text", "task_prefix": "private prompt"}], + "procedure_atlas": { + "opportunities": [{"title": "private opportunity"}], + "failure_signatures": [{"template": "private failure"}], + "recovery_targets": [{"failure_class": "tool_timeout"}], + "intervention_mappings": [ + { + "title": "private intervention", + "declaration_state": "unclaimed", + } + ], + }, + "intervention_evidence": { + "assignments": [ + { + "assignment": "shadow", + "session_id": "private-session", + } + ], + "experiments": [{"intervention_id": "private-id"}], + }, + } + + +def test_aggregate_payload_excludes_transcript_derived_values() -> None: + aggregate = aggregate_savings_payload(_rich_payload()) + + assert aggregate["privacy"]["aggregate_only"] is True + assert aggregate["summary"]["causal_savings_claimed"] is False + assert aggregate["opportunity_counts"]["historical_pattern_candidates"] == 1 + assert aggregate["intervention_state"]["declaration_states"] == {"unclaimed": 1} + serialized = json.dumps(aggregate) + for forbidden in ( + "secret shell text", + "private prompt", + "private opportunity", + "private failure", + "private intervention", + "private-session", + "private-id", + "nested prompt", + ): + assert forbidden not in serialized + + +def test_savings_aggregate_flag_emits_only_safe_envelope(monkeypatch) -> None: + monkeypatch.setattr(cmd_savings, "analyze_ledger", lambda _root: _rich_payload()) + + result = CliRunner().invoke( + cmd_savings.savings, + ["--aggregate"], + obj={"json": False}, + ) + + assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["command"] == "savings" + assert payload["privacy"]["aggregate_only"] is True + assert "historical_candidates" not in payload + assert "procedure_atlas" not in payload + assert "secret shell text" not in result.output diff --git a/tests/test_transcript_backfill.py b/tests/test_transcript_backfill.py new file mode 100644 index 000000000..287d0beeb --- /dev/null +++ b/tests/test_transcript_backfill.py @@ -0,0 +1,406 @@ +"""Privacy and known-answer tests for historical transcript backfill.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from roam.savings import analyze_ledger +from roam.transcript_backfill import ( + _action_outcome_tables, + _compressed_sequence, + _failure_class, + _friction_metrics, + _intent_archetypes, + _is_correction, + _project_scope, + backfill_transcripts, + sanitize_command_template, +) + + +def _write_claude_session(path: Path, cwd: Path, session_id: str, command: str) -> None: + rows = [ + { + "type": "user", + "timestamp": "2026-01-01T00:00:00Z", + "cwd": str(cwd), + "sessionId": session_id, + "message": {"role": "user", "content": "find the repeated customer login failure"}, + }, + { + "type": "assistant", + "timestamp": "2026-01-01T00:00:01Z", + "cwd": str(cwd), + "sessionId": session_id, + "message": { + "role": "assistant", + "usage": {"input_tokens": 100, "output_tokens": 20}, + "content": [ + { + "type": "tool_use", + "id": f"{session_id}-shell", + "name": "Bash", + "input": {"command": command}, + }, + { + "type": "tool_use", + "id": f"{session_id}-edit", + "name": "Edit", + "input": { + "file_path": "/private/customer/auth.py", + "old_string": "secret-old", + "new_string": "secret-new", + }, + }, + ], + }, + }, + { + "type": "user", + "timestamp": "2026-01-01T00:00:02Z", + "cwd": str(cwd), + "sessionId": session_id, + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": f"{session_id}-shell", + "content": "1 passed", + "is_error": False, + }, + { + "type": "tool_result", + "tool_use_id": f"{session_id}-edit", + "content": "updated /private/customer/auth.py", + "is_error": False, + }, + ], + }, + }, + ] + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def test_sanitized_command_retains_shape_and_removes_values() -> None: + raw = ( + "ROAM_MODE=safe AWS_SECRET_ACCESS_KEY=supersecret " + 'rg -n "customer-login-pattern" /private/repo/src ' + "&& git status --short && python -m pytest /private/repo/tests/test_auth.py -q" + ) + template = sanitize_command_template(raw) + assert template == ( + "= = rg -n && git status --short && python -m pytest -q" + ) + for secret in ("supersecret", "customer-login-pattern", "/private/repo", "AWS_SECRET"): + assert secret not in template + + +def test_sanitizer_respects_quoted_pipes_and_newline_boundaries() -> None: + template = sanitize_command_template( + "grep -E 'tests|failed|errors' /private/log\ncd /private/repo\ngit status --short" + ) + assert template == ("grep -e ; cd ; git status --short") + assert template.count("|") == 0 + + +def test_trajectory_compression_preserves_late_state_changes() -> None: + assert _compressed_sequence(["shell"] * 40 + ["edit"] + ["shell"] * 3) == ("shell*40>edit>shell*3") + + +def test_friction_metrics_count_observed_rework_without_claiming_savings() -> None: + actions = [ + {"phase": "orient", "template": "git status --short", "failed": False}, + {"phase": "search", "template": "rg -n ", "failed": False}, + {"phase": "inspect", "template": "sed -n ", "failed": False}, + {"phase": "search", "template": "rg -n ", "failed": True}, + {"phase": "search", "template": "rg -n ", "failed": False}, + {"phase": "edit", "template": "", "failed": False}, + {"phase": "inspect", "template": "sed -n ", "failed": False}, + {"phase": "verify", "template": "python -m pytest -q", "failed": False}, + ] + friction = _friction_metrics(actions, verification_attempts=2) + assert friction["exact_shell_replays"] == 3 + assert friction["failed_action_retries"] == 1 + assert friction["search_inspect_cycles"] == 1 + assert friction["post_edit_context_calls"] == 1 + assert friction["verification_retries"] == 1 + + +def test_action_outcomes_attribute_failures_without_result_content() -> None: + actions = [ + { + "phase": "verify", + "command_class": "verify", + "template": "python -m pytest -q", + "failed": True, + "result_size": 9000, + }, + { + "phase": "verify", + "command_class": "verify", + "template": "python -m pytest -q", + "failed": False, + "result_size": 1000, + }, + ] + outcomes = _action_outcome_tables(actions) + pytest_outcome = outcomes["shell_template_outcomes"]["python -m pytest -q"] + assert pytest_outcome == { + "attempts": 2, + "failures": 1, + "no_results": 0, + "retries_after_failure": 1, + "result_bytes_bucket": 12288, + "failure_classes": {}, + } + assert "9000" not in json.dumps(outcomes) + + +def test_failure_classification_is_closed_and_discards_result_values() -> None: + assert _failure_class("pytest: 3 failed, 10 passed") == "test_failure" + assert _failure_class("ModuleNotFoundError: No module named 'private_pkg'") == ("dependency_unavailable") + assert _failure_class("error: unrecognized argument '--private-flag'") == ("invalid_invocation") + assert _failure_class("opaque private failure detail") == "unknown" + + +def test_intent_archetypes_are_closed_labels_without_prompt_text() -> None: + assert _intent_archetypes("debug the auth failure and add a regression test") == [ + "debug", + "implement", + "verify", + "security", + ] + + +def test_project_scope_prefers_nearest_live_git_root(tmp_path: Path) -> None: + root = tmp_path / "repo" + nested = root / "src" / "feature" + (root / ".git").mkdir(parents=True) + nested.mkdir(parents=True) + _project_scope.cache_clear() + scope, basis = _project_scope(str(nested)) + assert scope == os.path.normcase(os.path.normpath(str(root))) + assert basis == "git_root" + + +def test_correction_detection_ignores_host_wrapper_blocks() -> None: + assert _is_correction("private host context\nActually, use the safer path") + + +def test_backfill_persists_templates_but_no_raw_text(tmp_path: Path) -> None: + root = tmp_path / "repo" + source = tmp_path / "transcripts" + (root / ".roam").mkdir(parents=True) + source.mkdir() + _write_claude_session( + source / "session.jsonl", + root, + "session-private", + 'rg -n "customer-login-token" /private/customer && pytest -q /private/tests', + ) + + result = backfill_transcripts(root, source, source="claude") + assert result["episodes"] == 1 + assert result["privacy_contract"]["sanitized_command_templates_persisted"] is True + raw = (root / ".roam" / "transcript-episodes.jsonl").read_text(encoding="utf-8") + assert "rg -n && pytest -q " in raw + for secret in ( + "find the repeated customer login failure", + "customer-login-token", + "/private/customer", + "secret-old", + "secret-new", + ): + assert secret not in raw + + +def test_repeated_shell_patterns_surface_without_unlocking_live_claims(tmp_path: Path) -> None: + root = tmp_path / "repo" + source = tmp_path / "transcripts" + (root / ".roam").mkdir(parents=True) + source.mkdir() + command = 'rg -n "login-latency" /private/customer && pytest -q /private/tests' + for index in range(3): + _write_claude_session( + source / f"session-{index}.jsonl", + root, + f"session-{index}", + command, + ) + + backfill_transcripts(root, source, source="claude") + result = analyze_ledger(root) + assert result["summary"]["measurement_admissible"] is False + assert result["coverage"]["prospective_prompt_starts"] == 0 + assert result["coverage"]["historical_prompt_starts"] == 3 + candidates = result["historical_candidates"] + assert any( + candidate["kind"] == "shell_template" + and candidate["episodes"] == 3 + and "rg -n && pytest -q " in candidate["pattern"] + for candidate in candidates + ) + assert all("observed_wall_ms" not in candidate for candidate in candidates) + assert all("associated_episode_wall_ms" in candidate for candidate in candidates) + + +def test_dry_run_does_not_create_backfill_files(tmp_path: Path) -> None: + root = tmp_path / "repo" + source = tmp_path / "transcripts" + (root / ".roam").mkdir(parents=True) + source.mkdir() + _write_claude_session(source / "session.jsonl", root, "session", "git status --short") + + result = backfill_transcripts(root, source, source="claude", dry_run=True) + assert result["state"] == "dry_run" + assert not (root / ".roam" / "transcript-episodes.jsonl").exists() + assert not (root / ".roam" / "savings-backfill.key").exists() + + +def test_codex_exec_control_code_is_not_mined_as_shell(tmp_path: Path) -> None: + root = tmp_path / "repo" + source = tmp_path / "sessions" + (root / ".roam").mkdir(parents=True) + source.mkdir() + rows = [ + { + "timestamp": "2026-01-01T00:00:00Z", + "type": "session_meta", + "payload": {"cwd": str(root)}, + }, + { + "timestamp": "2026-01-01T00:00:01Z", + "type": "event_msg", + "payload": {"type": "user_message", "message": "inspect the repository"}, + }, + { + "timestamp": "2026-01-01T00:00:02Z", + "type": "response_item", + "payload": { + "type": "custom_tool_call", + "name": "exec", + "call_id": "control", + "input": "const secretCustomer = await computer.getForegroundApp();", + }, + }, + { + "timestamp": "2026-01-01T00:00:03Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec_command", + "call_id": "shell", + "arguments": json.dumps({"cmd": "git status --short"}), + }, + }, + { + "timestamp": "2026-01-01T00:00:04Z", + "type": "event_msg", + "payload": { + "type": "token_count", + "info": { + "last_token_usage": { + "input_tokens": 100, + "cached_input_tokens": 80, + "output_tokens": 20, + "reasoning_output_tokens": 5, + } + }, + }, + }, + { + "timestamp": "2026-01-01T00:00:05Z", + "type": "event_msg", + "payload": {"type": "task_complete", "duration_ms": 3000}, + }, + ] + (source / "rollout.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + + backfill_transcripts(root, source, source="codex") + raw = (root / ".roam" / "transcript-episodes.jsonl").read_text(encoding="utf-8") + assert "git status --short" in raw + assert "secretCustomer" not in raw + assert "const " not in raw + terminal = json.loads(raw.splitlines()[-1]) + assert terminal["input_tokens"] == 100 + assert terminal["cached_input_tokens"] == 80 + assert terminal["output_tokens"] == 20 + assert terminal["reasoning_output_tokens"] == 5 + assert terminal["phase_sequence_template"] == "other>orient" + assert terminal["friction"]["orientation_calls"] == 1 + assert terminal["tool_result_bytes_bucket"] == 0 + + +def test_search_exit_one_is_no_results_not_tool_failure(tmp_path: Path) -> None: + root = tmp_path / "repo" + source = tmp_path / "sessions" + (root / ".roam").mkdir(parents=True) + source.mkdir() + rows = [ + { + "timestamp": "2026-01-01T00:00:00Z", + "type": "session_meta", + "payload": {"cwd": str(root)}, + }, + { + "timestamp": "2026-01-01T00:00:01Z", + "type": "event_msg", + "payload": {"type": "user_message", "message": "find a missing symbol"}, + }, + { + "timestamp": "2026-01-01T00:00:02Z", + "type": "response_item", + "payload": { + "type": "function_call", + "name": "exec_command", + "call_id": "search", + "arguments": json.dumps({"cmd": "rg -n missing_symbol src"}), + }, + }, + { + "timestamp": "2026-01-01T00:00:03Z", + "type": "response_item", + "payload": { + "type": "function_call_output", + "call_id": "search", + "output": "Process exited with code 1\nFinal output:\n", + }, + }, + { + "timestamp": "2026-01-01T00:00:04Z", + "type": "event_msg", + "payload": {"type": "task_complete", "duration_ms": 3000}, + }, + ] + (source / "rollout.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n", + encoding="utf-8", + ) + backfill_transcripts(root, source, source="codex") + terminal = json.loads((root / ".roam" / "transcript-episodes.jsonl").read_text(encoding="utf-8").splitlines()[-1]) + outcome = terminal["shell_template_outcomes"]["rg -n "] + assert terminal["tool_errors"] == 0 + assert outcome["failures"] == 0 + assert outcome["no_results"] == 1 + + +def test_multiple_transcript_roots_merge_into_one_snapshot(tmp_path: Path) -> None: + root = tmp_path / "repo" + first = tmp_path / "claude-a" + second = tmp_path / "claude-b" + (root / ".roam").mkdir(parents=True) + first.mkdir() + second.mkdir() + _write_claude_session(first / "a.jsonl", root, "a", "git status --short") + _write_claude_session(second / "b.jsonl", root, "b", "roam verify") + + result = backfill_transcripts(root, [first, second], source="claude") + assert result["episodes"] == 2 + assert result["files_considered"] == 2 + assert len(result["source_directories"]) == 2
    SurfaceCountNotes
    CLI commands279272 canonical + 7 aliases.
    CLI commands281274 canonical + 7 aliases.
    MCP tools registered244Full registry — pick a subset via preset.
    MCP tools in core preset16Default for new agent integrations.
    Languages28Tier-1 extractors plus tree-sitter fallback.
    commercial
    279 CLI commands281 CLI commands Yes Yes Yes