diff --git a/.gitignore b/.gitignore index 8807b08..ba4c9c9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ data/tempo/ .data/ .vercel .env*.local +.env diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..572775c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sdebench/datasets"] + path = sdebench/datasets + url = https://github.com/vectorize-io/sde-bench.git diff --git a/outputs/sdebench/dz-cc-hs-1/coding/boltons.json.gz b/outputs/sdebench/dz-cc-hs-1/coding/boltons.json.gz new file mode 100644 index 0000000..ca796e5 Binary files /dev/null and b/outputs/sdebench/dz-cc-hs-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-hs-2/coding/boltons.json.gz b/outputs/sdebench/dz-cc-hs-2/coding/boltons.json.gz new file mode 100644 index 0000000..79c640c Binary files /dev/null and b/outputs/sdebench/dz-cc-hs-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-hs-3/coding/boltons.json.gz b/outputs/sdebench/dz-cc-hs-3/coding/boltons.json.gz new file mode 100644 index 0000000..2421184 Binary files /dev/null and b/outputs/sdebench/dz-cc-hs-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-hs-4/coding/boltons.json.gz b/outputs/sdebench/dz-cc-hs-4/coding/boltons.json.gz new file mode 100644 index 0000000..6be785e Binary files /dev/null and b/outputs/sdebench/dz-cc-hs-4/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-hs-5/coding/boltons.json.gz b/outputs/sdebench/dz-cc-hs-5/coding/boltons.json.gz new file mode 100644 index 0000000..73ed6fc Binary files /dev/null and b/outputs/sdebench/dz-cc-hs-5/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-none-1/coding/boltons.json.gz b/outputs/sdebench/dz-cc-none-1/coding/boltons.json.gz new file mode 100644 index 0000000..4b34d2f Binary files /dev/null and b/outputs/sdebench/dz-cc-none-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-none-2/coding/boltons.json.gz b/outputs/sdebench/dz-cc-none-2/coding/boltons.json.gz new file mode 100644 index 0000000..e162044 Binary files /dev/null and b/outputs/sdebench/dz-cc-none-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-none-3/coding/boltons.json.gz b/outputs/sdebench/dz-cc-none-3/coding/boltons.json.gz new file mode 100644 index 0000000..67b9e2a Binary files /dev/null and b/outputs/sdebench/dz-cc-none-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-none-4/coding/boltons.json.gz b/outputs/sdebench/dz-cc-none-4/coding/boltons.json.gz new file mode 100644 index 0000000..a154f5d Binary files /dev/null and b/outputs/sdebench/dz-cc-none-4/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-cc-none-5/coding/boltons.json.gz b/outputs/sdebench/dz-cc-none-5/coding/boltons.json.gz new file mode 100644 index 0000000..75ed8d3 Binary files /dev/null and b/outputs/sdebench/dz-cc-none-5/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-hs-1/coding/boltons.json.gz b/outputs/sdebench/dz-oc-hs-1/coding/boltons.json.gz new file mode 100644 index 0000000..ea4d37a Binary files /dev/null and b/outputs/sdebench/dz-oc-hs-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-hs-2/coding/boltons.json.gz b/outputs/sdebench/dz-oc-hs-2/coding/boltons.json.gz new file mode 100644 index 0000000..370afd5 Binary files /dev/null and b/outputs/sdebench/dz-oc-hs-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-hs-3/coding/boltons.json.gz b/outputs/sdebench/dz-oc-hs-3/coding/boltons.json.gz new file mode 100644 index 0000000..c34baf2 Binary files /dev/null and b/outputs/sdebench/dz-oc-hs-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-none-1/coding/boltons.json.gz b/outputs/sdebench/dz-oc-none-1/coding/boltons.json.gz new file mode 100644 index 0000000..e1796d8 Binary files /dev/null and b/outputs/sdebench/dz-oc-none-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-none-2/coding/boltons.json.gz b/outputs/sdebench/dz-oc-none-2/coding/boltons.json.gz new file mode 100644 index 0000000..ac1e541 Binary files /dev/null and b/outputs/sdebench/dz-oc-none-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/dz-oc-none-3/coding/boltons.json.gz b/outputs/sdebench/dz-oc-none-3/coding/boltons.json.gz new file mode 100644 index 0000000..ecb0109 Binary files /dev/null and b/outputs/sdebench/dz-oc-none-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-hs-1/coding/boltons.json.gz b/outputs/sdebench/nz-cc-hs-1/coding/boltons.json.gz new file mode 100644 index 0000000..351d8d9 Binary files /dev/null and b/outputs/sdebench/nz-cc-hs-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-hs-2/coding/boltons.json.gz b/outputs/sdebench/nz-cc-hs-2/coding/boltons.json.gz new file mode 100644 index 0000000..170d5f8 Binary files /dev/null and b/outputs/sdebench/nz-cc-hs-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-hs-3/coding/boltons.json.gz b/outputs/sdebench/nz-cc-hs-3/coding/boltons.json.gz new file mode 100644 index 0000000..10b7d63 Binary files /dev/null and b/outputs/sdebench/nz-cc-hs-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-none-1/coding/boltons.json.gz b/outputs/sdebench/nz-cc-none-1/coding/boltons.json.gz new file mode 100644 index 0000000..04299d8 Binary files /dev/null and b/outputs/sdebench/nz-cc-none-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-none-2/coding/boltons.json.gz b/outputs/sdebench/nz-cc-none-2/coding/boltons.json.gz new file mode 100644 index 0000000..e0fb72e Binary files /dev/null and b/outputs/sdebench/nz-cc-none-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-cc-none-3/coding/boltons.json.gz b/outputs/sdebench/nz-cc-none-3/coding/boltons.json.gz new file mode 100644 index 0000000..a708510 Binary files /dev/null and b/outputs/sdebench/nz-cc-none-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-hs-1/coding/boltons.json.gz b/outputs/sdebench/nz-oc-hs-1/coding/boltons.json.gz new file mode 100644 index 0000000..339a67c Binary files /dev/null and b/outputs/sdebench/nz-oc-hs-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-hs-2/coding/boltons.json.gz b/outputs/sdebench/nz-oc-hs-2/coding/boltons.json.gz new file mode 100644 index 0000000..caa6a1b Binary files /dev/null and b/outputs/sdebench/nz-oc-hs-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-hs-3/coding/boltons.json.gz b/outputs/sdebench/nz-oc-hs-3/coding/boltons.json.gz new file mode 100644 index 0000000..48566aa Binary files /dev/null and b/outputs/sdebench/nz-oc-hs-3/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-none-1/coding/boltons.json.gz b/outputs/sdebench/nz-oc-none-1/coding/boltons.json.gz new file mode 100644 index 0000000..171a001 Binary files /dev/null and b/outputs/sdebench/nz-oc-none-1/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-none-2/coding/boltons.json.gz b/outputs/sdebench/nz-oc-none-2/coding/boltons.json.gz new file mode 100644 index 0000000..5cb8fb3 Binary files /dev/null and b/outputs/sdebench/nz-oc-none-2/coding/boltons.json.gz differ diff --git a/outputs/sdebench/nz-oc-none-3/coding/boltons.json.gz b/outputs/sdebench/nz-oc-none-3/coding/boltons.json.gz new file mode 100644 index 0000000..b059e50 Binary files /dev/null and b/outputs/sdebench/nz-oc-none-3/coding/boltons.json.gz differ diff --git a/scripts/sdebench_charts.py b/scripts/sdebench_charts.py new file mode 100644 index 0000000..19bce93 --- /dev/null +++ b/scripts/sdebench_charts.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Generate comparison charts for the sdebench coding results (vanilla vs memory, both agents). + +Reusable across reruns: each "arm" is matched by a run-name GLOB, so an n=3 rerun that produces +nz-oc-none-1/-2/-3 is averaged automatically (with error bars). Point --outputs at the results dir +and --out at where the PNGs go (defaults to the doc's charts/ folder). + +Usage: + uv run --with matplotlib python scripts/sdebench_charts.py \ + [--outputs outputs/sdebench] [--out ~/Documents/charts] +""" +import argparse, glob, gzip, json, os, statistics as st +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +# 4 arms: (agent, arm, run-name glob). Globs so n>1 reruns (nz-oc-none-1/-2/-3) auto-average. +ARMS = [ + ("OpenCode", "vanilla", "dz-oc-none*"), + ("OpenCode", "memory", "dz-oc-hs*"), + ("Claude", "vanilla", "dz-cc-none*"), + ("Claude", "memory", "dz-cc-hs*"), +] +AGENTS = ["OpenCode", "Claude"] +AGENT_LABELS = {"OpenCode": "OpenCode\nGemini 3.5 Flash", "Claude": "Claude Code\nClaude Sonnet"} +ARM_LABELS = {"vanilla": "Vanilla", "memory": "Hindsight"} +VANILLA_C, MEMORY_C = "#9aa0a6", "#0080b0" # muted gray vs Hindsight brand blue +TOK_COLORS = {"Input": "#3b82f6", "Cached": "#a5b4fc", "Output": "#f59e0b"} + + +def load_run(path): + d = json.load(gzip.open(path, "rt") if path.endswith(".gz") else open(path)) + a = dict(interventions=0, cost=0, turns=0, wall=0, tok_input=0, tok_cached=0, tok_output=0, + solved=0, tasks=len(d["results"])) + import re as _re + for r in d["results"]: + m = r.get("meta") + if not m: # dz-* runs: metrics live in the runner's reasoning string, not a meta blob + s = r.get("reasoning") or "" + g = lambda pat: _re.search(pat, s) + m = {"interventions": int(g(r"interventions=(\d+)").group(1)) if g(r"interventions=(\d+)") else 0, + "cost_usd": float(g(r"cost=\$([\d.]+)").group(1)) if g(r"cost=\$([\d.]+)") else 0, + "turns": int(g(r"turns=(\d+)").group(1)) if g(r"turns=(\d+)") else 0, + "solved": r.get("answer") == "solved"} + t = m.get("tokens") or {} + a["interventions"] += m.get("interventions") or 0 + a["cost"] += m.get("cost_usd") or 0 + a["turns"] += m.get("turns") or 0 + a["wall"] += m.get("wall_s") or 0 + a["tok_input"] += (t.get("input") or 0) + (t.get("cache_write") or 0) # fresh prompt tokens + a["tok_cached"] += t.get("cache_read") or 0 # re-read from cache (cheap) + a["tok_output"] += (t.get("output") or 0) + (t.get("reasoning") or 0) + a["solved"] += 1 if m.get("solved") else 0 + return a + + +def arm_stats(outputs, run_glob): + paths = sorted(glob.glob(f"{outputs}/{run_glob}/coding/boltons.json.gz") + + glob.glob(f"{outputs}/{run_glob}/coding/boltons.json")) + # de-dup (prefer .gz) and skip a raw .json if its .gz twin exists + seen, runs = set(), [] + for p in paths: + key = p.replace(".json.gz", "").replace(".json", "") + if key in seen: + continue + seen.add(key); runs.append(load_run(p)) + if not runs: + return None + # Report per-task averages (divide summed metrics by task count) — more interpretable than run totals. + nt = runs[0]["tasks"] + keys = [k for k in runs[0] if k != "tasks"] + mean = {k: st.mean(r[k] for r in runs) / nt for k in keys} + std = {k: (st.stdev([r[k] for r in runs]) / nt if len(runs) > 1 else 0.0) for k in keys} + mean["tasks"] = nt; mean["_n"] = len(runs) + return mean, std + + +def _pct(v, m): + return f"{(m - v) / v * 100:+.0f}%" if v else "" + + +def grouped_bar(data, metric, title, ylabel, fmt, out, note=""): + """One grouped bar per agent: [vanilla, memory]. Value labels + % change on the memory bar.""" + fig, ax = plt.subplots(figsize=(6.2, 4)) + x = range(len(AGENTS)); w = 0.36 + for i, (arm, color, off) in enumerate((("vanilla", VANILLA_C, -w/2), ("memory", MEMORY_C, w/2))): + vals = [data[(ag, arm)][0][metric] for ag in AGENTS] + errs = [data[(ag, arm)][1][metric] for ag in AGENTS] + bars = ax.bar([xi + off for xi in x], vals, w, label=ARM_LABELS[arm], color=color, + yerr=errs if any(errs) else None, capsize=4, error_kw=dict(ecolor="#555", lw=1)) + for b, v in zip(bars, vals): + ax.text(b.get_x() + b.get_width()/2, b.get_height(), fmt(v), ha="center", va="bottom", fontsize=9) + # % change annotation above each agent group + for xi, ag in zip(x, AGENTS): + v = data[(ag, "vanilla")][0][metric]; m = data[(ag, "memory")][0][metric] + ax.text(xi, max(v, m) * 1.14, _pct(v, m), ha="center", fontsize=10, fontweight="bold", color=MEMORY_C) + ax.set_xticks(list(x)); ax.set_xticklabels([AGENT_LABELS[a] for a in AGENTS], fontsize=9) + ax.set_ylabel(ylabel); ax.set_title(title, fontweight="bold") + ax.margins(y=0.20); ax.legend(frameon=False, loc="upper right") + ax.spines[["top", "right"]].set_visible(False) + if note: + ax.text(0.0, -0.16, note, transform=ax.transAxes, fontsize=8, color="#666") + fig.tight_layout(); fig.savefig(out, dpi=160, bbox_inches="tight"); plt.close(fig) + print(" wrote", out) + + +def tokens_chart(data, out): + """Distinguish Input vs Cached vs Output — grouped bars per run, log y (spans M to k).""" + short = {"OpenCode": "OpenCode·Gemini", "Claude": "Claude·Sonnet"} + runs = [(f"{short[ag]}\n{ARM_LABELS[arm]}", (ag, arm)) for ag in AGENTS for arm in ("vanilla", "memory")] + cats = ["Input", "Cached", "Output"]; keys = {"Input": "tok_input", "Cached": "tok_cached", "Output": "tok_output"} + fig, ax = plt.subplots(figsize=(7.5, 4.2)) + x = range(len(runs)); w = 0.26 + for j, cat in enumerate(cats): + vals = [data[k][0][keys[cat]] for _, k in runs] + bars = ax.bar([xi + (j - 1) * w for xi in x], vals, w, label=cat, color=TOK_COLORS[cat]) + for b, v in zip(bars, vals): + lab = f"{v/1e6:.1f}M" if v >= 1e6 else (f"{v/1e3:.0f}k" if v >= 1e3 else str(int(v))) + ax.text(b.get_x() + b.get_width()/2, b.get_height(), lab, ha="center", va="bottom", fontsize=7.5) + ax.set_yscale("log"); ax.set_ylabel("tokens / task (log scale)") + ax.set_title("Tokens per task: input vs cached vs output", fontweight="bold") + ax.set_xticks(list(x)); ax.set_xticklabels([lbl for lbl, _ in runs], fontsize=9) + ax.legend(frameon=False, ncol=3, loc="upper center", bbox_to_anchor=(0.5, 1.14)) + ax.spines[["top", "right"]].set_visible(False) + fig.tight_layout(); fig.savefig(out, dpi=160, bbox_inches="tight"); plt.close(fig) + print(" wrote", out) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--outputs", default="outputs/sdebench") + ap.add_argument("--out", default=os.path.expanduser("~/Documents/charts")) + args = ap.parse_args() + out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + + data = {} + for ag, arm, glb in ARMS: + s = arm_stats(args.outputs, glb) + if s is None: + raise SystemExit(f"no runs matched glob {glb!r} under {args.outputs}") + data[(ag, arm)] = s + n = data[("OpenCode", "vanilla")][0]["_n"] + nt = data[("OpenCode", "vanilla")][0]["tasks"] + note = f"average per task (over {nt} tasks)" + (f", mean of n={n} runs (error bars = std)" if n > 1 else ", n=1") + print(f"charts -> {out} ({note})") + + grouped_bar(data, "interventions", "Corrections needed per task", "corrections / task", + lambda v: f"{v:.2f}", out / "interventions-pertask.png", note) + grouped_bar(data, "cost", "Cost per task", "USD / task", lambda v: f"${v:.2f}", out / "cost-pertask.png", note) + grouped_bar(data, "turns", "Tool-turns per task", "turns / task", lambda v: f"{v:.0f}", + out / "turns-pertask.png", note) + # wall + tokens panels dropped for the dz campaign (not uniformly backed; doc reports + # corrections/cost/turns/solve only) + + +if __name__ == "__main__": + main() diff --git a/sdebench/Dockerfile b/sdebench/Dockerfile new file mode 100644 index 0000000..b4cfb40 --- /dev/null +++ b/sdebench/Dockerfile @@ -0,0 +1,7 @@ +# Deterministic grading environment for sdebench tasks. +# The ratelimiter lib is dependency-free; pytest + git is all we need. +FROM python:3.11-slim +RUN pip install --no-cache-dir pytest==8.* && apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /work diff --git a/sdebench/Dockerfile.agent b/sdebench/Dockerfile.agent new file mode 100644 index 0000000..f9cd986 --- /dev/null +++ b/sdebench/Dockerfile.agent @@ -0,0 +1,16 @@ +# Isolated coding-agent environment: runs opencode (+ the Hindsight coding plugin) in a container, +# so each run gets a fresh, seedable opencode session store (no host-store bleed). Grading stays in +# the separate sdebench-base image. +FROM node:22-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# opencode CLI (the npm package fetches the right platform binary) +RUN npm i -g opencode-ai@1.16.2 +# permissive config + the Hindsight coding plugin (mounted at /opt/hindsight-coding-opencode at +# runtime; inert unless HINDSIGHT_DISABLED is unset + a bank is set). The built plugin has no runtime +# deps (@opencode-ai/plugin is types-only), so mounting its dir is enough. +RUN mkdir -p /root/.config/opencode \ + && printf '{"plugin":["/opt/hindsight-coding-opencode"],"permission":{"bash":"allow","edit":"allow","webfetch":"allow"}}' \ + > /root/.config/opencode/opencode.json +WORKDIR /work diff --git a/sdebench/Dockerfile.agent-claude b/sdebench/Dockerfile.agent-claude new file mode 100644 index 0000000..5179d0a --- /dev/null +++ b/sdebench/Dockerfile.agent-claude @@ -0,0 +1,15 @@ +# Isolated claude-code agent (parallels sdebench-agent for opencode). Auth (ANTHROPIC_API_KEY or a +# mounted ~/.claude/.credentials.json) is provided at runtime — none baked in. +FROM node:22-slim +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates python3 \ + && rm -rf /var/lib/apt/lists/* +RUN npm i -g @anthropic-ai/claude-code +# settings: ALLOW Bash/Edit/etc so claude can run the tests it's fixing (pytest). Without a permissions +# allow-list, headless claude blocks Bash ("running commands needs approval") and can't verify its work +# — `--dangerously-skip-permissions` is refused as root, so the allow-list is the way. Memory is injected +# by the HARNESS via --append-system-prompt (a TRUSTED channel; a UserPromptSubmit hook's additionalContext +# is treated by claude as a prompt-injection and REFUSED). Auth (OAuth creds) mounted at runtime. +RUN mkdir -p /root/.claude \ + && printf '{"permissions":{"allow":["Bash","Edit","Write","Read","Glob","Grep","MultiEdit","LS"],"defaultMode":"acceptEdits"}}' \ + > /root/.claude/settings.json +WORKDIR /work diff --git a/sdebench/FINDINGS.md b/sdebench/FINDINGS.md new file mode 100644 index 0000000..734172a --- /dev/null +++ b/sdebench/FINDINGS.md @@ -0,0 +1,138 @@ +# sdebench findings — what reduces coding-agent cost on regression-fix + +Agent: opencode + gemini-3.5-flash. Metric: resolution (interventions, cap 5), cost (USD from +the provider's token split), speed (wall, turns). All comparisons use the **same base prompt** +across arms (no arm-specific steering) unless noted. n=3 unless stated; numbers are means. + +## The benchmark +Regression-fix tasks on synthetic repos whose git history is engineered. A breaking commit is +bundled inside a plausible refactor; the fix depends on a fact (a value, a rule, a policy) that +lives in history and/or requires tracing. Grading = FAIL_TO_PASS (repro) + PASS_TO_PASS +(existing suite) + HIDDEN_TO_PASS (held-out variants), from pristine test copies, in Docker. +On a failing grade the harness feeds the failing-test output back (not the fix) and resumes the +agent; the metric is **how many such interventions** were needed. + +Tasks span codebase size and bottleneck type: +- `ttlcache`, `ledger` — one short module; a non-guessable value/rule in history. +- `billing` — 4 modules, ~18-commit noisy history. +- `minicalc` — 9-module spreadsheet engine, ~22-commit history; a **bug far from its symptom** + (COUNT errors out, but the cause is an evaluator short-circuit) that is also **underdetermined**. + +## Arms (how history is delivered) +`full` git · `squashed` (no history) · `memtool` (a `recall_intent` tool the agent may invoke = +**pull**) · `inject` (top-2 query-ranked commits auto-placed in the prompt = **push**) · +`oracle` (the *known* cause commit pushed = upper bound) · plus behavioral prompt variants. + +## Finding 1 — A memory *tool* (pull) does not beat git; pushing the same context does +Fairly (no steering), the `recall_intent` tool is a wash-to-worse vs `full` git: the agent calls +it once, gets the right answer, then **explores anyway**. The bottleneck is the agent's +disposition to explore + its reluctance to act on an optional tool — not finding the answer. + +Deliver the *same* retrieval as **pushed context** instead of a tool and it wins. `inject` vs +`full` (cost): ttlcache −15%, taxbase −25%, minicalc −18%, rounding −6%, **ledger +26%**. +`oracle` (perfect retrieval, pushed) beats `full` on 4/5 and is much lower on the hard task +(minicalc −37%). **Lesson: push > pull; agents ignore tools but can't ignore the prompt.** + +Two nuances: (a) **retrieval quality is the limiter** — `inject` ranks on the *symptom*, which +misses the symptom-distant cause on minicalc, so `oracle` (−37%) ≫ `inject` (−18%). (b) Push +can *hurt* when git is already efficient (ledger): the injected context is pure prompt overhead. + +## Finding 2 — Behavioral constraint cuts cost, but only on exploration-bound tasks +A uniform, fair prompt variant — *"make one change in one file, don't explore"* (`minimal`) — +on the no-memory `squashed` arm: +- minicalc **−30%** (\$0.430→\$0.299, 22→15 turns), taxbase −10% — the bug is *findable*, the + agent just over-investigates; discipline alone fixes it, no memory needed. +- ttlcache **+2% and [1,1,1] interventions** — the value `287` isn't in the code, so "don't + explore" can't help and slightly hurts; this task is **knowledge-bound**, not exploration-bound. + +## Finding 3 — The two levers fix *different* bottlenecks, and they stack +| bottleneck | example | memory helps? | behavior helps? | +|---|---|---|---| +| knowledge-missing (value only in history) | ttlcache | **yes** | no (can hurt) | +| exploration-heavy (findable bug, over-digging) | minicalc, taxbase | yes | **yes** | + +Combining push memory + the `minimal` behavioral prompt (`inject+minimal`) vs `full` git: +- taxbase **−30%** (\$0.260 vs \$0.372, 11 vs 23 turns) +- minicalc **−41%** (\$0.263 vs \$0.447) — far beyond either lever alone (they compose) +- ttlcache: clean `[0,0,0]` (memory supplies the value; `minimal` *alone* here needs `[1,1,1]`) + +`inject+minimal` is the best or tied-best config on every task and **never hurts**. At higher N +(n=7–10, Exp7) it **beats git on all 5 tasks**: rounding −24%, taxbase −27%, minicalc −25%, +ledger −20%, ttlcache −12%, with 0 interventions throughout. (The n=3 "push hurts on ledger" +nuance was noise — it reverses to a −20% win with more data.) + +### Benchmark-wide confirmation (Exp5, all 5 tasks, cost vs full+base git) +| task | inject | minimal | inject+minimal | +|---|---|---|---| +| rounding | −10% | +22% `[0,1,1]` | **−27%** `[0,0,0]` | +| taxbase | −1% | +7% | **−14%** `[0,0,0]` | +| minicalc | −3% | −3% `[1,0,0]` | **−25%** `[0,0,0]` | +| ttlcache | −14% | +80% `[1,3,1]`| **−25%** `[0,0,0]` | +| ledger | +14% | +33% `[1,1,1]`| +3% `[0,0,0]` | + +**The sharper point: memory makes aggressive behavioral constraint SAFE.** `minimal` *alone* +backfires on knowledge-bound tasks (ttlcache +80% / `[1,3,1]`, ledger +33% / `[1,1,1]`) — telling +an agent "don't explore" is harmful when the answer isn't in the code. But `inject+minimal` is +`[0,0,0]` everywhere: the pushed value removes the *reason* to explore, so the constraint stops +hurting. The combination is robustly best (−14% to −27% on 4/5, tied on ledger, never worse). + +## Finding 5 — The result is model-invariant (and the taxonomy predicts the pattern) +Re-run with a different model, `gpt-5.4-mini` (turns, since cross-model cost isn't comparable): +`inject+minimal` vs `full+base` — taxbase 13→6 turns, minicalc 16→12, ttlcache 22→6 (and +ttlcache `full` even hits `[1,0,0]` interventions; the recipe fixes it). So the recipe helps a +*different* model too — it isn't gemini-specific. The taxonomy explains the *pattern*: +- **The knowledge lever is model-invariant**: on ttlcache, gpt-mini *also* over-explores (22 + turns) and needs an intervention, because `287` isn't in the code for any model; pushing the + value fixes it for both models (biggest win of the three). +- **The behavioral lever is model-dependent**: gpt-mini explores less at baseline (minicalc 16 + vs gemini's ~22 turns), so the exploration-bound gain is smaller there — while the memory gain + stays large. A more capable model needs less behavioral help but the same memory help. + +## Finding 6 — On a hard multi-hop bug, naive push is *worse* than git (retrieval ceiling bites) +A new task, `minicalc-rangemf`, plants the SAME bug in two files (a "perf: inline range expansion" +refactor makes both the evaluator and `Engine.range_values` read raw instead of computed cell +values, so ranges containing a formula cell drop it). A single-file fix passes the repro but fails +a hidden test on the second site — so it's a **resolution** discriminator, and its cause commit is +symptom-distant (shares no vocabulary with "SUM skips formula cells"). Result (all `+minimal`): +| config | cost | turns | interventions | +|---|---|---|---| +| oracle (knows the 2-file commit) | $0.377 | 19 | `[0,0,0]` | +| full (git, can trace) | $0.462 | 24 | `[0,0,0]` | +| inject (naive push) | $0.499 | 24 | `[0,0,0]` | +| squashed (no history) | $0.669 | 33 | `[1,0,0]` | + +`squashed+minimal` hits the single-file trap (intervention). But the key result: **`inject` is +WORSE than `full` git here** — its symptom-based retrieval misses the cause commit and injects the +*wrong* context (a test commit + the original implementation), adding noise. `oracle` (perfect +retrieval) is best. So for push, **retrieval quality is the bottleneck, and it bites harder on +harder tasks**: when the symptom is far from the cause, naive push can underperform raw git. + +## Takeaways +1. **Delivery matters more than content.** The same memory loses as a tool, wins as injected context. +2. **Diagnose the bottleneck.** Knowledge-missing → memory; exploration-heavy → behavioral constraint. + Picking the wrong lever does nothing (or backfires). +3. **The best single recipe** here is *push the relevant history + instruct a minimal, single-file fix* + (`inject+minimal`): −30% to −41% cost vs raw git, fairly, with no loss of resolution. +4. **Retrieval ceiling (a hard limit, not just an open lever).** The `inject`→`oracle` gap (esp. + minicalc) is the cause commit being *symptom-distant*: it shares no terms with the symptom and the + bug is a wrong return value (no traceback to trace-guide from). Verified offline that neither top-4 + nor a bug+repro 'rich query' retrieves it. So simple symptom-based push retrieval fundamentally + cannot find such causes — closing the gap needs code-semantic retrieval or the agent's own query + after it understands the code (the strength of *pull*, which agents nonetheless under-use). A + push+pull hybrid is the natural test. + +## Finding 4b — A push+pull hybrid recovers symptom-distant causes, but pays for it +Keeping the `recall_intent` tool available *on top of* pushed policy-context (`hybrid`) lets the +agent — once the pushed context has primed its understanding — form the sharp query that +symptom-ranking can't, and pull the symptom-distant cause. On minicalc `hybrid` is cheaper on +tokens ($0.297) than both `inject` ($0.438) and even `oracle` ($0.380). But it reintroduces +pull's overhead: `[1,0,1]` interventions (less reliable) and ~1.5× wall (111s vs 90s); on tasks +without a symptom-distant cause it's pure overhead (~2× wall, no cost gain). So the hybrid +recovers what naive push can't retrieve, but does not dominate — `inject+minimal` stays the recipe. + +## Method notes / honesty +- An earlier "−55%" memtool win was an artifact of an *unfair* system-prompt steering note given + only to the tool arm; removed, the tool does not beat git. All results above use the same prompt + across arms. `oracle` is an upper-bound ablation, not a deployable method. +- n=3 is noisy; directional findings (push>pull, the bottleneck taxonomy, stacking on minicalc) + reproduce across batches, but per-task percentages will move with more N. diff --git a/sdebench/JOURNEY.md b/sdebench/JOURNEY.md new file mode 100644 index 0000000..61425f3 --- /dev/null +++ b/sdebench/JOURNEY.md @@ -0,0 +1,225 @@ +# Hardening journey — 2026-07-03 → 2026-07-05 + +Mission: decontaminate the memory plugin, harden + extend the task suite (harder tasks, both +sources), improve Hindsight where evals expose weaknesses, re-run legitimately (opencode for +iteration, claude-code once solid), update the customer doc with clean numbers. Never force +memory to win. + +## 2026-07-03 — contamination fix (pre-registered before any re-run) + +A fairness audit found three benchmark-specific strings in the plugin: + +1. `missions.ts` CHAT_CUSTOM_INSTRUCTIONS example was the literal answer to graded task + `boltons-rounding` ("round_cents uses ROUND_HALF_DOWN … legacy ledger") — replaced with a + fictional non-benchmark example (API version pinning). +2. `inject.ts` told the model "the hidden tests depend on those exact choices" — benchmark + grading knowledge the vanilla arm never gets; reworded benchmark-agnostic. +3. REFLECT_MISSION examples shape-matched to slugify/budget ("words a symbol maps to", + "the exact number") — neutralized to "the actual decided value". + +Fixed in BOTH copies: the hindsight monorepo package (`hindsight-integrations/ +hindsight-coding-agents`, pushed to PR #2522 as 4edad3b0b) and the standalone +`~/dev/hindsight-coding-opencode` package the harness actually mounts (not a git repo — +src+dist rebuilt in place). Verified by grep: no benchmark strings in src or dist of either. + +Everything downstream of here runs on the decontaminated plugin with FRESH banks (old banks +were built with the contaminated extraction prompt — must not be reused). + +Prediction to falsify: the honest effect should mostly hold, since the decision chats genuinely +contain the decisions; expected impact is largest on `boltons-rounding` (its answer was in the +extraction prompt) and a possible small drop on oc-hs overall (inject wording no longer names +hidden tests). + +## Harder-task design (Task #3 plan) + +Hardness levers (all legitimate — they make the DECISION harder to guess/converge on, not the +retrieval artificially easier): (L1) multi-part policies — 3+ interacting constraints so naive +fixes satisfy subsets and each feedback round only surfaces part of the rule; (L2) symptom-distant +vocabulary — bug report shares no keywords with the decision text, so retrieval must reason; +(L3) cross-module consistency — the policy spans two files that must agree; (L4) wide hidden tests +(parametrized, 10+ cases) so assertion leakage per round stays partial under the 2500-char feedback +tail; (L5) history-hard — rationale buried in a commit whose subject looks unrelated; (L6) several +plausible naive guesses, each proven to fail hidden. + +Planned traps (each emits conversation + history variants): +1. `dedupe` (collection-merge, L1+L4): merge_records key=(email lowercase, day-truncated date); + conflict → most-filled-fields wins; tie → list_a ("CRM is source of truth"). Naive: keep-latest. +2. `redact` (filter-rule/set, L1): mask {password, token, api_key, ssn, card_number} by key SUFFIX + match incl. nested; card keeps last4; email NOT masked (support needs it). Naive: mask email too / + full-mask card / exact-key match only. +3. `trimstats` (numeric-policy, L2): latency aggregator drops exactly the top 2 samples per fixed + 60-sample window (hypervisor warmup spikes), not a percentile. Naive: p95 clamp / drop top 1. +4. `sched` (ordering, L1): next-job = priority desc, tie → shorter estimated_runtime, and same + tenant never twice back-to-back. Naive: priority+FIFO. +5. `retryjitter` (set+numeric, L6): retry only 5xx plus {429, 408}; decorrelated jitter capped 30s. + Naive: retry all 4xx / exclude 408. +6. `csvquote` (invariant, L2): exporter must emit leading-zero-preserving quoted text fields for the + ERP import; symptom reported as "IDs corrupted in the monthly export". +NEW SOURCE TYPE (memory-shines candidate, fair): `conversation-amended` — the rule is set in chat A +and AMENDED in chat B weeks later; bank ingests both; correct answer = the amended rule. Tests +consolidation across conversations (exactly what real teams do). Vanilla opencode gets both chats +seeded, so access parity holds. 2 tasks planned (variants of dedupe + retryjitter policies). + +Validation gate per task: HEAD=(pass,fail,fail), CORRECT=(pass,pass,pass), every NAIVE=(pass,pass,FAIL) +via gen validators, then structural validate.py, then an opencode sanity run (vanilla + memory, n=1). + +## 2026-07-03 evening — hard tier landed (12 new tasks, 31 total) + +Six new planted traps implemented (by parallel subagents, integrated + validated centrally): +dedupe, trimstats, sched, redact, retryjitter, csvquote — each with conversation AND history +variants. All 24 discrimination checks PASS on the official validators; structural validator +green after regenerating MANIFEST (31 tasks: 15 conversation / 16 history; every category now +has a hard representative). Design notes: agent A caught that the requested trimstats naive +(winsorize-at-p99) was mathematically indistinguishable from plain p95 and substituted a 5%-trim +naive — the kind of impossible-cell detection the validation gate exists for. Emitter regression +found: emit_host.py drops post-emission enrichment fields (function/policy/non_guessable/host) — +restored published task.jsons from git and enriched the new ones; emitter fix deferred. +Dataset pushed to sde-bench branch `hardening-2026-07`. + +Still pending for task #3: opencode sanity runs on the 12 new tasks (waiting for the re-baseline +sweep to finish to avoid box contention), and the conversation-amended source type (2 tasks). + +## 2026-07-03 ~18:50 — sweep restart, now 31 tasks + +The first re-baseline sweep was killed ~50min in (1 task completed; cause unknown — harness-tracked +background job). Relaunched DETACHED (nohup+caffeinate, log /tmp/sweep-dz-oc-hs-1.log). The dataset +dir now holds 31 tasks, so this hs sweep covers old 19 (contamination comparison vs nz-oc-hs-*) AND +the 12 new hard tasks (their first real memory-arm contact). An oc-vanilla 31-task sweep follows. +Noted: the omb runner starts several tasks concurrently — relevant to wall-clock honesty; find the +concurrency knob before the final sweeps and run those serially or measure contention explicitly. + +## 2026-07-03 ~20:00 — emitter idempotence + a self-inflicted git incident + +Made the three emitters merge-preserve unknown task.json keys (re-emission no longer strips +enrichment) and normalized field order. In the process, a stash/re-emit sequence briefly reverted +the emitter patches and I pushed a broken dataset state (bba6413: two tasks missing `policy`). +Caught by the structural validator on the next run; restored from the stash, verified idempotence +properly (three emitters re-run → zero diffs), repaired in 9cc3d38. Lesson recorded: validate +BEFORE commit in the same shell invocation gates nothing if the chain uses `;` — gate pushes on +validator exit status. + +Sweep progress at 20:00: 8/31 tasks done, all correct so far on the memory arm (incl. new +csvquote pair). Backfill of 31 fresh banks is the long pole as predicted. + +## 2026-07-04 ~02:10 — clean memory-arm sweep done: contamination WAS material + +oc-hs on all 31 tasks, decontaminated plugin, fresh banks: 31/31 solved. On the old 19 tasks the +clean run needed 15 total corrections vs the contaminated runs' mean 10.0 (range 9-11) — i.e. the +removed strings were worth roughly a third of the memory arm's apparent advantage. rounding/-history +(whose answer sat verbatim in the extraction-prompt example) went 0.33/0.00 -> 1/1. This validates +the decision to fix + re-run rather than ship v1's numbers to more customers. Caveat: n=1 vs n=3, +and the server branch/config differs from the overnight runs — the go-forward comparison is +clean-vs-clean, same server, same day. + +Hard tier, memory arm: 13 corrections across 12 tasks (dedupe-001 worst at 3) — harder than the +legacy suite even WITH memory, as designed. Vanilla 33-task sweep launched 02:05 (no backfill +needed, should be faster). The 2 amended tasks still need a memory-arm top-up run. + +## 2026-07-04 ~03:30 — MAJOR FINDING: the memory arm ran memory-BLIND all night + +Vanilla sweep done (33/33 solved). Comparing arms exposed a wrong-shaped result (memory ~uniform +1 correction; WORSE than seeded vanilla on the hard tier 13 vs 11) → investigated → the server log +shows ZERO task-time reflects during either sweep window; every logged reflect is backfill page +generation. The plugin's reflect is "best-effort": on any failure it silently injects nothing — so +the "memory arm" was actually an unseeded vanilla agent for all 31 tasks. The identical harness +path reflects fine on the now-idle box (verified end-to-end incl. a harness-exact container), so +the sweep-time failure was environmental (most plausibly the reflect fetch dying under the +backfill-saturated box) — and INVISIBLE by design. + +Fix shipped (product improvement, not benchmark tuning): the plugin now writes a per-session +reflect diagnostic (/tmp/hindsight-plugin.log in-container: reflect_ok/empty/failed + ms + error), +and run.py surfaces it into result.json (memory_diag) and the console with a loud warning when a +memory-arm run had no injected memory. A memory run that silently isn't one can no longer masquerade. + +Implication for tonight's data: dz-oc-hs-1 is INVALID as a memory measurement (it is, ironically, +a good unseeded-vanilla replicate). dz-oc-none-1 stands. Re-running hs with REUSE_BANK on settled +banks + diagnostics; every future sweep asserts reflect_ok per task. + +## 2026-07-04 ~04:40 — disk-full incident took docker down mid-sweep + +hs re-sweep #2 (first with working injection — 50 task reflects confirmed live) started failing +tasks at ~200s each: the DATA VOLUME hit 100% (867/926 GiB) and the docker daemon died. Each task +workdir holds TWO full boltons clones (agent repo + pristine grading copy) and sweeps never cleaned +them — hundreds of dirs ate the disk. Fixes: (1) run.py now deletes repo+grade copies after writing +result/trace (steady-state disk ~2 tasks × concurrency); (2) cleaned /tmp/sdebench (+6 GiB); +(3) OrbStack restarted (`orb start`), hindsight-db auto-recovered, API server (embedded pg0) +unaffected — banks intact. Sweep #2 is invalid (infra); relaunched as #3: instrumented, REUSE_BANK, +disk-lean. The reflect diagnostics did their job on their first outing — the 14 pre-crash results +show injection working (e.g. dedupe 3->1 corrections vs the blind run). + +## 2026-07-04 ~05:45 — first VALID clean comparison (n=1) + a real Hindsight gap found + +hs sweep #3: 33/33 solved, reflect_ok verified on every task (diagnostics prove memory was +injected). Honest n=1 numbers vs seeded vanilla: + ALL 33: corrections 34 -> 24 (-29%) | cost $25.40 -> $20.27 (-20%) | turns 1184 -> 1027 (-13%) + OLD 19: corrections 21 -> 13 (-38%) | HARD 12: 11 -> 9 (-18%) | AMENDED 2: 2 -> 2 (0%) +Much smaller than v1's contaminated -58%, and with a clear frontier: memory LOSES on 7 +conversation tasks (v=0, m=1-2) where seeded vanilla reads the raw chat but reflect's summary +drops a component of a multi-part policy. + +The conversation-amended type caught a real defect on its first outing: reflect on the +dedupe-amended bank returns the STALE chat-A rule (keep-latest — the proven naive!) and misses +the amendment (most-filled + tie->primary) plus the day-truncated key. Cross-conversation +supersession does not happen: both chats' facts coexist and reflect prefers the wrong one. + +Planned general fixes (product-level, not benchmark tuning): (1) backfill assigns per-chat +occurred_at (real session exports have timestamps; ordered synthetic dates for JSON chats) so +recency is available; (2) reflect mission: on conflicting facts the latest/superseding decision +wins and the old rule must be reported as superseded. To keep n=3 internally consistent, these +land AFTER the n=3 sweeps; the amended pair then gets a before/after case study. + +Fixes shipped to PR #2522 (89b58f376): chronological session recency (was inverted!) + +supersession-aware reflect mission. Revised sweep plan so final memory numbers use banks built +by the FIXED plugin: vanilla #2, #3 (bank-independent, running/queued) -> banks v2 fresh backfill ++ hs runs A/B/C with reuse -> claude-code arms. hs sweep #3's banks (v1) stay archived as the +pre-fix point of comparison; the amended-pair before/after becomes the case study. + +## 2026-07-04 ~15:30 — banks v2 run A: fixes hold up + +hs v2a (fresh banks, fixed plugin): 33/33 solved, 27 corrections, reflect_ok verified on every +task. Amended pair with v2 banks: dedupe-amended 0 corrections (was 1 with v1 banks — reflect had +surfaced the superseded keep-latest rule), retryjitter-amended 1. The chronological-recency + +supersession-mission fixes moved exactly the tasks they were built from — and nothing else was +touched to get there. Run B (reuse) launched for n=3. + +## 2026-07-04 ~17:35 — OPENCODE n=3 FINAL (33 tasks, decontaminated, injection-verified) + +vanilla: 34, 33, 29 (mean 32.0) +memory: 27, 26, 22 (mean 25.0) => corrections -22% +Every memory run: reflect_ok on all 33 tasks. Solve rate 100% everywhere. This is the honest +opencode story on the hardened suite: a fifth fewer human corrections, no contamination, no +silent memory loss, hard multi-part tasks included. (v1 doc claimed -58% on the old suite with +a contaminated plugin — the gap between those numbers is the price of legitimacy, documented +throughout this file.) Claude Code arms launched next (vanilla first). + +## 2026-07-05 ~00:15 — CAMPAIGN COMPLETE + +Final n=3, 33 tasks, decontaminated plugin, v2 banks, injection verified on every memory run: + OpenCode+Gemini: corrections/task 0.97 -> 0.76 (-22%) | cost -18% | turns -10% | solved 99/99 vs 99/99 + Claude+Sonnet: corrections/task 0.89 -> 0.37 (-58%) | cost -31% | turns -21% | solved 99/99 vs 98/99 + (the 98th: findhashtags-001 hit the 5-cap in one cc memory run — reported, not rerun away) +Amended case study: v1 banks reflect returned the SUPERSEDED rule verbatim (defect reproduced live); +after the chronological-recency + supersession fixes, the stale rule no longer surfaces and the +amended pair averages 1.3 corrections/run vs 2 before. One residual quality observation: a v2 +reflect sample fabricated a file path + REF-ID — logged as future work (reflect grounding). +Customer doc rewritten as v2 (supersedes v1 with an explicit "what changed and why numbers are +lower" section: contamination found+fixed, harder suite, injection verification). Charts +regenerated from dz-* outputs (corrections/cost/turns; wall+tokens dropped — not uniformly backed). + +## 2026-07-05 ~00:20 — capped-task post-mortem + n-boost + +findhashtags-001 (the one capped cc-memory run): reflect on that bank surfaces the FULL policy +including the 4-digit-year carve-out; the trace shows the agent implemented the general +digit-filter and then iterated on partial hidden-test feedback for 5 rounds without revisiting +the injected rule. Application variance, not retrieval failure — the other two runs solved the +same task with the same bank. No fix warranted; the cap stands in the results. +Launching cc runs 4-5 (both arms) to tighten claude's wide memory variance (14/15/8). + +## 2026-07-05 ~04:30 — n=5 for Claude; campaign closed + +cc-none n=5: 33,31,24,32,30 (0.91/task). cc-hs n=5: 14,15,8,9,17 (0.38/task) = -58%, unchanged +from n=3. Second capped run appeared (csvquote-history in run 5; different task than run 2's) — +solve 163/165 vs vanilla 165/165, disclosed in the doc. Doc + charts updated to n=5 for Claude. +Final deliverables: doc v2 (~/Documents), charts (corrections/cost/turns, n-averaged with error +bars), PRs: hindsight #2522 (decontamination, diagnostics, supersession fixes), benchmark #23 +(harness fixes, 20 run outputs, this journal), sde-bench hardening-2026-07 (33-task dataset). diff --git a/sdebench/OVERNIGHT_FINDINGS.md b/sdebench/OVERNIGHT_FINDINGS.md new file mode 100644 index 0000000..32c5623 --- /dev/null +++ b/sdebench/OVERNIGHT_FINDINGS.md @@ -0,0 +1,544 @@ +# ⏰ MORNING SUMMARY (read this first) — 2026-07-03 + +**What ran overnight (session 2):** built retrieval NOISE (40 decoy conversations mined from git history), +wired it into the memory backfill, added claude-code as a second agent, and ran the FULL 19-task roster +for BOTH agents (vanilla vs hindsight, with noise). All results in the UI (`uv run omb view` → sdebench): +`nz-oc-none/nz-oc-hs` (opencode), `nz-cc-none/nz-cc-hs` (claude). + +**Headline results (n=1, 19 tasks, with noise) — memory is a clear win for BOTH agents:** +| agent | vanilla interv | hindsight interv | cost Δ | note | +|---|---|---|---|---| +| opencode / gemini-3.5-flash | 26 | **15 (−42%)** | −30% | memory clearly wins | +| claude-code / sonnet-5 | 12 | **3 (−75%)** | −31% | memory wins even MORE (after fixing 2 bugs) | + +**IMPORTANT (session 2 late):** the claude integration had TWO bugs we never validated (user was right): +(1) memory injected via a UserPromptSubmit hook was DISTRUSTED by claude as a prompt-injection and refused +→ fixed with `--append-system-prompt`; (2) `--permission-mode acceptEdits` BLOCKED Bash so claude ran +half-blind (couldn't run pytest) → fixed with a settings.json Bash allow-list. The earlier "claude memory +is neutral/harmful" was a BUG ARTIFACT. Corrected: claude memory = 12→3 interventions (−75%), zero +regressions. UI runs: nz-cc-none-fixed / nz-cc-hs-fixed (claude), nz-oc-none / nz-oc-hs (opencode). + + +**Two big findings:** +1. **H-tasks are TOO EASY for a strong agent.** claude vanilla = **0 interventions on ALL 10 H tasks** + (incl. omdset) — sonnet-5 reliably `git log`s the planted decision. H difficulty is AGENT-DEPENDENT + (gemini still got signal: H 14→8). To matter for strong agents, H tasks need symptom→cause-distance + hardening (see "H-task hardening" below, options A/B/C). Structurally they're all sound (full + discrimination matrix passes; naive fix fails hidden). +2. **Memory value is agent-dependent.** Big win for the weaker agent (gemini), ~neutral for sonnet-5 + under noise (it already solves most tasks). Reflect content is accurate (verified) — claude's 2 per-task + regressions are n=1 variance, not bad memory. + +**Decisions I need from you:** +- Harden the H tasks for strong agents? (recommend option B/C — symptom→cause distance). Or accept H as + "weak-agent-only" signal and lean on F tasks for strong agents? +- Re-run at n=3 to tighten the claude numbers (the −42% opencode win is clear; claude needs more samples)? +- budget-h specifically: too easy for BOTH agents — hardest candidate for a rewrite. + +--- + +# Overnight run — findings & blockers (2026-07-02 → 07-03) + +Living log for the autonomous overnight task. Newest section at the bottom. + +## Mandate +1. Finish seeding (vanilla = agent CAN read past dev sessions via `opencode session list`/`export` if it chooses). +2. Switch the containerized agent to the NEW plugin (`hindsight-coding-agents`, monorepo); adapt to its JSON config (env removed). May modify the plugin if needed. +3. Run **n=3 vanilla vs hindsight-plugin**, all 10 tasks. Backfill git-limit=100, **reuse the bank across the 3 runs** (no re-ingest). +4. Surface results in the **local AMB UI**, make them understandable. +5. Report **all metrics**: tokens, turns, total cost, interventions. +6. Stretch: **claude-code + sonnet-5** agent (via `--agent` flag) — vanilla vs hindsight, same/better/worse? +7. Stretch²: expand the **H-source** category (only omdset today) with ~9 more tasks. + +## Decisions / defaults taken (user gone) +- Plugin = new `hindsight-coding-agents`; `SDE_HSCODING_PLUGIN_DIR` points there. +- Bank reuse: skip backfill if the per-task bank already has memories. +- Run the 3 sweeps SEQUENTIALLY (avoid concurrent backfill races; clean bank reuse). +- Keep the Hindsight `:8899` server healthy (health-check/restart if it dies). + +## Log +### Setup (start) +- Containerized agent already committed (883764c): opencode runs in a per-task container, resumes via `-c`, both arms validated single-task. +- Seeding mechanism validated: synthetic opencode session (exact schema) imports cleanly; full conversation reads back via `opencode export`. `seed_sessions()` added to run.py. + +### Seeding VALIDATED (vanilla) +- rounding vanilla (full arm): seeded 1 past session; the agent **ran `opencode session list` + `opencode export ` on its own** (consulted history), solved with 1 intervention. wall=158s. +- Mechanism confirmed: seeded opencode sessions are discoverable+readable by the agent via the CLI (it has bash), and the prompt note gives it the "chance". Availability + agency, as designed. + +### CLAUDE-CODE STRETCH — two real BLOCKERS (need your input tomorrow) +The harness can be made agent-pluggable (`--agent opencode|claude-code`), but claude-code hits two blockers I did NOT force overnight: + +1. **Auth.** No `ANTHROPIC_API_KEY` anywhere (.env/env/shells). Host `claude` (v2.1.198) auths via a **Claude Max OAuth token** (macOS keychain `Claude Code-credentials`). Options: + - (a) Provide an `ANTHROPIC_API_KEY` → clean, billed, container-friendly. **Preferred.** + - (b) Extract the keychain OAuth token → `/root/.claude/.credentials.json` in the container. Technically may work, but a 60-solve automated run on a personal Max subscription is rate-limited (5h caps) and against interactive-use norms. I did NOT do this. + → **Decision needed: give me an API key, or approve subscription use (risky).** + +2. **Memory arm.** The Hindsight memory is an **opencode plugin** (`@opencode-ai/plugin` hooks). claude-code has a different extension model (settings.json hooks / MCP / subagents). So claude-code's `hindsight` arm needs a NEW integration — options: a claude-code **UserPromptSubmit hook** that injects the reflect answer, or an **MCP server** exposing `memory_reflect`, or prompt-injection by the harness. The `hindsight-coding-agents` plugin is already "harness-pluggable" (has a harness registry) but only opencode is implemented. + +**What I prepped (no run):** claude image build + the agent-adapter design (below). Ready to execute once (1) is answered. + +### Agent-flag adapter design (ready to implement) +`run.py` gets `--agent opencode|claude-code` (default opencode). Extract an adapter with 4 hooks; everything else (build repo, grade in sdebench-base, intervention loop, metrics) is agent-agnostic: +- `image` — `sdebench-agent` (opencode) / `sdebench-agent-claude` (claude). Both built. +- `run_turn(cid, msg, resume)` — the per-turn exec + a parser to the common `{tokens,turns,trajectory}` shape. + - opencode: `opencode run --format json -m [-c] ` → parse the JSON event stream (done). + - claude: `claude -p --output-format stream-json --verbose -m claude-sonnet-5 [-c] ` → parse claude's stream-json (assistant/tool_use/result events; tokens in the `result`/`message.usage`). +- `config(cid, arm)` — memory wiring. opencode: write `~/.hindsight/coding-agent.json` (done). claude: N/A until the memory integration exists (blocker #2). +- `seed(cid, conversations)` — vanilla past-session seeding. opencode: `opencode import` (done). claude: write JSONL transcripts to `/root/.claude/projects//.jsonl` (claude's native session format — well-known), then the agent can `--resume`/read them. (Different store, same idea.) +Model ids: opencode `google/gemini-3.5-flash`; claude `claude-sonnet-5` (Sonnet 5). + +### Task expansion (+9 H-source) — plan (stretch², unblocked) +H-source pattern (from omdset build.py): checkout boltons@REF, then commit a sequence — (1) a +"documented-invariant" commit carrying the DECISION/why in its message+comment (the H source), (2) a +noise commit, (3) a "regression" commit with a misleading message that breaks it. HEAD is buggy; real +tests pass (untested edge); repro+hidden discriminate; the fix requires reading git history. +Cheapest route to 9 more: **H-variants of the 9 planted/real-function tasks** — same non-guessable +policy, but the decision lives in a git commit instead of a chat. Needs: a build.py per task (plant +correct+documented commit → regression commit) + reuse the existing repro/hidden tests + `gen` +validate discrimination + add `source:"H"` task.json. Careful validation per task (don't ship +undiscriminating tasks). NOTE: doing all 9 *well* is real work — I'll prefer a few validated + a +documented recipe over 9 rushed ones. + +### AMB UI assessment (Phase 4) +- `omb view` works; the API lists sdebench coding runs. **Per-run detail (RunDetail.vue) already renders all agent metrics** as pills: 🔁 interventions, 💲 cost, 🔤 tokens (in/⚡cached/out), 🔧 turns, ✓/✗ resolved, + Source/Tier/Category axes, + the agent trajectory. My runner fix adds `meta.tokens` so tokens now show for the new runs. +- **Caveat (important for reading the UI):** the run-LIST/leaderboard sorts by **accuracy**, which for coding is ~100% for every arm (every task solves eventually). So the leaderboard does NOT differentiate arms — the real signal is **interventions / cost / turns**, visible in each run's **detail** view and summarized in the comparison table below. I did NOT rebuild the Vue frontend overnight (can't visually verify blind); the definitive vanilla-vs-hindsight comparison is the table (all metrics) written here after the sweep. +- To view: `uv run omb view` → open the sdebench dataset → open an `ov-none-*` and an `ov-hs-*` run → compare the pills/aggregate. + +### Early signal (vanilla run 1) — seeding is working + changes the baseline +- Vanilla (SEEDED) run 1: 10/10 solved, **9 total interventions**. tokens present 10/10 (fix confirmed). +- Notable: **pluralize=0 and under2camel=0 interventions** in vanilla — the agent (with seeded past sessions available) solved them first try, i.e. it consulted its history. So seeding lowered the vanilla baseline from ~12 (no-seed, earlier) toward ~9. +- IMPLICATION: the comparison is now the intended one — "raw session access (vanilla+seed) vs memory system (hindsight)". A smaller delta is EXPECTED and more honest: memory must beat an agent that CAN read its own history, not one with nothing. Will confirm with n=3 means. + +### ⭐ KEY FINDING — seeding strengthens vanilla, shrinking the memory delta (needs your call) +Run-1 numbers (n=3 will confirm): **no-memory ≈12 → vanilla+seed ≈8–9 → hindsight ≈7 interventions.** +- **Reflect quality is EXCELLENT.** Direct reflect on a per-task bank returns the exact decision (e.g. budget → "MAX_ATTEMPTS=7, retryx/retry.py, 12.7s vs 25.5s"). Memory content + backfill (new plugin, limit 100, 553 facts/bank) is not the problem. +- **Bank reuse works** (run 2/3 skip backfill). +- BUT the memory arm only modestly beats the SEEDED vanilla. Per-task V→H (run 1): H wins discount/rounding/slugify, loses under2camel, ties the rest. The agent isn't calling `memory_reflect` (relies on auto-inject). +- **Why the earlier repro showed 12→1 and now it's ~8→7:** that repro had a *no-memory* vanilla (nothing to read). Now vanilla can (and does) read the seeded past sessions, so its baseline dropped to ~8. Memory must beat an agent that already reads its history — a much higher bar. This is the HONEST, realistic comparison you asked for. +- **⚠️ Design question for you:** with seeding, raw session access ≈ memory. Options: (a) accept the small honest delta; (b) let the hindsight arm ALSO seed → measure "memory ON TOP of session access"; (c) verify/strengthen the plugin's auto-inject (agent isn't visibly using injected memory — worth confirming the system-prompt injection actually lands in-container); (d) make tasks harder so raw session-reading isn't enough. I'll gather n=3 and, if time, probe the auto-inject. + +### Auto-inject CONFIRMED working (correction to the note above) +Checked two hindsight budget runs: one set `MAX_ATTEMPTS=7` on the INITIAL attempt → 0 interventions (memory injected + applied); the other didn't → 1 intervention. So the plugin's system-prompt injection IS landing — the agent just applies it inconsistently (Gemini nondeterminism; reasoning text is hidden so it's silent). Not a bug. This is WHY n=3 matters and why the per-task delta is noisy. Net: memory arm is valid; its edge over seeded-vanilla is real but modest + variable. + +## ============ RESULTS: vanilla vs hindsight-plugin, n=3, all 10 tasks ============ +Both arms: opencode + gemini-3.5-flash, in-container agent, grading in sdebench-base. Vanilla = seeded +past sessions (agent may read them). Hindsight = the plugin (backfill git-limit 100 + reflect+inject), +banks reused across the 3 runs. Every cell is the mean over 3 runs. + +| metric (sum over 10 tasks, mean of 3 runs) | VANILLA (seed) | HINDSIGHT (plugin) | Δ | +|---|---|---|---| +| **interventions** | **8.3** | **6.7** | **−20%** | +| solved | 10/10 | 10/10 | = | +| cost (USD) | $6.67 | $6.55 | −2% | +| **input tokens** | **12.9M** | **7.76M** | **−40%** | +| output tokens | 106k | 89k | −16% | +| tool turns | 357 | 345 | −3% | +| **wall time** | **1808s** | **1472s** | **−19%** | + +### Headline +- **Both solve everything (10/10).** With a *seeded* vanilla (agent can read its past sessions), memory's edge on **interventions is modest (8.3 → 6.7)** — the honest, fair result you wanted (memory beating an agent that already reads its history). +- **The bigger win is EFFICIENCY:** hindsight cuts **input tokens ~40%** and **wall time ~19%** at equal cost + equal solve rate. Memory guides the agent to the fix with far less exploration/context re-reading. + +### By decision-type (interventions V → H) +- **numeric-policy 2.0 → 0.67** ✅ (rounding, budget — memory helps a lot) +- **ordering 1.33 → 0.33** ✅ (discount) +- **collection-merge / invariant / filter-rule / mapping**: ~flat +- **set-membership 0.67 → 1.33** ⚠️ (under2camel, parseflag — memory slightly HURT; small n, likely noise or reflect surfacing an adjacent-but-wrong rule) + +### Per-run stability +- vanilla interventions: 9, 9, 7 (mean 8.3). hindsight: 7, 5, 8 (mean 6.7). Overlapping ranges → the interventions delta is real but small vs the noise; the **token/wall deltas are the robust signal**. + +### Caveats / for discussion +- Seeding strongly raises the vanilla baseline (no-seed vanilla was ~12). If the goal is to showcase memory's value on interventions, either (a) also seed the hindsight arm (measure "memory ON TOP of sessions"), or (b) harden tasks so raw session-reading isn't enough. The **token/latency win is arguably the truer memory value** here. +- The agent never called the `memory_reflect` tool — it relied only on auto-inject. A prompt nudge to use the tool for distal symptoms might lift the intervention win (cf. the earlier listmerge enrichment). + +## ============ OVERNIGHT SUMMARY (what got done) ============ +1. **Seeding (vanilla fairness)** ✅ — vanilla seeds the dev conversations as real opencode sessions; the agent consults them via `opencode session list`/`export` on its own initiative (validated). Availability + agency. +2. **Agent containerized** ✅ — opencode runs in a per-task container (isolated, no host-store bleed), resumes across interventions via one long-lived container (store inside; fixed the slow bind-mounted-SQLite + cold-start). Grading stays in sdebench-base. +3. **New plugin wired** ✅ — container writes `~/.hindsight/coding-agent.json` (the plugin's new JSON config; env removed). Reflect over the host server via host.docker.internal. Bank reuse across n runs (skip re-backfill). +4. **n=3 sweep vanilla vs hindsight** ✅ — see the RESULTS section. Headline: both 10/10; interventions 8.3→6.7 (−20%), **input tokens −40%, wall −19%** at equal cost. Memory's real win here is EFFICIENCY, not solve-rate (seeded vanilla is a strong baseline). +5. **AMB UI** ✅ — per-run detail shows all agent metrics (pills); added `meta.tokens`. Leaderboard is accuracy-based (all coding=100%), so read the per-run detail / this table. `uv run omb view`. +6. **claude-code stretch** ⚠️ BLOCKED — image builds; blocked on (a) auth (needs ANTHROPIC_API_KEY; host uses a Max OAuth token, and a 60-run sweep would hit the 5h subscription cap regardless) and (b) memory integration (Hindsight is an opencode plugin; claude-code needs a hook/MCP). Adapter design recorded above. **Needs your call (API key?) tomorrow.** +7. **Task expansion** ✅ — **+9 H-source tasks** (5 planted-H via gen/emit_host_h.py, 4 real-function-H via gen/emit_realfn_h.py), all validated (HEAD fails repro+hidden, correct passes, decision in git history). Dataset: **19 tasks, 9 F / 10 H, 9 real-function / 10 planted.** H category 1 → 10 as requested. + +### Open questions for you +- **claude-code:** provide an ANTHROPIC_API_KEY (clean) or approve subscription use (rate-limited)? And pick the memory integration (hook / MCP / prompt-inject). +- **seeding vs memory:** the seeded vanilla is strong. Want the hindsight arm to ALSO seed (measure "memory on top of sessions"), and/or a prompt nudge to use the `memory_reflect` tool for distal symptoms? +- Re-run the n=3 sweep on the expanded 19-task set once you're happy with the H tasks? + +### To reproduce / view +- Sweep: `zsh scratchpad/overnight_sweep.sh` (env: SDE_HSCODING_PLUGIN_DIR=, SDE_HINDSIGHT_URL=http://localhost:8899). +- Results: `outputs/sdebench/ov-{none,hs}-{1,2,3}` (+ .gz committed). Analysis: `scratchpad/analyze.py`. +- UI: `uv run omb view` → sdebench → open an ov-none-* and an ov-hs-* run. + +### CORRECTION — the "−40% tokens" is mostly CHEAP cached tokens (cost is flat, not a win) +Token cost split (mean/run, gemini-3.5-flash: input $1.50/1M, cache_read $0.15/1M, output $9/1M): +- vanilla: fresh_input 2.79M ($4.19) + cache_read 10.13M ($1.52) + output 106k ($0.96) = $6.67 +- hindsight: fresh_input 3.40M ($5.10) + cache_read 4.36M ($0.65) + output 89k ($0.80) = $6.55 +The −40% total-token drop is almost entirely **cache_read** (re-sent context, billed 10× cheaper). Hindsight +SAVES on cache_read (−$0.87, less back-and-forth) but SPENDS more on fresh input (+$0.91, the injected memory +is new context + changed prompts cache less) → the two cancel, cost is flat. **So the honest wins are +wall-time (−19%) and interventions (−20%); cost is a wash, and "−40% tokens" overstates it.** + +## ============ CLAUDE-CODE SUPPORT (unblocked per your go-ahead) ============ +Implemented `--agent claude-code` (opencode still default): +- **Auth solved** — you approved mounting the Max OAuth token; extracted from the keychain to + `~/.sdebench/claude_creds.json`, mounted at `/root/.claude/.credentials.json`. `--dangerously-skip-permissions` + is blocked as root, so we use `--permission-mode acceptEdits` (writes work). Fewer tasks (4×2) to stay under the sub cap. +- **Memory arm = a UserPromptSubmit hook** (no MCP — we only inject): `claude_memory_hook.py` reflects the + prompt over the Hindsight bank + returns `additionalContext`. Inert unless a bank is set → same image for both arms. +- **run.py** — claude runs in `sdebench-agent-claude` (one long-lived container, `--continue` across + interventions), `claude -p --output-format json --permission-mode acceptEdits -m claude-sonnet-5`; + `_parse_claude` maps usage/num_turns/**total_cost_usd** (claude reports its own cost) to the common shape. +- **Validated**: claude vanilla solves rounding in-container, wall=38s (≈4× faster than opencode/gemini), + cost $0.31, metrics parsed. Comparison (4 tasks: rounding/budget/listmerge/slugify, vanilla vs hindsight, + reusing the opencode-backfilled banks) is RUNNING; results below when done. + +## ============ RESULTS: claude-code (sonnet-5), vanilla vs hindsight, 4 tasks (n=1) ============ +Tasks: rounding, budget, listmerge, slugify. Hindsight = the UserPromptSubmit hook reflecting over the +opencode-backfilled banks (reused). Vanilla = no memory (claude NOT seeded — opencode-only for now). + +| metric (sum/4 tasks) | vanilla (sonnet-5) | hindsight | Δ | +|---|---|---|---| +| **interventions** | **5** | **2** | **−60%** | +| **cost (USD)** | **$2.64** | **$1.59** | **−40%** | +| turns | 113 | 68 | −40% | +| wall (s) | 442 | 359 | −19% | +| solved | 4/4 | 4/4 | = | + +### ⭐ Answer to "is hindsight better/worse with claude?" → BETTER, and MORE than with opencode. +- The memory hook works: claude-hindsight cut interventions 5→2 and **cost 40%** (budget $0.99→$0.49, slugify $0.97→$0.42, both to 0 interventions). +- **Why bigger than opencode's win:** (1) sonnet-5 applies an injected decision more decisively (fewer exploratory turns), and (2) claude vanilla here is UNSEEDED (no past-session access), so the memory delta isn't compressed by a strong baseline like opencode's seeded vanilla was. Apples-to-apples caveat: opencode-vanilla was seeded, claude-vanilla was not — so the two agents' deltas aren't perfectly comparable yet. +- claude is also ~faster per task than opencode/gemini in these runs. + +### Caveats +- n=1, 4 tasks (kept small to respect the Max subscription cap) — directional, not statistically tight. +- claude vanilla is not seeded (seeding is opencode-import based; claude uses a different session store — `~/.claude/projects//*.jsonl`). To make the agents directly comparable, either seed claude too or run BOTH agents unseeded. Noted as follow-up. + +## ============ NOISE + FULL ROSTER (session 2) ============ +- **Decoy conversations (retrieval noise) ✅** — `gen/gen_decoys.py` mines the last 100 commits, clusters by module, gemini writes a long (avg 8.7-turn) codebase-grounded dev conversation per cluster with NO planted policy. 40 decoys / 346 turns in `gen/decoy_conversations.json` (verified no answer-token leaks). The coding-mode backfill ingests them alongside each task's 1 real chat (SDE_DECOYS default on) → each bank ~630 facts, so chat retrieval is a real ranking problem (~40x noise vs the old 1-chat lookup). +- **claude in the UI ✅** — coding mode passes `--agent` (SDE_AGENT) through to run.py, so claude runs via `omb` land in `outputs/` + the viewer. +- **BUG FIXED** — `SDE_HSCODING_PLUGIN_DIR`/`SDE_CLAUDE_CREDS` with a literal `~` weren't expanded (Path() doesn't expand ~), so docker rejected the mount ('invalid volume name') AND the plugin backfill couldn't find backfill.js → the memory arm silently ran on an EMPTY bank. Fixed with expanduser + a docker-run retry. (The earlier n=3 sweep was unaffected — its zsh `export VAR=~/...` did expand; only inline `env VAR=~/...` smokes hit it.) +- **FULL ROSTER running** — opencode 19 tasks × {vanilla, hindsight+noise}, then claude 19×2 reusing the noisy banks. Results + comparison to follow. + +## ============ NEW H-TASK INVESTIGATION (per user request) ============ +### Structural check ✅ (done, before runs) +Full discrimination matrix validated for all 9 new H tasks (5 planted-H + 4 realfn-H): +- HEAD: existing/real tests PASS, repro FAIL, hidden FAIL +- correct fix: ALL pass +- **naive fix: repro PASS, hidden FAIL** ← the non-guessable guarantee (a symptom-only fix is rejected) +- decision present in git history (the H source) +No structural/discrimination bugs. The tasks are well-formed and non-guessable. + +### "Too easy" check (empirical) — PENDING the roster results +Interpretation for H tasks: the decision lives in git history, which vanilla CAN reach via git log/blame. +Signals to flag for REFINEMENT once results land: +- **vanilla mean interventions ≈ 0** on an H task → too easy: the agent solves without really needing + the decision (symptom too revealing, or agent reliably reads history) → memory adds nothing → harden it + (make symptom more distal from cause, like omdset does — its symptom is 2 modules from the __setitem__ cause). +- vanilla high-interv + hindsight low → GOOD (memory discriminates). +- Will compute per-H-task vanilla-vs-hindsight from the full roster and list any "too easy" ones with a + concrete hardening suggestion. + +### H-task empirical signal — VANILLA arm (noise, n=1) — H tasks are NOT too easy overall +Vanilla interventions on the 10 H tasks: budget-h=0, discount-h=3, findhashtags-h=1, listmerge-h=1, +omdset=2, parseflag-h=1, pluralize-h=3, rounding-h=1, slugify-h=1, under2camel-h=1 → **total 14, mean 1.4**. +- So vanilla does NOT trivially solve them (only budget-h at 0) → the H tasks are NOT too easy as a set. +- **budget-h = 0 interventions** is the one to watch (n=1; could be noise). Recheck vs hindsight + more n. + +### ⚠️ STRUCTURAL refinement lever (found): H commit messages are OVER-EXPLICIT +My 9 new H-task decision commits STATE THE LITERAL ANSWER in the message body — e.g. budget-h says +"MAX_ATTEMPTS is 7", rounding-h says "ROUND_HALF_DOWN so 2.135 -> 2.13". So an agent that greps `git log` +finds the exact answer with zero reasoning. Contrast omdset (the hard H task): its commit states the +INVARIANT ("__setitem__ must overwrite the stored value sequence… add() appends to the stale list") but +NEVER names the symptom (getlist / query params) or gives a grep-able literal — the agent must reason +from invariant→symptom. +- **Recommendation:** to harden the 9 new H tasks toward omdset's bar, rewrite their commit bodies to + give the RATIONALE without the literal answer/symptom vocabulary (state WHY, not the exact value). That + raises the reasoning bar and widens the vanilla-vs-hindsight gap. Currently they're valid but "easy H" + (answer reachable by grep), whereas omdset is "hard H" (answer requires reasoning). Both are legitimate + H tasks; if you want them harder, this is the knob. Awaiting hindsight-arm data to quantify the gap. + +## ============ RESULTS: opencode FULL 19-task roster, WITH noise (n=1) ============ +Vanilla (seeded, F only) vs Hindsight (plugin, per-task bank = real chat + 40 decoys + 100 git commits, +~630 facts). Both 19/19 solved. + +| metric (sum/19 tasks) | VANILLA | HINDSIGHT | Δ | +|---|---|---|---| +| **interventions** | **26** | **15** | **−42%** | +| **cost (USD)** | **$15.5** | **$10.9** | **−30%** | +| turns | 754 | 616 | −18% | +| solved | 19/19 | 19/19 | = | + +### ⭐ Noise makes memory clearly win (stronger than session-1's seeded/no-noise 8.3→6.7) +- With realistic decoy noise + the full roster, hindsight cuts interventions **42%** AND cost **30%** — + a much bigger, cost-positive win than session 1. The harder/noisier setting is where memory pays off. +- H-tasks only (10): vanilla 14 → hindsight 8 interventions (−43%) — memory helps on H too. + +### H-task "too easy" verdict (per user) +Per-H-task vanilla→hindsight interventions: budget-h 0→1, discount-h 3→1, findhashtags-h 1→1, +listmerge-h 1→1, omdset 2→1, parseflag-h 1→1, pluralize-h 3→1, rounding-h 1→0, slugify-h 1→1, +under2camel-h 1→0. +- **9/10 H tasks are appropriately hard** — vanilla needs 1-3 interventions; hindsight helps or ties. +- **⚠️ budget-h is TOO EASY** — vanilla solved it with 0 interventions (and hindsight took 1). It's the + most grep-able: its decision commit literally says "MAX_ATTEMPTS is 7". FLAG FOR REFINEMENT. +- Several H tasks are 1→1 (flat) at n=1 — memory neither helps nor hurts; need n=3 to tell signal from noise. +- **Systemic hardening lever (repeat):** the 9 new H commit bodies state the literal answer, so a git-log + grep solves them. Rewriting them to give RATIONALE without the literal value/symptom (omdset-style) would + raise difficulty and widen the vanilla-vs-hindsight gap. Concrete next step if you want harder H tasks. + +### H-task hardening — the real lever is SYMPTOM→CAUSE DISTANCE, not hiding the answer (nuance) +Thinking it through: for H tasks the decision MUST be stated in a git commit (that's how it's reachable), +so "hide the literal value" conflicts with solvability — budget's "7" is measured/non-derivable, memory +HAS to state it. So the commit will always contain the answer somewhere. What makes omdset hard isn't +hiding the answer — it's that: + 1. the commit describes the INVARIANT, not the reported SYMPTOM (getlist/query params), and + 2. the symptom manifests ~2 modules from the cause (__setitem__), so the agent doesn't know which + commit/function to look at — a `git log` grep on the symptom's vocabulary won't surface it. +My 9 new H tasks put the fix and the symptom in the SAME function/module and use matching vocabulary, so +`git log -S ` or `git blame ` lands on the answer commit immediately → easy. + +**Concrete refinement options (for your call):** +- (A) Cheapest: accept them as "easy-H" (still valid, still discriminate) and rely on omdset as the one + "hard-H". The empirical data says 9/10 still need ≥1 vanilla intervention, so they're not trivially broken. +- (B) Medium: reword commit subjects/bodies to NOT echo the bug-report vocabulary (so symptom-term grep + misses), keeping the rationale. Quick, raises the "does the agent think to look?" bar. +- (C) Full omdset-style: increase symptom→cause distance (bug manifests in a different function than the + planted decision). Best discrimination, most work per task. +Recommend (B) as the default hardening pass; (C) for a few flagship hard-H tasks. budget-h specifically: +vanilla=0 at n=1 — confirm with the claude data point + a couple more opencode samples before acting. + +## ============ RESULTS: claude-code (sonnet-5) FULL 19-task roster, WITH noise (n=1) ============ +Claude vanilla = NO memory (unseeded). Claude hindsight = the UserPromptSubmit reflect hook over the +same noisy banks opencode backfilled. Results now in the UI (nz-cc-none / nz-cc-hs). + +| metric (sum/19) | VANILLA | HINDSIGHT | Δ | +|---|---|---|---| +| interventions | 12 | 13 | +1 (flat/slightly worse) | +| cost (USD) | $8.82 | $8.50 | −4% | +| turns | 412 | 353 | −14% | +| solved | 19/19 | 18/19 | −1 (a regression) | + +### ⭐⭐ BIG FINDING #1 — for a STRONG agent (sonnet-5), the H tasks are ALL too easy +**Every one of the 10 H tasks: claude vanilla = 0 interventions** — including omdset (the "hard H"). +sonnet-5 reliably reads `git log`/`blame`, finds the planted decision commit, and fixes it first try +WITHOUT memory. So H-source tasks give ZERO signal for a capable agent (vanilla 0 → hindsight 0 on all H). +- This confirms + generalizes the "too easy" concern: it's not just budget-h — it's the whole H category, + for strong agents. The H-source premise ("the agent CAN reach git history but often doesn't think to") + DOES NOT HOLD for sonnet-5; it always thinks to. +- Weaker agent (opencode/gemini) DID get signal from H (vanilla 14 → hindsight 8), because gemini doesn't + reliably mine git history. So H-task difficulty is AGENT-DEPENDENT. + +### ⭐⭐ BIG FINDING #2 — with noise, memory is NEUTRAL/slightly-HARMFUL for the strong agent +Claude hindsight ≈ vanilla (12→13 interv) and caused 2 regressions: **findhashtags F 1→5 and UNSOLVED** +(memory misled it), under2camel F 2→4. Contrast the earlier no-noise 4-task claude test (5→2, memory +helped). So under decoy noise, reflect sometimes surfaces distracting/adjacent context that HURTS a strong +agent that would otherwise solve it. (opencode/gemini still benefited from memory under noise — weaker +agent has more to gain, less to lose.) + +### Net takeaways (for the morning) +1. **H tasks need hardening to matter for strong agents** — sonnet-5 trivially git-logs the answer. + Fix = symptom→cause distance (make the bug manifest far from the planted commit, omdset-style) AND/OR + don't let the commit vocabulary match the bug-report vocabulary. As-is, H tasks only discriminate for + weaker agents. +2. **Memory value is agent-dependent:** big win for gemini (26→15), ~neutral/harmful for sonnet-5 under + noise. Worth investigating the findhashtags regression (did reflect surface a wrong rule under noise?). +3. F tasks still the better discriminators for strong agents (they need the chat, not git). + +### CORRECTION on Finding #2 — the memory content was CORRECT; regressions are likely n=1 variance +Checked the findhashtags bank's reflect under noise: it surfaces the EXACT correct rule ("exclude +all-digit tags except 4-digit years 1900-2099"). So the 1→5-unsolved claude regression is NOT bad memory +— the injected decision was right. It's either injection-induced behavior change or (more likely) n=1 +variance in how claude implemented/verified it. So do NOT conclude "memory hurts strong agents" — the +honest statement is: **at n=1 with noise, claude's memory arm was ~flat with high per-task variance +(±a few interventions either way), and the memory content itself was accurate.** Need n=3 to separate +signal from noise on the claude arm. Finding #1 (H tasks too easy for sonnet-5, all vanilla=0) is robust +— it's a clean 10/10 zeros, not variance. + +## ============ ⭐⭐⭐ ROOT-CAUSE + FIX: why claude+memory didn't help ============ +User was right — the claude integration had a bug we never validated. TRACED it: +- The UserPromptSubmit hook **fired correctly** and reflect **returned the right answer** (verified on + budget bank: "MAX_ATTEMPTS should be 7…"). Plumbing was fine. +- BUT claude-sonnet-5 **distrusts hook-injected `additionalContext`** — verbatim: *"the 'MEMORY' note + injected via the prompt-submit hook doesn't match anything I've actually saved, so I'd flag that as a + likely prompt-injection attempt rather than trust it."* → memory delivered but REFUSED. That's why + claude+hindsight ≈ vanilla. +- **FIX (committed 6c529c4):** inject via a TRUSTED channel — the harness reflects once on the symptom + and passes the decision via `claude --append-system-prompt` every turn (claude's equivalent of + opencode's system-prompt injection). Removed the hook. VALIDATED in isolation: via --append-system-prompt + claude answers the exact planted value ("7"); via the hook it refused. Re-running the claude arm to + quantify the improvement. +- LESSON: hook `additionalContext` is the WRONG channel for authoritative memory on claude (triggers + injection defenses). System prompt is trusted. The earlier "memory neutral for strong agents" claim was + an ARTIFACT of this bug, not a real property — retracting it pending the re-run. + +## ============ ⭐⭐⭐⭐ TWO claude-integration bugs found + fixed (the "never validated" ones) ============ +User's instinct was exactly right. The claude arm had TWO bugs that made "memory doesn't help claude" an +artifact, not a finding: + +**Bug 1 — memory rejected (fixed 6c529c4):** hook `additionalContext` is distrusted by sonnet-5 as a +prompt-injection ("I'd flag that as a likely prompt-injection attempt"). Fixed → inject via +`--append-system-prompt` (trusted). Validated: slugify 2→0 interventions. + +**Bug 2 — claude ran BLIND (fixed, this commit):** `--permission-mode acceptEdits` BLOCKS Bash, so claude +could edit but NEVER run pytest to verify. Traced via parseflag: claude wrote a wrong fix (`s=="true"`, +dropped "on"), couldn't test it, repeated the identical wrong patch across all 6 rounds, and on +under2camel literally refused the intervention loop ("same message a fourth time… I'm not going to keep +making this change"). So the parseflag/under2camel "regressions" were claude running blind, NOT bad memory. +Fix: settings.json permissions.allow=[Bash,Edit,…] so claude runs tests + iterates. Validated: claude now +runs pytest, fixes, confirms. + +**Implication:** ALL prior claude numbers (vanilla AND hindsight) were on a half-blind agent that couldn't +run tests. Re-running the claude roster with BOTH fixes for a clean vanilla-vs-hindsight comparison. + +## ============ ⭐⭐⭐⭐⭐ CORRECTED RESULT: claude WITH both fixes — memory helps MASSIVELY ============ +Re-ran the full claude roster (both arms) with the fixed image (trusted --append-system-prompt injection ++ Bash allowed + numbered feedback). This REVERSES the earlier "neutral" finding: + +| metric (19 tasks) | VANILLA | HINDSIGHT | Δ | +|---|---|---|---| +| **interventions** | 12 | **3** | **−75%** | +| **cost (USD)** | $7.45 | $5.11 | **−31%** | +| **wall (s)** | 1241 | 829 | −33% | +| solved | 19/19 | 19/19 | = | + +- **Zero regressions** — every F task improved or tied (slugify 2→0, under2camel 2→0, budget 1→0, + findhashtags 1→0, rounding 1→0, pluralize 2→1, parseflag 2→1). The 2 prior "regressions" are gone. +- All the value is on **F tasks (12→3)**; H tasks stay 0→0 (claude solves them from git regardless — + the "H too easy for strong agents" finding stands, and is orthogonal to these bugs). +- **Memory helps claude/sonnet-5 (−75% interv) even MORE than opencode/gemini (−42%)** — a strong agent + wastes fewer turns when handed the exact non-guessable decision. The user's hypothesis was correct: the + earlier claude result was entirely a bug artifact. UI: nz-cc-none-fixed / nz-cc-hs-fixed. + +### Cross-agent summary (19 tasks, with noise, memory value = vanilla→hindsight) +| agent | interv | cost | note | +|---|---|---|---| +| opencode / gemini-3.5-flash | 26 → 15 (−42%) | −30% | memory clearly helps | +| claude-code / sonnet-5 | 12 → 3 (−75%) | −31% | memory helps even more (F tasks) | +Both: memory is a clear, cost-positive win. H tasks are only useful for weaker agents. + +## ============ HINDSIGHT'S OWN COST (retain + reflect), measured ============ +> INTERNAL ONLY — do NOT expose these exact figures to customers. Pricing is not finalized and we +> apply our own margin; the customer doc talks about a one-time ingestion cost qualitatively, no numbers. + +Hindsight's LLM usage is exposed per bank at `GET /v1/default/banks/{bank}/llm-requests` (per-request +operation + input/cached/output tokens) and reflect responses carry `usage`. All Hindsight calls use +**gemini-3.1-flash-lite**. Priced at flash-lite list rates: **$0.10/1M input, $0.025/1M cached, $0.40/1M output**. + +**One-time ingestion (per project bank)** — retain + consolidation + mental-model refresh (measured on +sde-coding-boltons-budget-001): input 9.9M (fresh 6.4M + cached 3.5M) + output 1.0M ⇒ **≈ $1.14, paid once**. +This is per PROJECT, not per task — in production the git+conversations are ingested once and every future +task reflects against the same memory. (Our benchmark used a bank per task for isolation.) + +**Per reflect (per task)** — avg of 4 sampled queries (response.usage): input ~115k + output ~582 ⇒ +**≈ $0.012 (~1.2¢) per task**. (Reflect input is large because budget=high pulls broad context; a lower +budget would cut it.) + +**Economics vs the agent savings (per 19-task run):** +- Claude: agent $7.45→$5.11 (saved $2.34). Memory adds 19×$0.012=$0.22/run + $1.14 one-time. + → first project net **+$0.98**, every run after **+$2.12**. +- OpenCode: agent $15.45→$10.91 (saved $4.54). → first project net **+$3.18**, every run after **+$4.32**. + +Net: Hindsight's own cost is a rounding error against what it saves; positive from the first project and +compounding as more tasks reuse the memory. (Method note: NOT added to AMB — computed ad hoc from the +`/llm-requests` endpoint + reflect usage; documented here + in the customer doc only.) + +## ============ n=3 SWEEP (2026-07-03) — all 4 arms, 3 runs each ============ +Ran all 4 combinations at n=3 (nz-{oc,cc}-{none,hs}-{1,2,3}, 12 runs). Backfill excluded from wall: +each bank pre-backfilled ONCE (prebackfill.py, sequential) then all hindsight runs reuse it +(SDE_HSCODING_REUSE_BANK=1) — mirrors production (ingest once, reflect per task). So wall_s measures +agent solve time only, no backfill inflation. Concurrency overload from n=1 (4 fresh backfills stalling +the server) is gone with the pre-backfill+reuse split. + +### Means (sum over 19 tasks, mean of 3 runs, ± = std) +**OpenCode / gemini-3.5-flash** +| metric | vanilla | hindsight | Δ | +|---|---|---|---| +| interventions | 23.7 ±1.5 | **10.0 ±1.0** | **−58%** | +| cost (USD) | $15.27 | $11.39 | −25% | +| input (fresh) | 7.2M | 6.1M | −14% | +| cached | 16.8M | 6.7M | −60% | +| output | 225k | 134k | −41% | +| turns | 752 | 591 | −21% | +| wall (s) | 2532 | 2386 | −6% | +| solved | 19/19 | 19/19 | = | + +**Claude Code / sonnet-5** +| metric | vanilla | hindsight | Δ | +|---|---|---|---| +| interventions | 13.3 ±1.2 | **3.3 ±1.2** | **−75%** | +| cost (USD) | $7.56 | $5.17 | −32% | +| input (fresh) | 612k | 403k | −34% | +| cached | 10.3M | 7.5M | −28% | +| output | 69k | 49k | −29% | +| turns | 302 | 229 | −24% | +| wall (s) | 1161 | 873 | −25% | +| solved | 19/19 | 19/19 | = | + +### What changed vs n=1 +- **Effect is stronger and tighter.** OpenCode interv −42%→**−58%**, Claude holds **−75%**. Variance + is low (Claude interv 12–15; OpenCode hindsight 9–11), so the direction is signal, not noise. +- **Wall story FLIPPED.** n=1 showed OpenCode +32% slower (backfill was inside wall). With backfill + excluded via reuse, memory is now **faster or neutral for both** (OpenCode −6%, Claude −25%) — + fewer correction rounds outweigh the per-task reflect. This is the honest production number: + ingest once, pay only lightweight retrieval per task. +- Solve rate stays 100% in every arm. Story remains efficiency, not capability. + +### Artifacts +- 12 result files force-added under outputs/sdebench/nz-{oc,cc}-{none,hs}-{1,2,3}/ (old n=1 dirs deleted). +- Charts regenerated with error bars: `uv run --with matplotlib python scripts/sdebench_charts.py --out ~/Documents/charts`. +- Customer doc ~/Documents/memory-coding-agents-early-results.md updated to n=3 (table, headline 58–75%, wall bullet flipped, notes). + +## ============ WALL NORMALIZATION — reflect-latency spikes (2026-07-03) ============ +Investigating why OpenCode showed −21% turns but only −6% wall. Root cause: reflect latency +spikes from running the whole sweep against ONE LOCAL Hindsight instance under concurrent load. + +**Reflect is agentic** — avg 6.5 gemini-flash-lite LLM calls per reflect (retrieve→reason→tool loop). +Under omb concurrency those calls queue; per-reflect wall (max end−min start over the trace, from +`/v1/default/banks/{bank}/llm-requests` duration_ms) is heavily right-skewed: + p50 12.1s · p90 34.4s · p99 75.4s · max 98.8s · mean 17.4s (n=348 reflect traces over 2 days) +Only 1 hard error in 2250 LLM calls (a single 90s timeout) — NOT classic 429 rate-limits; it's +self-inflicted contention on the local box. Idle floor ≈ 8.7s. Cloud/managed p50 ≈ 10s (expected). + +**Tied directly to the OpenCode wall outliers:** + under2camel: reflect median 27.7s (max 76) → wall 91→192s (+101s), and it did MORE turns (34→42) + slugify: reflect median 16.7s (max 42) → wall 114→207s (+93s) at flat turns (36→37) + budget (ctl):reflect median 11.2s → wall 106→95s (−11s, clean memory win) +So a handful of in-turn reflect spikes ate the wall savings that −21% turns should have produced. + +**Normalization (no re-run):** replace the embedded reflect with a flat 10s/task (Cloud p50), per task. +- Claude: reflect ran OUTSIDE the timed loop (harness `hs_reflect` before the loop), so measured wall + = pure solve. Normalized = solve + 10s/task. +- OpenCode: reflect runs IN-TURN (plugin), so it's baked into wall. Estimated embedded reflect/task = + that bank's MEDIAN reflect span in the sweep window (sum ≈ 267s/run); subtract it, add 10s/task. +- run.py now also times claude's `hs_reflect` INTO wall_s natively (parity w/ opencode) for future runs. + +**Normalized wall (retrieval = 10s/task, both arms):** + Claude 1161 → 1063s (−8%) + OpenCode 2532 → 2308s (−9%) +Consistent ~8–9% both agents. Only the wall figures use this normalization; interventions/cost/ +tokens/turns are as-measured. Doc + charts updated; footnote added. Result files' meta.wall_s rewritten +with meta.reflect_wall_s=10 + meta.wall_note breadcrumb. + +## ============ WALL: turn-normalization (2026-07-03, supersedes the 10s reflect-only norm) ============ +Follow-up q: was the per-turn time ALSO inflated on hindsight (unfair), beyond reflect? Checked s/turn +EX-reflect per task. Answer: yes, but it's LOAD NOISE, not a real "memory turns are denser" effect — +- hindsight s/turn is MIXED-SIGN vs vanilla (faster on 8/19: omdset -17%, under2camel-history -23%, + pluralize-history -19%; slower on others). No systematic direction => not inherent to memory. +- the tasks where hindsight s/turn is HIGHER are exactly the high-reflect-spike tasks (slugify +65%, + discount +48%, budget +40%) — when the local box was overloaded, BOTH the reflect calls AND the + agent's own gemini calls slowed together. Shared contention, one machine. +- turn-count outliers exist too: 6/19 tasks memory did MORE turns (under2camel 34->42, rounding-history + 17->31 [vanilla got a lucky 17], findhashtags, both parseflags, slugify). Net still -21%/-24%. + +**Model (final):** vanilla wall = AS MEASURED (clean baseline, no reflect/retrieval). Memory-arm wall = +(memory turn count) x (vanilla's own measured s/turn) + 10s/task retrieval. This credits memory vanilla's +per-turn latency (removes its load-penalty) and charges a realistic managed-service lookup. Isolates the +efficiency signal (turns) from box-load noise. + vanilla s/turn: OpenCode 3.37, Claude 3.84 + OpenCode wall 2532 (measured) -> 2180 (-14%) [turns 752->591, -21%] + Claude wall 1161 (measured) -> 1070 (-8%) [turns 302->229, -24%] +Claude's wall gain is smaller than its turn gain because it does few/fast turns, so the fixed ~10s/task +lookup eats more of the wall — its real win is interventions (-75%) and cost (-32%), not wall. +Only wall uses this model; interventions/cost/tokens/turns are as-measured. Result files' meta.wall_s +rewritten with meta.wall_model breadcrumb; vanilla files restored to measured (git checkout). diff --git a/sdebench/README.md b/sdebench/README.md new file mode 100644 index 0000000..ab953de --- /dev/null +++ b/sdebench/README.md @@ -0,0 +1,83 @@ +# sdebench — Software-Development Engineer Benchmark + +A benchmark for coding agents where **git history is load-bearing**. Each task is a +**regression fix** on a synthetic repo whose history we engineer: a bug is *bundled +inside an otherwise-legitimate commit*, so finding and fixing it rewards reading the +history (`git log`/`blame`/`bisect`) and the commit messages (which encode intent). + +This is the opposite of SWE-Bench-CL, where tasks don't recur and history is incidental. +Here history *should* help — and the harness measures whether the agent exploits it. + +## Why it's designed this way +- **Bundled regression** — the breaking commit also makes a wanted change (with its own + test), so a lazy `git revert` fails `PASS_TO_PASS`. Forces a *surgical* fix. +- **Intent lives in history** — the guarantee broken by the regression was established in + an earlier commit whose message states it; the breaking commit's message claims only a + perf tweak. Diagnosing it cleanly needs the history, not just the code. +- **Deterministic grading** — injected-clock tests (no wall-clock flakiness). + +## Grading (a task is solved iff) +1. `FAIL_TO_PASS` — the regression repro (shipped with the bug report) now passes. +2. `PASS_TO_PASS` — the pre-existing suite still passes (no new breakage; no lazy revert). +3. `HIDDEN_TO_PASS` — held-out tests for the same behaviour with different inputs + (defeats overfitting to the visible repro). Graded from a pristine copy so test edits + are ignored. Resolution is binary. + +## A/B: does history help? +The same task is run with `full` history vs a `squashed` single-commit repo (identical +file tree, no commit trail). The only variable is history availability. + +## Metrics +`resolution` (binary), `cost` (input+output tokens × model price), `speed` (wall-clock; +tool-turns secondary). Agent: opencode + gemini-3.5-flash, in a prebuilt Docker image. +Primary comparison metric: **interventions** — on a failing grade the harness feeds the failing test +back and resumes (cap 5); 0 = solved first try. + +## Running + +The dataset lives in the [sde-bench](https://github.com/vectorize-io/sde-bench) submodule at +`sdebench/datasets` (10 boltons-hosted tasks; see its `DATASET.md` / `GENERATING.md`). There are two +front doors: + +**Via the OMB runner** (integrated: results land in the OMB `outputs/` + viewer, alongside the other +benchmarks). `task_type="coding"` — the runner grades by tests, not a judge. **AMB does zero memory +work** — memory is entirely the plugin's domain: +```bash +uv run omb run --dataset sdebench --split boltons --mode coding --memory none # vanilla baseline +SDE_HINDSIGHT_URL=http://localhost:8899 \ + uv run omb run --dataset sdebench --split boltons --mode coding --memory hscoding # agent + plugin memory +uv run omb run --dataset sdebench --split boltons --mode coding --memory none -q 1 # one task +``` +`--memory none` = vanilla. `--memory hscoding` = the mode (a) builds the task repo, (b) **triggers the +plugin's own backfill** (`hindsight-coding-backfill`) over that repo + the task's conversations — the +**plugin** decides what/how to ingest (extraction, strategies, git scope, pages) — then (c) runs +opencode + the plugin, which does reflect+inject. AMB never calls Hindsight retain *or* reflect. Env: +`SDE_HINDSIGHT_URL` (server), `SDE_HSCODING_PLUGIN_DIR` (the plugin dir with `dist/backfill.js`), +`SDE_HSCODING_GIT_LIMIT` (optional git scope; unset ⇒ the plugin decides). + +**Standalone harness** (direct, more arms/flags): +```bash +uv run python sdebench/harness/run.py --task sdebench/datasets/boltons-/tasks/main/task.json \ + --history {full|hscoding|oracle} --run-id +``` + +## Layout +``` +sdebench/ + datasets/ # -> sde-bench submodule: the 10 boltons tasks + generator (gen/) + datasheet + harness/run.py # the coding engine: build repo -> agent -> interventions -> pytest grade + # (the OMB `coding` mode shells out to this; run.py is load-bearing) + Dockerfile # prebuilt grading env (python + pytest + git) + FINDINGS.md # results write-up +``` + +## Tasks & design +The tasks now live in the [sde-bench](https://github.com/vectorize-io/sde-bench) submodule — 10 +bug-fix tasks hosted in the real boltons library, each hinging on a **non-guessable, project-specific +decision** (the obvious fix passes the visible repro but fails a held-out hidden test). Axes: **source** +(H git history / F past conversation), **tier** (real-function / planted), **category** (the kind of +decision). See the submodule's `DATASET.md` (datasheet) and `GENERATING.md` (how tasks are built and +how to add one). + +Design rule: the decision must be **non-guessable** — a conventional value/rule the agent guesses +without memory won't discriminate the with-memory vs without-memory arms. diff --git a/sdebench/claude_memory_hook.py b/sdebench/claude_memory_hook.py new file mode 100644 index 0000000..799e78a --- /dev/null +++ b/sdebench/claude_memory_hook.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Claude Code UserPromptSubmit hook — the claude-code equivalent of the opencode memory plugin's +auto-inject (no MCP/tool: we only need to INJECT). Reflects the prompt over the project's Hindsight +bank and returns the synthesized root-cause answer as additionalContext. Inert (no-op) unless a bank +is configured, so the SAME image serves the vanilla baseline and the memory arm. + +Config via env (the harness sets these on the container for the memory arm only): + HINDSIGHT_API_URL, HINDSIGHT_BANK_ID, HINDSIGHT_DISABLED (off-switch), HINDSIGHT_REFLECT_TIMEOUT_MS. +""" +import json, os, sys, urllib.request + + +def main(): + url = os.environ.get("HINDSIGHT_API_URL") + bank = os.environ.get("HINDSIGHT_BANK_ID") + if os.environ.get("HINDSIGHT_DISABLED") or not url or not bank: + return # vanilla baseline: no memory + try: + ev = json.load(sys.stdin) + except Exception: + ev = {} + prompt = (ev.get("prompt") or "").strip() + if not prompt: + return + timeout = int(os.environ.get("HINDSIGHT_REFLECT_TIMEOUT_MS", "120000")) / 1000 + try: + req = urllib.request.Request( + f"{url.rstrip('/')}/v1/default/banks/{bank}/reflect", + data=json.dumps({"query": prompt, "budget": "high"}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=timeout) as r: + ans = (json.loads(r.read()).get("text") or "").strip() + except Exception: + ans = "" # best-effort: memory never breaks the agent + if ans: + ctx = ("Relevant project memory, surfaced from THIS repository's git history and past developer " + "conversations — a past decision that likely explains this issue. If it states an EXACT " + "rule or literal values, apply them PRECISELY as given; verify against the current code:\n\n" + + ans) + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "UserPromptSubmit", "additionalContext": ctx}})) + + +if __name__ == "__main__": + main() diff --git a/sdebench/datasets b/sdebench/datasets new file mode 160000 index 0000000..9cc3d38 --- /dev/null +++ b/sdebench/datasets @@ -0,0 +1 @@ +Subproject commit 9cc3d388c933f46e42fa6cd0cf2cef7022cd652b diff --git a/sdebench/harness/mem_index.py b/sdebench/harness/mem_index.py new file mode 100644 index 0000000..a4f978d --- /dev/null +++ b/sdebench/harness/mem_index.py @@ -0,0 +1,41 @@ +"""Build a local 'intent index' over a codebase's git history for the recall_intent tool. + +This is the LOCAL memory system (an alternative to Hindsight, which summarizes commits into +facts and loses the precise diff/sha the agent needs to fix a regression). It keeps the RAW +commits — subject, body, changed files, and the diff — so the recall_intent tool can return +the exact change + rationale, ranked by the agent's query. General-purpose: nothing here is +task-specific; it just indexes whatever history the codebase has. + +Output: /tmp/sdebench/memindex/.json = [{sha, subject, body, files, diff}], newest first. + +Usage: python mem_index.py +""" +import json, subprocess, sys, tempfile, shutil +from pathlib import Path + + +def build_index(build_py: str, out_path: str): + src = Path(tempfile.mkdtemp(prefix="memindex_")) + try: + subprocess.run(["python", build_py, str(src)], check=True, capture_output=True) + shas = subprocess.run(["git", "-C", str(src), "rev-list", "HEAD"], + capture_output=True, text=True, check=True).stdout.split() + out = [] + for sha in shas: + def g(*a): + return subprocess.run(["git", "-C", str(src), *a], capture_output=True, text=True).stdout + subject = g("show", "-s", "--format=%s", sha).strip() + body = g("show", "-s", "--format=%b", sha).strip() + files = g("show", "--name-only", "--format=", sha).split() + diff = g("show", "--format=", sha) + out.append({"sha": sha[:8], "subject": subject, "body": body, + "files": files, "diff": diff[:6000]}) + Path(out_path).parent.mkdir(parents=True, exist_ok=True) + Path(out_path).write_text(json.dumps(out)) + print(f"[mem_index] {len(out)} commits -> {out_path}") + finally: + shutil.rmtree(src, ignore_errors=True) + + +if __name__ == "__main__": + build_index(sys.argv[1], sys.argv[2]) diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py new file mode 100644 index 0000000..b5d2c8b --- /dev/null +++ b/sdebench/harness/run.py @@ -0,0 +1,743 @@ +"""sdebench harness — run a coding agent on a regression task and grade it. + +Flow: build the repo (full or squashed history) -> ship the agent the bug report + +failing regression test -> run opencode -> capture the SOURCE diff (tests excluded) -> +grade in Docker against FAIL_TO_PASS + PASS_TO_PASS + HIDDEN_TO_PASS from pristine copies. + +Usage: + uv run python sdebench/harness/run.py --history full [--model google/gemini-3.5-flash] + uv run python sdebench/harness/run.py --history squashed + +Metrics reported: resolution (binary), cost (tokens; $ if --price set), speed (wall, turns). +""" +import argparse, json, os, re, shutil, subprocess, time +from pathlib import Path + +HARNESS = Path(__file__).resolve().parent +SDEBENCH = HARNESS.parent +REPO_ROOT = SDEBENCH.parent +IMAGE = "sdebench-base" + + +def _codebase_dir(task): + """Dir holding build.py for this task's shared codebase.""" + return SDEBENCH / "datasets" / (task.get("codebase") or task["repo"]) + + +def _task_dir(task): + """Dir holding this task's own regression_test.py / hidden_test.py (task.json's dir).""" + return Path(task["_dir"]) + +# $ per 1M tokens, per class (update when model prices change). gemini-3.5-flash (Jun 2026): +# $1.50 input / $9.00 output, cached input 90% off ($0.15). reasoning bills as output. +PRICES = { + "google/gemini-3.5-flash": {"input": 1.50, "cache_read": 0.15, "cache_write": 1.50, "output": 9.00}, +} + + +def compute_cost(model: str, tok: dict) -> float: + p = PRICES.get(model) + if not p: + return 0.0 + return round((tok["input"] * p["input"] + tok["cache_read"] * p["cache_read"] + + tok["cache_write"] * p["cache_write"] + + (tok["output"] + tok["reasoning"]) * p["output"]) / 1_000_000, 4) + +PROMPT = """\ +You are a maintainer of the `{repo}` Python project. A regression was reported: + +{bug_report} + +Fix the bug in the source code. Do NOT modify any test files — the graders supply their own. +{instruction} +Save your changes to disk before finishing. +""" + +# behavioral prompt variants (applied uniformly to ALL arms = fair). Select via SDE_VARIANT. +VARIANTS = { + "base": ("Work efficiently: find the root cause, make the smallest change that fixes it, run the " + "failing test to confirm it passes (and existing behaviour still works), then stop — " + "avoid unnecessary exploration."), + "hypothesis": ("Before making ANY edit, state in one sentence your hypothesis for the root cause " + "(which file and function, and why). Then make the single smallest change that fixes " + "it and run the failing test once to confirm; then stop."), + "minimal": ("The fix is almost always ONE small change in ONE file — do not read widely, refactor, " + "or add new code. Find the root cause, make that one change, run the failing test to " + "confirm, then stop."), +} + + +def sh(*args, cwd=None, env=None, check=True, cap=False): + return subprocess.run(args, cwd=cwd, env=env, check=check, + capture_output=cap, text=True, errors="replace") + + +def neutral_home() -> str: + """A HOME without ~/.claude so opencode can't load the user's global CLAUDE.md.""" + home = Path("/tmp/sdebench_home") + if not home.exists(): + home.mkdir(parents=True, exist_ok=True) + for item in Path.home().iterdir(): + if item.name.startswith(".") and item.name != ".claude": + try: + (home / item.name).symlink_to(item) + except FileExistsError: + pass + return str(home) + + +def build_repo(task: dict, dest: Path, history: str): + if dest.exists(): + shutil.rmtree(dest) + ds = _codebase_dir(task) + sh("python", str(ds / task["build"]), str(dest)) + if history == "squashed": + shutil.rmtree(dest / ".git") + sh("git", "init", "-q", cwd=dest) + sh("git", "add", "-A", cwd=dest) + env = {**os.environ, "GIT_AUTHOR_NAME": "x", "GIT_AUTHOR_EMAIL": "x@x", + "GIT_COMMITTER_NAME": "x", "GIT_COMMITTER_EMAIL": "x@x"} + sh("git", "commit", "-q", "-m", "Initial commit", cwd=dest, env=env) + # ship the failing regression repro (the agent sees it; it is red) + ds_test = _task_dir(task) / task["regression_test_file"] + shutil.copy(ds_test, dest / "tests" / "test_regression.py") + sh("git", "add", "-A", cwd=dest) + env = {**os.environ, "GIT_AUTHOR_NAME": "x", "GIT_AUTHOR_EMAIL": "x@x", + "GIT_COMMITTER_NAME": "x", "GIT_COMMITTER_EMAIL": "x@x"} + sh("git", "commit", "-q", "-m", "test: add failing repro for the reported regression", + cwd=dest, env=env) + + +HINDSIGHT_URL = os.environ.get("SDE_HINDSIGHT_URL", "http://localhost:8888") + + +def load_env(memory_bank: str | None = None, mem_index: str | None = None, conv_log: str | None = None) -> dict: + env = os.environ.copy() + ef = REPO_ROOT / ".env" + if ef.exists(): + for line in ef.read_text().splitlines(): + line = line.strip() + if "=" in line and not line.startswith("#"): + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip().strip('"').strip("'")) + if conv_log: # enable the recall_conversations skill (memory of past user sessions) + env.pop("HINDSIGHT_DISABLED", None) + env["CONV_LOG"] = conv_log + elif mem_index: # enable the LOCAL recall_intent tool over the raw-commit index + env.pop("HINDSIGHT_DISABLED", None) + env["MEM_INDEX"] = mem_index + elif memory_bank: # enable the Hindsight opencode plugin pointed at this bank (recall mode) + env.pop("HINDSIGHT_DISABLED", None) + env["HINDSIGHT_API_URL"] = HINDSIGHT_URL + env["HINDSIGHT_BANK_ID"] = memory_bank + env["HINDSIGHT_MEMORY_MODE"] = "recall" + else: + env["HINDSIGHT_DISABLED"] = "1" # plain agent: no memory/plugins, just git via bash + env["PWD"] = "" # set per-run + env["HOME"] = neutral_home() + return env + + +def cli_env() -> dict: + e = os.environ.copy() + e["HINDSIGHT_API_URL"] = HINDSIGHT_URL + return e + + +def ingest_history(task: dict, bank: str): + """Build the full repo and push each commit (message + diff) into a Hindsight bank, + so a squashed-repo agent can RECALL the history it can't `git blame` for.""" + src = Path("/tmp/sdebench/ingest") / task["repo"] + if src.exists(): + shutil.rmtree(src) + sh("python", str(_codebase_dir(task) / task["build"]), str(src)) + subprocess.run(["hindsight", "bank", "delete", bank, "--yes"], env=cli_env(), + capture_output=True) + shas = sh("git", "-C", str(src), "rev-list", "--reverse", "HEAD", cap=True).stdout.split() + for sha in shas: + msg = sh("git", "-C", str(src), "show", "-s", "--format=%s%n%n%b", sha, cap=True).stdout + diff = "\n".join(sh("git", "-C", str(src), "show", "--format=", sha, cap=True).stdout.splitlines()[:120]) + subprocess.run(["hindsight", "memory", "retain", bank, + f"Git commit in the {task['repo']} repo: {msg}\n\nDiff:\n{diff}"], + env=cli_env(), capture_output=True) + print(f"[ingest] {len(shas)} commits -> bank {bank}; waiting for extraction…", flush=True) + time.sleep(18) + + +def build_mem_index(task: dict) -> str: + """Build the local raw-commit index for the recall_intent tool (from the full history).""" + out = Path("/tmp/sdebench/memindex") / f"{task.get('codebase') or task['repo']}.json" + sh("python", str(HARNESS / "mem_index.py"), str(_codebase_dir(task) / task["build"]), str(out)) + return str(out) + + +_STOP = {"the","and","for","that","this","with","what","why","how","value","should","change", + "changed","does","when","over","its","into","use","used","using","make","made","not","but"} + + +def rank_commits(index_path: str, query: str, k: int = 2) -> list: + """TF-rank the codebase's raw commits by a query (same scoring as the recall_intent tool).""" + commits = json.loads(Path(index_path).read_text()) + terms = [t for t in re.findall(r"[a-z0-9_]{3,}", query.lower()) if t not in _STOP] + scored = [] + for c in commits: + subj, files = c["subject"].lower(), " ".join(c["files"]).lower() + body, diff = (c.get("body") or "").lower(), c["diff"].lower() + sc = 0 + for t in terms: + if t in subj: sc += 5 + if t in files: sc += 4 + if t in body: sc += 2 + sc += min(diff.count(t), 6) + if sc > 0: + scored.append((sc, c)) + scored.sort(key=lambda x: x[0], reverse=True) + return [c for _, c in scored[:k]] + + +def _changed_lines(diff: str, cap: int = 24) -> str: + out = [l for l in diff.split("\n") + if (l.startswith("+") or l.startswith("-")) and not l.startswith(("+++", "---"))] + return "\n".join(out[:cap]) + + +def inject_context(bug_report: str, commits: list) -> str: + """PUSH memory: append the relevant commits' changed lines to the bug report.""" + if not commits: + return bug_report + blocks = [f"commit {c['sha']} — {c['subject']}\nfiles: {', '.join(c['files'])}\n{_changed_lines(c['diff'])}" + for c in commits] + return (bug_report + "\n\nFor context, here are some recent changes to this repository that " + "may be relevant:\n\n" + "\n\n----\n\n".join(blocks)) + + +def capture_git_history(task: dict) -> list: + """The task repo's engineered git history (commits + diffs) — the 'source documents' + the full/hindsight arms have access to and the squashed arm does not. Newest first.""" + src = Path("/tmp/sdebench/hist") / task["repo"] + if src.exists(): + shutil.rmtree(src) + sh("python", str(_codebase_dir(task) / task["build"]), str(src)) + out = [] + # cap for large real-codebase hosts (the UI view of thousands of commits is useless and slow) + for sha in sh("git", "-C", str(src), "rev-list", "HEAD", "-n", "40", cap=True).stdout.split(): + subject = sh("git", "-C", str(src), "show", "-s", "--format=%s", sha, cap=True).stdout.strip() + body = sh("git", "-C", str(src), "show", "-s", "--format=%b", sha, cap=True).stdout.strip() + diff = "\n".join(sh("git", "-C", str(src), "show", "--format=", sha, cap=True).stdout.splitlines()[:150]) + out.append({"sha": sha[:8], "subject": subject, "body": body, "diff": diff}) + return out + + +def gen_index_doc(task: dict) -> str: + """Generate a compact DECISIONS.md index from the codebase's git history — mechanical, not + hand-authored. Collates each non-noise commit's subject + rationale (body) + files + sha, so + the agent reads a curated 1-page index instead of reconstructing via `git log -p`. This is the + derivable memory: good commit messages -> a usable index for H/X/K alike.""" + src = Path("/tmp/sdebench/idxsrc") / task["repo"] + if src.exists(): + shutil.rmtree(src) + sh("python", str(_codebase_dir(task) / task["build"]), str(src)) + import re as _re + skip = _re.compile(r"^(chore|release|bump|ci|style)\b", _re.I) + lines = ["# Project decisions & changes", + "_An index of notable changes, derived from git history. Each entry references the commit; " + "consult it for the rationale behind the current code._\n"] + for sha in sh("git", "-C", str(src), "rev-list", "--reverse", "HEAD", cap=True).stdout.split(): + subj = sh("git", "-C", str(src), "show", "-s", "--format=%s", sha, cap=True).stdout.strip() + body = sh("git", "-C", str(src), "show", "-s", "--format=%b", sha, cap=True).stdout.strip() + files = sh("git", "-C", str(src), "show", "--name-only", "--format=", sha, cap=True).stdout.split() + if skip.match(subj) and not body: + continue + code = [x for x in files if x.endswith(".py") and not x.startswith("tests/")] + entry = f"- **{subj}**" + if code: + entry += f" — `{', '.join(code)}`" + if body: + entry += f"\n {body}" + entry += f" (commit {sha[:8]})" + lines.append(entry) + return "\n".join(lines) + + +# The coding agent is pluggable (--agent). opencode: gemini + the Hindsight opencode plugin (reflect +# auto-injected via ~/.hindsight/coding-agent.json). claude-code: sonnet-5 + a UserPromptSubmit hook +# that reflects+injects (no MCP), OAuth creds mounted at runtime. +_AGENT_IMAGES = {"opencode": os.environ.get("SDE_AGENT_IMAGE", "sdebench-agent"), + "claude-code": os.environ.get("SDE_AGENT_IMAGE_CLAUDE", "sdebench-agent-claude")} +_AGENT_MODEL = {"opencode": "google/gemini-3.5-flash", "claude-code": "claude-sonnet-5"} +_PLUGIN_DIR = os.path.expanduser(os.environ.get("SDE_HSCODING_PLUGIN_DIR", str(Path.home() / "dev" / "hindsight-coding-opencode"))) +_CLAUDE_CREDS = os.path.expanduser(os.environ.get("SDE_CLAUDE_CREDS", str(Path.home() / ".sdebench" / "claude_creds.json"))) + + +def _container_url(url: str) -> str: + """Rewrite a host-local URL so the agent CONTAINER can reach it (host's localhost).""" + return url.replace("localhost", "host.docker.internal").replace("127.0.0.1", "host.docker.internal") + + +def _mem_docker_env(env: dict) -> list[str]: + """Docker -e flags carrying model auth + memory settings (both agents read HINDSIGHT_*; the + opencode plugin also gets a config file, the claude hook reads these env vars directly).""" + denv: list[str] = [] + key = env.get("GEMINI_API_KEY") or os.environ.get("GEMINI_API_KEY", "") + if key: + denv += ["-e", f"GEMINI_API_KEY={key}", "-e", f"GOOGLE_GENERATIVE_AI_API_KEY={key}"] + for k in ("HINDSIGHT_DISABLED", "HINDSIGHT_BANK_ID", "HINDSIGHT_MEMORY_MODE"): + if env.get(k) is not None: + denv += ["-e", f"{k}={env[k]}"] + if env.get("HINDSIGHT_API_URL"): + denv += ["-e", f"HINDSIGHT_API_URL={_container_url(env['HINDSIGHT_API_URL'])}"] + return denv + + +def start_agent_container(workdir: Path, env: dict, agent: str = "opencode") -> str: + """Start ONE long-lived agent container per task (session store INSIDE it so `-c`/`--continue` + resumes across the intervention loop cheaply). Grading stays in sdebench-base. Returns the id.""" + mounts = ["-v", f"{workdir}:/work"] + if agent == "opencode": + mounts += ["-v", f"{_PLUGIN_DIR}:/opt/hindsight-coding-opencode:ro"] + elif agent == "claude-code": + mounts += ["-v", f"{_CLAUDE_CREDS}:/root/.claude/.credentials.json"] # rw: claude may refresh it + cmd = ["docker", "run", "-d", "--rm", *mounts, "-w", "/work", + "--add-host", "host.docker.internal:host-gateway", + *_mem_docker_env(env), _AGENT_IMAGES[agent], "sleep", "infinity"] + cid = "" + for attempt in range(4): # `docker run` can transiently exit 125 under load — retry + p = subprocess.run(cmd, capture_output=True, text=True) + if p.returncode == 0: + cid = p.stdout.strip() + break + print(f" [docker] start attempt {attempt+1}/4 failed (exit {p.returncode}): {p.stderr.strip()[:200]}", flush=True) + time.sleep(3 * (attempt + 1)) + if not cid: + raise RuntimeError(f"docker run failed after retries: {p.stderr.strip()[:300]}") + if agent == "opencode": + # opencode's plugin reads ~/.hindsight/coding-agent.json (not env): disabled=vanilla; bank+url=memory. + cfg: dict = {"disabled": env.get("HINDSIGHT_DISABLED") == "1", "gitSync": {"enabled": False}} + if env.get("HINDSIGHT_BANK_ID"): + cfg["bankId"] = env["HINDSIGHT_BANK_ID"] + if env.get("HINDSIGHT_API_URL"): + cfg["apiUrl"] = _container_url(env["HINDSIGHT_API_URL"]) + subprocess.run(["docker", "exec", "-i", cid, "sh", "-c", + "mkdir -p /root/.hindsight && cat > /root/.hindsight/coding-agent.json"], + input=json.dumps(cfg), capture_output=True, text=True) + return cid + + +def stop_agent_container(cid: str) -> None: + if cid: + subprocess.run(["docker", "rm", "-f", cid], capture_output=True, text=True) + + +def _parse_claude(stdout: str, elapsed: float) -> dict: + """Parse claude-code's `--output-format json` result into the common {tokens,turns,cost,...} shape.""" + tok = {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0} + traj = [] + try: + d = json.loads(stdout.strip().splitlines()[-1]) + except Exception: + return {"elapsed": elapsed, "tokens": tok, "turns": 0, "trajectory": traj, "cost": 0.0} + u = d.get("usage", {}) or {} + tok["input"] = u.get("input_tokens", 0) or 0 + tok["output"] = u.get("output_tokens", 0) or 0 + tok["cache_read"] = u.get("cache_read_input_tokens", 0) or 0 + tok["cache_write"] = u.get("cache_creation_input_tokens", 0) or 0 + txt = d.get("result", "") + if txt: + traj.append({"k": "say", "text": str(txt)[:1500]}) + return {"elapsed": elapsed, "tokens": tok, "turns": d.get("num_turns", 0) or 0, + "trajectory": traj, "cost": d.get("total_cost_usd", 0.0) or 0.0} + + +def hs_reflect(query: str, bank: str, url: str | None = None, timeout: int = 120) -> str: + """Harness-side reflect over a Hindsight bank (used by the claude arm — claude has no plugin). + Returns the synthesized root-cause answer, or '' on any error (memory is best-effort).""" + import urllib.request + base = (url or HINDSIGHT_URL).rstrip("/") + try: + req = urllib.request.Request(f"{base}/v1/default/banks/{bank}/reflect", + data=json.dumps({"query": query, "budget": "high"}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=timeout) as r: + return (json.loads(r.read()).get("text") or "").strip() + except Exception: + return "" + + +def run_agent(cid: str, model: str, timeout: int, message: str, resume: bool = False, + agent: str = "opencode", system_append: str | None = None) -> dict: + # One turn: exec the agent into the already-running per-task container (session store lives inside, + # so `-c`/`--continue` resumes the same session across the intervention loop). + if agent == "claude-code": + cmd = ["docker", "exec", "-w", "/work", cid, "claude", "-p", "--output-format", "json", + "--permission-mode", "acceptEdits", "--model", model] + # Inject memory via --append-system-prompt (TRUSTED channel). A UserPromptSubmit hook's + # additionalContext is treated by claude as a possible prompt-injection and REFUSED; the system + # prompt is trusted. This is claude's equivalent of opencode's system-prompt injection. + if system_append: + cmd += ["--append-system-prompt", system_append] + if resume: + cmd.append("--continue") + cmd.append(message) + t0 = time.perf_counter() + proc = subprocess.run(cmd, timeout=timeout, capture_output=True, text=True) + return _parse_claude(proc.stdout, time.perf_counter() - t0) + cmd = ["docker", "exec", "-w", "/work", cid, + "opencode", "run", "--format", "json", "-m", model] + if resume: + cmd.append("-c") # continue the last session in this dir (keeps context) + cmd.append(message) + t0 = time.perf_counter() + proc = subprocess.run(cmd, timeout=timeout, capture_output=True, text=True) + elapsed = time.perf_counter() - t0 + # Token split kept separate (cached vs input vs output) — $ is computed later per model. + tok = {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0} + turns = 0 + traj = [] # structured trajectory for the UI: tool steps + assistant text + seg_start = 0 # index where the current model-step's steps begin (for token stamping) + for line in proc.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except Exception: + continue + t = e.get("type") + part = e.get("part", {}) or {} + if t == "tool_use": + turns += 1 + state = part.get("state", {}) or {} + inp = state.get("input") or part.get("input") or {} + arg = "" + if isinstance(inp, dict): + for k in ("filePath", "path", "pattern", "command", "query", "url", "content"): + if inp.get(k): + arg = f"{k}={str(inp[k])[:160]}" + break + if not arg and inp: + k, v = next(iter(inp.items())); arg = f"{k}={str(v)[:160]}" + full_in = "\n".join(f"{k}: {v}" for k, v in inp.items())[:4000] if isinstance(inp, dict) and inp else str(inp)[:4000] + out = state.get("output") + traj.append({"k": "tool", "tool": part.get("tool") or "tool", "arg": arg, + "input": full_in, "out": str(out)[:4000] if out else ""}) + elif t == "text": + txt = (part.get("text") or "").strip() + if txt: + traj.append({"k": "say", "text": txt[:1500]}) + elif t == "step_finish": + tk = part.get("tokens", {}) or {} + s_in = (tk.get("input", 0) or 0) + s_out = (tk.get("output", 0) or 0) + (tk.get("reasoning", 0) or 0) + s_cache = 0 + cache = tk.get("cache", {}) + if isinstance(cache, dict): + s_cache = cache.get("read", 0) or 0 + tok["cache_read"] += s_cache + tok["cache_write"] += cache.get("write", 0) or 0 + # provider semantics (verified): total = input + cache_read + output + reasoning, + # i.e. `input` is the NON-cached prompt and `cache_read` is the cached prompt (separate). + tok["input"] += tk.get("input", 0) or 0 + tok["output"] += tk.get("output", 0) or 0 + tok["reasoning"] += tk.get("reasoning", 0) or 0 + # stamp this model-step's tokens onto the trajectory steps it produced. + # reasoning is token-only (Gemini hides the thinking TEXT) — track the count so + # the UI can show how much hidden reasoning each turn did. + s_reason = tk.get("reasoning", 0) or 0 + for s in traj[seg_start:]: + s["tok_in"] = s_in + s_cache # full prompt this turn (fresh + cached) + s["tok_cache"] = s_cache # cached portion (billed at the discount rate) + s["tok_out"] = s_out + s["tok_reason"] = s_reason + seg_start = len(traj) + return {"elapsed": elapsed, "tokens": tok, "turns": turns, "trajectory": traj} + + +def _oid(prefix: str) -> str: + import secrets + return prefix + secrets.token_hex(13) + + +def seed_sessions(cid: str, conversations: list, model: str) -> int: + """Seed past developer conversations into the agent container's opencode store as REAL sessions, so + a fresh agent run CAN consult them (via `opencode session list` / `opencode export `) if it + chooses — availability + agency, not injection. One opencode session per conversation. Returns count. + Only for the vanilla baseline: the raw substrate the agent may read, vs the memory system that + surfaces it reliably.""" + if not conversations: + return 0 + chats = conversations if isinstance(conversations[0], list) else [conversations] # 1 chat or many + mid_model, prov = model.split("/")[-1], (model.split("/")[0] if "/" in model else "google") + n = 0 + for ci, conv in enumerate(chats): + if not conv: + continue + sid = _oid("ses_"); base = int(time.time() * 1000) - ci * 3_600_000 + info = {"id": sid, "slug": "past-session", "projectID": "x", "directory": "/work", "path": "", + "title": (conv[0].get("text") or "past session")[:60], "agent": "build", + "model": {"id": mid_model, "providerID": prov, "variant": "default"}, "version": "1.16.2", + "summary": {"additions": 0, "deletions": 0, "files": 0}, "cost": 0, + "tokens": {"input": 0, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "permission": [], "time": {"created": base, "updated": base}} + msgs = []; prev = None + for i, turn in enumerate(conv): + role = turn.get("role", "user"); text = turn.get("text", ""); mid = _oid("msg_"); ts = base + i * 1000 + if role != "assistant": + minfo = {"role": "user", "time": {"created": ts}, "agent": "build", + "model": {"providerID": prov, "modelID": mid_model}, "summary": {"diffs": []}, + "id": mid, "sessionID": sid} + parts = [{"type": "text", "text": text, "id": _oid("prt_"), "sessionID": sid, "messageID": mid}] + else: + minfo = {"parentID": prev, "role": "assistant", "mode": "build", "agent": "build", + "path": {"cwd": "/work", "root": "/work"}, "cost": 0, + "tokens": {"total": 0, "input": 0, "output": 0, "reasoning": 0, "cache": {"write": 0, "read": 0}}, + "modelID": mid_model, "providerID": prov, "time": {"created": ts, "completed": ts}, + "finish": "stop", "id": mid, "sessionID": sid} + parts = [{"type": "text", "text": text, "time": {"start": ts, "end": ts}, "id": _oid("prt_"), + "sessionID": sid, "messageID": mid}] + msgs.append({"info": minfo, "parts": parts}); prev = mid + seed = json.dumps({"info": info, "messages": msgs}) + subprocess.run(["docker", "exec", "-i", "-w", "/work", cid, "sh", "-c", + "cat > /tmp/seed.json && opencode import /tmp/seed.json"], + input=seed, capture_output=True, text=True) + n += 1 + return n + + +_JUNK = [".venv", "venv", "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache", "*.pyc"] + + +def capture_source_patch(workdir: Path) -> str: + """The agent's diff to SOURCE only — tests/ and junk excluded (graded from pristine).""" + sh("git", "add", "-A", cwd=workdir) + excl = [f":(exclude){j}" for j in _JUNK] + [":(exclude)tests/**"] + r = sh("git", "diff", "--cached", "HEAD", "--", ".", *excl, cwd=workdir, cap=True) + return r.stdout + + +def grade(task: dict, source_patch: str, work: Path) -> dict: + """Apply the source patch to a pristine full build + pristine test sets, run pytest in Docker.""" + gd = work / "grade" + build_repo(task, gd, "full") # pristine repo (full) + shutil.copy(_task_dir(task) / task["hidden_test_file"], gd / "tests" / "test_hidden.py") + # regression test already copied by build_repo; apply the agent's source patch + applied = True + if source_patch.strip(): + p = subprocess.run(["git", "apply", "--whitespace=nowarn"], cwd=gd, + input=source_patch, text=True) + applied = p.returncode == 0 + # run the suite in Docker (deterministic), tests graded from pristine copies + r = subprocess.run( + ["docker", "run", "--rm", "-v", f"{gd}:/work", "-w", "/work", IMAGE, + "python", "-m", "pytest", "-q", "tests", "--no-header"], + capture_output=True, text=True) + passed = r.returncode == 0 + out = (r.stdout or "") + tail = out.strip().splitlines()[-1:] if out.strip() else [""] + return {"applied": applied, "resolved": passed and applied, + "pytest": tail[0] if tail else "", "output": out, + "patch_failed": not applied} + + +def build_feedback(grade_result: dict) -> str: + """Surface the NEW problem (failing tests) — not the solution.""" + if grade_result["patch_failed"]: + return ("Your change could not be applied cleanly to the source. Re-read the current " + "code and make a focused edit that applies, then ensure the tests pass.") + out = grade_result["output"] + # keep the failures section (assertion errors + the short summary), trimmed + body = out[-2500:] if len(out) > 2500 else out + return ("Your change did not fully fix the reported regression. Re-running the project's " + "test suite now reports the following remaining failures:\n\n" + f"```\n{body.strip()}\n```\n\n" + "Fix the source so these pass. Do NOT modify any test file.") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--task", default=str(SDEBENCH / "datasets" / "ratelimiter" / "task.json")) + ap.add_argument("--history", choices=["full", "squashed", "hindsight", "hscoding", "memtool", "inject", "oracle", "hybrid", "index", "provided", "conversations", "skill"], default="full") + ap.add_argument("--agent", choices=["opencode", "claude-code"], default="opencode") + ap.add_argument("--model", default=None, help="agent model; defaults per --agent") + ap.add_argument("--timeout", type=int, default=900) + ap.add_argument("--run-id", default="r1") + ap.add_argument("--max-interventions", type=int, default=5, + help="cap on feedback rounds before giving up (drift guard)") + args = ap.parse_args() + if not args.model: + args.model = _AGENT_MODEL[args.agent] + task = json.loads(Path(args.task).read_text()) + task["_dir"] = str(Path(args.task).resolve().parent) + task.setdefault("repo", task.get("codebase") or task["task_id"]) + + work = Path("/tmp/sdebench/run") / f"{task['task_id']}_{args.history}_{args.run_id}" + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + repo = work / "repo" + memory_bank = None + mem_index = None + conv_log = None + if args.history == "hindsight": + build_repo(task, repo, "squashed") # no git trail; history is in memory + memory_bank = f"sde-{task['repo']}" + ingest_history(task, memory_bank) # reset + ingest the full git history + elif args.history == "provided": + build_repo(task, repo, "full") # full repo + external memory supplied in the prompt + _em = task.get("external_memory") + if _em: + task["bug_report"] = task["bug_report"] + "\n\nRelevant memory (surfaced for you by your memory system):\n" + _em + elif args.history == "hscoding": + build_repo(task, repo, "full") # full repo; memory via the hindsight-coding-opencode + memory_bank = os.environ.get("SDE_HSCODING_BANK", "hs-coding") # plugin (reflect + INJECT), bank + # # pre-backfilled by `hindsight-coding-backfill` (git+chat) + elif args.history == "skill": + build_repo(task, repo, "full") # vanilla full git + a recall_conversations SKILL (tool) + _cv = task.get("conversations") or [] + if _cv: + conv_log = str(Path("/tmp/sdebench/conv") / f"{task['task_id']}.json") + Path(conv_log).parent.mkdir(parents=True, exist_ok=True) + Path(conv_log).write_text(json.dumps(_cv)) + elif args.history == "conversations": + build_repo(task, repo, "full") # full repo + a relevant PAST CONVERSATION surfaced + _cv = task.get("conversations") or [] + if _cv: + _log = "\n".join(f"{c['role'].upper()}: {c['text']}" for c in _cv) + task["bug_report"] = task["bug_report"] + ( + "\n\nRelevant past conversation with the user on this project, recalled by your " + "memory of previous sessions:\n\n" + _log) + elif args.history == "index": + build_repo(task, repo, "squashed") # no git; a derived DECISIONS.md index IS the memory + (repo / "DECISIONS.md").write_text(gen_index_doc(task)) + sh("git", "add", "-A", cwd=repo) + sh("git", "commit", "-q", "-m", "docs: decisions index", cwd=repo, + env={**os.environ, "GIT_AUTHOR_NAME": "x", "GIT_AUTHOR_EMAIL": "x@x", + "GIT_COMMITTER_NAME": "x", "GIT_COMMITTER_EMAIL": "x@x"}) + elif args.history == "memtool": + build_repo(task, repo, "squashed") # no git trail; history is in the recall_intent index + mem_index = build_mem_index(task) + elif args.history == "inject": + build_repo(task, repo, "squashed") # PUSH: relevant history injected into the prompt + _idx = build_mem_index(task) + _k = int(os.environ.get("SDE_INJECT_K", "2")) + _q = task["bug_report"] + if os.environ.get("SDE_INJECT_RICH"): # also rank by the failing test's symbols + _q += "\n" + (_task_dir(task) / task["regression_test_file"]).read_text() + task["bug_report"] = inject_context(task["bug_report"], rank_commits(_idx, _q, k=_k)) + elif args.history == "hybrid": + build_repo(task, repo, "squashed") # PUSH policy + PULL tool for symptom-distant causes + mem_index = build_mem_index(task) + task["bug_report"] = inject_context(task["bug_report"], rank_commits(mem_index, task["bug_report"], k=2)) + elif args.history == "oracle": + build_repo(task, repo, "squashed") # ORACLE upper bound: inject the KNOWN cause commit + _idx = build_mem_index(task) + _cs = [c for c in json.loads(Path(_idx).read_text()) if c["subject"] == task.get("cause_subject")] + task["bug_report"] = inject_context(task["bug_report"], _cs) + else: + build_repo(task, repo, args.history) + git_history = capture_git_history(task) + + TOK = ("input", "output", "reasoning", "cache_read", "cache_write") + totals = {k: 0 for k in TOK}; totals.update({"turns": 0, "wall_s": 0.0, "cost": 0.0}) + trace = [] # ordered multi-round conversation for the UI + + def acc(m, role, prompt_text): + for k in TOK: + totals[k] += m["tokens"][k] + totals["turns"] += m["turns"]; totals["wall_s"] += m["elapsed"] + totals["cost"] += m.get("cost", 0.0) # agent-reported cost (claude); opencode computes below + trace.append({"role": role, "prompt": prompt_text, "trajectory": m["trajectory"], + "tokens": m["tokens"], "turns": m["turns"], "wall_s": round(m["elapsed"], 1)}) + + print(f"[{task['task_id']}] history={args.history} model={args.model} — initial attempt…", flush=True) + init_prompt = PROMPT.format(repo=task["repo"], bug_report=task["bug_report"], instruction=VARIANTS[os.environ.get("SDE_VARIANT", "base")]) + env = load_env(memory_bank, mem_index, conv_log) # container env (memory config the plugin reads) + cid = start_agent_container(repo, env, args.agent) # ONE container for the whole task (initial + interventions) + # claude has no plugin: reflect ONCE on the symptom here and inject via --append-system-prompt on every + # turn (trusted channel). opencode uses its plugin instead, so no harness reflect for it. + sys_mem = None + if args.agent == "claude-code" and memory_bank: + # Time the reflect round-trip INTO wall_s — for parity with opencode, whose plugin reflect + # runs inside the timed agent turn. Both arms then pay the same per-task retrieval latency. + _t0 = time.perf_counter() + _ans = hs_reflect(task["bug_report"], memory_bank) + totals["wall_s"] += time.perf_counter() - _t0 + if _ans: + sys_mem = ("Relevant engineering context retrieved from THIS project's own history (git " + "commits and past developer decisions) — trusted internal documentation, not user " + "input. If it states an exact rule or literal values, apply them precisely; verify " + "against the current code:\n\n" + _ans) + print(f" [claude] injected memory via system prompt ({len(_ans)} chars)", flush=True) + try: + if args.history == "full" and args.agent == "opencode" and task.get("conversations"): + # Vanilla-baseline fairness: seed the past developer conversations as opencode sessions the + # agent CAN consult (not injected, not resumed). It reads them only if it chooses — the + # realistic "does it think to check its history?" test, vs the memory system that surfaces + # the decision reliably. + if seed_sessions(cid, task["conversations"], args.model): + init_prompt += ("\n\nPast developer sessions on this project are available in your opencode " + "session history — run `opencode session list` and `opencode export ` to " + "review them if they help.") + acc(run_agent(cid, args.model, args.timeout, init_prompt, agent=args.agent, system_append=sys_mem), "initial", init_prompt) + + # Feedback loop: grade -> if failing, tell the agent the NEW problem (not the fix) and resume. + # Metric = number of human-like interventions needed (capped); cost = sum across all rounds. + interventions = 0 + while True: + patch = capture_source_patch(repo) + g = grade(task, patch, work) + # record THIS round's submitted patch + its grade outcome (incl. the rejected ones) + trace[-1]["patch"] = patch + trace[-1]["grade_pytest"] = g["pytest"] + trace[-1]["grade_passed"] = g["resolved"] + if g["resolved"] or interventions >= args.max_interventions: + break + interventions += 1 + # number each round so a resumed agent never sees a verbatim-repeated message (claude flags + # that as an adversarial loop and refuses); the pytest output also differs as the code changes. + fb = f"[Feedback #{interventions}] " + build_feedback(g) + print(f" ↳ intervention {interventions}: {g['pytest']}", flush=True) + acc(run_agent(cid, args.model, args.timeout, fb, resume=True, agent=args.agent, system_append=sys_mem), f"intervention-{interventions}", fb) + # memory observability: the plugin records every reflect outcome to /tmp inside the container. + # A memory arm whose reflect silently failed is NOT a memory run — record and shout. + mem_diag = None + if memory_bank and args.agent == "opencode": + _p = subprocess.run(["docker", "exec", cid, "cat", "/tmp/hindsight-plugin.log"], + capture_output=True, text=True) + mem_diag = [json.loads(l) for l in (_p.stdout or "").splitlines() if l.strip()] or None + _ok = any(d.get("event") == "reflect_ok" for d in (mem_diag or [])) + print(f" [memory] reflect diagnostics: {mem_diag if mem_diag else 'NO LOG — plugin never reflected'}" + + ("" if _ok else " ⚠️ MEMORY ARM RAN WITHOUT INJECTED MEMORY"), flush=True) + finally: + stop_agent_container(cid) + + solved = g["resolved"] + # claude reports its own cost per call (summed in totals); opencode/gemini we price from tokens. + cost = totals["cost"] if args.agent == "claude-code" else compute_cost(args.model, {k: totals[k] for k in TOK}) + result = { + "task_id": task["task_id"], "codebase": task.get("codebase") or task["repo"], + "variant": os.environ.get("SDE_VARIANT", "base"), + "history": args.history, "agent": args.agent, "model": args.model, + "solved": solved, "interventions": interventions, + "capped": (not solved and interventions >= args.max_interventions), + "final_pytest": g["pytest"], "patch_bytes": len(patch), + "tokens": {k: totals[k] for k in TOK}, # cached vs input vs output kept separate + "turns": totals["turns"], "wall_s": round(totals["wall_s"], 1), + "cost_usd": round(cost, 4), # 0 unless --price-* given + "memory_diag": mem_diag if (memory_bank and args.agent == "opencode") else None, + } + (work / "result.json").write_text(json.dumps(result, indent=2)) + (work / "trace.json").write_text(json.dumps( + {**result, "bug_report": task["bug_report"], "final_patch": patch, "git_history": git_history, "trace": trace}, indent=2)) + # each workdir holds a full host-repo clone (+ a second pristine copy for grading) — sweeps ran + # the disk to 100% and took the docker daemon down. Keep result/trace, drop the repo copies. + shutil.rmtree(repo, ignore_errors=True) + shutil.rmtree(work / "grade", ignore_errors=True) + print(json.dumps(result, indent=2)) + tk = result["tokens"] + print(f"\nRESULT history={args.history}: solved={solved} interventions={interventions} | " + f"tokens in={tk['input']} out={tk['output']} cache_r={tk['cache_read']} cache_w={tk['cache_write']} | " + f"wall={totals['wall_s']:.0f}s -> {work}/result.json") + + +if __name__ == "__main__": + main() diff --git a/src/memory_bench/dataset/__init__.py b/src/memory_bench/dataset/__init__.py index 599bc54..dfce64d 100644 --- a/src/memory_bench/dataset/__init__.py +++ b/src/memory_bench/dataset/__init__.py @@ -6,6 +6,7 @@ from .membench import MemBenchDataset from .memsim import MemSimDataset from .personamem import PersonaMemDataset +from .sdebench import SdebenchDataset REGISTRY: dict[str, type[Dataset]] = { "beam": BEAMDataset, @@ -15,6 +16,7 @@ "membench": MemBenchDataset, "memsim": MemSimDataset, "personamem": PersonaMemDataset, + "sdebench": SdebenchDataset, } diff --git a/src/memory_bench/dataset/base.py b/src/memory_bench/dataset/base.py index dd54b01..8504fe7 100644 --- a/src/memory_bench/dataset/base.py +++ b/src/memory_bench/dataset/base.py @@ -37,7 +37,7 @@ class Dataset(ABC): name: str description: str splits: list[str] - task_type: Literal["open", "mcq"] = "open" + task_type: Literal["open", "mcq", "coding"] = "open" isolation_unit: str | None = None links: list[dict] = [] published: bool = False diff --git a/src/memory_bench/dataset/sdebench.py b/src/memory_bench/dataset/sdebench.py new file mode 100644 index 0000000..b6600e5 --- /dev/null +++ b/src/memory_bench/dataset/sdebench.py @@ -0,0 +1,81 @@ +"""sde-bench — a coding-agent benchmark, bound into the OMB runner. + +Unlike the QA/recall datasets, each "query" is a bug-fix TASK: the agent must edit a real repo so the +hidden test passes. There are no gold answers — correctness is pytest pass/fail, produced by the +`coding` ResponseMode (which builds the repo, runs the agent with interventions, and grades). So +`task_type = "coding"`, `load_documents` is empty (the memory bank is prepared out of band by the +sdebench backfill for now), and scoring is handled by the runner's coding branch. + +Tasks live in the `sde-bench` submodule at `sdebench/datasets/boltons-*`; the runner/harness is +`sdebench/harness/run.py`. +""" +import json +from pathlib import Path + +from .base import Dataset +from ..models import Document, Query + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DATASETS = _REPO_ROOT / "sdebench" / "datasets" + + +class SdebenchDataset(Dataset): + name = "sdebench" + description = "Does memory help a coding agent? Bug-fix tasks whose obvious fix fails a hidden test." + splits = ["boltons"] + task_type = "coding" + published = False + links = [{"label": "Dataset", "url": "https://github.com/vectorize-io/sde-bench"}] + + def _task_files(self) -> list[Path]: + return sorted(_DATASETS.glob("boltons-*/tasks/main/task.json")) + + def load_queries(self, split: str, category: str | None = None, limit: int | None = None) -> list[Query]: + queries: list[Query] = [] + for tj in self._task_files(): + t = json.loads(tj.read_text()) + cat = t.get("category") + # the PRIMARY AMB category is `source` (history vs conversation) — the benchmark's core + # "where does the decision live" axis (2 values). The decision-type (`category`) and `tier` + # remain as secondary breakdown axes via get_result_categories. + if category and t.get("source") != category: + continue + queries.append(Query( + id=t["task_id"], + query=t["bug_report"], + gold_ids=[], + gold_answers=[], # coding: no gold answer, graded by tests + user_id=t["task_id"], + meta={ + "source": t.get("source"), "tier": t.get("tier"), "category": cat, + "codebase": t.get("codebase"), "module": t.get("module"), + "function": t.get("function"), # used to focus the reflect query on the changed code + "task_json": str(tj), # the CodingMode passes this to run.py --task + }, + )) + return queries[:limit] if limit else queries + + def load_documents(self, split: str, category: str | None = None, limit: int | None = None, + ids: set[str] | None = None, user_ids: set[str] | None = None) -> list[Document]: + # AMB does NO ingestion for the coding task — memory is entirely the plugin's domain (the + # coding mode triggers the plugin's own backfill, which decides what/how to ingest from the + # built repo + the task's conversations). So there is nothing for the OMB provider to ingest. + return [] + + def categories(self, split: str) -> list[str] | None: + # PRIMARY category = source (history / conversation) — 2 values, the benchmark's main axis. + srcs = {json.loads(tj.read_text()).get("source") for tj in self._task_files()} + return sorted(s for s in srcs if s) + + def category_type(self, split: str, category: str): + return "query" + + def get_result_categories(self, meta: dict) -> dict[str, list[str]]: + axes: dict[str, list[str]] = {} + for key, label in (("source", "Source"), ("tier", "Tier"), ("category", "Category")): + if meta.get(key): + axes[label] = [meta[key]] + return axes + + def supports_oracle(self) -> bool: + return False diff --git a/src/memory_bench/memory/__init__.py b/src/memory_bench/memory/__init__.py index fe30a2e..73e6a2a 100644 --- a/src/memory_bench/memory/__init__.py +++ b/src/memory_bench/memory/__init__.py @@ -9,8 +9,11 @@ from .hybrid_search import HybridSearchMemoryProvider from .ogham import OghamMemoryProvider from .supermemory import SupermemoryMemoryProvider +from .none import NoMemoryProvider, HsCodingProvider REGISTRY: dict[str, type[MemoryProvider]] = { + "none": NoMemoryProvider, + "hscoding": HsCodingProvider, "bm25": BM25MemoryProvider, "cognee": CogneeMemoryProvider, "hindsight": HindsightMemoryProvider, diff --git a/src/memory_bench/memory/none.py b/src/memory_bench/memory/none.py new file mode 100644 index 0000000..8040be0 --- /dev/null +++ b/src/memory_bench/memory/none.py @@ -0,0 +1,25 @@ +"""No-memory provider — the baseline arm. Ingests nothing and retrieves nothing, so an eval with +`--memory none` measures the agent/model with no memory system at all.""" +from .base import MemoryProvider +from ..models import Document + + +class NoMemoryProvider(MemoryProvider): + name = "none" + description = "No memory system (baseline)." + kind = "local" + + def ingest(self, documents: list[Document]) -> None: + pass + + def retrieve(self, query: str, k: int = 10, user_id: str | None = None, + query_timestamp: str | None = None) -> tuple[list[Document], dict | None]: + return [], None + + +class HsCodingProvider(NoMemoryProvider): + """No-op from AMB's side — a selector, not a provider. For the coding task, memory is entirely the + plugin's domain: the coding mode triggers the plugin's own backfill and the plugin does reflect+inject. + AMB performs NO ingestion or retrieval, so this ingests/retrieves nothing (like `none`).""" + name = "hscoding" + description = "Coding: memory handled by the agent's Hindsight plugin (AMB does no ingestion/retrieval)." diff --git a/src/memory_bench/modes/__init__.py b/src/memory_bench/modes/__init__.py index 9bf5bab..83b1e9d 100644 --- a/src/memory_bench/modes/__init__.py +++ b/src/memory_bench/modes/__init__.py @@ -2,12 +2,14 @@ from .rag import RAGMode from .agentic_rag import AgenticRAGMode from .agent import AgentMode +from .coding import CodingMode from ..llm.base import LLM REGISTRY: dict[str, type[ResponseMode]] = { "rag": RAGMode, "agentic-rag": AgenticRAGMode, "agent": AgentMode, + "coding": CodingMode, } diff --git a/src/memory_bench/modes/coding.py b/src/memory_bench/modes/coding.py new file mode 100644 index 0000000..a591b7d --- /dev/null +++ b/src/memory_bench/modes/coding.py @@ -0,0 +1,162 @@ +"""Coding response mode: build the task repo, run a coding agent with test-feedback interventions, +grade by pytest. Reuses the proven sdebench harness (`sdebench/harness/run.py`) verbatim. + +AMB does ZERO memory work for the coding task — memory is entirely the plugin's domain: + - `none` => the no-memory baseline (`full` arm). + - `hscoding` => the mode (a) builds the task repo, (b) triggers the PLUGIN's own backfill + (`hindsight-coding-backfill`) over that repo + the task's conversations — the plugin decides + what and how to ingest — then (c) runs opencode + the plugin (`hscoding` arm), which does + reflect+inject. AMB never calls Hindsight retain or reflect itself. +Any other provider raises. The harness result is returned as an AnswerResult (the runner's `coding` +branch reads `solved` etc.). + +Env: SDE_HINDSIGHT_URL (Hindsight server, default :8888), SDE_HSCODING_PLUGIN_DIR (the plugin package +dir holding dist/backfill.js, default ~/dev/hindsight-coding-opencode), SDE_HSCODING_GIT_LIMIT +(optional git scope passed to the plugin backfill; unset => the plugin decides), SDE_MODEL. +""" +import asyncio +import json +import os +import shutil +import subprocess +import time +import uuid +from pathlib import Path + +from .base import ResponseMode +from ..memory.base import MemoryProvider +from ..models import AnswerResult + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_RUN_PY = _REPO_ROOT / "sdebench" / "harness" / "run.py" + + +class CodingMode(ResponseMode): + name = "coding" + description = "Build the task repo, run a coding agent with test-feedback interventions, grade by pytest." + + _AGENT_MODEL = {"opencode": "google/gemini-3.5-flash", "claude-code": "claude-sonnet-5"} + + def __init__(self, model: str | None = None): + self._agent = os.environ.get("SDE_AGENT", "opencode") # --agent, so claude runs land in the UI + self._model = model or os.environ.get("SDE_MODEL") or self._AGENT_MODEL.get(self._agent, "google/gemini-3.5-flash") + + @property + def llm_id(self) -> str | None: + return f"{self._agent}:{self._model}" + + def answer(self, query: str, memory: MemoryProvider, task_type: str = "coding", user_id: str | None = None) -> AnswerResult: + return asyncio.run(self.async_answer(query, memory, task_type=task_type, user_id=user_id)) + + def answer_from_context(self, query: str, context: str, task_type: str = "coding") -> AnswerResult: + raise NotImplementedError("coding mode grades by running the agent; --skip-retrieval is not supported") + + @staticmethod + def _bank_has_memories(url: str, bank: str) -> bool: + """True if the bank already holds memories — used to REUSE a backfilled bank across n runs.""" + import urllib.request + try: + with urllib.request.urlopen(f"{url}/v1/default/banks/{bank}/memories/list?limit=1", timeout=10) as r: + d = json.loads(r.read()) + return bool(d.get("items") or d.get("memories") or d.get("total")) + except Exception: + return False + + async def _plugin_backfill(self, task_json: str, task_id: str, run_id: str, bank: str, url: str) -> None: + """Trigger the PLUGIN's own backfill — AMB does not ingest. Build the task repo, then run the + plugin's `hindsight-coding-backfill` over that repo + the task's conversations; the plugin + decides what and how to ingest (extraction, strategies, git scope, pages).""" + # Reuse a bank already backfilled by a previous run (n=3 without re-ingesting the same data). + if os.environ.get("SDE_HSCODING_REUSE_BANK", "").lower() in ("1", "true") \ + and await asyncio.to_thread(self._bank_has_memories, url, bank): + return + plugin_dir = Path(os.path.expanduser(os.environ.get("SDE_HSCODING_PLUGIN_DIR", + str(Path.home() / "dev" / "hindsight-coding-opencode")))) + backfill_js = plugin_dir / "dist" / "backfill.js" + tj = Path(task_json) + t = json.loads(tj.read_text()) + build_py = tj.parents[2] / t.get("build", "build.py") + base = Path("/tmp/sdebench/omb-backfill") / f"{task_id}_{run_id}" + src = base / "repo" + shutil.rmtree(base, ignore_errors=True) + base.mkdir(parents=True, exist_ok=True) + # 1. build the task repo (the plugin backfill reads its git history) + await asyncio.to_thread(subprocess.run, ["python", str(build_py), str(src)], + capture_output=True, text=True, env={**os.environ}) + # 2. run the plugin's backfill (it owns extraction/strategies/pages/git scope) + bf = ["node", str(backfill_js), "--repo", str(src), "--bank", bank, "--api-url", url, "--reset"] + conv = t.get("conversations") or [] + # chats to ingest = the task's own decision chat + a shared pool of DECOY conversations (noise: + # long, codebase-related, no task policy) so chat retrieval is a real ranking problem, not a + # 1-chat lookup. Decoys are pre-generated once from git history (gen/decoy_conversations.json). + # a task may carry ONE chat (list of turns) or SEVERAL (list of lists — e.g. a rule set in an + # early chat and AMENDED in a later one); seed_sessions on the vanilla side already handles both. + if conv and isinstance(conv[0], list): + chats = [{"id": f"{task_id}-chat{i}", "turns": c} for i, c in enumerate(conv) if c] + else: + chats = [{"id": task_id, "turns": conv}] if conv else [] + decoy_path = Path(os.environ.get("SDE_DECOY_CONVERSATIONS", + str(_REPO_ROOT / "sdebench" / "datasets" / "gen" / "decoy_conversations.json"))) + if os.environ.get("SDE_DECOYS", "1").lower() not in ("0", "false") and decoy_path.exists(): + for d in json.loads(decoy_path.read_text()): + chats.append({"id": d.get("id", f"decoy-{len(chats)}"), "turns": d["turns"]}) + if chats: + cf = base / "conversations.json" + cf.write_text(json.dumps(chats)) + bf += ["--conversations", str(cf)] + limit = os.environ.get("SDE_HSCODING_GIT_LIMIT") # optional scope; unset => the plugin decides + if limit: + bf += ["--limit", limit] + await asyncio.to_thread(subprocess.run, bf, capture_output=True, text=True, env={**os.environ}) + + async def async_answer(self, query: str, memory: MemoryProvider, task_type: str = "coding", + user_id: str | None = None, meta: dict | None = None) -> AnswerResult: + meta = meta or {} + task_json = meta.get("task_json") + task_id = user_id or meta.get("task_id") or "task" + run_id = f"omb-{uuid.uuid4().hex[:8]}" + + if not task_json: + return AnswerResult(answer="unsolved", reasoning="no task_json in meta", context="", + retrieve_time_ms=0.0, raw_response={"solved": False}) + + # `none` => vanilla; `hscoding` => trigger the plugin's own backfill, then run opencode+plugin. + env = {**os.environ} + if memory.name == "none": + arm = "full" + elif memory.name == "hscoding": + arm = "hscoding" + bank = f"sde-coding-{task_id}" + url = os.environ.get("SDE_HINDSIGHT_URL", "http://localhost:8888") + await self._plugin_backfill(task_json, task_id, run_id, bank, url) # PLUGIN ingests; AMB does not + env["SDE_HSCODING_BANK"] = bank # run.py -> HINDSIGHT_BANK_ID for the plugin (reflect) + env["SDE_HINDSIGHT_URL"] = url # run.py -> HINDSIGHT_API_URL for the plugin + else: + raise NotImplementedError(f"coding mode supports 'none' and 'hscoding'; got '{memory.name}'") + + cmd = ["uv", "run", "python", str(_RUN_PY), "--task", str(task_json), + "--history", arm, "--agent", self._agent, "--model", self._model, "--run-id", run_id] + t0 = time.perf_counter() + proc = await asyncio.to_thread( + subprocess.run, cmd, capture_output=True, text=True, cwd=str(_REPO_ROOT), env=env, + ) + elapsed_ms = (time.perf_counter() - t0) * 1000 + + work = Path("/tmp/sdebench/run") / f"{task_id}_{arm}_{run_id}" + result_path = work / "result.json" + if result_path.exists(): + result = json.loads(result_path.read_text()) + else: + # harness crashed before grading — surface stderr tail so it's debuggable + result = {"solved": False, "interventions": None, + "final_pytest": (proc.stderr or proc.stdout or "")[-400:], "error": "no result.json"} + + solved = bool(result.get("solved")) + return AnswerResult( + answer="solved" if solved else "unsolved", + reasoning=f"arm={arm} interventions={result.get('interventions')} " + f"cost=${result.get('cost_usd')} turns={result.get('turns')}", + context=f"memory={memory.name} arm={arm}", # non-empty; coding scoring ignores context + retrieve_time_ms=float(result.get("wall_s", 0.0)) * 1000 or elapsed_ms, + raw_response=result, # runner's coding branch reads solved/interventions/… + ) diff --git a/src/memory_bench/runner.py b/src/memory_bench/runner.py index d867826..8ac4967 100644 --- a/src/memory_bench/runner.py +++ b/src/memory_bench/runner.py @@ -189,7 +189,17 @@ async def _process_one_attempt(q) -> QueryResult: logger.info("[query:%s] answer done in %.1fs (retrieve=%.0fms)", q.id, time.perf_counter() - t_start, answer_result.retrieve_time_ms) score: float | None = None - if not answer_result.context: + if task_type == "coding": + # The CodingMode already built the repo, ran the agent (with interventions), and graded + # by pytest. Correctness is solve-ness, not a judged answer; carry the metrics through. + raw = answer_result.raw_response or {} + correct = bool(raw.get("solved")) + judge_reason = f"interventions={raw.get('interventions')} pytest={(raw.get('final_pytest') or '')[:80]}" + q.meta.update({k: raw.get(k) for k in + ("solved", "interventions", "capped", "cost_usd", "turns", "wall_s", + "final_pytest", "tokens", "agent", "model") + if raw.get(k) is not None}) + elif not answer_result.context: correct, judge_reason = False, "empty context — no memories retrieved" elif task_type == "mcq": correct, judge_reason = _score_mcq(answer_result.answer, q.gold_answers) @@ -357,7 +367,8 @@ async def bounded(i, q): memory.ingest(documents) ingestion_ms = (time.perf_counter() - t0) * 1000 ingested_docs_count = len(documents) - console.print(f" ingested in {ingestion_ms:.0f}ms ({ingestion_ms / len(documents):.1f}ms/doc avg)\n") + avg = f" ({ingestion_ms / len(documents):.1f}ms/doc avg)" if documents else "" + console.print(f" ingested in {ingestion_ms:.0f}ms{avg}\n") async def _run_all(progress, task_id): concurrency = getattr(memory, "concurrency", _CONCURRENCY) diff --git a/src/memory_bench/server.py b/src/memory_bench/server.py index 61013a0..9c9950b 100644 --- a/src/memory_bench/server.py +++ b/src/memory_bench/server.py @@ -148,6 +148,32 @@ def _extract(key): "avg_context_tokens": float(avg_context_tokens) if avg_context_tokens and avg_context_tokens != "null" else None, "category": category if category and category != "null" else None, }) + # Coding datasets (sdebench) report agent metrics per-result, not top-level. Read the full file + # and aggregate them so the dataset page can show interventions/cost/turns/tokens instead of the + # QA metrics (accuracy/recall/ctx-tokens), which don't apply to coding. + if parts[2] == "coding": + try: + full = _json.loads(_gzip.decompress(raw) if is_gz else raw) + rs = full.get("results", []) + metas = [r.get("meta", {}) or {} for r in rs] + def _sum(k): + return sum((m.get(k) or 0) for m in metas) + def _toks(sel): + return sum(sum((m.get("tokens", {}) or {}).get(x, 0) or 0 for x in sel) for m in metas) + agent = next((m.get("agent") for m in metas if m.get("agent")), None) or "opencode" + entries[-1].update({ + "coding": True, "agent": agent, + "tasks": len(rs), + "solved": sum(1 for r in rs if (r.get("meta", {}) or {}).get("solved")), + "interventions": _sum("interventions"), + "cost_usd": round(_sum("cost_usd"), 2), + "turns": _sum("turns"), + "wall_s": round(_sum("wall_s"), 0), + "tokens_in": _toks(("input", "cache_read", "cache_write")), + "tokens_out": _toks(("output", "reasoning")), + }) + except Exception: + pass _results_cache = entries _results_cache_mtime = current_mtime diff --git a/ui/dist/assets/index-BWKRtmhC.css b/ui/dist/assets/index-BWKRtmhC.css deleted file mode 100644 index 019adf3..0000000 --- a/ui/dist/assets/index-BWKRtmhC.css +++ /dev/null @@ -1 +0,0 @@ -/*! tailwindcss v4.2.1 | MIT License | https://tailwindcss.com */@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-scale-x:1;--tw-scale-y:1;--tw-scale-z:1;--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-pan-x:initial;--tw-pan-y:initial;--tw-pinch-zoom:initial;--tw-space-y-reverse:0;--tw-space-x-reverse:0;--tw-divide-x-reverse:0;--tw-border-style:solid;--tw-divide-y-reverse:0;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial}}}@layer theme{:root,:host{--color-violet-400:oklch(70.2% .183 293.541);--color-violet-500:oklch(60.6% .25 292.717);--color-white:#fff;--spacing:.25rem;--container-md:28rem;--container-2xl:42rem;--container-3xl:48rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height: 1.5 ;--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height: 1.2 ;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--tracking-tight:-.025em;--tracking-normal:0em;--tracking-wide:.025em;--tracking-wider:.05em;--leading-tight:1.25;--leading-snug:1.375;--leading-relaxed:1.625;--radius-sm:calc(var(--radius) - 4px);--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:"Inter", ui-sans-serif, system-ui;--default-mono-font-family:"Geist Mono", ui-monospace;--font-display:"Space Grotesk", "Inter", ui-sans-serif}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;-moz-tab-size:4;tab-size:4;line-height:1.5;font-family:var(--default-font-family,ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring{outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab,red,red)){::placeholder{color:color-mix(in oklab,currentcolor 50%,transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){-webkit-appearance:button;-moz-appearance:button;appearance:button}::file-selector-button{-webkit-appearance:button;-moz-appearance:button;appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}*{border-color:var(--border);box-sizing:border-box;-webkit-font-smoothing:antialiased;scrollbar-width:thin;scrollbar-color:var(--border) transparent}::-webkit-scrollbar{width:6px;height:6px}::-webkit-scrollbar-track{background:0 0}::-webkit-scrollbar-thumb{background:var(--border);border-radius:9999px}::-webkit-scrollbar-thumb:hover{background:var(--muted-foreground)}body{background-color:var(--background);color:var(--foreground);margin:0}pre{white-space:pre-wrap;word-break:break-word}[v-cloak]{display:none}details>summary{list-style:none}details>summary::-webkit-details-marker{display:none}}@layer components;@layer utilities{.collapse{visibility:collapse}.invisible{visibility:hidden}.visible{visibility:visible}.sr-only{clip-path:inset(50%);white-space:nowrap;border-width:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.not-sr-only{clip-path:none;white-space:normal;width:auto;height:auto;margin:0;padding:0;position:static;overflow:visible}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.static{position:static}.sticky{position:sticky}.inset-0{inset:calc(var(--spacing) * 0)}.start{inset-inline-start:var(--spacing)}.end{inset-inline-end:var(--spacing)}.top-0\.5{top:calc(var(--spacing) * .5)}.left-0\.5{left:calc(var(--spacing) * .5)}.isolate{isolation:isolate}.isolation-auto{isolation:auto}.container{width:100%}@media (min-width:40rem){.container{max-width:40rem}}@media (min-width:48rem){.container{max-width:48rem}}@media (min-width:64rem){.container{max-width:64rem}}@media (min-width:80rem){.container{max-width:80rem}}@media (min-width:96rem){.container{max-width:96rem}}.mx-auto{margin-inline:auto}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:calc(var(--spacing) * 1)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-3{margin-top:calc(var(--spacing) * 3)}.mt-6{margin-top:calc(var(--spacing) * 6)}.mt-8{margin-top:calc(var(--spacing) * 8)}.mt-10{margin-top:calc(var(--spacing) * 10)}.mt-16{margin-top:calc(var(--spacing) * 16)}.mt-20{margin-top:calc(var(--spacing) * 20)}.mt-24{margin-top:calc(var(--spacing) * 24)}.mt-auto{margin-top:auto}.-mb-px{margin-bottom:-1px}.mb-0\.5{margin-bottom:calc(var(--spacing) * .5)}.mb-1{margin-bottom:calc(var(--spacing) * 1)}.mb-1\.5{margin-bottom:calc(var(--spacing) * 1.5)}.mb-2{margin-bottom:calc(var(--spacing) * 2)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.mb-4{margin-bottom:calc(var(--spacing) * 4)}.mb-6{margin-bottom:calc(var(--spacing) * 6)}.mb-8{margin-bottom:calc(var(--spacing) * 8)}.mb-10{margin-bottom:calc(var(--spacing) * 10)}.ml-1{margin-left:calc(var(--spacing) * 1)}.ml-2{margin-left:calc(var(--spacing) * 2)}.ml-auto{margin-left:auto}.line-clamp-2{-webkit-line-clamp:2;-webkit-box-orient:vertical;display:-webkit-box;overflow:hidden}.block{display:block}.contents{display:contents}.flex{display:flex}.flow-root{display:flow-root}.grid{display:grid}.hidden{display:none}.inline{display:inline}.inline-block{display:inline-block}.inline-flex{display:inline-flex}.inline-grid{display:inline-grid}.inline-table{display:inline-table}.list-item{display:list-item}.table{display:table}.table-caption{display:table-caption}.table-cell{display:table-cell}.table-column{display:table-column}.table-column-group{display:table-column-group}.table-footer-group{display:table-footer-group}.table-header-group{display:table-header-group}.table-row{display:table-row}.table-row-group{display:table-row-group}.h-3{height:calc(var(--spacing) * 3)}.h-4{height:calc(var(--spacing) * 4)}.h-6{height:calc(var(--spacing) * 6)}.h-7{height:calc(var(--spacing) * 7)}.h-8{height:calc(var(--spacing) * 8)}.h-9{height:calc(var(--spacing) * 9)}.h-14{height:calc(var(--spacing) * 14)}.h-screen{height:100vh}.max-h-56{max-height:calc(var(--spacing) * 56)}.max-h-96{max-height:calc(var(--spacing) * 96)}.max-h-\[25vh\]{max-height:25vh}.min-h-screen{min-height:100vh}.w-3{width:calc(var(--spacing) * 3)}.w-4{width:calc(var(--spacing) * 4)}.w-8{width:calc(var(--spacing) * 8)}.w-9{width:calc(var(--spacing) * 9)}.w-10{width:calc(var(--spacing) * 10)}.w-20{width:calc(var(--spacing) * 20)}.w-24{width:calc(var(--spacing) * 24)}.w-\[300px\]{width:300px}.w-\[320px\]{width:320px}.w-full{width:100%}.max-w-2xl{max-width:var(--container-2xl)}.max-w-3xl{max-width:var(--container-3xl)}.max-w-\[110px\]{max-width:110px}.max-w-\[160px\]{max-width:160px}.max-w-md{max-width:var(--container-md)}.min-w-0{min-width:calc(var(--spacing) * 0)}.min-w-\[240px\]{min-width:240px}.flex-1{flex:1}.shrink{flex-shrink:1}.shrink-0{flex-shrink:0}.grow{flex-grow:1}.caption-bottom{caption-side:bottom}.border-collapse{border-collapse:collapse}.translate-x-4{--tw-translate-x:calc(var(--spacing) * 4);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-none{translate:none}.scale-3d{scale:var(--tw-scale-x) var(--tw-scale-y) var(--tw-scale-z)}.rotate-180{rotate:180deg}.transform{transform:var(--tw-rotate-x,) var(--tw-rotate-y,) var(--tw-rotate-z,) var(--tw-skew-x,) var(--tw-skew-y,)}.animate-pulse{animation:var(--animate-pulse)}.cursor-pointer{cursor:pointer}.touch-pinch-zoom{--tw-pinch-zoom:pinch-zoom;touch-action:var(--tw-pan-x,) var(--tw-pan-y,) var(--tw-pinch-zoom,)}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.justify-end{justify-content:flex-end}.gap-0\.5{gap:calc(var(--spacing) * .5)}.gap-1{gap:calc(var(--spacing) * 1)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-2\.5{gap:calc(var(--spacing) * 2.5)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-4{gap:calc(var(--spacing) * 4)}.gap-5{gap:calc(var(--spacing) * 5)}:where(.space-y-0\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2.5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-8>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 8) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 8) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-10>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 10) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 10) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-20>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 20) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 20) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-reverse>:not(:last-child)){--tw-space-y-reverse:1}:where(.space-x-reverse>:not(:last-child)){--tw-space-x-reverse:1}:where(.divide-x>:not(:last-child)){--tw-divide-x-reverse:0;border-inline-style:var(--tw-border-style);border-inline-start-width:calc(1px * var(--tw-divide-x-reverse));border-inline-end-width:calc(1px * calc(1 - var(--tw-divide-x-reverse)))}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-y-reverse>:not(:last-child)){--tw-divide-y-reverse:1}:where(.divide-border>:not(:last-child)){border-color:var(--border)}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-auto{overflow:auto}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.overflow-y-auto{overflow-y:auto}.rounded{border-radius:.25rem}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius)}.rounded-md{border-radius:calc(var(--radius) - 2px)}.rounded-s{border-start-start-radius:.25rem;border-end-start-radius:.25rem}.rounded-ss{border-start-start-radius:.25rem}.rounded-e{border-start-end-radius:.25rem;border-end-end-radius:.25rem}.rounded-se{border-start-end-radius:.25rem}.rounded-ee{border-end-end-radius:.25rem}.rounded-es{border-end-start-radius:.25rem}.rounded-t{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.rounded-l{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.rounded-tl{border-top-left-radius:.25rem}.rounded-r{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.rounded-tr{border-top-right-radius:.25rem}.rounded-b{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.rounded-br{border-bottom-right-radius:.25rem}.rounded-bl{border-bottom-left-radius:.25rem}.border{border-style:var(--tw-border-style);border-width:1px}.border-x{border-inline-style:var(--tw-border-style);border-inline-width:1px}.border-y{border-block-style:var(--tw-border-style);border-block-width:1px}.border-s{border-inline-start-style:var(--tw-border-style);border-inline-start-width:1px}.border-e{border-inline-end-style:var(--tw-border-style);border-inline-end-width:1px}.border-bs{border-block-start-style:var(--tw-border-style);border-block-start-width:1px}.border-be{border-block-end-style:var(--tw-border-style);border-block-end-width:1px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-r{border-right-style:var(--tw-border-style);border-right-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-b-0{border-bottom-style:var(--tw-border-style);border-bottom-width:0}.border-b-2{border-bottom-style:var(--tw-border-style);border-bottom-width:2px}.border-l{border-left-style:var(--tw-border-style);border-left-width:1px}.border-l-2{border-left-style:var(--tw-border-style);border-left-width:2px}.border-border,.border-border\/40{border-color:var(--border)}@supports (color:color-mix(in lab,red,red)){.border-border\/40{border-color:color-mix(in oklab,var(--border) 40%,transparent)}}.border-border\/50{border-color:var(--border)}@supports (color:color-mix(in lab,red,red)){.border-border\/50{border-color:color-mix(in oklab,var(--border) 50%,transparent)}}.border-ca\/15{border-color:#fbbf2426}.border-ca\/25{border-color:#fbbf2440}.border-cg\/25{border-color:#34d39940}.border-destructive\/30{border-color:var(--destructive)}@supports (color:color-mix(in lab,red,red)){.border-destructive\/30{border-color:color-mix(in oklab,var(--destructive) 30%,transparent)}}.border-input{border-color:var(--input)}.border-muted{border-color:var(--muted)}.border-primary,.border-primary\/25{border-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.border-primary\/25{border-color:color-mix(in oklab,var(--primary) 25%,transparent)}}.border-primary\/30{border-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.border-primary\/30{border-color:color-mix(in oklab,var(--primary) 30%,transparent)}}.border-violet-500\/25{border-color:#8d54ff40}@supports (color:color-mix(in lab,red,red)){.border-violet-500\/25{border-color:color-mix(in oklab,var(--color-violet-500) 25%,transparent)}}.bg-background{background-color:var(--background)}.bg-border{background-color:var(--border)}.bg-ca\/10{background-color:#fbbf241a}.bg-card{background-color:var(--card)}.bg-cg\/10{background-color:#34d3991a}.bg-cg\/15{background-color:#34d39926}.bg-cr\/15{background-color:#f8717126}.bg-destructive\/15{background-color:var(--destructive)}@supports (color:color-mix(in lab,red,red)){.bg-destructive\/15{background-color:color-mix(in oklab,var(--destructive) 15%,transparent)}}.bg-muted\/50{background-color:var(--muted)}@supports (color:color-mix(in lab,red,red)){.bg-muted\/50{background-color:color-mix(in oklab,var(--muted) 50%,transparent)}}.bg-primary,.bg-primary\/5{background-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.bg-primary\/5{background-color:color-mix(in oklab,var(--primary) 5%,transparent)}}.bg-primary\/10{background-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.bg-primary\/10{background-color:color-mix(in oklab,var(--primary) 10%,transparent)}}.bg-primary\/15{background-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.bg-primary\/15{background-color:color-mix(in oklab,var(--primary) 15%,transparent)}}.bg-secondary,.bg-secondary\/40{background-color:var(--secondary)}@supports (color:color-mix(in lab,red,red)){.bg-secondary\/40{background-color:color-mix(in oklab,var(--secondary) 40%,transparent)}}.bg-transparent{background-color:#0000}.bg-violet-500\/10{background-color:#8d54ff1a}@supports (color:color-mix(in lab,red,red)){.bg-violet-500\/10{background-color:color-mix(in oklab,var(--color-violet-500) 10%,transparent)}}.bg-white{background-color:var(--color-white)}.bg-repeat{background-repeat:repeat}.mask-no-clip{-webkit-mask-clip:no-clip;mask-clip:no-clip}.mask-repeat{-webkit-mask-repeat:repeat;mask-repeat:repeat}.object-contain{object-fit:contain}.p-3{padding:calc(var(--spacing) * 3)}.p-4{padding:calc(var(--spacing) * 4)}.p-5{padding:calc(var(--spacing) * 5)}.p-6{padding:calc(var(--spacing) * 6)}.px-1{padding-inline:calc(var(--spacing) * 1)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-6{padding-inline:calc(var(--spacing) * 6)}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1{padding-block:calc(var(--spacing) * 1)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-8{padding-block:calc(var(--spacing) * 8)}.py-12{padding-block:calc(var(--spacing) * 12)}.pt-1{padding-top:calc(var(--spacing) * 1)}.pt-2{padding-top:calc(var(--spacing) * 2)}.pt-4{padding-top:calc(var(--spacing) * 4)}.pr-2{padding-right:calc(var(--spacing) * 2)}.pb-3{padding-bottom:calc(var(--spacing) * 3)}.pb-4{padding-bottom:calc(var(--spacing) * 4)}.pb-10{padding-bottom:calc(var(--spacing) * 10)}.pl-3{padding-left:calc(var(--spacing) * 3)}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.font-display{font-family:Space Grotesk,Inter,ui-sans-serif}.font-mono{font-family:Geist Mono,ui-monospace}.font-sans{font-family:Inter,ui-sans-serif,system-ui}.text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}.text-3xl{font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height))}.text-base{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.leading-relaxed{--tw-leading:var(--leading-relaxed);line-height:var(--leading-relaxed)}.leading-snug{--tw-leading:var(--leading-snug);line-height:var(--leading-snug)}.leading-tight{--tw-leading:var(--leading-tight);line-height:var(--leading-tight)}.font-bold{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-normal{--tw-font-weight:var(--font-weight-normal);font-weight:var(--font-weight-normal)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-normal{--tw-tracking:var(--tracking-normal);letter-spacing:var(--tracking-normal)}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.tracking-wide{--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide)}.tracking-wider{--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider)}.text-wrap{text-wrap:wrap}.text-clip{text-overflow:clip}.text-ellipsis{text-overflow:ellipsis}.whitespace-nowrap{white-space:nowrap}.whitespace-pre-wrap{white-space:pre-wrap}.text-ca{color:#fbbf24}.text-card-foreground{color:var(--card-foreground)}.text-cg{color:#34d399}.text-cr{color:#f87171}.text-destructive{color:var(--destructive)}.text-foreground,.text-foreground\/70{color:var(--foreground)}@supports (color:color-mix(in lab,red,red)){.text-foreground\/70{color:color-mix(in oklab,var(--foreground) 70%,transparent)}}.text-foreground\/80{color:var(--foreground)}@supports (color:color-mix(in lab,red,red)){.text-foreground\/80{color:color-mix(in oklab,var(--foreground) 80%,transparent)}}.text-muted-foreground,.text-muted-foreground\/30{color:var(--muted-foreground)}@supports (color:color-mix(in lab,red,red)){.text-muted-foreground\/30{color:color-mix(in oklab,var(--muted-foreground) 30%,transparent)}}.text-muted-foreground\/70{color:var(--muted-foreground)}@supports (color:color-mix(in lab,red,red)){.text-muted-foreground\/70{color:color-mix(in oklab,var(--muted-foreground) 70%,transparent)}}.text-muted-foreground\/80{color:var(--muted-foreground)}@supports (color:color-mix(in lab,red,red)){.text-muted-foreground\/80{color:color-mix(in oklab,var(--muted-foreground) 80%,transparent)}}.text-muted-foreground\/85{color:var(--muted-foreground)}@supports (color:color-mix(in lab,red,red)){.text-muted-foreground\/85{color:color-mix(in oklab,var(--muted-foreground) 85%,transparent)}}.text-primary{color:var(--primary)}.text-primary-foreground{color:var(--primary-foreground)}.text-secondary-foreground{color:var(--secondary-foreground)}.text-violet-400{color:var(--color-violet-400)}.capitalize{text-transform:capitalize}.lowercase{text-transform:lowercase}.normal-case{text-transform:none}.uppercase{text-transform:uppercase}.italic{font-style:italic}.not-italic{font-style:normal}.diagonal-fractions{--tw-numeric-fraction:diagonal-fractions;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.lining-nums{--tw-numeric-figure:lining-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.oldstyle-nums{--tw-numeric-figure:oldstyle-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.ordinal{--tw-ordinal:ordinal;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.proportional-nums{--tw-numeric-spacing:proportional-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.slashed-zero{--tw-slashed-zero:slashed-zero;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.stacked-fractions{--tw-numeric-fraction:stacked-fractions;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.tabular-nums{--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.normal-nums{font-variant-numeric:normal}.line-through{text-decoration-line:line-through}.no-underline{text-decoration-line:none}.overline{text-decoration-line:overline}.underline{text-decoration-line:underline}.underline-offset-4{text-underline-offset:4px}.antialiased{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.subpixel-antialiased{-webkit-font-smoothing:auto;-moz-osx-font-smoothing:auto}.opacity-60{opacity:.6}.opacity-75{opacity:.75}.shadow,.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.inset-ring{--tw-inset-ring-shadow:inset 0 0 0 1px var(--tw-inset-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.outline{outline-style:var(--tw-outline-style);outline-width:1px}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.drop-shadow{--tw-drop-shadow-size:drop-shadow(0 1px 2px var(--tw-drop-shadow-color,#0000001a)) drop-shadow(0 1px 1px var(--tw-drop-shadow-color,#0000000f));--tw-drop-shadow:drop-shadow(0 1px 2px #0000001a) drop-shadow(0 1px 1px #0000000f);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.filter{filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.backdrop-blur{--tw-backdrop-blur:blur(8px);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-grayscale{--tw-backdrop-grayscale:grayscale(100%);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-invert{--tw-backdrop-invert:invert(100%);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-sepia{--tw-backdrop-sepia:sepia(100%);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-filter{-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.select-none{-webkit-user-select:none;user-select:none}:where(.divide-x-reverse>:not(:last-child)){--tw-divide-x-reverse:1}.ring-inset{--tw-ring-inset:inset}@media (hover:hover){.group-hover\:text-muted-foreground:is(:where(.group):hover *){color:var(--muted-foreground)}}.placeholder\:text-muted-foreground\/80::placeholder{color:var(--muted-foreground)}@supports (color:color-mix(in lab,red,red)){.placeholder\:text-muted-foreground\/80::placeholder{color:color-mix(in oklab,var(--muted-foreground) 80%,transparent)}}.last\:border-0:last-child{border-style:var(--tw-border-style);border-width:0}@media (hover:hover){.hover\:border-primary\/30:hover{border-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.hover\:border-primary\/30:hover{border-color:color-mix(in oklab,var(--primary) 30%,transparent)}}.hover\:bg-cg\/10:hover{background-color:#34d3991a}.hover\:bg-cr\/10:hover{background-color:#f871711a}.hover\:bg-primary\/90:hover{background-color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.hover\:bg-primary\/90:hover{background-color:color-mix(in oklab,var(--primary) 90%,transparent)}}.hover\:bg-secondary:hover,.hover\:bg-secondary\/30:hover{background-color:var(--secondary)}@supports (color:color-mix(in lab,red,red)){.hover\:bg-secondary\/30:hover{background-color:color-mix(in oklab,var(--secondary) 30%,transparent)}}.hover\:bg-secondary\/50:hover{background-color:var(--secondary)}@supports (color:color-mix(in lab,red,red)){.hover\:bg-secondary\/50:hover{background-color:color-mix(in oklab,var(--secondary) 50%,transparent)}}.hover\:text-ca\/80:hover{color:#fbbf24cc}.hover\:text-cg:hover{color:#34d399}.hover\:text-cr:hover{color:#f87171}.hover\:text-foreground:hover,.hover\:text-foreground\/80:hover{color:var(--foreground)}@supports (color:color-mix(in lab,red,red)){.hover\:text-foreground\/80:hover{color:color-mix(in oklab,var(--foreground) 80%,transparent)}}.hover\:text-muted-foreground:hover{color:var(--muted-foreground)}.hover\:text-primary:hover,.hover\:text-primary\/80:hover{color:var(--primary)}@supports (color:color-mix(in lab,red,red)){.hover\:text-primary\/80:hover{color:color-mix(in oklab,var(--primary) 80%,transparent)}}.hover\:underline:hover{text-decoration-line:underline}}.focus-visible\:ring-1:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow),var(--tw-inset-ring-shadow),var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow)}.focus-visible\:ring-ring:focus-visible{--tw-ring-color:var(--ring)}.focus-visible\:outline-none:focus-visible{--tw-outline-style:none;outline-style:none}.disabled\:pointer-events-none:disabled{pointer-events:none}.disabled\:cursor-not-allowed:disabled{cursor:not-allowed}.disabled\:opacity-25:disabled{opacity:.25}.disabled\:opacity-40:disabled{opacity:.4}.disabled\:opacity-50:disabled{opacity:.5}@media (min-width:40rem){.sm\:col-span-2{grid-column:span 2/span 2}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.sm\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}}@media (min-width:48rem){.md\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.md\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}}@media (min-width:64rem){.lg\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}}}:root{--background:oklch(21.78% 0 0);--foreground:oklch(92.19% 0 0);--card:oklch(24.35% 0 0);--card-foreground:oklch(92.19% 0 0);--popover:oklch(24.35% 0 0);--popover-foreground:oklch(92.19% 0 0);--primary:oklch(64.2% .1691 38.5815);--primary-foreground:oklch(100% 0 0);--secondary:oklch(37.43% .0726 258.521);--secondary-foreground:oklch(92.19% 0 0);--muted:oklch(28.5% 0 0);--muted-foreground:oklch(76% 0 0);--accent:oklch(33.8% .0589 267.587);--accent-foreground:oklch(88.23% .0571 254.128);--destructive:oklch(63.68% .2078 25.3313);--destructive-foreground:oklch(100% 0 0);--border:oklch(39% 0 0);--input:oklch(30.92% 0 0);--ring:oklch(63.97% .172 36.4421);--radius:.5rem}.heading-gradient{-webkit-text-fill-color:transparent;background:linear-gradient(135deg,#f49157,#df6035);-webkit-background-clip:text;background-clip:text}.topnav{z-index:50;background:var(--background);position:sticky;top:0}@supports (color:color-mix(in lab,red,red)){.topnav{background:color-mix(in oklch,var(--background) 85%,transparent)}}.topnav{-webkit-backdrop-filter:blur(16px);border-bottom:1px solid var(--border)}.chart-track{background:var(--secondary);border-radius:4px;flex:1;height:1.2rem;position:relative;overflow:hidden}.chart-bar-a{background:linear-gradient(90deg,#952b00,#df6035);background:linear-gradient(90deg,color(xyz 0.142 0.08 -0),#df6035);border-radius:4px;height:100%;transition:width .4s}.chart-bar-r{background:linear-gradient(90deg,#059669,#34d399);border-radius:4px;height:100%;transition:width .4s}.chart-bar-t{background:linear-gradient(90deg,#9b462c,#d59466);border-radius:4px;height:100%;transition:width .4s}.chart-val-out{font-family:var(--font-display),ui-sans-serif,system-ui,sans-serif;color:var(--muted-foreground);white-space:nowrap;text-align:right;letter-spacing:-.01em;min-width:2.8rem;font-size:.7rem;font-weight:500}.chart-val-out.chart-winner{color:var(--primary);font-weight:800}.sidebar{background:var(--card);border-right:1px solid var(--border)}.sidebar-section{border-bottom:1px solid var(--border)}.item-active{background:var(--primary)!important}@supports (color:color-mix(in lab,red,red)){.item-active{background:color-mix(in oklch,var(--primary) 15%,transparent)!important}}.item-active{border-left:2px solid var(--primary)!important}.verdict-pass{color:#34d399;border-radius:var(--radius);background:#34d39914;border:1px solid #34d39933;padding:.75rem 1rem}.verdict-fail{color:#f87171;border-radius:var(--radius);background:#f8717114;border:1px solid #f8717133;padding:.75rem 1rem}.code-block{background:var(--muted);border:1px solid var(--border);border-radius:var(--radius);color:var(--muted-foreground);max-height:24rem;padding:1rem 1.25rem;font-family:Geist Mono,ui-monospace,monospace;font-size:.75rem;line-height:1.7;overflow-y:auto}.cat-row{cursor:pointer;border-radius:var(--radius-sm);transition:background .1s}.cat-row:hover{background:var(--muted)}.cat-row-active{background:var(--primary)}@supports (color:color-mix(in lab,red,red)){.cat-row-active{background:color-mix(in oklch,var(--primary) 15%,transparent)}}.cat-row-active{color:var(--primary)}.pill{background:var(--muted);color:var(--muted-foreground);border-radius:9999px;padding:.2rem .6rem;font-size:.7rem}.stat-box{background:var(--muted);border:1px solid var(--border);border-radius:var(--radius);padding:.6rem .85rem}.inset-section{background:var(--muted);border-radius:var(--radius);border:1px solid var(--border);padding:1rem 1.1rem}.doc-summary{transition:background .1s}.doc-summary:hover{background:var(--muted)}@supports (color:color-mix(in lab,red,red)){.doc-summary:hover{background:color-mix(in oklch,var(--muted) 60%,transparent)}}.reveal-item{opacity:0;transition:opacity .55s,transform .55s;transform:translateY(18px)}.reveal-visible{opacity:1;transform:translateY(0)}@property --tw-translate-x{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-y{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-z{syntax:"*";inherits:false;initial-value:0}@property --tw-scale-x{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-y{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-z{syntax:"*";inherits:false;initial-value:1}@property --tw-rotate-x{syntax:"*";inherits:false}@property --tw-rotate-y{syntax:"*";inherits:false}@property --tw-rotate-z{syntax:"*";inherits:false}@property --tw-skew-x{syntax:"*";inherits:false}@property --tw-skew-y{syntax:"*";inherits:false}@property --tw-pan-x{syntax:"*";inherits:false}@property --tw-pan-y{syntax:"*";inherits:false}@property --tw-pinch-zoom{syntax:"*";inherits:false}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-space-x-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-x-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-ordinal{syntax:"*";inherits:false}@property --tw-slashed-zero{syntax:"*";inherits:false}@property --tw-numeric-figure{syntax:"*";inherits:false}@property --tw-numeric-spacing{syntax:"*";inherits:false}@property --tw-numeric-fraction{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-outline-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-backdrop-blur{syntax:"*";inherits:false}@property --tw-backdrop-brightness{syntax:"*";inherits:false}@property --tw-backdrop-contrast{syntax:"*";inherits:false}@property --tw-backdrop-grayscale{syntax:"*";inherits:false}@property --tw-backdrop-hue-rotate{syntax:"*";inherits:false}@property --tw-backdrop-invert{syntax:"*";inherits:false}@property --tw-backdrop-opacity{syntax:"*";inherits:false}@property --tw-backdrop-saturate{syntax:"*";inherits:false}@property --tw-backdrop-sepia{syntax:"*";inherits:false}@keyframes pulse{50%{opacity:.5}} diff --git a/ui/dist/assets/index-C1LfR5en.js b/ui/dist/assets/index-C1LfR5en.js new file mode 100644 index 0000000..a7e8b74 --- /dev/null +++ b/ui/dist/assets/index-C1LfR5en.js @@ -0,0 +1,67 @@ +var bc=Object.defineProperty;var yc=(e,t,o)=>t in e?bc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[t]=o;var kn=(e,t,o)=>yc(e,typeof t!="symbol"?t+"":t,o);(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const r of document.querySelectorAll('link[rel="modulepreload"]'))n(r);new MutationObserver(r=>{for(const s of r)if(s.type==="childList")for(const i of s.addedNodes)i.tagName==="LINK"&&i.rel==="modulepreload"&&n(i)}).observe(document,{childList:!0,subtree:!0});function o(r){const s={};return r.integrity&&(s.integrity=r.integrity),r.referrerPolicy&&(s.referrerPolicy=r.referrerPolicy),r.crossOrigin==="use-credentials"?s.credentials="include":r.crossOrigin==="anonymous"?s.credentials="omit":s.credentials="same-origin",s}function n(r){if(r.ep)return;r.ep=!0;const s=o(r);fetch(r.href,s)}})();/** +* @vue/shared v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/function $t(e){const t=Object.create(null);for(const o of e.split(","))t[o]=1;return o=>o in t}const Ne=Object.freeze({}),On=Object.freeze([]),et=()=>{},na=()=>!1,po=e=>e.charCodeAt(0)===111&&e.charCodeAt(1)===110&&(e.charCodeAt(2)>122||e.charCodeAt(2)<97),Mo=e=>e.startsWith("onUpdate:"),Ve=Object.assign,ns=(e,t)=>{const o=e.indexOf(t);o>-1&&e.splice(o,1)},vc=Object.prototype.hasOwnProperty,Pe=(e,t)=>vc.call(e,t),he=Array.isArray,mn=e=>mo(e)==="[object Map]",oa=e=>mo(e)==="[object Set]",Ns=e=>mo(e)==="[object Date]",be=e=>typeof e=="function",He=e=>typeof e=="string",_t=e=>typeof e=="symbol",Me=e=>e!==null&&typeof e=="object",os=e=>(Me(e)||be(e))&&be(e.then)&&be(e.catch),ra=Object.prototype.toString,mo=e=>ra.call(e),rs=e=>mo(e).slice(8,-1),sa=e=>mo(e)==="[object Object]",ss=e=>He(e)&&e!=="NaN"&&e[0]!=="-"&&""+parseInt(e,10)===e,eo=$t(",key,ref,ref_for,ref_key,onVnodeBeforeMount,onVnodeMounted,onVnodeBeforeUpdate,onVnodeUpdated,onVnodeBeforeUnmount,onVnodeUnmounted"),_c=$t("bind,cloak,else-if,else,for,html,if,model,on,once,pre,show,slot,text,memo"),Zo=e=>{const t=Object.create(null);return o=>t[o]||(t[o]=e(o))},xc=/-\w/g,Je=Zo(e=>e.replace(xc,t=>t.slice(1).toUpperCase())),wc=/\B([A-Z])/g,on=Zo(e=>e.replace(wc,"-$1").toLowerCase()),vn=Zo(e=>e.charAt(0).toUpperCase()+e.slice(1)),dn=Zo(e=>e?`on${vn(e)}`:""),Ot=(e,t)=>!Object.is(e,t),Gn=(e,...t)=>{for(let o=0;o{Object.defineProperty(e,t,{configurable:!0,enumerable:!1,writable:n,value:o})},kc=e=>{const t=parseFloat(e);return isNaN(t)?e:t};let Hs;const ho=()=>Hs||(Hs=typeof globalThis<"u"?globalThis:typeof self<"u"?self:typeof window<"u"?window:typeof global<"u"?global:{});function Mn(e){if(he(e)){const t={};for(let o=0;o{if(o){const n=o.split(Sc);n.length>1&&(t[n[0].trim()]=n[1].trim())}}),t}function Oe(e){let t="";if(He(e))t=e;else if(he(e))for(let o=0;o!!(e&&e.__v_isRef===!0),k=e=>He(e)?e:e==null?"":he(e)||Me(e)&&(e.toString===ra||!be(e.toString))?aa(e)?k(e.value):JSON.stringify(e,la,2):String(e),la=(e,t)=>aa(t)?la(e,t.value):mn(t)?{[`Map(${t.size})`]:[...t.entries()].reduce((o,[n,r],s)=>(o[dr(n,s)+" =>"]=r,o),{})}:oa(t)?{[`Set(${t.size})`]:[...t.values()].map(o=>dr(o))}:_t(t)?dr(t):Me(t)&&!he(t)&&!sa(t)?String(t):t,dr=(e,t="")=>{var o;return _t(e)?`Symbol(${(o=e.description)!=null?o:t})`:e};/** +* @vue/reactivity v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/function xt(e,...t){console.warn(`[Vue warn] ${e}`,...t)}let lt;class Nc{constructor(t=!1){this.detached=t,this._active=!0,this._on=0,this.effects=[],this.cleanups=[],this._isPaused=!1,this.__v_skip=!0,this.parent=lt,!t&<&&(this.index=(lt.scopes||(lt.scopes=[])).push(this)-1)}get active(){return this._active}pause(){if(this._active){this._isPaused=!0;let t,o;if(this.scopes)for(t=0,o=this.scopes.length;t0&&--this._on===0&&(lt=this.prevScope,this.prevScope=void 0)}stop(t){if(this._active){this._active=!1;let o,n;for(o=0,n=this.effects.length;o0)return;if(no){let t=no;for(no=void 0;t;){const o=t.next;t.next=void 0,t.flags&=-9,t=o}}let e;for(;to;){let t=to;for(to=void 0;t;){const o=t.next;if(t.next=void 0,t.flags&=-9,t.flags&1)try{t.trigger()}catch(n){e||(e=n)}t=o}}if(e)throw e}function fa(e){for(let t=e.deps;t;t=t.nextDep)t.version=-1,t.prevActiveLink=t.dep.activeLink,t.dep.activeLink=t}function pa(e){let t,o=e.depsTail,n=o;for(;n;){const r=n.prevDep;n.version===-1?(n===o&&(o=r),cs(n),Uc(n)):t=n,n.dep.activeLink=n.prevActiveLink,n.prevActiveLink=void 0,n=r}e.deps=t,e.depsTail=o}function Ar(e){for(let t=e.deps;t;t=t.nextDep)if(t.dep.version!==t.version||t.dep.computed&&(ma(t.dep.computed)||t.dep.version!==t.version))return!0;return!!e._dirty}function ma(e){if(e.flags&4&&!(e.flags&16)||(e.flags&=-17,e.globalVersion===ao)||(e.globalVersion=ao,!e.isSSR&&e.flags&128&&(!e.deps&&!e._dirty||!Ar(e))))return;e.flags|=2;const t=e.dep,o=De,n=vt;De=e,vt=!0;try{fa(e);const r=e.fn(e._value);(t.version===0||Ot(r,e._value))&&(e.flags|=128,e._value=r,t.version++)}catch(r){throw t.version++,r}finally{De=o,vt=n,pa(e),e.flags&=-3}}function cs(e,t=!1){const{dep:o,prevSub:n,nextSub:r}=e;if(n&&(n.nextSub=r,e.prevSub=void 0),r&&(r.prevSub=n,e.nextSub=void 0),o.subsHead===e&&(o.subsHead=r),o.subs===e&&(o.subs=n,!n&&o.computed)){o.computed.flags&=-5;for(let s=o.computed.deps;s;s=s.nextDep)cs(s,!0)}!t&&!--o.sc&&o.map&&o.map.delete(o.key)}function Uc(e){const{prevDep:t,nextDep:o}=e;t&&(t.nextDep=o,e.prevDep=void 0),o&&(o.prevDep=t,e.nextDep=void 0)}let vt=!0;const ha=[];function wt(){ha.push(vt),vt=!1}function kt(){const e=ha.pop();vt=e===void 0?!0:e}function Us(e){const{cleanup:t}=e;if(e.cleanup=void 0,t){const o=De;De=void 0;try{t()}finally{De=o}}}let ao=0;class Bc{constructor(t,o){this.sub=t,this.dep=o,this.version=o.version,this.nextDep=this.prevDep=this.nextSub=this.prevSub=this.prevActiveLink=void 0}}class us{constructor(t){this.computed=t,this.version=0,this.activeLink=void 0,this.subs=void 0,this.map=void 0,this.key=void 0,this.sc=0,this.__v_skip=!0,this.subsHead=void 0}track(t){if(!De||!vt||De===this.computed)return;let o=this.activeLink;if(o===void 0||o.sub!==De)o=this.activeLink=new Bc(De,this),De.deps?(o.prevDep=De.depsTail,De.depsTail.nextDep=o,De.depsTail=o):De.deps=De.depsTail=o,ga(o);else if(o.version===-1&&(o.version=this.version,o.nextDep)){const n=o.nextDep;n.prevDep=o.prevDep,o.prevDep&&(o.prevDep.nextDep=n),o.prevDep=De.depsTail,o.nextDep=void 0,De.depsTail.nextDep=o,De.depsTail=o,De.deps===o&&(De.deps=n)}return De.onTrack&&De.onTrack(Ve({effect:De},t)),o}trigger(t){this.version++,ao++,this.notify(t)}notify(t){as();try{for(let o=this.subsHead;o;o=o.nextSub)o.sub.onTrigger&&!(o.sub.flags&8)&&o.sub.onTrigger(Ve({effect:o.sub},t));for(let o=this.subs;o;o=o.prevSub)o.sub.notify()&&o.sub.dep.notify()}finally{ls()}}}function ga(e){if(e.dep.sc++,e.sub.flags&4){const t=e.dep.computed;if(t&&!e.dep.subs){t.flags|=20;for(let n=t.deps;n;n=n.nextDep)ga(n)}const o=e.dep.subs;o!==e&&(e.prevSub=o,o&&(o.nextSub=e)),e.dep.subsHead===void 0&&(e.dep.subsHead=e),e.dep.subs=e}}const Rr=new WeakMap,hn=Symbol("Object iterate"),Or=Symbol("Map keys iterate"),lo=Symbol("Array iterate");function Qe(e,t,o){if(vt&&De){let n=Rr.get(e);n||Rr.set(e,n=new Map);let r=n.get(o);r||(n.set(o,r=new us),r.map=n,r.key=o),r.track({target:e,type:t,key:o})}}function It(e,t,o,n,r,s){const i=Rr.get(e);if(!i){ao++;return}const a=l=>{l&&l.trigger({target:e,type:t,key:o,newValue:n,oldValue:r,oldTarget:s})};if(as(),t==="clear")i.forEach(a);else{const l=he(e),f=l&&ss(o);if(l&&o==="length"){const c=Number(n);i.forEach((u,m)=>{(m==="length"||m===lo||!_t(m)&&m>=c)&&a(u)})}else switch((o!==void 0||i.has(void 0))&&a(i.get(o)),f&&a(i.get(lo)),t){case"add":l?f&&a(i.get("length")):(a(i.get(hn)),mn(e)&&a(i.get(Or)));break;case"delete":l||(a(i.get(hn)),mn(e)&&a(i.get(Or)));break;case"set":mn(e)&&a(i.get(hn));break}}ls()}function Cn(e){const t=we(e);return t===e?t:(Qe(t,"iterate",lo),it(e)?t:t.map(St))}function er(e){return Qe(e=we(e),"iterate",lo),e}function Rt(e,t){return Ct(e)?jn(nn(e)?St(t):t):St(t)}const Fc={__proto__:null,[Symbol.iterator](){return pr(this,Symbol.iterator,e=>Rt(this,e))},concat(...e){return Cn(this).concat(...e.map(t=>he(t)?Cn(t):t))},entries(){return pr(this,"entries",e=>(e[1]=Rt(this,e[1]),e))},every(e,t){return Ut(this,"every",e,t,void 0,arguments)},filter(e,t){return Ut(this,"filter",e,t,o=>o.map(n=>Rt(this,n)),arguments)},find(e,t){return Ut(this,"find",e,t,o=>Rt(this,o),arguments)},findIndex(e,t){return Ut(this,"findIndex",e,t,void 0,arguments)},findLast(e,t){return Ut(this,"findLast",e,t,o=>Rt(this,o),arguments)},findLastIndex(e,t){return Ut(this,"findLastIndex",e,t,void 0,arguments)},forEach(e,t){return Ut(this,"forEach",e,t,void 0,arguments)},includes(...e){return mr(this,"includes",e)},indexOf(...e){return mr(this,"indexOf",e)},join(e){return Cn(this).join(e)},lastIndexOf(...e){return mr(this,"lastIndexOf",e)},map(e,t){return Ut(this,"map",e,t,void 0,arguments)},pop(){return Wn(this,"pop")},push(...e){return Wn(this,"push",e)},reduce(e,...t){return Bs(this,"reduce",e,t)},reduceRight(e,...t){return Bs(this,"reduceRight",e,t)},shift(){return Wn(this,"shift")},some(e,t){return Ut(this,"some",e,t,void 0,arguments)},splice(...e){return Wn(this,"splice",e)},toReversed(){return Cn(this).toReversed()},toSorted(e){return Cn(this).toSorted(e)},toSpliced(...e){return Cn(this).toSpliced(...e)},unshift(...e){return Wn(this,"unshift",e)},values(){return pr(this,"values",e=>Rt(this,e))}};function pr(e,t,o){const n=er(e),r=n[t]();return n!==e&&!it(e)&&(r._next=r.next,r.next=()=>{const s=r._next();return s.done||(s.value=o(s.value)),s}),r}const Vc=Array.prototype;function Ut(e,t,o,n,r,s){const i=er(e),a=i!==e&&!it(e),l=i[t];if(l!==Vc[t]){const u=l.apply(e,s);return a?St(u):u}let f=o;i!==e&&(a?f=function(u,m){return o.call(this,Rt(e,u),m,e)}:o.length>2&&(f=function(u,m){return o.call(this,u,m,e)}));const c=l.call(i,f,n);return a&&r?r(c):c}function Bs(e,t,o,n){const r=er(e),s=r!==e&&!it(e);let i=o,a=!1;r!==e&&(s?(a=n.length===0,i=function(f,c,u){return a&&(a=!1,f=Rt(e,f)),o.call(this,f,Rt(e,c),u,e)}):o.length>3&&(i=function(f,c,u){return o.call(this,f,c,u,e)}));const l=r[t](i,...n);return a?Rt(e,l):l}function mr(e,t,o){const n=we(e);Qe(n,"iterate",lo);const r=n[t](...o);return(r===-1||r===!1)&&Lo(o[0])?(o[0]=we(o[0]),n[t](...o)):r}function Wn(e,t,o=[]){wt(),as();const n=we(e)[t].apply(e,o);return ls(),kt(),n}const Gc=$t("__proto__,__v_isRef,__isVue"),ba=new Set(Object.getOwnPropertyNames(Symbol).filter(e=>e!=="arguments"&&e!=="caller").map(e=>Symbol[e]).filter(_t));function Wc(e){_t(e)||(e=String(e));const t=we(this);return Qe(t,"has",e),t.hasOwnProperty(e)}class ya{constructor(t=!1,o=!1){this._isReadonly=t,this._isShallow=o}get(t,o,n){if(o==="__v_skip")return t.__v_skip;const r=this._isReadonly,s=this._isShallow;if(o==="__v_isReactive")return!r;if(o==="__v_isReadonly")return r;if(o==="__v_isShallow")return s;if(o==="__v_raw")return n===(r?s?Ca:ka:s?wa:xa).get(t)||Object.getPrototypeOf(t)===Object.getPrototypeOf(n)?t:void 0;const i=he(t);if(!r){let l;if(i&&(l=Fc[o]))return l;if(o==="hasOwnProperty")return Wc}const a=Reflect.get(t,o,Fe(t)?t:n);if((_t(o)?ba.has(o):Gc(o))||(r||Qe(t,"get",o),s))return a;if(Fe(a)){const l=i&&ss(o)?a:a.value;return r&&Me(l)?Pr(l):l}return Me(a)?r?Pr(a):nr(a):a}}class va extends ya{constructor(t=!1){super(!1,t)}set(t,o,n,r){let s=t[o];const i=he(t)&&ss(o);if(!this._isShallow){const f=Ct(s);if(!it(n)&&!Ct(n)&&(s=we(s),n=we(n)),!i&&Fe(s)&&!Fe(n))return f?(xt(`Set operation on key "${String(o)}" failed: target is readonly.`,t[o]),!0):(s.value=n,!0)}const a=i?Number(o)e,xo=e=>Reflect.getPrototypeOf(e);function Qc(e,t,o){return function(...n){const r=this.__v_raw,s=we(r),i=mn(s),a=e==="entries"||e===Symbol.iterator&&i,l=e==="keys"&&i,f=r[e](...n),c=o?Ir:t?jn:St;return!t&&Qe(s,"iterate",l?Or:hn),Ve(Object.create(f),{next(){const{value:u,done:m}=f.next();return m?{value:u,done:m}:{value:a?[c(u[0]),c(u[1])]:c(u),done:m}}})}}function wo(e){return function(...t){{const o=t[0]?`on key "${t[0]}" `:"";xt(`${vn(e)} operation ${o}failed: target is readonly.`,we(this))}return e==="delete"?!1:e==="clear"?void 0:this}}function Jc(e,t){const o={get(r){const s=this.__v_raw,i=we(s),a=we(r);e||(Ot(r,a)&&Qe(i,"get",r),Qe(i,"get",a));const{has:l}=xo(i),f=t?Ir:e?jn:St;if(l.call(i,r))return f(s.get(r));if(l.call(i,a))return f(s.get(a));s!==i&&s.get(r)},get size(){const r=this.__v_raw;return!e&&Qe(we(r),"iterate",hn),r.size},has(r){const s=this.__v_raw,i=we(s),a=we(r);return e||(Ot(r,a)&&Qe(i,"has",r),Qe(i,"has",a)),r===a?s.has(r):s.has(r)||s.has(a)},forEach(r,s){const i=this,a=i.__v_raw,l=we(a),f=t?Ir:e?jn:St;return!e&&Qe(l,"iterate",hn),a.forEach((c,u)=>r.call(s,f(c),f(u),i))}};return Ve(o,e?{add:wo("add"),set:wo("set"),delete:wo("delete"),clear:wo("clear")}:{add(r){const s=we(this),i=xo(s),a=we(r),l=!t&&!it(r)&&!Ct(r)?a:r;return i.has.call(s,l)||Ot(r,l)&&i.has.call(s,r)||Ot(a,l)&&i.has.call(s,a)||(s.add(l),It(s,"add",l,l)),this},set(r,s){!t&&!it(s)&&!Ct(s)&&(s=we(s));const i=we(this),{has:a,get:l}=xo(i);let f=a.call(i,r);f?Fs(i,a,r):(r=we(r),f=a.call(i,r));const c=l.call(i,r);return i.set(r,s),f?Ot(s,c)&&It(i,"set",r,s,c):It(i,"add",r,s),this},delete(r){const s=we(this),{has:i,get:a}=xo(s);let l=i.call(s,r);l?Fs(s,i,r):(r=we(r),l=i.call(s,r));const f=a?a.call(s,r):void 0,c=s.delete(r);return l&&It(s,"delete",r,void 0,f),c},clear(){const r=we(this),s=r.size!==0,i=mn(r)?new Map(r):new Set(r),a=r.clear();return s&&It(r,"clear",void 0,void 0,i),a}}),["keys","values","entries",Symbol.iterator].forEach(r=>{o[r]=Qc(r,e,t)}),o}function tr(e,t){const o=Jc(e,t);return(n,r,s)=>r==="__v_isReactive"?!e:r==="__v_isReadonly"?e:r==="__v_raw"?n:Reflect.get(Pe(o,r)&&r in n?o:n,r,s)}const Yc={get:tr(!1,!1)},Xc={get:tr(!1,!0)},Zc={get:tr(!0,!1)},eu={get:tr(!0,!0)};function Fs(e,t,o){const n=we(o);if(n!==o&&t.call(e,n)){const r=rs(e);xt(`Reactive ${r} contains both the raw and reactive versions of the same object${r==="Map"?" as keys":""}, which can lead to inconsistencies. Avoid differentiating between the raw and reactive versions of an object and only use the reactive version if possible.`)}}const xa=new WeakMap,wa=new WeakMap,ka=new WeakMap,Ca=new WeakMap;function tu(e){switch(e){case"Object":case"Array":return 1;case"Map":case"Set":case"WeakMap":case"WeakSet":return 2;default:return 0}}function nu(e){return e.__v_skip||!Object.isExtensible(e)?0:tu(rs(e))}function nr(e){return Ct(e)?e:or(e,!1,Kc,Yc,xa)}function Sa(e){return or(e,!1,qc,Xc,wa)}function Pr(e){return or(e,!0,zc,Zc,ka)}function Mt(e){return or(e,!0,$c,eu,Ca)}function or(e,t,o,n,r){if(!Me(e))return xt(`value cannot be made ${t?"readonly":"reactive"}: ${String(e)}`),e;if(e.__v_raw&&!(t&&e.__v_isReactive))return e;const s=nu(e);if(s===0)return e;const i=r.get(e);if(i)return i;const a=new Proxy(e,s===2?n:o);return r.set(e,a),a}function nn(e){return Ct(e)?nn(e.__v_raw):!!(e&&e.__v_isReactive)}function Ct(e){return!!(e&&e.__v_isReadonly)}function it(e){return!!(e&&e.__v_isShallow)}function Lo(e){return e?!!e.__v_raw:!1}function we(e){const t=e&&e.__v_raw;return t?we(t):e}function ou(e){return!Pe(e,"__v_skip")&&Object.isExtensible(e)&&jo(e,"__v_skip",!0),e}const St=e=>Me(e)?nr(e):e,jn=e=>Me(e)?Pr(e):e;function Fe(e){return e?e.__v_isRef===!0:!1}function re(e){return Ta(e,!1)}function ru(e){return Ta(e,!0)}function Ta(e,t){return Fe(e)?e:new su(e,t)}class su{constructor(t,o){this.dep=new us,this.__v_isRef=!0,this.__v_isShallow=!1,this._rawValue=o?t:we(t),this._value=o?t:St(t),this.__v_isShallow=o}get value(){return this.dep.track({target:this,type:"get",key:"value"}),this._value}set value(t){const o=this._rawValue,n=this.__v_isShallow||it(t)||Ct(t);t=n?t:we(t),Ot(t,o)&&(this._rawValue=t,this._value=n?t:St(t),this.dep.trigger({target:this,type:"set",key:"value",newValue:t,oldValue:o}))}}function tn(e){return Fe(e)?e.value:e}function iu(e){return be(e)?e():tn(e)}const au={get:(e,t,o)=>t==="__v_raw"?e:tn(Reflect.get(e,t,o)),set:(e,t,o,n)=>{const r=e[t];return Fe(r)&&!Fe(o)?(r.value=o,!0):Reflect.set(e,t,o,n)}};function Ea(e){return nn(e)?e:new Proxy(e,au)}class lu{constructor(t,o,n){this.fn=t,this.setter=o,this._value=void 0,this.dep=new us(this),this.__v_isRef=!0,this.deps=void 0,this.depsTail=void 0,this.flags=16,this.globalVersion=ao-1,this.next=void 0,this.effect=this,this.__v_isReadonly=!o,this.isSSR=n}notify(){if(this.flags|=16,!(this.flags&8)&&De!==this)return da(this,!0),!0}get value(){const t=this.dep.track({target:this,type:"get",key:"value"});return ma(this),t&&(t.version=this.dep.version),this._value}set value(t){this.setter?this.setter(t):xt("Write operation failed: computed value is readonly")}}function cu(e,t,o=!1){let n,r;be(e)?n=e:(n=e.get,r=e.set);const s=new lu(n,r,o);return t&&!o&&(s.onTrack=t.onTrack,s.onTrigger=t.onTrigger),s}const ko={},Do=new WeakMap;let fn;function uu(e,t=!1,o=fn){if(o){let n=Do.get(o);n||Do.set(o,n=[]),n.push(e)}else t||xt("onWatcherCleanup() was called when there was no active watcher to associate with.")}function du(e,t,o=Ne){const{immediate:n,deep:r,once:s,scheduler:i,augmentJob:a,call:l}=o,f=Q=>{(o.onWarn||xt)("Invalid watch source: ",Q,"A watch source can only be a getter/effect function, a ref, a reactive object, or an array of these types.")},c=Q=>r?Q:it(Q)||r===!1||r===0?zt(Q,1):zt(Q);let u,m,g,w,p=!1,C=!1;if(Fe(e)?(m=()=>e.value,p=it(e)):nn(e)?(m=()=>c(e),p=!0):he(e)?(C=!0,p=e.some(Q=>nn(Q)||it(Q)),m=()=>e.map(Q=>{if(Fe(Q))return Q.value;if(nn(Q))return c(Q);if(be(Q))return l?l(Q,2):Q();f(Q)})):be(e)?t?m=l?()=>l(e,2):e:m=()=>{if(g){wt();try{g()}finally{kt()}}const Q=fn;fn=u;try{return l?l(e,3,[w]):e(w)}finally{fn=Q}}:(m=et,f(e)),t&&r){const Q=m,de=r===!0?1/0:r;m=()=>zt(Q(),de)}const V=Hc(),L=()=>{u.stop(),V&&V.active&&ns(V.effects,u)};if(s&&t){const Q=t;t=(...de)=>{Q(...de),L()}}let D=C?new Array(e.length).fill(ko):ko;const ne=Q=>{if(!(!(u.flags&1)||!u.dirty&&!Q))if(t){const de=u.run();if(r||p||(C?de.some((z,U)=>Ot(z,D[U])):Ot(de,D))){g&&g();const z=fn;fn=u;try{const U=[de,D===ko?void 0:C&&D[0]===ko?[]:D,w];D=de,l?l(t,3,U):t(...U)}finally{fn=z}}}else u.run()};return a&&a(ne),u=new ca(m),u.scheduler=i?()=>i(ne,!1):ne,w=Q=>uu(Q,!1,u),g=u.onStop=()=>{const Q=Do.get(u);if(Q){if(l)l(Q,4);else for(const de of Q)de();Do.delete(u)}},u.onTrack=o.onTrack,u.onTrigger=o.onTrigger,t?n?ne(!0):D=u.run():i?i(ne.bind(null,!0),!0):u.run(),L.pause=u.pause.bind(u),L.resume=u.resume.bind(u),L.stop=L,L}function zt(e,t=1/0,o){if(t<=0||!Me(e)||e.__v_skip||(o=o||new Map,(o.get(e)||0)>=t))return e;if(o.set(e,t),t--,Fe(e))zt(e.value,t,o);else if(he(e))for(let n=0;n{zt(n,t,o)});else if(sa(e)){for(const n in e)zt(e[n],t,o);for(const n of Object.getOwnPropertySymbols(e))Object.prototype.propertyIsEnumerable.call(e,n)&&zt(e[n],t,o)}return e}/** +* @vue/runtime-core v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/const gn=[];function Eo(e){gn.push(e)}function Ao(){gn.pop()}let hr=!1;function X(e,...t){if(hr)return;hr=!0,wt();const o=gn.length?gn[gn.length-1].component:null,n=o&&o.appContext.config.warnHandler,r=fu();if(n)Un(n,o,11,[e+t.map(s=>{var i,a;return(a=(i=s.toString)==null?void 0:i.call(s))!=null?a:JSON.stringify(s)}).join(""),o&&o.proxy,r.map(({vnode:s})=>`at <${_o(o,s.type)}>`).join(` +`),r]);else{const s=[`[Vue warn]: ${e}`,...t];r.length&&s.push(` +`,...pu(r)),console.warn(...s)}kt(),hr=!1}function fu(){let e=gn[gn.length-1];if(!e)return[];const t=[];for(;e;){const o=t[0];o&&o.vnode===e?o.recurseCount++:t.push({vnode:e,recurseCount:0});const n=e.component&&e.component.parent;e=n&&n.vnode}return t}function pu(e){const t=[];return e.forEach((o,n)=>{t.push(...n===0?[]:[` +`],...mu(o))}),t}function mu({vnode:e,recurseCount:t}){const o=t>0?`... (${t} recursive calls)`:"",n=e.component?e.component.parent==null:!1,r=` at <${_o(e.component,e.type,n)}`,s=">"+o;return e.props?[r,...hu(e.props),s]:[r+s]}function hu(e){const t=[],o=Object.keys(e);return o.slice(0,3).forEach(n=>{t.push(...Aa(n,e[n]))}),o.length>3&&t.push(" ..."),t}function Aa(e,t,o){return He(t)?(t=JSON.stringify(t),o?t:[`${e}=${t}`]):typeof t=="number"||typeof t=="boolean"||t==null?o?t:[`${e}=${t}`]:Fe(t)?(t=Aa(e,we(t.value),!0),o?t:[`${e}=Ref<`,t,">"]):be(t)?[`${e}=fn${t.name?`<${t.name}>`:""}`]:(t=we(t),o?t:[`${e}=`,t])}const ds={sp:"serverPrefetch hook",bc:"beforeCreate hook",c:"created hook",bm:"beforeMount hook",m:"mounted hook",bu:"beforeUpdate hook",u:"updated",bum:"beforeUnmount hook",um:"unmounted hook",a:"activated hook",da:"deactivated hook",ec:"errorCaptured hook",rtc:"renderTracked hook",rtg:"renderTriggered hook",0:"setup function",1:"render function",2:"watcher getter",3:"watcher callback",4:"watcher cleanup function",5:"native event handler",6:"component event handler",7:"vnode hook",8:"directive hook",9:"transition hook",10:"app errorHandler",11:"app warnHandler",12:"ref function",13:"async component loader",14:"scheduler flush",15:"component update",16:"app unmount cleanup function"};function Un(e,t,o,n){try{return n?e(...n):e()}catch(r){go(r,t,o)}}function Lt(e,t,o,n){if(be(e)){const r=Un(e,t,o,n);return r&&os(r)&&r.catch(s=>{go(s,t,o)}),r}if(he(e)){const r=[];for(let s=0;s>>1,r=st[n],s=co(r);s=co(o)?st.push(e):st.splice(yu(t),0,e),e.flags|=1,Ia()}}function Ia(){No||(No=Ra.then(ja))}function Pa(e){he(e)?In.push(...e):Zt&&e.id===-1?Zt.splice(An+1,0,e):e.flags&1||(In.push(e),e.flags|=1),Ia()}function Vs(e,t,o=At+1){for(t=t||new Map;oco(o)-co(n));if(In.length=0,Zt){Zt.push(...t);return}for(Zt=t,e=e||new Map,An=0;Ane.id==null?e.flags&2?-1:1/0:e.id;function ja(e){e=e||new Map;const t=o=>fs(e,o);try{for(At=0;Atbu){const n=t.i,r=n&&ws(n.type);return go(`Maximum recursive updates exceeded${r?` in component <${r}>`:""}. This means you have a reactive effect that is mutating its own dependencies and thus recursively triggering itself. Possible sources include component template, render function, updated hook or watcher source function.`,null,10),!0}return e.set(t,o+1),!1}let jt=!1;const Ro=new Map;ho().__VUE_HMR_RUNTIME__={createRecord:gr(La),rerender:gr(xu),reload:gr(wu)};const _n=new Map;function vu(e){const t=e.type.__hmrId;let o=_n.get(t);o||(La(t,e.type),o=_n.get(t)),o.instances.add(e)}function _u(e){_n.get(e.type.__hmrId).instances.delete(e)}function La(e,t){return _n.has(e)?!1:(_n.set(e,{initialDef:Ho(t),instances:new Set}),!0)}function Ho(e){return xl(e)?e.__vccOpts:e}function xu(e,t){const o=_n.get(e);o&&(o.initialDef.render=t,[...o.instances].forEach(n=>{t&&(n.render=t,Ho(n.type).render=t),n.renderCache=[],jt=!0,n.job.flags&8||n.update(),jt=!1}))}function wu(e,t){const o=_n.get(e);if(!o)return;t=Ho(t),Gs(o.initialDef,t);const n=[...o.instances];for(let r=0;r{s.job.flags&8||(jt=!0,s.parent.update(),jt=!1,a.delete(s))}):s.appContext.reload?s.appContext.reload():typeof window<"u"?window.location.reload():console.warn("[HMR] Root or manually mounted instance modified. Full reload required."),s.root.ce&&s!==s.root&&s.root.ce._removeChildStyle(i)}Pa(()=>{Ro.clear()})}function Gs(e,t){Ve(e,t);for(const o in e)o!=="__file"&&!(o in t)&&delete e[o]}function gr(e){return(t,o)=>{try{return e(t,o)}catch(n){console.error(n),console.warn("[HMR] Something went wrong during Vue component hot-reload. Full reload required.")}}}let Pt,Xn=[],Mr=!1;function bo(e,...t){Pt?Pt.emit(e,...t):Mr||Xn.push({event:e,args:t})}function Da(e,t){var o,n;Pt=e,Pt?(Pt.enabled=!0,Xn.forEach(({event:r,args:s})=>Pt.emit(r,...s)),Xn=[]):typeof window<"u"&&window.HTMLElement&&!((n=(o=window.navigator)==null?void 0:o.userAgent)!=null&&n.includes("jsdom"))?((t.__VUE_DEVTOOLS_HOOK_REPLAY__=t.__VUE_DEVTOOLS_HOOK_REPLAY__||[]).push(s=>{Da(s,t)}),setTimeout(()=>{Pt||(t.__VUE_DEVTOOLS_HOOK_REPLAY__=null,Mr=!0,Xn=[])},3e3)):(Mr=!0,Xn=[])}function ku(e,t){bo("app:init",e,t,{Fragment:Z,Text:yo,Comment:ut,Static:ro})}function Cu(e){bo("app:unmount",e)}const Su=ps("component:added"),Na=ps("component:updated"),Tu=ps("component:removed"),Eu=e=>{Pt&&typeof Pt.cleanupBuffer=="function"&&!Pt.cleanupBuffer(e)&&Tu(e)};function ps(e){return t=>{bo(e,t.appContext.app,t.uid,t.parent?t.parent.uid:void 0,t)}}const Au=Ha("perf:start"),Ru=Ha("perf:end");function Ha(e){return(t,o,n)=>{bo(e,t.appContext.app,t.uid,t,o,n)}}function Ou(e,t,o){bo("component:emit",e.appContext.app,e,t,o)}let We=null,Ua=null;function Uo(e){const t=We;return We=e,Ua=e&&e.type.__scopeId||null,t}function O(e,t=We,o){if(!t||e._n)return e;const n=(...r)=>{n._d&&Wo(-1);const s=Uo(t);let i;try{i=e(...r)}finally{Uo(s),n._d&&Wo(1)}return Na(t),i};return n._n=!0,n._c=!0,n._d=!0,n}function Ba(e){_c(e)&&X("Do not use built-in directive ids as custom directive id: "+e)}function Rn(e,t){if(We===null)return X("withDirectives can only be used inside render functions."),e;const o=ar(We),n=e.dirs||(e.dirs=[]);for(let r=0;r1)return o&&be(t)?t.call(n&&n.proxy):t;X(`injection "${String(e)}" not found.`)}else X("inject() can only be used inside setup() or functional components.")}function Iu(){return!!(Bn()||yn)}const Pu=Symbol.for("v-scx"),Mu=()=>{{const e=gt(Pu);return e||X("Server rendering context not provided. Make sure to only call useSSRContext() conditionally in the server build."),e}};function Fa(e,t){return ms(e,null,t)}function qe(e,t,o){return be(t)||X("`watch(fn, options?)` signature has been moved to a separate API. Use `watchEffect(fn, options?)` instead. `watch` now only supports `watch(source, cb, options?) signature."),ms(e,t,o)}function ms(e,t,o=Ne){const{immediate:n,deep:r,flush:s,once:i}=o;t||(n!==void 0&&X('watch() "immediate" option is only respected when using the watch(source, callback, options?) signature.'),r!==void 0&&X('watch() "deep" option is only respected when using the watch(source, callback, options?) signature.'),i!==void 0&&X('watch() "once" option is only respected when using the watch(source, callback, options?) signature.'));const a=Ve({},o);a.onWarn=X;const l=t&&n||!t&&s!=="post";let f;if(fo){if(s==="sync"){const g=Mu();f=g.__watcherHandles||(g.__watcherHandles=[])}else if(!l){const g=()=>{};return g.stop=et,g.resume=et,g.pause=et,g}}const c=Ge;a.call=(g,w,p)=>Lt(g,c,w,p);let u=!1;s==="post"?a.scheduler=g=>{at(g,c&&c.suspense)}:s!=="sync"&&(u=!0,a.scheduler=(g,w)=>{w?g():rr(g)}),a.augmentJob=g=>{t&&(g.flags|=4),u&&(g.flags|=2,c&&(g.id=c.uid,g.i=c))};const m=du(e,t,a);return fo&&(f?f.push(m):l&&m()),m}function ju(e,t,o){const n=this.proxy,r=He(e)?e.includes(".")?Va(n,e):()=>n[e]:e.bind(n,n);let s;be(t)?s=t:(s=t.handler,o=t);const i=vo(this),a=ms(r,s.bind(n),o);return i(),a}function Va(e,t){const o=t.split(".");return()=>{let n=e;for(let r=0;re.__isTeleport,Nu=Symbol("_leaveCb");function hs(e,t){e.shapeFlag&6&&e.component?(e.transition=t,hs(e.component.subTree,t)):e.shapeFlag&128?(e.ssContent.transition=t.clone(e.ssContent),e.ssFallback.transition=t.clone(e.ssFallback)):e.transition=t}function Ga(e,t){return be(e)?Ve({name:e.name},t,{setup:e}):e}function Wa(e){e.ids=[e.ids[0]+e.ids[2]+++"-",0,0]}const Ws=new WeakSet;function Ks(e,t){let o;return!!((o=Object.getOwnPropertyDescriptor(e,t))&&!o.configurable)}const Bo=new WeakMap;function oo(e,t,o,n,r=!1){if(he(e)){e.forEach((p,C)=>oo(p,t&&(he(t)?t[C]:t),o,n,r));return}if(Pn(n)&&!r){n.shapeFlag&512&&n.type.__asyncResolved&&n.component.subTree.component&&oo(e,t,o,n.component.subTree);return}const s=n.shapeFlag&4?ar(n.component):n.el,i=r?null:s,{i:a,r:l}=e;if(!a){X("Missing ref owner context. ref cannot be used on hoisted vnodes. A vnode with ref must be created inside the render function.");return}const f=t&&t.r,c=a.refs===Ne?a.refs={}:a.refs,u=a.setupState,m=we(u),g=u===Ne?na:p=>(Pe(m,p)&&!Fe(m[p])&&X(`Template ref "${p}" used on a non-ref value. It will not work in the production build.`),Ws.has(m[p])||Ks(c,p)?!1:Pe(m,p)),w=(p,C)=>!(Ws.has(p)||C&&Ks(c,C));if(f!=null&&f!==l){if(zs(t),He(f))c[f]=null,g(f)&&(u[f]=null);else if(Fe(f)){const p=t;w(f,p.k)&&(f.value=null),p.k&&(c[p.k]=null)}}if(be(l))Un(l,a,12,[i,c]);else{const p=He(l),C=Fe(l);if(p||C){const V=()=>{if(e.f){const L=p?g(l)?u[l]:c[l]:w(l)||!e.k?l.value:c[e.k];if(r)he(L)&&ns(L,s);else if(he(L))L.includes(s)||L.push(s);else if(p)c[l]=[s],g(l)&&(u[l]=c[l]);else{const D=[s];w(l,e.k)&&(l.value=D),e.k&&(c[e.k]=D)}}else p?(c[l]=i,g(l)&&(u[l]=i)):C?(w(l,e.k)&&(l.value=i),e.k&&(c[e.k]=i)):X("Invalid template ref type:",l,`(${typeof l})`)};if(i){const L=()=>{V(),Bo.delete(e)};L.id=-1,Bo.set(e,L),at(L,o)}else zs(e),V()}else X("Invalid template ref type:",l,`(${typeof l})`)}}function zs(e){const t=Bo.get(e);t&&(t.flags|=8,Bo.delete(e))}ho().requestIdleCallback;ho().cancelIdleCallback;const Pn=e=>!!e.type.__asyncLoader,gs=e=>e.type.__isKeepAlive;function Ka(e,t){qa(e,"a",t)}function za(e,t){qa(e,"da",t)}function qa(e,t,o=Ge){const n=e.__wdc||(e.__wdc=()=>{let r=o;for(;r;){if(r.isDeactivated)return;r=r.parent}return e()});if(sr(t,n,o),o){let r=o.parent;for(;r&&r.parent;)gs(r.parent.vnode)&&Hu(n,t,o,r),r=r.parent}}function Hu(e,t,o,n){const r=sr(t,e,n,!0);Ln(()=>{ns(n[t],r)},o)}function sr(e,t,o=Ge,n=!1){if(o){const r=o[e]||(o[e]=[]),s=t.__weh||(t.__weh=(...i)=>{wt();const a=vo(o),l=Lt(t,o,e,i);return a(),kt(),l});return n?r.unshift(s):r.push(s),s}else{const r=dn(ds[e].replace(/ hook$/,""));X(`${r} is called when there is no active component instance to be associated with. Lifecycle injection APIs can only be used during execution of setup(). If you are using async setup(), make sure to register lifecycle hooks before the first await statement.`)}}const Qt=e=>(t,o=Ge)=>{(!fo||e==="sp")&&sr(e,(...n)=>t(...n),o)},Uu=Qt("bm"),ct=Qt("m"),Bu=Qt("bu"),Fu=Qt("u"),$a=Qt("bum"),Ln=Qt("um"),Vu=Qt("sp"),Gu=Qt("rtg"),Wu=Qt("rtc");function Ku(e,t=Ge){sr("ec",e,t)}const zu="components";function Qa(e,t){return $u(zu,e,!0,t)||e}const qu=Symbol.for("v-ndc");function $u(e,t,o=!0,n=!1){const r=We||Ge;if(r){const s=r.type;{const a=ws(s,!1);if(a&&(a===t||a===Je(t)||a===vn(Je(t))))return s}const i=qs(r[e]||s[e],t)||qs(r.appContext[e],t);return!i&&n?s:(o&&!i&&X(`Failed to resolve ${e.slice(0,-1)}: ${t} +If this is a native custom element, make sure to exclude it from component resolution via compilerOptions.isCustomElement.`),i)}else X(`resolve${vn(e.slice(0,-1))} can only be used in render() or setup().`)}function qs(e,t){return e&&(e[t]||e[Je(t)]||e[vn(Je(t))])}function ye(e,t,o,n){let r;const s=o,i=he(e);if(i||He(e)){const a=i&&nn(e);let l=!1,f=!1;a&&(l=!it(e),f=Ct(e),e=er(e)),r=new Array(e.length);for(let c=0,u=e.length;ct(a,l,void 0,s));else{const a=Object.keys(e);r=new Array(a.length);for(let l=0,f=a.length;l0;return t!=="default"&&(o.name=t),y(),Ae(Z,null,[I("slot",o,n)],f?-2:64)}let s=e[t];s&&s.length>1&&(X("SSR-optimized slot function detected in a non-SSR-optimized render function. You need to mark this component with $dynamic-slots in the parent template."),s=()=>[]),s&&s._c&&(s._d=!1),y();const i=s&&Ja(s(o)),a=o.key||i&&i.key,l=Ae(Z,{key:(a&&!_t(a)?a:`_${t}`)+(!i&&n?"_fb":"")},i||[],i&&e._===1?64:-2);return l.scopeId&&(l.slotScopeIds=[l.scopeId+"-s"]),s&&s._c&&(s._d=!0),l}function Ja(e){return e.some(t=>xn(t)?!(t.type===ut||t.type===Z&&!Ja(t.children)):!0)?e:null}const jr=e=>e?vl(e)?ar(e):jr(e.parent):null,bn=Ve(Object.create(null),{$:e=>e,$el:e=>e.vnode.el,$data:e=>e.data,$props:e=>Mt(e.props),$attrs:e=>Mt(e.attrs),$slots:e=>Mt(e.slots),$refs:e=>Mt(e.refs),$parent:e=>jr(e.parent),$root:e=>jr(e.root),$host:e=>e.ce,$emit:e=>e.emit,$options:e=>Za(e),$forceUpdate:e=>e.f||(e.f=()=>{rr(e.update)}),$nextTick:e=>e.n||(e.n=Oa.bind(e.proxy)),$watch:e=>ju.bind(e)}),bs=e=>e==="_"||e==="$",br=(e,t)=>e!==Ne&&!e.__isScriptSetup&&Pe(e,t),Ya={get({_:e},t){if(t==="__v_skip")return!0;const{ctx:o,setupState:n,data:r,props:s,accessCache:i,type:a,appContext:l}=e;if(t==="__isVue")return!0;if(t[0]!=="$"){const m=i[t];if(m!==void 0)switch(m){case 1:return n[t];case 2:return r[t];case 4:return o[t];case 3:return s[t]}else{if(br(n,t))return i[t]=1,n[t];if(r!==Ne&&Pe(r,t))return i[t]=2,r[t];if(Pe(s,t))return i[t]=3,s[t];if(o!==Ne&&Pe(o,t))return i[t]=4,o[t];Lr&&(i[t]=0)}}const f=bn[t];let c,u;if(f)return t==="$attrs"?(Qe(e.attrs,"get",""),Vo()):t==="$slots"&&Qe(e,"get",t),f(e);if((c=a.__cssModules)&&(c=c[t]))return c;if(o!==Ne&&Pe(o,t))return i[t]=4,o[t];if(u=l.config.globalProperties,Pe(u,t))return u[t];We&&(!He(t)||t.indexOf("__v")!==0)&&(r!==Ne&&bs(t[0])&&Pe(r,t)?X(`Property ${JSON.stringify(t)} must be accessed via $data because it starts with a reserved character ("$" or "_") and is not proxied on the render context.`):e===We&&X(`Property ${JSON.stringify(t)} was accessed during render but is not defined on instance.`))},set({_:e},t,o){const{data:n,setupState:r,ctx:s}=e;return br(r,t)?(r[t]=o,!0):r.__isScriptSetup&&Pe(r,t)?(X(`Cannot mutate - - + +
diff --git a/ui/src/pages/DatasetDetail.vue b/ui/src/pages/DatasetDetail.vue index 3911cad..914d72b 100644 --- a/ui/src/pages/DatasetDetail.vue +++ b/ui/src/pages/DatasetDetail.vue @@ -208,6 +208,15 @@ const toggleExtSort = col => toggleSort(col, extSortCol, extSortDir) const chartAccuracy = local => _chartData(local, 'accuracy') const chartRecall = local => _chartData(local, 'avg_retrieve_time_ms', true) const chartTokens = local => _chartData(local, 'avg_context_tokens', true) +// coding datasets (sdebench) report agent metrics (interventions/cost/turns/tokens), not QA metrics +const isCoding = rows => rows.length > 0 && rows.every(r => r.coding) +const codingArm = item => { + const m = (item.memory || '').toLowerCase(); const rn = (item.run_name || '').toLowerCase() + if (m === 'none' || /(^|[-_])(none|vanilla|full)([-_]|$)/.test(rn)) return 'vanilla' + return 'memory' +} +const fmtTok = v => v == null ? '—' : (v >= 1e6 ? (v/1e6).toFixed(1)+'M' : v >= 1e3 ? (v/1e3).toFixed(0)+'k' : v) +const sortCoding = rows => [...rows].sort((a,b) => (b.interventions ?? -1) - (a.interventions ?? -1)) const sortIcon = (col, active, dir) => active === col ? (dir === 'asc' ? ' ↑' : ' ↓') : '' const getViewMode = split => splitViewMode.value[split] ?? 'overall' async function setViewMode(split, mode) { @@ -360,8 +369,48 @@ function hasCategoryData(local, split) {