From 19e28c290ea6ef1cc54e468b95c026c1a48aba6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 25 Jun 2026 17:23:53 +0200 Subject: [PATCH 001/142] =?UTF-8?q?feat(sdebench):=20first=20regression=20?= =?UTF-8?q?task=20=E2=80=94=20ratelimiter,=20validated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - synthetic repo built with engineered git history (build.py); a perf commit bundles an int() refill floor (regression) with a legit available() accessor - FAIL_TO_PASS regression repro + HIDDEN_TO_PASS held-out variants + PASS_TO_PASS suite - validated: suite green at HEAD, regression+hidden red, surgical fix -> all green, git revert of the regression commit conflicts (forces surgical fix) - README documents the benchmark design + grading + history A/B --- sdebench/README.md | 47 ++++ sdebench/datasets/ratelimiter/build.py | 261 ++++++++++++++++++ sdebench/datasets/ratelimiter/hidden_test.py | 28 ++ .../datasets/ratelimiter/regression_test.py | 16 ++ sdebench/datasets/ratelimiter/task.json | 15 + 5 files changed, 367 insertions(+) create mode 100644 sdebench/README.md create mode 100644 sdebench/datasets/ratelimiter/build.py create mode 100644 sdebench/datasets/ratelimiter/hidden_test.py create mode 100644 sdebench/datasets/ratelimiter/regression_test.py create mode 100644 sdebench/datasets/ratelimiter/task.json diff --git a/sdebench/README.md b/sdebench/README.md new file mode 100644 index 0000000..0720bb2 --- /dev/null +++ b/sdebench/README.md @@ -0,0 +1,47 @@ +# sdebench — Software-Development Engineer Benchmark + +A benchmark for coding agents where **git history is load-bearing**. Each task is a +**regression fix** on a synthetic repo whose history we engineer: a bug is *bundled +inside an otherwise-legitimate commit*, so finding and fixing it rewards reading the +history (`git log`/`blame`/`bisect`) and the commit messages (which encode intent). + +This is the opposite of SWE-Bench-CL, where tasks don't recur and history is incidental. +Here history *should* help — and the harness measures whether the agent exploits it. + +## Why it's designed this way +- **Bundled regression** — the breaking commit also makes a wanted change (with its own + test), so a lazy `git revert` fails `PASS_TO_PASS`. Forces a *surgical* fix. +- **Intent lives in history** — the guarantee broken by the regression was established in + an earlier commit whose message states it; the breaking commit's message claims only a + perf tweak. Diagnosing it cleanly needs the history, not just the code. +- **Deterministic grading** — injected-clock tests (no wall-clock flakiness). + +## Grading (a task is solved iff) +1. `FAIL_TO_PASS` — the regression repro (shipped with the bug report) now passes. +2. `PASS_TO_PASS` — the pre-existing suite still passes (no new breakage; no lazy revert). +3. `HIDDEN_TO_PASS` — held-out tests for the same behaviour with different inputs + (defeats overfitting to the visible repro). Graded from a pristine copy so test edits + are ignored. Resolution is binary. + +## A/B: does history help? +The same task is run with `full` history vs a `squashed` single-commit repo (identical +file tree, no commit trail). The only variable is history availability. + +## Metrics +`resolution` (binary), `cost` (input+output tokens × model price), `speed` (wall-clock; +tool-turns secondary). Agent: opencode + gemini-3.5-flash, in a prebuilt Docker image. + +## Layout +``` +sdebench/ + datasets//build.py # builds the repo with engineered git history + datasets//regression_test.py # FAIL_TO_PASS (shipped to the agent, red at HEAD) + datasets//hidden_test.py # HIDDEN_TO_PASS (held out) + datasets//task.json # task definition + test sets + harness/ # runner (full vs squashed), grading, metrics + Dockerfile # prebuilt env (python + pytest + git) +``` + +## Tasks +- `ratelimiter-regression-001` — token-bucket limiter; a `perf:` commit floors partial + token refill (`int(elapsed*rate)`) while adding `available()`. Fix = drop the floor. diff --git a/sdebench/datasets/ratelimiter/build.py b/sdebench/datasets/ratelimiter/build.py new file mode 100644 index 0000000..f191308 --- /dev/null +++ b/sdebench/datasets/ratelimiter/build.py @@ -0,0 +1,261 @@ +"""Build the `ratelimiter` synthetic repo as a clean, incremental git history. + +The history is engineered so a *regression* is bundled inside an otherwise-legit +commit ("perf: ... add available()"): commit C8 introduces an `int()` floor in the +refill path (silently dropping partial tokens) WHILE also adding the `available()` +accessor. A lazy `git revert` of C8 would lose `available()` and fail PASS_TO_PASS, +forcing a surgical fix — and making the git history load-bearing. + +Usage: python build.py (default: /tmp/sdebench/ratelimiter) +""" +import os, subprocess, sys +from pathlib import Path + +OUT = Path(sys.argv[1] if len(sys.argv) > 1 else "/tmp/sdebench/ratelimiter") + +# ── limiter.py evolves across commits ─────────────────────────────────────── +LIMITER_V1 = '''\ +"""A small, dependency-free token-bucket rate limiter.""" +import time + + +class TokenBucket: + """A token bucket that refills continuously at `rate` tokens/second. + + Tokens accrue continuously, so PARTIAL tokens carry over between calls: + at rate=0.5, half a token is available one second after the bucket empties. + """ + + def __init__(self, rate, capacity): + if rate <= 0 or capacity <= 0: + raise ValueError("rate and capacity must be positive") + self.rate = rate + self.capacity = capacity + self._tokens = float(capacity) + self._last = None + + def _refill(self, now): + if self._last is None: + self._last = now + return + elapsed = now - self._last + if elapsed > 0: + self._tokens = self._tokens + elapsed * self.rate + self._last = now + + def try_acquire(self, cost=1, now=None): + """Consume `cost` tokens if available; return True on success.""" + now = time.monotonic() if now is None else now + self._refill(now) + if self._tokens >= cost: + self._tokens -= cost + return True + return False +''' + +# C4: cap refill at capacity +LIMITER_V2 = LIMITER_V1.replace( + "self._tokens = self._tokens + elapsed * self.rate", + "self._tokens = min(self.capacity, self._tokens + elapsed * self.rate)", +) + +# C5: add RateLimiter (per-key buckets) +LIMITER_V3 = LIMITER_V2 + ''' + +class RateLimiter: + """Manage one TokenBucket per key (e.g. per user / per IP).""" + + def __init__(self, rate, capacity): + self.rate = rate + self.capacity = capacity + self._buckets = {} + + def try_acquire(self, key, cost=1, now=None): + bucket = self._buckets.get(key) + if bucket is None: + bucket = self._buckets[key] = TokenBucket(self.rate, self.capacity) + return bucket.try_acquire(cost, now) +''' + +# C8 (REGRESSION): int() floor in _refill (drops partial tokens) + available() +LIMITER_V4 = LIMITER_V3.replace( + " self._tokens = min(self.capacity, self._tokens + elapsed * self.rate)\n" + " self._last = now\n", + " # perf: accumulate whole tokens to avoid floating-point drift\n" + " self._tokens = min(self.capacity, self._tokens + int(elapsed * self.rate))\n" + " self._last = now\n", +).replace( + " if self._tokens >= cost:\n self._tokens -= cost\n return True\n return False\n", + " if self._tokens >= cost:\n self._tokens -= cost\n return True\n return False\n\n" + " def available(self, now=None):\n" + ' """Return the number of tokens currently available."""\n' + " now = time.monotonic() if now is None else now\n" + " self._refill(now)\n" + " return self._tokens\n", +) + +PYPROJECT = '''\ +[project] +name = "ratelimiter" +version = "{ver}" +description = "A small token-bucket rate limiter." +requires-python = ">=3.9" + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +''' + +README = '''\ +# ratelimiter + +A tiny, dependency-free token-bucket rate limiter. + +```python +from ratelimiter import TokenBucket +b = TokenBucket(rate=5, capacity=10) # 5 tokens/sec, burst of 10 +b.try_acquire() # -> True while tokens remain +``` + +Tokens refill **continuously**; partial tokens carry over between calls. +''' + +TEST_BASIC = '''\ +from ratelimiter import TokenBucket + + +def test_starts_full(): + b = TokenBucket(rate=1, capacity=5) + assert b.try_acquire(5, now=0) is True + + +def test_denies_when_empty(): + b = TokenBucket(rate=1, capacity=2) + assert b.try_acquire(2, now=0) is True + assert b.try_acquire(1, now=0) is False +''' + +TEST_CAP = '''\ +from ratelimiter import TokenBucket + + +def test_refill_caps_at_capacity(): + b = TokenBucket(rate=10, capacity=3) + b.try_acquire(3, now=0) # empty it + # after a long time it should refill only up to capacity, not beyond + assert b.try_acquire(3, now=100) is True + assert b.try_acquire(1, now=100) is False +''' + +TEST_RL = '''\ +from ratelimiter import RateLimiter + + +def test_keys_are_independent(): + rl = RateLimiter(rate=1, capacity=1) + assert rl.try_acquire("a", now=0) is True + assert rl.try_acquire("a", now=0) is False + assert rl.try_acquire("b", now=0) is True +''' + +TEST_REFILL = '''\ +from ratelimiter import TokenBucket + + +def test_integer_rate_refills_over_time(): + b = TokenBucket(rate=2, capacity=10) + b.try_acquire(10, now=0) # empty + assert b.try_acquire(1, now=0) is False + # 2 tokens/sec for 3s -> 6 tokens available + assert b.try_acquire(6, now=3) is True +''' + +TEST_AVAILABLE = '''\ +from ratelimiter import TokenBucket + + +def test_available_reports_tokens(): + b = TokenBucket(rate=2, capacity=10) + assert b.available(now=0) == 10 + b.try_acquire(4, now=0) + assert b.available(now=0) == 6 +''' + +CHANGELOG = '''\ +# Changelog + +## 0.3.0 +- perf: refill accumulates whole tokens to avoid float drift +- feat: `TokenBucket.available()` accessor + +## 0.2.0 +- feat: `RateLimiter` with a bucket per key + +## 0.1.0 +- initial token bucket +''' + + +def main(): + if OUT.exists(): + subprocess.run(["rm", "-rf", str(OUT)], check=True) + OUT.mkdir(parents=True) + subprocess.run(["git", "init", "-q"], cwd=OUT, check=True) + subprocess.run(["git", "branch", "-M", "main"], cwd=OUT, check=True) + + def write(path, content): + p = OUT / path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + + def commit(msg, day): + date = f"2024-03-{day:02d}T10:00:00" + env = {**os.environ, + "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date, + "GIT_AUTHOR_NAME": "Dana Dev", "GIT_AUTHOR_EMAIL": "dana@example.com", + "GIT_COMMITTER_NAME": "Dana Dev", "GIT_COMMITTER_EMAIL": "dana@example.com"} + subprocess.run(["git", "add", "-A"], cwd=OUT, check=True) + subprocess.run(["git", "commit", "-q", "-m", msg], cwd=OUT, env=env, check=True) + + # C1 + write("pyproject.toml", PYPROJECT.format(ver="0.1.0")) + write("README.md", "# ratelimiter\n\nA tiny token-bucket rate limiter.\n") + write("ratelimiter/__init__.py", '"""ratelimiter package."""\n') + commit("chore: scaffold project", 1) + # C2 + write("ratelimiter/limiter.py", LIMITER_V1) + write("ratelimiter/__init__.py", '"""ratelimiter package."""\nfrom .limiter import TokenBucket\n\n__all__ = ["TokenBucket"]\n') + commit("feat: TokenBucket with continuous refill (partial tokens carry over)", 3) + # C3 + write("tests/test_basic.py", TEST_BASIC) + commit("test: cover capacity, consume, and deny", 4) + # C4 + write("ratelimiter/limiter.py", LIMITER_V2) + write("tests/test_cap.py", TEST_CAP) + commit("feat: cap tokens at capacity on refill", 6) + # C5 + write("ratelimiter/limiter.py", LIMITER_V3) + write("ratelimiter/__init__.py", '"""ratelimiter package."""\nfrom .limiter import TokenBucket, RateLimiter\n\n__all__ = ["TokenBucket", "RateLimiter"]\n') + write("tests/test_ratelimiter.py", TEST_RL) + commit("feat: RateLimiter manages a bucket per key", 9) + # C6 + write("tests/test_refill.py", TEST_REFILL) + commit("test: integer-rate refill recovers tokens over time", 11) + # C7 + write("README.md", README) + commit("docs: usage example and refill semantics", 13) + # C8 <-- REGRESSION bundled with a legit accessor + write("ratelimiter/limiter.py", LIMITER_V4) + write("tests/test_available.py", TEST_AVAILABLE) + commit("perf: accumulate whole tokens to avoid float drift; add available()", 18) + # C9 + write("pyproject.toml", PYPROJECT.format(ver="0.3.0")) + write("CHANGELOG.md", CHANGELOG) + commit("chore: release 0.3.0", 20) + + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=OUT, capture_output=True, text=True).stdout.strip() + print(f"built {OUT} @ {head[:10]} ({subprocess.run(['git','rev-list','--count','HEAD'],cwd=OUT,capture_output=True,text=True).stdout.strip()} commits)") + + +if __name__ == "__main__": + main() diff --git a/sdebench/datasets/ratelimiter/hidden_test.py b/sdebench/datasets/ratelimiter/hidden_test.py new file mode 100644 index 0000000..9ad1d5e --- /dev/null +++ b/sdebench/datasets/ratelimiter/hidden_test.py @@ -0,0 +1,28 @@ +"""Held-out tests (HIDDEN_TO_PASS) — never shown to the agent. + +Same behaviour (partial-token refill) exercised with different rates/intervals and +through RateLimiter, so a fix that special-cases the visible repro's inputs still fails. +""" +from ratelimiter import TokenBucket, RateLimiter + + +def test_quarter_rate_accrues(): + b = TokenBucket(rate=0.25, capacity=2) + b.try_acquire(2, now=0) + assert b.available(now=2) == 0.5 # 0.25 * 2s + assert b.try_acquire(1, now=4) is True # 0.25 * 4s = 1.0 + + +def test_fractional_partial_cost(): + b = TokenBucket(rate=0.1, capacity=1) + b.try_acquire(1, now=0) + assert b.try_acquire(0.5, now=5) is True # 0.1 * 5s = 0.5 + assert b.try_acquire(0.1, now=5) is False + + +def test_ratelimiter_fractional_rate(): + rl = RateLimiter(rate=0.5, capacity=1) + assert rl.try_acquire("u", now=0) is True + rl.try_acquire("u", now=1) # a partial refill step (0.5 should accrue) + # second 0.5 accrues over the next second -> 1.0 total; int() floors each step to 0 + assert rl.try_acquire("u", now=2) is True diff --git a/sdebench/datasets/ratelimiter/regression_test.py b/sdebench/datasets/ratelimiter/regression_test.py new file mode 100644 index 0000000..a3d57b8 --- /dev/null +++ b/sdebench/datasets/ratelimiter/regression_test.py @@ -0,0 +1,16 @@ +"""Regression repro filed with the bug report (FAIL_TO_PASS). + +Bug: with a FRACTIONAL rate, the bucket never refills — partial tokens are dropped. +This test is RED on the broken HEAD and GREEN once the refill floor is removed. +""" +from ratelimiter import TokenBucket + + +def test_partial_tokens_carry_over(): + b = TokenBucket(rate=0.5, capacity=1) + assert b.try_acquire(1, now=0) is True # consume the one token + assert b.try_acquire(1, now=0) is False # empty + # at 0.5 tokens/sec, half a token has accrued after 1s ... + assert b.available(now=1) == 0.5 + # ... and a whole token after 2s, so a request now succeeds + assert b.try_acquire(1, now=2) is True diff --git a/sdebench/datasets/ratelimiter/task.json b/sdebench/datasets/ratelimiter/task.json new file mode 100644 index 0000000..c33b200 --- /dev/null +++ b/sdebench/datasets/ratelimiter/task.json @@ -0,0 +1,15 @@ +{ + "task_id": "ratelimiter-regression-001", + "repo": "ratelimiter", + "type": "regression-fix", + "build": "build.py", + "base_ref": "HEAD", + "regression_intro_subject": "perf: accumulate whole tokens to avoid float drift; add available()", + "guarantee_subject": "feat: TokenBucket with continuous refill (partial tokens carry over)", + "bug_report": "Users on a low fractional rate report the limiter NEVER lets requests through after the bucket empties. Example: a TokenBucket(rate=0.5, capacity=1) stays empty forever even though half a token should accrue each second. A failing repro test is included at tests/test_regression.py. Fix the bug so partial tokens accrue again, without breaking existing behaviour.", + "fail_to_pass": ["tests/test_regression.py"], + "pass_to_pass": ["tests/test_basic.py", "tests/test_cap.py", "tests/test_ratelimiter.py", "tests/test_refill.py", "tests/test_available.py"], + "hidden_to_pass": ["tests/test_hidden.py"], + "regression_test_file": "regression_test.py", + "hidden_test_file": "hidden_test.py" +} From aa6dc7861adb8dfd523f76da285b5ed27d492d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 25 Jun 2026 17:31:11 +0200 Subject: [PATCH 002/142] feat(sdebench): Docker grading image + harness (full/squashed A/B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Dockerfile: deterministic python+pytest+git grading env - harness/run.py: build repo (full|squashed history) -> ship bug report + failing repro -> run opencode (gemini-3.5-flash) -> capture SOURCE diff (tests excluded) -> grade in Docker vs FAIL_TO_PASS+PASS_TO_PASS+HIDDEN from pristine copies - reports resolution/cost(tokens)/speed(wall,turns) - smoke test (full history): agent solved it — 591B fix, 10 passed, 20 turns, 176s --- sdebench/Dockerfile | 7 ++ sdebench/harness/run.py | 185 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 sdebench/Dockerfile create mode 100644 sdebench/harness/run.py diff --git a/sdebench/Dockerfile b/sdebench/Dockerfile new file mode 100644 index 0000000..b4cfb40 --- /dev/null +++ b/sdebench/Dockerfile @@ -0,0 +1,7 @@ +# Deterministic grading environment for sdebench tasks. +# The ratelimiter lib is dependency-free; pytest + git is all we need. +FROM python:3.11-slim +RUN pip install --no-cache-dir pytest==8.* && apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /work diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py new file mode 100644 index 0000000..f74aa5f --- /dev/null +++ b/sdebench/harness/run.py @@ -0,0 +1,185 @@ +"""sdebench harness — run a coding agent on a regression task and grade it. + +Flow: build the repo (full or squashed history) -> ship the agent the bug report + +failing regression test -> run opencode -> capture the SOURCE diff (tests excluded) -> +grade in Docker against FAIL_TO_PASS + PASS_TO_PASS + HIDDEN_TO_PASS from pristine copies. + +Usage: + uv run python sdebench/harness/run.py --history full [--model google/gemini-3.5-flash] + uv run python sdebench/harness/run.py --history squashed + +Metrics reported: resolution (binary), cost (tokens; $ if --price set), speed (wall, turns). +""" +import argparse, json, os, shutil, subprocess, time +from pathlib import Path + +HARNESS = Path(__file__).resolve().parent +SDEBENCH = HARNESS.parent +REPO_ROOT = SDEBENCH.parent +IMAGE = "sdebench-base" + +PROMPT = """\ +You are a maintainer of the `{repo}` Python project. A regression was reported: + +{bug_report} + +Fix the bug in the source code. Do NOT modify any test files — the graders supply their own. +Save your changes to disk before finishing. +""" + + +def sh(*args, cwd=None, env=None, check=True, cap=False): + return subprocess.run(args, cwd=cwd, env=env, check=check, + capture_output=cap, text=True) + + +def neutral_home() -> str: + """A HOME without ~/.claude so opencode can't load the user's global CLAUDE.md.""" + home = Path("/tmp/sdebench_home") + if not home.exists(): + home.mkdir(parents=True, exist_ok=True) + for item in Path.home().iterdir(): + if item.name.startswith(".") and item.name != ".claude": + try: + (home / item.name).symlink_to(item) + except FileExistsError: + pass + return str(home) + + +def build_repo(task: dict, dest: Path, history: str): + if dest.exists(): + shutil.rmtree(dest) + ds = SDEBENCH / "datasets" / task["repo"] + sh("python", str(ds / task["build"]), str(dest)) + if history == "squashed": + shutil.rmtree(dest / ".git") + sh("git", "init", "-q", cwd=dest) + sh("git", "add", "-A", cwd=dest) + env = {**os.environ, "GIT_AUTHOR_NAME": "x", "GIT_AUTHOR_EMAIL": "x@x", + "GIT_COMMITTER_NAME": "x", "GIT_COMMITTER_EMAIL": "x@x"} + sh("git", "commit", "-q", "-m", "Initial commit", cwd=dest, env=env) + # ship the failing regression repro (the agent sees it; it is red) + ds_test = ds / task["regression_test_file"] + shutil.copy(ds_test, dest / "tests" / "test_regression.py") + sh("git", "add", "-A", cwd=dest) + env = {**os.environ, "GIT_AUTHOR_NAME": "x", "GIT_AUTHOR_EMAIL": "x@x", + "GIT_COMMITTER_NAME": "x", "GIT_COMMITTER_EMAIL": "x@x"} + sh("git", "commit", "-q", "-m", "test: add failing repro for the reported regression", + cwd=dest, env=env) + + +def load_env() -> dict: + env = os.environ.copy() + ef = REPO_ROOT / ".env" + if ef.exists(): + for line in ef.read_text().splitlines(): + line = line.strip() + if "=" in line and not line.startswith("#"): + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip().strip('"').strip("'")) + env["HINDSIGHT_DISABLED"] = "1" # plain agent: no memory/plugins, just git via bash + env["PWD"] = "" # set per-run + env["HOME"] = neutral_home() + return env + + +def run_agent(workdir: Path, task: dict, model: str, timeout: int) -> dict: + env = load_env() + env["PWD"] = str(workdir) + prompt = PROMPT.format(repo=task["repo"], bug_report=task["bug_report"]) + t0 = time.perf_counter() + proc = subprocess.run(["opencode", "run", "--format", "json", "-m", model, prompt], + cwd=str(workdir), env=env, timeout=timeout, + capture_output=True, text=True) + elapsed = time.perf_counter() - t0 + out_tok = in_tok = turns = 0 + for line in proc.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + e = json.loads(line) + except Exception: + continue + if e.get("type") == "tool_use": + turns += 1 + elif e.get("type") == "step_finish": + tk = e.get("part", {}).get("tokens", {}) or {} + out_tok += (tk.get("output", 0) or 0) + (tk.get("reasoning", 0) or 0) + in_tok += tk.get("input", 0) or 0 + return {"elapsed": elapsed, "out_tokens": out_tok, "in_tokens": in_tok, "turns": turns} + + +_JUNK = [".venv", "venv", "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache", "*.pyc"] + + +def capture_source_patch(workdir: Path) -> str: + """The agent's diff to SOURCE only — tests/ and junk excluded (graded from pristine).""" + sh("git", "add", "-A", cwd=workdir) + excl = [f":(exclude){j}" for j in _JUNK] + [":(exclude)tests/**"] + r = sh("git", "diff", "--cached", "HEAD", "--", ".", *excl, cwd=workdir, cap=True) + return r.stdout + + +def grade(task: dict, source_patch: str, work: Path) -> dict: + """Apply the source patch to a pristine full build + pristine test sets, run pytest in Docker.""" + gd = work / "grade" + build_repo(task, gd, "full") # pristine repo (full) + ds = SDEBENCH / "datasets" / task["repo"] + shutil.copy(ds / task["hidden_test_file"], gd / "tests" / "test_hidden.py") + # regression test already copied by build_repo; apply the agent's source patch + applied = True + if source_patch.strip(): + p = subprocess.run(["git", "apply", "--whitespace=nowarn"], cwd=gd, + input=source_patch, text=True) + applied = p.returncode == 0 + # run the suite in Docker (deterministic), tests graded from pristine copies + r = subprocess.run( + ["docker", "run", "--rm", "-v", f"{gd}:/work", "-w", "/work", IMAGE, + "python", "-m", "pytest", "-q", "tests", "--no-header"], + capture_output=True, text=True) + passed = r.returncode == 0 + tail = (r.stdout or "").strip().splitlines()[-1:] if r.stdout else [""] + return {"applied": applied, "resolved": passed and applied, "pytest": tail[0] if tail else ""} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--task", default=str(SDEBENCH / "datasets" / "ratelimiter" / "task.json")) + ap.add_argument("--history", choices=["full", "squashed"], default="full") + ap.add_argument("--model", default="google/gemini-3.5-flash") + ap.add_argument("--timeout", type=int, default=900) + ap.add_argument("--price-in", type=float, default=0.0, help="$ per 1M input tokens") + ap.add_argument("--price-out", type=float, default=0.0, help="$ per 1M output tokens") + ap.add_argument("--run-id", default="r1") + args = ap.parse_args() + task = json.loads(Path(args.task).read_text()) + + work = Path("/tmp/sdebench/run") / f"{task['task_id']}_{args.history}_{args.run_id}" + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + repo = work / "repo" + build_repo(task, repo, args.history) + print(f"[{task['task_id']}] history={args.history} model={args.model} — running agent…", flush=True) + m = run_agent(repo, task, args.model, args.timeout) + patch = capture_source_patch(repo) + g = grade(task, patch, work) + cost = (m["in_tokens"] * args.price_in + m["out_tokens"] * args.price_out) / 1_000_000 + result = { + "task_id": task["task_id"], "history": args.history, "model": args.model, + "resolved": g["resolved"], "applied": g["applied"], "pytest": g["pytest"], + "patch_bytes": len(patch), "turns": m["turns"], + "out_tokens": m["out_tokens"], "in_tokens": m["in_tokens"], + "wall_s": round(m["elapsed"], 1), "cost_usd": round(cost, 4), + } + out = work / "result.json" + out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + print(f"\nRESULT history={args.history}: resolved={g['resolved']} " + f"turns={m['turns']} out_tok={m['out_tokens']} wall={m['elapsed']:.0f}s -> {out}") + + +if __name__ == "__main__": + main() From 77490bcb2a1d13094b2ea3fa9405ca1d65e51e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 25 Jun 2026 18:23:36 +0200 Subject: [PATCH 003/142] =?UTF-8?q?feat(sdebench):=20feedback-loop=20eval?= =?UTF-8?q?=20=E2=80=94=20measure=20interventions,=20not=20binary=20pass/f?= =?UTF-8?q?ail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - if grading fails, surface the NEW problem (failing tests' assertion output, NOT the fix) back to the agent and RESUME its session (opencode -c) to continue its work - metric = number of human-like interventions needed (capped at 5 = drift guard) - cost/turns/wall accumulate across all rounds; solved = passed within the cap - realistic: feedback is what CI/QA would report; agent must still generalize the fix --- sdebench/harness/run.py | 74 +++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py index f74aa5f..58a5d4b 100644 --- a/sdebench/harness/run.py +++ b/sdebench/harness/run.py @@ -84,13 +84,15 @@ def load_env() -> dict: return env -def run_agent(workdir: Path, task: dict, model: str, timeout: int) -> dict: +def run_agent(workdir: Path, model: str, timeout: int, message: str, resume: bool = False) -> dict: env = load_env() env["PWD"] = str(workdir) - prompt = PROMPT.format(repo=task["repo"], bug_report=task["bug_report"]) + cmd = ["opencode", "run", "--format", "json", "-m", model] + if resume: + cmd.append("-c") # continue the last session in this dir (keeps context) + cmd.append(message) t0 = time.perf_counter() - proc = subprocess.run(["opencode", "run", "--format", "json", "-m", model, prompt], - cwd=str(workdir), env=env, timeout=timeout, + proc = subprocess.run(cmd, cwd=str(workdir), env=env, timeout=timeout, capture_output=True, text=True) elapsed = time.perf_counter() - t0 out_tok = in_tok = turns = 0 @@ -140,8 +142,25 @@ def grade(task: dict, source_patch: str, work: Path) -> dict: "python", "-m", "pytest", "-q", "tests", "--no-header"], capture_output=True, text=True) passed = r.returncode == 0 - tail = (r.stdout or "").strip().splitlines()[-1:] if r.stdout else [""] - return {"applied": applied, "resolved": passed and applied, "pytest": tail[0] if tail else ""} + out = (r.stdout or "") + tail = out.strip().splitlines()[-1:] if out.strip() else [""] + return {"applied": applied, "resolved": passed and applied, + "pytest": tail[0] if tail else "", "output": out, + "patch_failed": not applied} + + +def build_feedback(grade_result: dict) -> str: + """Surface the NEW problem (failing tests) — not the solution.""" + if grade_result["patch_failed"]: + return ("Your change could not be applied cleanly to the source. Re-read the current " + "code and make a focused edit that applies, then ensure the tests pass.") + out = grade_result["output"] + # keep the failures section (assertion errors + the short summary), trimmed + body = out[-2500:] if len(out) > 2500 else out + return ("Your change did not fully fix the reported regression. Re-running the project's " + "test suite now reports the following remaining failures:\n\n" + f"```\n{body.strip()}\n```\n\n" + "Fix the source so these pass. Do NOT modify any test file.") def main(): @@ -153,6 +172,8 @@ def main(): ap.add_argument("--price-in", type=float, default=0.0, help="$ per 1M input tokens") ap.add_argument("--price-out", type=float, default=0.0, help="$ per 1M output tokens") ap.add_argument("--run-id", default="r1") + ap.add_argument("--max-interventions", type=int, default=5, + help="cap on feedback rounds before giving up (drift guard)") args = ap.parse_args() task = json.loads(Path(args.task).read_text()) @@ -162,23 +183,42 @@ def main(): work.mkdir(parents=True) repo = work / "repo" build_repo(task, repo, args.history) - print(f"[{task['task_id']}] history={args.history} model={args.model} — running agent…", flush=True) - m = run_agent(repo, task, args.model, args.timeout) - patch = capture_source_patch(repo) - g = grade(task, patch, work) - cost = (m["in_tokens"] * args.price_in + m["out_tokens"] * args.price_out) / 1_000_000 + + totals = {"out_tokens": 0, "in_tokens": 0, "turns": 0, "wall_s": 0.0} + def acc(m): + totals["out_tokens"] += m["out_tokens"]; totals["in_tokens"] += m["in_tokens"] + totals["turns"] += m["turns"]; totals["wall_s"] += m["elapsed"] + + print(f"[{task['task_id']}] history={args.history} model={args.model} — initial attempt…", flush=True) + acc(run_agent(repo, args.model, args.timeout, PROMPT.format(repo=task["repo"], bug_report=task["bug_report"]))) + + # Feedback loop: grade -> if failing, tell the agent the NEW problem (not the fix) and resume. + # Metric = number of human-like interventions needed (capped); cost = sum across all rounds. + interventions = 0 + while True: + patch = capture_source_patch(repo) + g = grade(task, patch, work) + if g["resolved"] or interventions >= args.max_interventions: + break + interventions += 1 + print(f" ↳ intervention {interventions}: {g['pytest']}", flush=True) + acc(run_agent(repo, args.model, args.timeout, build_feedback(g), resume=True)) + + solved = g["resolved"] + cost = (totals["in_tokens"] * args.price_in + totals["out_tokens"] * args.price_out) / 1_000_000 result = { "task_id": task["task_id"], "history": args.history, "model": args.model, - "resolved": g["resolved"], "applied": g["applied"], "pytest": g["pytest"], - "patch_bytes": len(patch), "turns": m["turns"], - "out_tokens": m["out_tokens"], "in_tokens": m["in_tokens"], - "wall_s": round(m["elapsed"], 1), "cost_usd": round(cost, 4), + "solved": solved, "interventions": interventions, + "capped": (not solved and interventions >= args.max_interventions), + "final_pytest": g["pytest"], "patch_bytes": len(patch), + "turns": totals["turns"], "out_tokens": totals["out_tokens"], "in_tokens": totals["in_tokens"], + "wall_s": round(totals["wall_s"], 1), "cost_usd": round(cost, 4), } out = work / "result.json" out.write_text(json.dumps(result, indent=2)) print(json.dumps(result, indent=2)) - print(f"\nRESULT history={args.history}: resolved={g['resolved']} " - f"turns={m['turns']} out_tok={m['out_tokens']} wall={m['elapsed']:.0f}s -> {out}") + print(f"\nRESULT history={args.history}: solved={solved} interventions={interventions} " + f"turns={totals['turns']} out_tok={totals['out_tokens']} wall={totals['wall_s']:.0f}s -> {out}") if __name__ == "__main__": From f0cc8b7b33c804635a4c8a74428958eb2f2c902f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 26 Jun 2026 10:07:09 +0200 Subject: [PATCH 004/142] =?UTF-8?q?feat(sdebench):=20task=20#2=20ttlcache?= =?UTF-8?q?=20=E2=80=94=20history-dependent,=20underdetermined=20regressio?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a refactor changed DEFAULT_TTL 300->600 and dropped the rationale comment; 300 now lives ONLY in git history (blame/log the constant). bundled with a legit clear(). - repro proves entries expire too late but CANNOT reveal the value; hidden tests pin it to EXACTLY 300. validated: TTL=450 passes repro but fails hidden (underdetermined). - with history: read 300 off blame -> 1-shot. without: binary-search via feedback rounds. this is the task designed to make the intervention-count A/B diverge. --- sdebench/datasets/ttlcache/build.py | 203 ++++++++++++++++++ sdebench/datasets/ttlcache/hidden_test.py | 19 ++ sdebench/datasets/ttlcache/regression_test.py | 12 ++ sdebench/datasets/ttlcache/task.json | 15 ++ 4 files changed, 249 insertions(+) create mode 100644 sdebench/datasets/ttlcache/build.py create mode 100644 sdebench/datasets/ttlcache/hidden_test.py create mode 100644 sdebench/datasets/ttlcache/regression_test.py create mode 100644 sdebench/datasets/ttlcache/task.json diff --git a/sdebench/datasets/ttlcache/build.py b/sdebench/datasets/ttlcache/build.py new file mode 100644 index 0000000..d8cbea1 --- /dev/null +++ b/sdebench/datasets/ttlcache/build.py @@ -0,0 +1,203 @@ +"""Build the `ttlcache` synthetic repo — a HISTORY-DEPENDENT regression. + +A refactor ("centralize cache constants; add clear()") changed DEFAULT_TTL from 300 to +600 AND dropped the comment explaining *why* it was 300 (matches the upstream auth token +lifetime). The regression repro only proves entries expire too LATE — it cannot reveal the +correct value. The hidden tests pin DEFAULT_TTL to EXACTLY 300. The number 300 now lives +ONLY in git history (the original commit's message + the pre-refactor file), so: + - with history: `git blame`/`log` the constant -> 300 -> fixed in one shot. + - without history: the agent must binary-search the value via feedback rounds. +The refactor also bundles a legit `clear()` (with its own test), so a lazy `git revert` +fails PASS_TO_PASS -> forces a surgical one-line fix. + +Usage: python build.py (default: /tmp/sdebench/ttlcache) +""" +import os, subprocess, sys +from pathlib import Path + +OUT = Path(sys.argv[1] if len(sys.argv) > 1 else "/tmp/sdebench/ttlcache") + +CACHE_V1 = '''\ +"""A tiny in-memory cache with per-entry time-to-live.""" + +# Entries expire after DEFAULT_TTL seconds. 300 matches the upstream auth token +# lifetime — keep these in sync, or cached values can outlive the tokens they were +# fetched with. Do NOT change without updating the auth layer. +DEFAULT_TTL = 300 + + +class TTLCache: + """A cache whose entries expire DEFAULT_TTL seconds after they are set. + + Time is passed in explicitly (`now`, seconds) so behaviour is deterministic. + """ + + def __init__(self, ttl=DEFAULT_TTL): + self.ttl = ttl + self._store = {} + + def set(self, key, value, now): + self._store[key] = (value, now + self.ttl) + + def get(self, key, now): + item = self._store.get(key) + if item is None: + return None + value, expires_at = item + if now >= expires_at: + del self._store[key] + return None + return value +''' + +# C6 refactor: DEFAULT_TTL 300 -> 600 (REGRESSION), rationale comment dropped, + clear() +CACHE_V2 = '''\ +"""A tiny in-memory cache with per-entry time-to-live.""" + +# Default entry lifetime, in seconds. +DEFAULT_TTL = 600 + + +class TTLCache: + """A cache whose entries expire DEFAULT_TTL seconds after they are set. + + Time is passed in explicitly (`now`, seconds) so behaviour is deterministic. + """ + + def __init__(self, ttl=DEFAULT_TTL): + self.ttl = ttl + self._store = {} + + def set(self, key, value, now): + self._store[key] = (value, now + self.ttl) + + def get(self, key, now): + item = self._store.get(key) + if item is None: + return None + value, expires_at = item + if now >= expires_at: + del self._store[key] + return None + return value + + def clear(self): + """Remove all entries.""" + self._store.clear() +''' + +PYPROJECT = '[project]\nname = "ttlcache"\nversion = "{ver}"\nrequires-python = ">=3.9"\n' + +TEST_BASIC = '''\ +from ttlcache import TTLCache + + +def test_set_then_get(): + c = TTLCache() + c.set("k", "v", now=0) + assert c.get("k", now=10) == "v" # present shortly after + + +def test_missing_key(): + assert TTLCache().get("nope", now=0) is None +''' + +TEST_EXPIRY = '''\ +from ttlcache import TTLCache + + +def test_eventually_expires(): + c = TTLCache() + c.set("k", "v", now=0) + assert c.get("k", now=100000) is None # long gone, for any sane TTL + + +def test_overwrite_resets(): + c = TTLCache() + c.set("k", "a", now=0) + c.set("k", "b", now=5) + assert c.get("k", now=10) == "b" +''' + +TEST_CUSTOM = '''\ +from ttlcache import TTLCache + + +def test_explicit_ttl(): + c = TTLCache(ttl=50) + c.set("k", "v", now=0) + assert c.get("k", now=40) == "v" + assert c.get("k", now=60) is None +''' + +TEST_CLEAR = '''\ +from ttlcache import TTLCache + + +def test_clear_removes_all(): + c = TTLCache() + c.set("a", "1", now=0) + c.set("b", "2", now=0) + c.clear() + assert c.get("a", now=1) is None + assert c.get("b", now=1) is None +''' + + +def main(): + if OUT.exists(): + subprocess.run(["rm", "-rf", str(OUT)], check=True) + OUT.mkdir(parents=True) + subprocess.run(["git", "init", "-q"], cwd=OUT, check=True) + subprocess.run(["git", "branch", "-M", "main"], cwd=OUT, check=True) + + def write(path, content): + p = OUT / path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content) + + def commit(msg, day): + date = f"2024-05-{day:02d}T10:00:00" + env = {**os.environ, + "GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date, + "GIT_AUTHOR_NAME": "Sam Maintainer", "GIT_AUTHOR_EMAIL": "sam@example.com", + "GIT_COMMITTER_NAME": "Sam Maintainer", "GIT_COMMITTER_EMAIL": "sam@example.com"} + subprocess.run(["git", "add", "-A"], cwd=OUT, check=True) + subprocess.run(["git", "commit", "-q", "-m", msg], cwd=OUT, env=env, check=True) + + write("pyproject.toml", PYPROJECT.format(ver="0.1.0")) + write("README.md", "# ttlcache\n\nA tiny in-memory TTL cache.\n") + write("ttlcache/__init__.py", '"""ttlcache package."""\n') + commit("chore: scaffold project", 2) + + write("ttlcache/limiter.py" if False else "ttlcache/cache.py", CACHE_V1) + write("ttlcache/__init__.py", '"""ttlcache package."""\nfrom .cache import TTLCache, DEFAULT_TTL\n\n__all__ = ["TTLCache", "DEFAULT_TTL"]\n') + commit("feat: TTLCache with default 300s lifetime (matches upstream auth token TTL)", 4) + + write("tests/test_basic.py", TEST_BASIC) + commit("test: basic set/get and missing keys", 5) + + write("tests/test_expiry.py", TEST_EXPIRY) + commit("test: expiry and overwrite", 7) + + write("tests/test_custom.py", TEST_CUSTOM) + commit("feat: support an explicit per-cache ttl", 9) + + write("README.md", "# ttlcache\n\nA tiny in-memory TTL cache.\n\n```python\nfrom ttlcache import TTLCache\nc = TTLCache()\nc.set('k', 'v', now=time.monotonic())\n```\n") + commit("docs: usage example", 11) + + # C6: the regression, bundled with clear() + write("ttlcache/cache.py", CACHE_V2) + write("tests/test_clear.py", TEST_CLEAR) + commit("refactor: centralize cache constants; add clear()", 16) + + write("pyproject.toml", PYPROJECT.format(ver="0.4.0")) + commit("chore: release 0.4.0", 18) + + head = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=OUT, capture_output=True, text=True).stdout.strip() + n = subprocess.run(["git", "rev-list", "--count", "HEAD"], cwd=OUT, capture_output=True, text=True).stdout.strip() + print(f"built {OUT} @ {head} ({n} commits)") + + +if __name__ == "__main__": + main() diff --git a/sdebench/datasets/ttlcache/hidden_test.py b/sdebench/datasets/ttlcache/hidden_test.py new file mode 100644 index 0000000..0315b04 --- /dev/null +++ b/sdebench/datasets/ttlcache/hidden_test.py @@ -0,0 +1,19 @@ +"""Held-out tests (HIDDEN_TO_PASS): the default lifetime must be EXACTLY 300s. + +A fix that merely makes the repro pass (any TTL <= ~480) fails here unless it is 300. +""" +from ttlcache import TTLCache + + +def test_default_ttl_exact_boundary(): + c = TTLCache() + c.set("k", "v", now=0) + assert c.get("k", now=299) == "v" # valid just before expiry + assert c.get("k", now=300) is None # expired exactly at 300s + + +def test_default_ttl_relative_to_set_time(): + c = TTLCache() + c.set("k", "v", now=1000) + assert c.get("k", now=1299) == "v" + assert c.get("k", now=1300) is None diff --git a/sdebench/datasets/ttlcache/regression_test.py b/sdebench/datasets/ttlcache/regression_test.py new file mode 100644 index 0000000..5376248 --- /dev/null +++ b/sdebench/datasets/ttlcache/regression_test.py @@ -0,0 +1,12 @@ +"""Regression repro (FAIL_TO_PASS): default-TTL entries linger far past their lifetime. + +Note: this proves the entry expires too LATE — it does NOT reveal the correct lifetime. +""" +from ttlcache import TTLCache + + +def test_default_entry_expires_promptly(): + c = TTLCache() + c.set("session", "data", now=0) + # An entry created over eight minutes ago must already be evicted. + assert c.get("session", now=500) is None diff --git a/sdebench/datasets/ttlcache/task.json b/sdebench/datasets/ttlcache/task.json new file mode 100644 index 0000000..6153ff7 --- /dev/null +++ b/sdebench/datasets/ttlcache/task.json @@ -0,0 +1,15 @@ +{ + "task_id": "ttlcache-regression-001", + "repo": "ttlcache", + "type": "regression-fix", + "build": "build.py", + "base_ref": "HEAD", + "regression_intro_subject": "refactor: centralize cache constants; add clear()", + "guarantee_subject": "feat: TTLCache with default 300s lifetime (matches upstream auth token TTL)", + "bug_report": "We're serving stale data: cached entries live far longer than they should. An entry created over EIGHT MINUTES ago is still being returned, even though cache entries are meant to expire well before that. This started recently. A failing repro test is at tests/test_regression.py. Fix the source so entries expire when they should, without breaking existing behaviour.", + "fail_to_pass": ["tests/test_regression.py"], + "pass_to_pass": ["tests/test_basic.py", "tests/test_expiry.py", "tests/test_custom.py", "tests/test_clear.py"], + "hidden_to_pass": ["tests/test_hidden.py"], + "regression_test_file": "regression_test.py", + "hidden_test_file": "hidden_test.py" +} From 9682226ed6ad54bfc0977c57cfa649f58d4a2c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 26 Jun 2026 10:18:56 +0200 Subject: [PATCH 005/142] fix(sdebench): ttlcache uses non-guessable 287 (was 300) so the metric discriminates - 300 is a conventional TTL the agent guesses without history (A/B showed 0 interventions both, though full used ~half the tokens). 287 (a 'measured p99' value) can't be guessed: validated TTL=300 now passes repro but FAILS hidden. only history states 287. - expectation: full reads 287 off git blame (0 interv); squashed must discover it via feedback rounds (>0 interv) -> intervention metric diverges. --- sdebench/datasets/ttlcache/build.py | 8 ++++---- sdebench/datasets/ttlcache/hidden_test.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sdebench/datasets/ttlcache/build.py b/sdebench/datasets/ttlcache/build.py index d8cbea1..a3fd483 100644 --- a/sdebench/datasets/ttlcache/build.py +++ b/sdebench/datasets/ttlcache/build.py @@ -20,10 +20,10 @@ CACHE_V1 = '''\ """A tiny in-memory cache with per-entry time-to-live.""" -# Entries expire after DEFAULT_TTL seconds. 300 matches the upstream auth token -# lifetime — keep these in sync, or cached values can outlive the tokens they were +# Entries expire after DEFAULT_TTL seconds. 287 is the measured p99 auth-token refresh +# interval — keep these in sync, or cached values can outlive the tokens they were # fetched with. Do NOT change without updating the auth layer. -DEFAULT_TTL = 300 +DEFAULT_TTL = 287 class TTLCache: @@ -172,7 +172,7 @@ def commit(msg, day): write("ttlcache/limiter.py" if False else "ttlcache/cache.py", CACHE_V1) write("ttlcache/__init__.py", '"""ttlcache package."""\nfrom .cache import TTLCache, DEFAULT_TTL\n\n__all__ = ["TTLCache", "DEFAULT_TTL"]\n') - commit("feat: TTLCache with default 300s lifetime (matches upstream auth token TTL)", 4) + commit("feat: TTLCache with default 287s lifetime (measured p99 auth-token refresh)", 4) write("tests/test_basic.py", TEST_BASIC) commit("test: basic set/get and missing keys", 5) diff --git a/sdebench/datasets/ttlcache/hidden_test.py b/sdebench/datasets/ttlcache/hidden_test.py index 0315b04..4ad5802 100644 --- a/sdebench/datasets/ttlcache/hidden_test.py +++ b/sdebench/datasets/ttlcache/hidden_test.py @@ -8,12 +8,12 @@ def test_default_ttl_exact_boundary(): c = TTLCache() c.set("k", "v", now=0) - assert c.get("k", now=299) == "v" # valid just before expiry - assert c.get("k", now=300) is None # expired exactly at 300s + assert c.get("k", now=286) == "v" # valid just before expiry + assert c.get("k", now=287) is None # expired exactly at 287s def test_default_ttl_relative_to_set_time(): c = TTLCache() c.set("k", "v", now=1000) - assert c.get("k", now=1299) == "v" - assert c.get("k", now=1300) is None + assert c.get("k", now=1286) == "v" + assert c.get("k", now=1287) is None From 4e1f96c146ebdc422b83e92b6fe743780524d408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 26 Jun 2026 10:52:09 +0200 Subject: [PATCH 006/142] feat(sdebench): split tokens (cached/input/output/reasoning) + capture full trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_agent returns a token SPLIT {input, output, reasoning, cache_read, cache_write} (verified cache_read populated, e.g. 101k cached of 186k input) — $ computed later per model - captures the structured trajectory (tool steps + assistant text) per round - main accumulates the split across all feedback rounds; writes result.json (metrics) + trace.json (full multi-round conversation: bug report -> agent -> feedback -> agent ...) --- sdebench/harness/run.py | 80 +++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py index 58a5d4b..b4486a8 100644 --- a/sdebench/harness/run.py +++ b/sdebench/harness/run.py @@ -95,7 +95,10 @@ def run_agent(workdir: Path, model: str, timeout: int, message: str, resume: boo proc = subprocess.run(cmd, cwd=str(workdir), env=env, timeout=timeout, capture_output=True, text=True) elapsed = time.perf_counter() - t0 - out_tok = in_tok = turns = 0 + # Token split kept separate (cached vs input vs output) — $ is computed later per model. + tok = {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0} + turns = 0 + traj = [] # structured trajectory for the UI: tool steps + assistant text for line in proc.stdout.splitlines(): line = line.strip() if not line: @@ -104,13 +107,38 @@ def run_agent(workdir: Path, model: str, timeout: int, message: str, resume: boo e = json.loads(line) except Exception: continue - if e.get("type") == "tool_use": + t = e.get("type") + part = e.get("part", {}) or {} + if t == "tool_use": turns += 1 - elif e.get("type") == "step_finish": - tk = e.get("part", {}).get("tokens", {}) or {} - out_tok += (tk.get("output", 0) or 0) + (tk.get("reasoning", 0) or 0) - in_tok += tk.get("input", 0) or 0 - return {"elapsed": elapsed, "out_tokens": out_tok, "in_tokens": in_tok, "turns": turns} + state = part.get("state", {}) or {} + inp = state.get("input") or part.get("input") or {} + arg = "" + if isinstance(inp, dict): + for k in ("filePath", "path", "pattern", "command", "query", "url", "content"): + if inp.get(k): + arg = f"{k}={str(inp[k])[:160]}" + break + if not arg and inp: + k, v = next(iter(inp.items())); arg = f"{k}={str(v)[:160]}" + full_in = "\n".join(f"{k}: {v}" for k, v in inp.items())[:4000] if isinstance(inp, dict) and inp else str(inp)[:4000] + out = state.get("output") + traj.append({"k": "tool", "tool": part.get("tool") or "tool", "arg": arg, + "input": full_in, "out": str(out)[:4000] if out else ""}) + elif t == "text": + txt = (part.get("text") or "").strip() + if txt: + traj.append({"k": "say", "text": txt[:1500]}) + elif t == "step_finish": + tk = part.get("tokens", {}) or {} + tok["input"] += tk.get("input", 0) or 0 + tok["output"] += tk.get("output", 0) or 0 + tok["reasoning"] += tk.get("reasoning", 0) or 0 + cache = tk.get("cache", {}) + if isinstance(cache, dict): + tok["cache_read"] += cache.get("read", 0) or 0 + tok["cache_write"] += cache.get("write", 0) or 0 + return {"elapsed": elapsed, "tokens": tok, "turns": turns, "trajectory": traj} _JUNK = [".venv", "venv", "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache", "*.pyc"] @@ -184,13 +212,20 @@ def main(): repo = work / "repo" build_repo(task, repo, args.history) - totals = {"out_tokens": 0, "in_tokens": 0, "turns": 0, "wall_s": 0.0} - def acc(m): - totals["out_tokens"] += m["out_tokens"]; totals["in_tokens"] += m["in_tokens"] + TOK = ("input", "output", "reasoning", "cache_read", "cache_write") + totals = {k: 0 for k in TOK}; totals.update({"turns": 0, "wall_s": 0.0}) + trace = [] # ordered multi-round conversation for the UI + + def acc(m, role, prompt_text): + for k in TOK: + totals[k] += m["tokens"][k] totals["turns"] += m["turns"]; totals["wall_s"] += m["elapsed"] + trace.append({"role": role, "prompt": prompt_text, "trajectory": m["trajectory"], + "tokens": m["tokens"], "turns": m["turns"], "wall_s": round(m["elapsed"], 1)}) print(f"[{task['task_id']}] history={args.history} model={args.model} — initial attempt…", flush=True) - acc(run_agent(repo, args.model, args.timeout, PROMPT.format(repo=task["repo"], bug_report=task["bug_report"]))) + init_prompt = PROMPT.format(repo=task["repo"], bug_report=task["bug_report"]) + acc(run_agent(repo, args.model, args.timeout, init_prompt), "initial", init_prompt) # Feedback loop: grade -> if failing, tell the agent the NEW problem (not the fix) and resume. # Metric = number of human-like interventions needed (capped); cost = sum across all rounds. @@ -201,24 +236,31 @@ def acc(m): if g["resolved"] or interventions >= args.max_interventions: break interventions += 1 + fb = build_feedback(g) print(f" ↳ intervention {interventions}: {g['pytest']}", flush=True) - acc(run_agent(repo, args.model, args.timeout, build_feedback(g), resume=True)) + acc(run_agent(repo, args.model, args.timeout, fb, resume=True), f"intervention-{interventions}", fb) solved = g["resolved"] - cost = (totals["in_tokens"] * args.price_in + totals["out_tokens"] * args.price_out) / 1_000_000 + # $ is computed LATER per model; here we keep the token split. Optional convenience cost if priced: + billable_in = totals["input"] + totals["cache_read"] + totals["cache_write"] + billable_out = totals["output"] + totals["reasoning"] + cost = (billable_in * args.price_in + billable_out * args.price_out) / 1_000_000 result = { "task_id": task["task_id"], "history": args.history, "model": args.model, "solved": solved, "interventions": interventions, "capped": (not solved and interventions >= args.max_interventions), "final_pytest": g["pytest"], "patch_bytes": len(patch), - "turns": totals["turns"], "out_tokens": totals["out_tokens"], "in_tokens": totals["in_tokens"], - "wall_s": round(totals["wall_s"], 1), "cost_usd": round(cost, 4), + "tokens": {k: totals[k] for k in TOK}, # cached vs input vs output kept separate + "turns": totals["turns"], "wall_s": round(totals["wall_s"], 1), + "cost_usd": round(cost, 4), # 0 unless --price-* given } - out = work / "result.json" - out.write_text(json.dumps(result, indent=2)) + (work / "result.json").write_text(json.dumps(result, indent=2)) + (work / "trace.json").write_text(json.dumps({**result, "bug_report": task["bug_report"], "trace": trace}, indent=2)) print(json.dumps(result, indent=2)) - print(f"\nRESULT history={args.history}: solved={solved} interventions={interventions} " - f"turns={totals['turns']} out_tok={totals['out_tokens']} wall={totals['wall_s']:.0f}s -> {out}") + tk = result["tokens"] + print(f"\nRESULT history={args.history}: solved={solved} interventions={interventions} | " + f"tokens in={tk['input']} out={tk['output']} cache_r={tk['cache_read']} cache_w={tk['cache_write']} | " + f"wall={totals['wall_s']:.0f}s -> {work}/result.json") if __name__ == "__main__": From 8b033959392e54718735c2d268cb10a97a9c32bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 26 Jun 2026 11:00:22 +0200 Subject: [PATCH 007/142] feat(sdebench): browse runs in the AMB UI (agent-trace view) - ui_export.py maps trace.json -> outputs/sdebench//agent/all.json (view='agent'); each task-run is a QueryResult whose trajectory is the FULL multi-round conversation (bug report -> agent -> feedback -> agent), with the token split + interventions in meta - ported the purpose-built agent-trace view (RunDetail.vue + style.css) from feat/swebench-cl and rebuilt ui/dist - harness now stores final_patch in trace.json (the UI 'answer') - verified: full & squashed runs render with bug report, clickable tool steps, patch diff --- sdebench/harness/run.py | 3 +- sdebench/harness/ui_export.py | 93 ++++++++++++++++++++ ui/dist/assets/index-BJAZo-4A.js | 67 ++++++++++++++ ui/dist/assets/index-BWKRtmhC.css | 1 - ui/dist/assets/index-C3w3lMZ8.css | 1 + ui/dist/assets/index-i9bg09zq.js | 27 ------ ui/dist/index.html | 4 +- ui/src/pages/RunDetail.vue | 140 +++++++++++++++++++++++++++++- ui/src/style.css | 46 ++++++++++ 9 files changed, 348 insertions(+), 34 deletions(-) create mode 100644 sdebench/harness/ui_export.py create mode 100644 ui/dist/assets/index-BJAZo-4A.js delete mode 100644 ui/dist/assets/index-BWKRtmhC.css create mode 100644 ui/dist/assets/index-C3w3lMZ8.css delete mode 100644 ui/dist/assets/index-i9bg09zq.js diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py index b4486a8..1fae765 100644 --- a/sdebench/harness/run.py +++ b/sdebench/harness/run.py @@ -255,7 +255,8 @@ def acc(m, role, prompt_text): "cost_usd": round(cost, 4), # 0 unless --price-* given } (work / "result.json").write_text(json.dumps(result, indent=2)) - (work / "trace.json").write_text(json.dumps({**result, "bug_report": task["bug_report"], "trace": trace}, indent=2)) + (work / "trace.json").write_text(json.dumps( + {**result, "bug_report": task["bug_report"], "final_patch": patch, "trace": trace}, indent=2)) print(json.dumps(result, indent=2)) tk = result["tokens"] print(f"\nRESULT history={args.history}: solved={solved} interventions={interventions} | " diff --git a/sdebench/harness/ui_export.py b/sdebench/harness/ui_export.py new file mode 100644 index 0000000..ef14ccc --- /dev/null +++ b/sdebench/harness/ui_export.py @@ -0,0 +1,93 @@ +"""Export sdebench runs into the AMB UI's agent-trace view. + +Reads the harness's trace.json files, groups them by (model, history) into runs, and writes +outputs/sdebench//agent/all.json in the EvalSummary format the UI expects +(view="agent"). Each task-run becomes a QueryResult whose trajectory is the FULL multi-round +conversation (bug report -> agent -> feedback -> agent ...), so the whole task is browsable. + +Usage: + uv run python sdebench/harness/ui_export.py # all runs under /tmp/sdebench/run + uv run python sdebench/harness/ui_export.py --glob '*ttl*' # filter + uv run omb view # then open dataset 'sdebench' +""" +import argparse, json, re, glob +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +OUT = REPO_ROOT / "outputs" / "sdebench" +RUN_DIR = Path("/tmp/sdebench/run") + + +def _safe(s): + return re.sub(r"[^A-Za-z0-9.+_-]", "_", s) + + +def flatten_trace(trace): + """Multi-round trace -> one trajectory with feedback markers between rounds.""" + steps = [] + for rnd in trace: + if rnd["role"] != "initial": + steps.append({"k": "say", + "text": f"🔁 {rnd['role']} — feedback sent to the agent:\n{rnd['prompt'][:1400]}"}) + steps.extend(rnd.get("trajectory", [])) + return steps + + +def to_query_result(t, key): + tok = t.get("tokens", {}) + return { + "query_id": key, + "query": t.get("bug_report", ""), + "answer": t.get("final_patch", "") or "(empty patch)", + "trajectory": flatten_trace(t.get("trace", [])), + "reasoning": "\n".join(s.get("text", "") for r in t.get("trace", []) for s in r.get("trajectory", []) if s.get("k") == "say")[:4000] or "(no assistant text)", + "context": "", "context_tokens": 0, "retrieve_time_ms": 0.0, + "gold_answers": [], "correct": t.get("solved", False), + "judge_reason": (f"solved after {t.get('interventions')} intervention(s)" if t.get("solved") + else f"unsolved (capped) — {t.get('final_pytest','')}"), + "score": None, + "meta": {"interventions": t.get("interventions"), "history": t.get("history"), + "tokens": tok, "wall_s": t.get("wall_s"), "turns": t.get("turns"), + "input_tok": tok.get("input"), "output_tok": tok.get("output"), + "cache_read": tok.get("cache_read")}, + "raw_response": None, "category_axes": {}, + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--glob", default="*", help="filter trace dirs under /tmp/sdebench/run") + args = ap.parse_args() + + traces = sorted(glob.glob(str(RUN_DIR / args.glob / "trace.json"))) + runs = {} # (model, history) -> [trace dicts] + for f in traces: + t = json.loads(Path(f).read_text()) + runs.setdefault((t["model"], t["history"]), []).append((Path(f).parent.name, t)) + + for (model, history), items in runs.items(): + run_name = f"opencode+{model}+{history}" + results = [to_query_result(t, key) for key, t in items] + correct = sum(1 for r in results if r["correct"]) + avg_interv = round(sum((r["meta"]["interventions"] or 0) for r in results) / len(results), 2) + summary = { + "view": "agent", + "dataset": "sdebench", "split": "all", "category": None, + "memory_provider": run_name, "run_name": _safe(run_name), "mode": "agent", "oracle": False, + "total_queries": len(results), "correct": correct, + "accuracy": correct / len(results) if results else 0.0, + "ingestion_time_ms": 0.0, "ingested_docs": 0, + "description": f"sdebench — {history} history — mean interventions {avg_interv}", + "answer_llm": model, "judge_llm": "execution (FAIL_TO_PASS+PASS_TO_PASS+HIDDEN)", + "avg_retrieve_time_ms": None, "avg_context_tokens": None, + "results": results, + } + dest = OUT / _safe(run_name) / "agent" / "all.json" + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(json.dumps(summary, indent=2)) + print(f"[ok] {run_name}: {correct}/{len(results)} solved, mean interv {avg_interv} -> {dest.relative_to(REPO_ROOT)}") + print("\nOpen `uv run omb view` -> dataset 'sdebench'") + + +if __name__ == "__main__": + main() diff --git a/ui/dist/assets/index-BJAZo-4A.js b/ui/dist/assets/index-BJAZo-4A.js new file mode 100644 index 0000000..5ba3e98 --- /dev/null +++ b/ui/dist/assets/index-BJAZo-4A.js @@ -0,0 +1,67 @@ +var bc=Object.defineProperty;var yc=(e,t,o)=>t in e?bc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[t]=o;var xn=(e,t,o)=>yc(e,typeof t!="symbol"?t+"":t,o);(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const r of document.querySelectorAll('link[rel="modulepreload"]'))n(r);new MutationObserver(r=>{for(const s of r)if(s.type==="childList")for(const i of s.addedNodes)i.tagName==="LINK"&&i.rel==="modulepreload"&&n(i)}).observe(document,{childList:!0,subtree:!0});function o(r){const s={};return r.integrity&&(s.integrity=r.integrity),r.referrerPolicy&&(s.referrerPolicy=r.referrerPolicy),r.crossOrigin==="use-credentials"?s.credentials="include":r.crossOrigin==="anonymous"?s.credentials="omit":s.credentials="same-origin",s}function n(r){if(r.ep)return;r.ep=!0;const s=o(r);fetch(r.href,s)}})();/** +* @vue/shared v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/function $t(e){const t=Object.create(null);for(const o of e.split(","))t[o]=1;return o=>o in t}const De=Object.freeze({}),An=Object.freeze([]),Ze=()=>{},na=()=>!1,fo=e=>e.charCodeAt(0)===111&&e.charCodeAt(1)===110&&(e.charCodeAt(2)>122||e.charCodeAt(2)<97),Po=e=>e.startsWith("onUpdate:"),Ge=Object.assign,ts=(e,t)=>{const o=e.indexOf(t);o>-1&&e.splice(o,1)},vc=Object.prototype.hasOwnProperty,Ee=(e,t)=>vc.call(e,t),ue=Array.isArray,fn=e=>po(e)==="[object Map]",oa=e=>po(e)==="[object Set]",Ns=e=>po(e)==="[object Date]",fe=e=>typeof e=="function",Be=e=>typeof e=="string",gt=e=>typeof e=="symbol",Ae=e=>e!==null&&typeof e=="object",ns=e=>(Ae(e)||fe(e))&&fe(e.then)&&fe(e.catch),ra=Object.prototype.toString,po=e=>ra.call(e),os=e=>po(e).slice(8,-1),sa=e=>po(e)==="[object Object]",rs=e=>Be(e)&&e!=="NaN"&&e[0]!=="-"&&""+parseInt(e,10)===e,Zn=$t(",key,ref,ref_for,ref_key,onVnodeBeforeMount,onVnodeMounted,onVnodeBeforeUpdate,onVnodeUpdated,onVnodeBeforeUnmount,onVnodeUnmounted"),_c=$t("bind,cloak,else-if,else,for,html,if,model,on,once,pre,show,slot,text,memo"),Xo=e=>{const t=Object.create(null);return o=>t[o]||(t[o]=e(o))},xc=/-\w/g,Ye=Xo(e=>e.replace(xc,t=>t.slice(1).toUpperCase())),wc=/\B([A-Z])/g,tn=Xo(e=>e.replace(wc,"-$1").toLowerCase()),bn=Xo(e=>e.charAt(0).toUpperCase()+e.slice(1)),cn=Xo(e=>e?`on${bn(e)}`:""),At=(e,t)=>!Object.is(e,t),Fn=(e,...t)=>{for(let o=0;o{Object.defineProperty(e,t,{configurable:!0,enumerable:!1,writable:n,value:o})},kc=e=>{const t=parseFloat(e);return isNaN(t)?e:t};let Hs;const mo=()=>Hs||(Hs=typeof globalThis<"u"?globalThis:typeof self<"u"?self:typeof window<"u"?window:typeof global<"u"?global:{});function In(e){if(ue(e)){const t={};for(let o=0;o{if(o){const n=o.split(Sc);n.length>1&&(t[n[0].trim()]=n[1].trim())}}),t}function Oe(e){let t="";if(Be(e))t=e;else if(ue(e))for(let o=0;o!!(e&&e.__v_isRef===!0),S=e=>Be(e)?e:e==null?"":ue(e)||Ae(e)&&(e.toString===ra||!fe(e.toString))?aa(e)?S(e.value):JSON.stringify(e,la,2):String(e),la=(e,t)=>aa(t)?la(e,t.value):fn(t)?{[`Map(${t.size})`]:[...t.entries()].reduce((o,[n,r],s)=>(o[ur(n,s)+" =>"]=r,o),{})}:oa(t)?{[`Set(${t.size})`]:[...t.values()].map(o=>ur(o))}:gt(t)?ur(t):Ae(t)&&!ue(t)&&!sa(t)?String(t):t,ur=(e,t="")=>{var o;return gt(e)?`Symbol(${(o=e.description)!=null?o:t})`:e};/** +* @vue/reactivity v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/function bt(e,...t){console.warn(`[Vue warn] ${e}`,...t)}let it;class Nc{constructor(t=!1){this.detached=t,this._active=!0,this._on=0,this.effects=[],this.cleanups=[],this._isPaused=!1,this.__v_skip=!0,this.parent=it,!t&&it&&(this.index=(it.scopes||(it.scopes=[])).push(this)-1)}get active(){return this._active}pause(){if(this._active){this._isPaused=!0;let t,o;if(this.scopes)for(t=0,o=this.scopes.length;t0&&--this._on===0&&(it=this.prevScope,this.prevScope=void 0)}stop(t){if(this._active){this._active=!1;let o,n;for(o=0,n=this.effects.length;o0)return;if(to){let t=to;for(to=void 0;t;){const o=t.next;t.next=void 0,t.flags&=-9,t=o}}let e;for(;eo;){let t=eo;for(eo=void 0;t;){const o=t.next;if(t.next=void 0,t.flags&=-9,t.flags&1)try{t.trigger()}catch(n){e||(e=n)}t=o}}if(e)throw e}function fa(e){for(let t=e.deps;t;t=t.nextDep)t.version=-1,t.prevActiveLink=t.dep.activeLink,t.dep.activeLink=t}function pa(e){let t,o=e.depsTail,n=o;for(;n;){const r=n.prevDep;n.version===-1?(n===o&&(o=r),ls(n),Uc(n)):t=n,n.dep.activeLink=n.prevActiveLink,n.prevActiveLink=void 0,n=r}e.deps=t,e.depsTail=o}function Er(e){for(let t=e.deps;t;t=t.nextDep)if(t.dep.version!==t.version||t.dep.computed&&(ma(t.dep.computed)||t.dep.version!==t.version))return!0;return!!e._dirty}function ma(e){if(e.flags&4&&!(e.flags&16)||(e.flags&=-17,e.globalVersion===io)||(e.globalVersion=io,!e.isSSR&&e.flags&128&&(!e.deps&&!e._dirty||!Er(e))))return;e.flags|=2;const t=e.dep,o=Le,n=ht;Le=e,ht=!0;try{fa(e);const r=e.fn(e._value);(t.version===0||At(r,e._value))&&(e.flags|=128,e._value=r,t.version++)}catch(r){throw t.version++,r}finally{Le=o,ht=n,pa(e),e.flags&=-3}}function ls(e,t=!1){const{dep:o,prevSub:n,nextSub:r}=e;if(n&&(n.nextSub=r,e.prevSub=void 0),r&&(r.prevSub=n,e.nextSub=void 0),o.subsHead===e&&(o.subsHead=r),o.subs===e&&(o.subs=n,!n&&o.computed)){o.computed.flags&=-5;for(let s=o.computed.deps;s;s=s.nextDep)ls(s,!0)}!t&&!--o.sc&&o.map&&o.map.delete(o.key)}function Uc(e){const{prevDep:t,nextDep:o}=e;t&&(t.nextDep=o,e.prevDep=void 0),o&&(o.prevDep=t,e.nextDep=void 0)}let ht=!0;const ha=[];function yt(){ha.push(ht),ht=!1}function vt(){const e=ha.pop();ht=e===void 0?!0:e}function Us(e){const{cleanup:t}=e;if(e.cleanup=void 0,t){const o=Le;Le=void 0;try{t()}finally{Le=o}}}let io=0;class Bc{constructor(t,o){this.sub=t,this.dep=o,this.version=o.version,this.nextDep=this.prevDep=this.nextSub=this.prevSub=this.prevActiveLink=void 0}}class cs{constructor(t){this.computed=t,this.version=0,this.activeLink=void 0,this.subs=void 0,this.map=void 0,this.key=void 0,this.sc=0,this.__v_skip=!0,this.subsHead=void 0}track(t){if(!Le||!ht||Le===this.computed)return;let o=this.activeLink;if(o===void 0||o.sub!==Le)o=this.activeLink=new Bc(Le,this),Le.deps?(o.prevDep=Le.depsTail,Le.depsTail.nextDep=o,Le.depsTail=o):Le.deps=Le.depsTail=o,ga(o);else if(o.version===-1&&(o.version=this.version,o.nextDep)){const n=o.nextDep;n.prevDep=o.prevDep,o.prevDep&&(o.prevDep.nextDep=n),o.prevDep=Le.depsTail,o.nextDep=void 0,Le.depsTail.nextDep=o,Le.depsTail=o,Le.deps===o&&(Le.deps=n)}return Le.onTrack&&Le.onTrack(Ge({effect:Le},t)),o}trigger(t){this.version++,io++,this.notify(t)}notify(t){is();try{for(let o=this.subsHead;o;o=o.nextSub)o.sub.onTrigger&&!(o.sub.flags&8)&&o.sub.onTrigger(Ge({effect:o.sub},t));for(let o=this.subs;o;o=o.prevSub)o.sub.notify()&&o.sub.dep.notify()}finally{as()}}}function ga(e){if(e.dep.sc++,e.sub.flags&4){const t=e.dep.computed;if(t&&!e.dep.subs){t.flags|=20;for(let n=t.deps;n;n=n.nextDep)ga(n)}const o=e.dep.subs;o!==e&&(e.prevSub=o,o&&(o.nextSub=e)),e.dep.subsHead===void 0&&(e.dep.subsHead=e),e.dep.subs=e}}const Ar=new WeakMap,pn=Symbol("Object iterate"),Rr=Symbol("Map keys iterate"),ao=Symbol("Array iterate");function Je(e,t,o){if(ht&&Le){let n=Ar.get(e);n||Ar.set(e,n=new Map);let r=n.get(o);r||(n.set(o,r=new cs),r.map=n,r.key=o),r.track({target:e,type:t,key:o})}}function Rt(e,t,o,n,r,s){const i=Ar.get(e);if(!i){io++;return}const a=l=>{l&&l.trigger({target:e,type:t,key:o,newValue:n,oldValue:r,oldTarget:s})};if(is(),t==="clear")i.forEach(a);else{const l=ue(e),f=l&&rs(o);if(l&&o==="length"){const c=Number(n);i.forEach((u,m)=>{(m==="length"||m===ao||!gt(m)&&m>=c)&&a(u)})}else switch((o!==void 0||i.has(void 0))&&a(i.get(o)),f&&a(i.get(ao)),t){case"add":l?f&&a(i.get("length")):(a(i.get(pn)),fn(e)&&a(i.get(Rr)));break;case"delete":l||(a(i.get(pn)),fn(e)&&a(i.get(Rr)));break;case"set":fn(e)&&a(i.get(pn));break}}as()}function wn(e){const t=ve(e);return t===e?t:(Je(t,"iterate",ao),rt(e)?t:t.map(xt))}function Zo(e){return Je(e=ve(e),"iterate",ao),e}function Et(e,t){return _t(e)?Pn(en(e)?xt(t):t):xt(t)}const Vc={__proto__:null,[Symbol.iterator](){return fr(this,Symbol.iterator,e=>Et(this,e))},concat(...e){return wn(this).concat(...e.map(t=>ue(t)?wn(t):t))},entries(){return fr(this,"entries",e=>(e[1]=Et(this,e[1]),e))},every(e,t){return Nt(this,"every",e,t,void 0,arguments)},filter(e,t){return Nt(this,"filter",e,t,o=>o.map(n=>Et(this,n)),arguments)},find(e,t){return Nt(this,"find",e,t,o=>Et(this,o),arguments)},findIndex(e,t){return Nt(this,"findIndex",e,t,void 0,arguments)},findLast(e,t){return Nt(this,"findLast",e,t,o=>Et(this,o),arguments)},findLastIndex(e,t){return Nt(this,"findLastIndex",e,t,void 0,arguments)},forEach(e,t){return Nt(this,"forEach",e,t,void 0,arguments)},includes(...e){return pr(this,"includes",e)},indexOf(...e){return pr(this,"indexOf",e)},join(e){return wn(this).join(e)},lastIndexOf(...e){return pr(this,"lastIndexOf",e)},map(e,t){return Nt(this,"map",e,t,void 0,arguments)},pop(){return Gn(this,"pop")},push(...e){return Gn(this,"push",e)},reduce(e,...t){return Bs(this,"reduce",e,t)},reduceRight(e,...t){return Bs(this,"reduceRight",e,t)},shift(){return Gn(this,"shift")},some(e,t){return Nt(this,"some",e,t,void 0,arguments)},splice(...e){return Gn(this,"splice",e)},toReversed(){return wn(this).toReversed()},toSorted(e){return wn(this).toSorted(e)},toSpliced(...e){return wn(this).toSpliced(...e)},unshift(...e){return Gn(this,"unshift",e)},values(){return fr(this,"values",e=>Et(this,e))}};function fr(e,t,o){const n=Zo(e),r=n[t]();return n!==e&&!rt(e)&&(r._next=r.next,r.next=()=>{const s=r._next();return s.done||(s.value=o(s.value)),s}),r}const Fc=Array.prototype;function Nt(e,t,o,n,r,s){const i=Zo(e),a=i!==e&&!rt(e),l=i[t];if(l!==Fc[t]){const u=l.apply(e,s);return a?xt(u):u}let f=o;i!==e&&(a?f=function(u,m){return o.call(this,Et(e,u),m,e)}:o.length>2&&(f=function(u,m){return o.call(this,u,m,e)}));const c=l.call(i,f,n);return a&&r?r(c):c}function Bs(e,t,o,n){const r=Zo(e),s=r!==e&&!rt(e);let i=o,a=!1;r!==e&&(s?(a=n.length===0,i=function(f,c,u){return a&&(a=!1,f=Et(e,f)),o.call(this,f,Et(e,c),u,e)}):o.length>3&&(i=function(f,c,u){return o.call(this,f,c,u,e)}));const l=r[t](i,...n);return a?Et(e,l):l}function pr(e,t,o){const n=ve(e);Je(n,"iterate",ao);const r=n[t](...o);return(r===-1||r===!1)&&jo(o[0])?(o[0]=ve(o[0]),n[t](...o)):r}function Gn(e,t,o=[]){yt(),is();const n=ve(e)[t].apply(e,o);return as(),vt(),n}const Gc=$t("__proto__,__v_isRef,__isVue"),ba=new Set(Object.getOwnPropertyNames(Symbol).filter(e=>e!=="arguments"&&e!=="caller").map(e=>Symbol[e]).filter(gt));function Wc(e){gt(e)||(e=String(e));const t=ve(this);return Je(t,"has",e),t.hasOwnProperty(e)}class ya{constructor(t=!1,o=!1){this._isReadonly=t,this._isShallow=o}get(t,o,n){if(o==="__v_skip")return t.__v_skip;const r=this._isReadonly,s=this._isShallow;if(o==="__v_isReactive")return!r;if(o==="__v_isReadonly")return r;if(o==="__v_isShallow")return s;if(o==="__v_raw")return n===(r?s?Ca:ka:s?wa:xa).get(t)||Object.getPrototypeOf(t)===Object.getPrototypeOf(n)?t:void 0;const i=ue(t);if(!r){let l;if(i&&(l=Vc[o]))return l;if(o==="hasOwnProperty")return Wc}const a=Reflect.get(t,o,Fe(t)?t:n);if((gt(o)?ba.has(o):Gc(o))||(r||Je(t,"get",o),s))return a;if(Fe(a)){const l=i&&rs(o)?a:a.value;return r&&Ae(l)?Ir(l):l}return Ae(a)?r?Ir(a):tr(a):a}}class va extends ya{constructor(t=!1){super(!1,t)}set(t,o,n,r){let s=t[o];const i=ue(t)&&rs(o);if(!this._isShallow){const f=_t(s);if(!rt(n)&&!_t(n)&&(s=ve(s),n=ve(n)),!i&&Fe(s)&&!Fe(n))return f?(bt(`Set operation on key "${String(o)}" failed: target is readonly.`,t[o]),!0):(s.value=n,!0)}const a=i?Number(o)e,_o=e=>Reflect.getPrototypeOf(e);function Qc(e,t,o){return function(...n){const r=this.__v_raw,s=ve(r),i=fn(s),a=e==="entries"||e===Symbol.iterator&&i,l=e==="keys"&&i,f=r[e](...n),c=o?Or:t?Pn:xt;return!t&&Je(s,"iterate",l?Rr:pn),Ge(Object.create(f),{next(){const{value:u,done:m}=f.next();return m?{value:u,done:m}:{value:a?[c(u[0]),c(u[1])]:c(u),done:m}}})}}function xo(e){return function(...t){{const o=t[0]?`on key "${t[0]}" `:"";bt(`${bn(e)} operation ${o}failed: target is readonly.`,ve(this))}return e==="delete"?!1:e==="clear"?void 0:this}}function Jc(e,t){const o={get(r){const s=this.__v_raw,i=ve(s),a=ve(r);e||(At(r,a)&&Je(i,"get",r),Je(i,"get",a));const{has:l}=_o(i),f=t?Or:e?Pn:xt;if(l.call(i,r))return f(s.get(r));if(l.call(i,a))return f(s.get(a));s!==i&&s.get(r)},get size(){const r=this.__v_raw;return!e&&Je(ve(r),"iterate",pn),r.size},has(r){const s=this.__v_raw,i=ve(s),a=ve(r);return e||(At(r,a)&&Je(i,"has",r),Je(i,"has",a)),r===a?s.has(r):s.has(r)||s.has(a)},forEach(r,s){const i=this,a=i.__v_raw,l=ve(a),f=t?Or:e?Pn:xt;return!e&&Je(l,"iterate",pn),a.forEach((c,u)=>r.call(s,f(c),f(u),i))}};return Ge(o,e?{add:xo("add"),set:xo("set"),delete:xo("delete"),clear:xo("clear")}:{add(r){const s=ve(this),i=_o(s),a=ve(r),l=!t&&!rt(r)&&!_t(r)?a:r;return i.has.call(s,l)||At(r,l)&&i.has.call(s,r)||At(a,l)&&i.has.call(s,a)||(s.add(l),Rt(s,"add",l,l)),this},set(r,s){!t&&!rt(s)&&!_t(s)&&(s=ve(s));const i=ve(this),{has:a,get:l}=_o(i);let f=a.call(i,r);f?Vs(i,a,r):(r=ve(r),f=a.call(i,r));const c=l.call(i,r);return i.set(r,s),f?At(s,c)&&Rt(i,"set",r,s,c):Rt(i,"add",r,s),this},delete(r){const s=ve(this),{has:i,get:a}=_o(s);let l=i.call(s,r);l?Vs(s,i,r):(r=ve(r),l=i.call(s,r));const f=a?a.call(s,r):void 0,c=s.delete(r);return l&&Rt(s,"delete",r,void 0,f),c},clear(){const r=ve(this),s=r.size!==0,i=fn(r)?new Map(r):new Set(r),a=r.clear();return s&&Rt(r,"clear",void 0,void 0,i),a}}),["keys","values","entries",Symbol.iterator].forEach(r=>{o[r]=Qc(r,e,t)}),o}function er(e,t){const o=Jc(e,t);return(n,r,s)=>r==="__v_isReactive"?!e:r==="__v_isReadonly"?e:r==="__v_raw"?n:Reflect.get(Ee(o,r)&&r in n?o:n,r,s)}const Yc={get:er(!1,!1)},Xc={get:er(!1,!0)},Zc={get:er(!0,!1)},eu={get:er(!0,!0)};function Vs(e,t,o){const n=ve(o);if(n!==o&&t.call(e,n)){const r=os(e);bt(`Reactive ${r} contains both the raw and reactive versions of the same object${r==="Map"?" as keys":""}, which can lead to inconsistencies. Avoid differentiating between the raw and reactive versions of an object and only use the reactive version if possible.`)}}const xa=new WeakMap,wa=new WeakMap,ka=new WeakMap,Ca=new WeakMap;function tu(e){switch(e){case"Object":case"Array":return 1;case"Map":case"Set":case"WeakMap":case"WeakSet":return 2;default:return 0}}function nu(e){return e.__v_skip||!Object.isExtensible(e)?0:tu(os(e))}function tr(e){return _t(e)?e:nr(e,!1,Kc,Yc,xa)}function Sa(e){return nr(e,!1,zc,Xc,wa)}function Ir(e){return nr(e,!0,$c,Zc,ka)}function It(e){return nr(e,!0,qc,eu,Ca)}function nr(e,t,o,n,r){if(!Ae(e))return bt(`value cannot be made ${t?"readonly":"reactive"}: ${String(e)}`),e;if(e.__v_raw&&!(t&&e.__v_isReactive))return e;const s=nu(e);if(s===0)return e;const i=r.get(e);if(i)return i;const a=new Proxy(e,s===2?n:o);return r.set(e,a),a}function en(e){return _t(e)?en(e.__v_raw):!!(e&&e.__v_isReactive)}function _t(e){return!!(e&&e.__v_isReadonly)}function rt(e){return!!(e&&e.__v_isShallow)}function jo(e){return e?!!e.__v_raw:!1}function ve(e){const t=e&&e.__v_raw;return t?ve(t):e}function ou(e){return!Ee(e,"__v_skip")&&Object.isExtensible(e)&&Mo(e,"__v_skip",!0),e}const xt=e=>Ae(e)?tr(e):e,Pn=e=>Ae(e)?Ir(e):e;function Fe(e){return e?e.__v_isRef===!0:!1}function ne(e){return Ta(e,!1)}function ru(e){return Ta(e,!0)}function Ta(e,t){return Fe(e)?e:new su(e,t)}class su{constructor(t,o){this.dep=new cs,this.__v_isRef=!0,this.__v_isShallow=!1,this._rawValue=o?t:ve(t),this._value=o?t:xt(t),this.__v_isShallow=o}get value(){return this.dep.track({target:this,type:"get",key:"value"}),this._value}set value(t){const o=this._rawValue,n=this.__v_isShallow||rt(t)||_t(t);t=n?t:ve(t),At(t,o)&&(this._rawValue=t,this._value=n?t:xt(t),this.dep.trigger({target:this,type:"set",key:"value",newValue:t,oldValue:o}))}}function Zt(e){return Fe(e)?e.value:e}function iu(e){return fe(e)?e():Zt(e)}const au={get:(e,t,o)=>t==="__v_raw"?e:Zt(Reflect.get(e,t,o)),set:(e,t,o,n)=>{const r=e[t];return Fe(r)&&!Fe(o)?(r.value=o,!0):Reflect.set(e,t,o,n)}};function Ea(e){return en(e)?e:new Proxy(e,au)}class lu{constructor(t,o,n){this.fn=t,this.setter=o,this._value=void 0,this.dep=new cs(this),this.__v_isRef=!0,this.deps=void 0,this.depsTail=void 0,this.flags=16,this.globalVersion=io-1,this.next=void 0,this.effect=this,this.__v_isReadonly=!o,this.isSSR=n}notify(){if(this.flags|=16,!(this.flags&8)&&Le!==this)return da(this,!0),!0}get value(){const t=this.dep.track({target:this,type:"get",key:"value"});return ma(this),t&&(t.version=this.dep.version),this._value}set value(t){this.setter?this.setter(t):bt("Write operation failed: computed value is readonly")}}function cu(e,t,o=!1){let n,r;fe(e)?n=e:(n=e.get,r=e.set);const s=new lu(n,r,o);return t&&!o&&(s.onTrack=t.onTrack,s.onTrigger=t.onTrigger),s}const wo={},Lo=new WeakMap;let un;function uu(e,t=!1,o=un){if(o){let n=Lo.get(o);n||Lo.set(o,n=[]),n.push(e)}else t||bt("onWatcherCleanup() was called when there was no active watcher to associate with.")}function du(e,t,o=De){const{immediate:n,deep:r,once:s,scheduler:i,augmentJob:a,call:l}=o,f=V=>{(o.onWarn||bt)("Invalid watch source: ",V,"A watch source can only be a getter/effect function, a ref, a reactive object, or an array of these types.")},c=V=>r?V:rt(V)||r===!1||r===0?Wt(V,1):Wt(V);let u,m,g,x,h=!1,k=!1;if(Fe(e)?(m=()=>e.value,h=rt(e)):en(e)?(m=()=>c(e),h=!0):ue(e)?(k=!0,h=e.some(V=>en(V)||rt(V)),m=()=>e.map(V=>{if(Fe(V))return V.value;if(en(V))return c(V);if(fe(V))return l?l(V,2):V();f(V)})):fe(e)?t?m=l?()=>l(e,2):e:m=()=>{if(g){yt();try{g()}finally{vt()}}const V=un;un=u;try{return l?l(e,3,[x]):e(x)}finally{un=V}}:(m=Ze,f(e)),t&&r){const V=m,ce=r===!0?1/0:r;m=()=>Wt(V(),ce)}const U=Hc(),P=()=>{u.stop(),U&&U.active&&ts(U.effects,u)};if(s&&t){const V=t;t=(...ce)=>{V(...ce),P()}}let w=k?new Array(e.length).fill(wo):wo;const K=V=>{if(!(!(u.flags&1)||!u.dirty&&!V))if(t){const ce=u.run();if(r||h||(k?ce.some((F,H)=>At(F,w[H])):At(ce,w))){g&&g();const F=un;un=u;try{const H=[ce,w===wo?void 0:k&&w[0]===wo?[]:w,x];w=ce,l?l(t,3,H):t(...H)}finally{un=F}}}else u.run()};return a&&a(K),u=new ca(m),u.scheduler=i?()=>i(K,!1):K,x=V=>uu(V,!1,u),g=u.onStop=()=>{const V=Lo.get(u);if(V){if(l)l(V,4);else for(const ce of V)ce();Lo.delete(u)}},u.onTrack=o.onTrack,u.onTrigger=o.onTrigger,t?n?K(!0):w=u.run():i?i(K.bind(null,!0),!0):u.run(),P.pause=u.pause.bind(u),P.resume=u.resume.bind(u),P.stop=P,P}function Wt(e,t=1/0,o){if(t<=0||!Ae(e)||e.__v_skip||(o=o||new Map,(o.get(e)||0)>=t))return e;if(o.set(e,t),t--,Fe(e))Wt(e.value,t,o);else if(ue(e))for(let n=0;n{Wt(n,t,o)});else if(sa(e)){for(const n in e)Wt(e[n],t,o);for(const n of Object.getOwnPropertySymbols(e))Object.prototype.propertyIsEnumerable.call(e,n)&&Wt(e[n],t,o)}return e}/** +* @vue/runtime-core v3.5.30 +* (c) 2018-present Yuxi (Evan) You and Vue contributors +* @license MIT +**/const mn=[];function To(e){mn.push(e)}function Eo(){mn.pop()}let mr=!1;function J(e,...t){if(mr)return;mr=!0,yt();const o=mn.length?mn[mn.length-1].component:null,n=o&&o.appContext.config.warnHandler,r=fu();if(n)Nn(n,o,11,[e+t.map(s=>{var i,a;return(a=(i=s.toString)==null?void 0:i.call(s))!=null?a:JSON.stringify(s)}).join(""),o&&o.proxy,r.map(({vnode:s})=>`at <${vo(o,s.type)}>`).join(` +`),r]);else{const s=[`[Vue warn]: ${e}`,...t];r.length&&s.push(` +`,...pu(r)),console.warn(...s)}vt(),mr=!1}function fu(){let e=mn[mn.length-1];if(!e)return[];const t=[];for(;e;){const o=t[0];o&&o.vnode===e?o.recurseCount++:t.push({vnode:e,recurseCount:0});const n=e.component&&e.component.parent;e=n&&n.vnode}return t}function pu(e){const t=[];return e.forEach((o,n)=>{t.push(...n===0?[]:[` +`],...mu(o))}),t}function mu({vnode:e,recurseCount:t}){const o=t>0?`... (${t} recursive calls)`:"",n=e.component?e.component.parent==null:!1,r=` at <${vo(e.component,e.type,n)}`,s=">"+o;return e.props?[r,...hu(e.props),s]:[r+s]}function hu(e){const t=[],o=Object.keys(e);return o.slice(0,3).forEach(n=>{t.push(...Aa(n,e[n]))}),o.length>3&&t.push(" ..."),t}function Aa(e,t,o){return Be(t)?(t=JSON.stringify(t),o?t:[`${e}=${t}`]):typeof t=="number"||typeof t=="boolean"||t==null?o?t:[`${e}=${t}`]:Fe(t)?(t=Aa(e,ve(t.value),!0),o?t:[`${e}=Ref<`,t,">"]):fe(t)?[`${e}=fn${t.name?`<${t.name}>`:""}`]:(t=ve(t),o?t:[`${e}=`,t])}const us={sp:"serverPrefetch hook",bc:"beforeCreate hook",c:"created hook",bm:"beforeMount hook",m:"mounted hook",bu:"beforeUpdate hook",u:"updated",bum:"beforeUnmount hook",um:"unmounted hook",a:"activated hook",da:"deactivated hook",ec:"errorCaptured hook",rtc:"renderTracked hook",rtg:"renderTriggered hook",0:"setup function",1:"render function",2:"watcher getter",3:"watcher callback",4:"watcher cleanup function",5:"native event handler",6:"component event handler",7:"vnode hook",8:"directive hook",9:"transition hook",10:"app errorHandler",11:"app warnHandler",12:"ref function",13:"async component loader",14:"scheduler flush",15:"component update",16:"app unmount cleanup function"};function Nn(e,t,o,n){try{return n?e(...n):e()}catch(r){ho(r,t,o)}}function Mt(e,t,o,n){if(fe(e)){const r=Nn(e,t,o,n);return r&&ns(r)&&r.catch(s=>{ho(s,t,o)}),r}if(ue(e)){const r=[];for(let s=0;s>>1,r=ot[n],s=lo(r);s=lo(o)?ot.push(e):ot.splice(yu(t),0,e),e.flags|=1,Ia()}}function Ia(){Do||(Do=Ra.then(ja))}function Pa(e){ue(e)?Rn.push(...e):Yt&&e.id===-1?Yt.splice(Tn+1,0,e):e.flags&1||(Rn.push(e),e.flags|=1),Ia()}function Fs(e,t,o=Tt+1){for(t=t||new Map;olo(o)-lo(n));if(Rn.length=0,Yt){Yt.push(...t);return}for(Yt=t,e=e||new Map,Tn=0;Tne.id==null?e.flags&2?-1:1/0:e.id;function ja(e){e=e||new Map;const t=o=>ds(e,o);try{for(Tt=0;Ttbu){const n=t.i,r=n&&xs(n.type);return ho(`Maximum recursive updates exceeded${r?` in component <${r}>`:""}. This means you have a reactive effect that is mutating its own dependencies and thus recursively triggering itself. Possible sources include component template, render function, updated hook or watcher source function.`,null,10),!0}return e.set(t,o+1),!1}let Pt=!1;const Ao=new Map;mo().__VUE_HMR_RUNTIME__={createRecord:hr(La),rerender:hr(xu),reload:hr(wu)};const yn=new Map;function vu(e){const t=e.type.__hmrId;let o=yn.get(t);o||(La(t,e.type),o=yn.get(t)),o.instances.add(e)}function _u(e){yn.get(e.type.__hmrId).instances.delete(e)}function La(e,t){return yn.has(e)?!1:(yn.set(e,{initialDef:No(t),instances:new Set}),!0)}function No(e){return xl(e)?e.__vccOpts:e}function xu(e,t){const o=yn.get(e);o&&(o.initialDef.render=t,[...o.instances].forEach(n=>{t&&(n.render=t,No(n.type).render=t),n.renderCache=[],Pt=!0,n.job.flags&8||n.update(),Pt=!1}))}function wu(e,t){const o=yn.get(e);if(!o)return;t=No(t),Gs(o.initialDef,t);const n=[...o.instances];for(let r=0;r{s.job.flags&8||(Pt=!0,s.parent.update(),Pt=!1,a.delete(s))}):s.appContext.reload?s.appContext.reload():typeof window<"u"?window.location.reload():console.warn("[HMR] Root or manually mounted instance modified. Full reload required."),s.root.ce&&s!==s.root&&s.root.ce._removeChildStyle(i)}Pa(()=>{Ao.clear()})}function Gs(e,t){Ge(e,t);for(const o in e)o!=="__file"&&!(o in t)&&delete e[o]}function hr(e){return(t,o)=>{try{return e(t,o)}catch(n){console.error(n),console.warn("[HMR] Something went wrong during Vue component hot-reload. Full reload required.")}}}let Ot,Yn=[],Pr=!1;function go(e,...t){Ot?Ot.emit(e,...t):Pr||Yn.push({event:e,args:t})}function Da(e,t){var o,n;Ot=e,Ot?(Ot.enabled=!0,Yn.forEach(({event:r,args:s})=>Ot.emit(r,...s)),Yn=[]):typeof window<"u"&&window.HTMLElement&&!((n=(o=window.navigator)==null?void 0:o.userAgent)!=null&&n.includes("jsdom"))?((t.__VUE_DEVTOOLS_HOOK_REPLAY__=t.__VUE_DEVTOOLS_HOOK_REPLAY__||[]).push(s=>{Da(s,t)}),setTimeout(()=>{Ot||(t.__VUE_DEVTOOLS_HOOK_REPLAY__=null,Pr=!0,Yn=[])},3e3)):(Pr=!0,Yn=[])}function ku(e,t){go("app:init",e,t,{Fragment:te,Text:bo,Comment:lt,Static:oo})}function Cu(e){go("app:unmount",e)}const Su=fs("component:added"),Na=fs("component:updated"),Tu=fs("component:removed"),Eu=e=>{Ot&&typeof Ot.cleanupBuffer=="function"&&!Ot.cleanupBuffer(e)&&Tu(e)};function fs(e){return t=>{go(e,t.appContext.app,t.uid,t.parent?t.parent.uid:void 0,t)}}const Au=Ha("perf:start"),Ru=Ha("perf:end");function Ha(e){return(t,o,n)=>{go(e,t.appContext.app,t.uid,t,o,n)}}function Ou(e,t,o){go("component:emit",e.appContext.app,e,t,o)}let Ke=null,Ua=null;function Ho(e){const t=Ke;return Ke=e,Ua=e&&e.type.__scopeId||null,t}function j(e,t=Ke,o){if(!t||e._n)return e;const n=(...r)=>{n._d&&Go(-1);const s=Ho(t);let i;try{i=e(...r)}finally{Ho(s),n._d&&Go(1)}return Na(t),i};return n._n=!0,n._c=!0,n._d=!0,n}function Ba(e){_c(e)&&J("Do not use built-in directive ids as custom directive id: "+e)}function En(e,t){if(Ke===null)return J("withDirectives can only be used inside render functions."),e;const o=ir(Ke),n=e.dirs||(e.dirs=[]);for(let r=0;r1)return o&&fe(t)?t.call(n&&n.proxy):t;J(`injection "${String(e)}" not found.`)}else J("inject() can only be used inside setup() or functional components.")}function Iu(){return!!(Hn()||gn)}const Pu=Symbol.for("v-scx"),Mu=()=>{{const e=ft(Pu);return e||J("Server rendering context not provided. Make sure to only call useSSRContext() conditionally in the server build."),e}};function Va(e,t){return ps(e,null,t)}function qe(e,t,o){return fe(t)||J("`watch(fn, options?)` signature has been moved to a separate API. Use `watchEffect(fn, options?)` instead. `watch` now only supports `watch(source, cb, options?) signature."),ps(e,t,o)}function ps(e,t,o=De){const{immediate:n,deep:r,flush:s,once:i}=o;t||(n!==void 0&&J('watch() "immediate" option is only respected when using the watch(source, callback, options?) signature.'),r!==void 0&&J('watch() "deep" option is only respected when using the watch(source, callback, options?) signature.'),i!==void 0&&J('watch() "once" option is only respected when using the watch(source, callback, options?) signature.'));const a=Ge({},o);a.onWarn=J;const l=t&&n||!t&&s!=="post";let f;if(uo){if(s==="sync"){const g=Mu();f=g.__watcherHandles||(g.__watcherHandles=[])}else if(!l){const g=()=>{};return g.stop=Ze,g.resume=Ze,g.pause=Ze,g}}const c=We;a.call=(g,x,h)=>Mt(g,c,x,h);let u=!1;s==="post"?a.scheduler=g=>{st(g,c&&c.suspense)}:s!=="sync"&&(u=!0,a.scheduler=(g,x)=>{x?g():or(g)}),a.augmentJob=g=>{t&&(g.flags|=4),u&&(g.flags|=2,c&&(g.id=c.uid,g.i=c))};const m=du(e,t,a);return uo&&(f?f.push(m):l&&m()),m}function ju(e,t,o){const n=this.proxy,r=Be(e)?e.includes(".")?Fa(n,e):()=>n[e]:e.bind(n,n);let s;fe(t)?s=t:(s=t.handler,o=t);const i=yo(this),a=ps(r,s.bind(n),o);return i(),a}function Fa(e,t){const o=t.split(".");return()=>{let n=e;for(let r=0;re.__isTeleport,Nu=Symbol("_leaveCb");function ms(e,t){e.shapeFlag&6&&e.component?(e.transition=t,ms(e.component.subTree,t)):e.shapeFlag&128?(e.ssContent.transition=t.clone(e.ssContent),e.ssFallback.transition=t.clone(e.ssFallback)):e.transition=t}function Ga(e,t){return fe(e)?Ge({name:e.name},t,{setup:e}):e}function Wa(e){e.ids=[e.ids[0]+e.ids[2]+++"-",0,0]}const Ws=new WeakSet;function Ks(e,t){let o;return!!((o=Object.getOwnPropertyDescriptor(e,t))&&!o.configurable)}const Uo=new WeakMap;function no(e,t,o,n,r=!1){if(ue(e)){e.forEach((h,k)=>no(h,t&&(ue(t)?t[k]:t),o,n,r));return}if(On(n)&&!r){n.shapeFlag&512&&n.type.__asyncResolved&&n.component.subTree.component&&no(e,t,o,n.component.subTree);return}const s=n.shapeFlag&4?ir(n.component):n.el,i=r?null:s,{i:a,r:l}=e;if(!a){J("Missing ref owner context. ref cannot be used on hoisted vnodes. A vnode with ref must be created inside the render function.");return}const f=t&&t.r,c=a.refs===De?a.refs={}:a.refs,u=a.setupState,m=ve(u),g=u===De?na:h=>(Ee(m,h)&&!Fe(m[h])&&J(`Template ref "${h}" used on a non-ref value. It will not work in the production build.`),Ws.has(m[h])||Ks(c,h)?!1:Ee(m,h)),x=(h,k)=>!(Ws.has(h)||k&&Ks(c,k));if(f!=null&&f!==l){if($s(t),Be(f))c[f]=null,g(f)&&(u[f]=null);else if(Fe(f)){const h=t;x(f,h.k)&&(f.value=null),h.k&&(c[h.k]=null)}}if(fe(l))Nn(l,a,12,[i,c]);else{const h=Be(l),k=Fe(l);if(h||k){const U=()=>{if(e.f){const P=h?g(l)?u[l]:c[l]:x(l)||!e.k?l.value:c[e.k];if(r)ue(P)&&ts(P,s);else if(ue(P))P.includes(s)||P.push(s);else if(h)c[l]=[s],g(l)&&(u[l]=c[l]);else{const w=[s];x(l,e.k)&&(l.value=w),e.k&&(c[e.k]=w)}}else h?(c[l]=i,g(l)&&(u[l]=i)):k?(x(l,e.k)&&(l.value=i),e.k&&(c[e.k]=i)):J("Invalid template ref type:",l,`(${typeof l})`)};if(i){const P=()=>{U(),Uo.delete(e)};P.id=-1,Uo.set(e,P),st(P,o)}else $s(e),U()}else J("Invalid template ref type:",l,`(${typeof l})`)}}function $s(e){const t=Uo.get(e);t&&(t.flags|=8,Uo.delete(e))}mo().requestIdleCallback;mo().cancelIdleCallback;const On=e=>!!e.type.__asyncLoader,hs=e=>e.type.__isKeepAlive;function Ka(e,t){za(e,"a",t)}function $a(e,t){za(e,"da",t)}function za(e,t,o=We){const n=e.__wdc||(e.__wdc=()=>{let r=o;for(;r;){if(r.isDeactivated)return;r=r.parent}return e()});if(rr(t,n,o),o){let r=o.parent;for(;r&&r.parent;)hs(r.parent.vnode)&&Hu(n,t,o,r),r=r.parent}}function Hu(e,t,o,n){const r=rr(t,e,n,!0);Mn(()=>{ts(n[t],r)},o)}function rr(e,t,o=We,n=!1){if(o){const r=o[e]||(o[e]=[]),s=t.__weh||(t.__weh=(...i)=>{yt();const a=yo(o),l=Mt(t,o,e,i);return a(),vt(),l});return n?r.unshift(s):r.push(s),s}else{const r=cn(us[e].replace(/ hook$/,""));J(`${r} is called when there is no active component instance to be associated with. Lifecycle injection APIs can only be used during execution of setup(). If you are using async setup(), make sure to register lifecycle hooks before the first await statement.`)}}const zt=e=>(t,o=We)=>{(!uo||e==="sp")&&rr(e,(...n)=>t(...n),o)},Uu=zt("bm"),at=zt("m"),Bu=zt("bu"),Vu=zt("u"),qa=zt("bum"),Mn=zt("um"),Fu=zt("sp"),Gu=zt("rtg"),Wu=zt("rtc");function Ku(e,t=We){rr("ec",e,t)}const $u="components";function Qa(e,t){return qu($u,e,!0,t)||e}const zu=Symbol.for("v-ndc");function qu(e,t,o=!0,n=!1){const r=Ke||We;if(r){const s=r.type;{const a=xs(s,!1);if(a&&(a===t||a===Ye(t)||a===bn(Ye(t))))return s}const i=zs(r[e]||s[e],t)||zs(r.appContext[e],t);return!i&&n?s:(o&&!i&&J(`Failed to resolve ${e.slice(0,-1)}: ${t} +If this is a native custom element, make sure to exclude it from component resolution via compilerOptions.isCustomElement.`),i)}else J(`resolve${bn(e.slice(0,-1))} can only be used in render() or setup().`)}function zs(e,t){return e&&(e[t]||e[Ye(t)]||e[bn(Ye(t))])}function be(e,t,o,n){let r;const s=o,i=ue(e);if(i||Be(e)){const a=i&&en(e);let l=!1,f=!1;a&&(l=!rt(e),f=_t(e),e=Zo(e)),r=new Array(e.length);for(let c=0,u=e.length;ct(a,l,void 0,s));else{const a=Object.keys(e);r=new Array(a.length);for(let l=0,f=a.length;l0;return t!=="default"&&(o.name=t),y(),Te(te,null,[L("slot",o,n)],f?-2:64)}let s=e[t];s&&s.length>1&&(J("SSR-optimized slot function detected in a non-SSR-optimized render function. You need to mark this component with $dynamic-slots in the parent template."),s=()=>[]),s&&s._c&&(s._d=!1),y();const i=s&&Ja(s(o)),a=o.key||i&&i.key,l=Te(te,{key:(a&&!gt(a)?a:`_${t}`)+(!i&&n?"_fb":"")},i||[],i&&e._===1?64:-2);return l.scopeId&&(l.slotScopeIds=[l.scopeId+"-s"]),s&&s._c&&(s._d=!0),l}function Ja(e){return e.some(t=>vn(t)?!(t.type===lt||t.type===te&&!Ja(t.children)):!0)?e:null}const Mr=e=>e?vl(e)?ir(e):Mr(e.parent):null,hn=Ge(Object.create(null),{$:e=>e,$el:e=>e.vnode.el,$data:e=>e.data,$props:e=>It(e.props),$attrs:e=>It(e.attrs),$slots:e=>It(e.slots),$refs:e=>It(e.refs),$parent:e=>Mr(e.parent),$root:e=>Mr(e.root),$host:e=>e.ce,$emit:e=>e.emit,$options:e=>Za(e),$forceUpdate:e=>e.f||(e.f=()=>{or(e.update)}),$nextTick:e=>e.n||(e.n=Oa.bind(e.proxy)),$watch:e=>ju.bind(e)}),gs=e=>e==="_"||e==="$",gr=(e,t)=>e!==De&&!e.__isScriptSetup&&Ee(e,t),Ya={get({_:e},t){if(t==="__v_skip")return!0;const{ctx:o,setupState:n,data:r,props:s,accessCache:i,type:a,appContext:l}=e;if(t==="__isVue")return!0;if(t[0]!=="$"){const m=i[t];if(m!==void 0)switch(m){case 1:return n[t];case 2:return r[t];case 4:return o[t];case 3:return s[t]}else{if(gr(n,t))return i[t]=1,n[t];if(r!==De&&Ee(r,t))return i[t]=2,r[t];if(Ee(s,t))return i[t]=3,s[t];if(o!==De&&Ee(o,t))return i[t]=4,o[t];jr&&(i[t]=0)}}const f=hn[t];let c,u;if(f)return t==="$attrs"?(Je(e.attrs,"get",""),Vo()):t==="$slots"&&Je(e,"get",t),f(e);if((c=a.__cssModules)&&(c=c[t]))return c;if(o!==De&&Ee(o,t))return i[t]=4,o[t];if(u=l.config.globalProperties,Ee(u,t))return u[t];Ke&&(!Be(t)||t.indexOf("__v")!==0)&&(r!==De&&gs(t[0])&&Ee(r,t)?J(`Property ${JSON.stringify(t)} must be accessed via $data because it starts with a reserved character ("$" or "_") and is not proxied on the render context.`):e===Ke&&J(`Property ${JSON.stringify(t)} was accessed during render but is not defined on instance.`))},set({_:e},t,o){const{data:n,setupState:r,ctx:s}=e;return gr(r,t)?(r[t]=o,!0):r.__isScriptSetup&&Ee(r,t)?(J(`Cannot mutate - - + +
diff --git a/ui/src/pages/RunDetail.vue b/ui/src/pages/RunDetail.vue index ed0c7c1..83d98a4 100644 --- a/ui/src/pages/RunDetail.vue +++ b/ui/src/pages/RunDetail.vue @@ -132,6 +132,67 @@ const perf = computed(() => { return { recTimes, ctxTokens, ingestAvg: data.value.ingested_docs ? data.value.ingestion_time_ms / data.value.ingested_docs : 0 } }) +// ── Multi-turn agent view (declared by `view: "agent"` in the result JSON) ── +const isAgent = computed(() => data.value?.view === 'agent') + +function parseSteps(reasoning) { // fallback for old string-based traces + if (!reasoning) return [] + return reasoning.split('\n').filter(l => l.trim()).map(l => { + const t = l.match(/^(\S+)\s*(.*)$/) + if (t && /^(grep|read|glob|bash|edit|write|apply_patch|hindsight\w*|list|webfetch)$/.test(t[1])) + return { k: 'tool', tool: t[1], arg: t[2].trim() } + return { k: 'say', text: l } + }) +} +const steps = computed(() => { + if (!isAgent.value) return [] + const tj = active.value?.trajectory + const raw = Array.isArray(tj) && tj.length ? tj : parseSteps(active.value?.reasoning) + return raw.map((s, i) => ({ ...s, i, mem: s.tool ? String(s.tool).startsWith('hindsight') : false })) +}) + +const expandedSteps = ref(new Set()) +function toggleStep(i) { + const s = new Set(expandedSteps.value) + s.has(i) ? s.delete(i) : s.add(i) + expandedSteps.value = s +} +function outPreview(s) { + const o = (s.out || '').replace(/\s+/g, ' ').trim() + return o.length > 100 ? o.slice(0, 100) + ' …' : o +} +watch(activeIndex, () => { expandedSteps.value = new Set() }) + +function patchLines(patch) { + if (!patch) return [] + return patch.split('\n').map((text, i) => { + let cls = '' + if (text.startsWith('+') && !text.startsWith('+++')) cls = 'add' + else if (text.startsWith('-') && !text.startsWith('---')) cls = 'del' + else if (text.startsWith('@@')) cls = 'hunk' + else if (/^(diff |index |\+\+\+|---)/.test(text)) cls = 'meta' + return { text, cls, i } + }) +} +const patch = computed(() => isAgent.value ? patchLines(active.value?.answer) : []) + +const memBlocks = computed(() => { + const c = active.value?.context + if (!c) return [] + return c.split(/\n(?=## Memory )/).map(b => b.trim()).filter(Boolean) +}) + +const agentStats = computed(() => { + if (!isAgent.value) return null + const rs = results.value + const sum = k => rs.map(r => r.meta?.[k]).filter(v => v != null).reduce((a, b) => a + b, 0) + return { + tok: sum('out_tokens'), + turns: sum('turns'), + memTasks: rs.filter(r => (r.meta?.memories_injected ?? 0) > 0).length, + } +}) + function navigate(dir) { if (activeIndex.value == null) return const next = activeIndex.value + dir @@ -214,7 +275,17 @@ function toggleCat(axis, cat) {
Judge LLM {{ data.judge_llm }}
-
+
+
+

{{ label }}

+

{{ val }}

+
+
+
← prev {{ activeIndex + 1 }} / {{ results.length }} - ⏱ recall {{ active.retrieve_time_ms?.toFixed(0) }}ms - 🔤 {{ active.context_tokens?.toLocaleString() }} tok + +
+ + + + +
diff --git a/ui/src/style.css b/ui/src/style.css index 4549b15..3c69c9c 100644 --- a/ui/src/style.css +++ b/ui/src/style.css @@ -131,6 +131,52 @@ color: var(--muted-foreground); max-height: 24rem; overflow-y: auto; line-height: 1.7; } +/* ── Multi-turn agent view: trajectory + diff ────────────────────────── */ +.trajectory { + background: var(--muted); border: 1px solid var(--border); border-radius: var(--radius); + padding: 0.5rem 0; max-height: 28rem; overflow-y: auto; + font-family: 'Geist Mono', ui-monospace, monospace; font-size: 0.72rem; +} +.traj-step { display: flex; gap: 0.6rem; padding: 0.18rem 1rem; align-items: baseline; line-height: 1.5; } +.traj-step:hover { background: color-mix(in oklch, var(--foreground) 4%, transparent); } +.traj-tool { + flex-shrink: 0; min-width: 5.5rem; color: var(--primary); font-weight: 600; + background: color-mix(in oklch, var(--primary) 12%, transparent); + border-radius: var(--radius-sm); padding: 0 0.4rem; text-align: center; +} +.traj-tool-mem { color: #c084fc !important; background: rgba(192,132,252,0.14) !important; } +.traj-arg { color: var(--muted-foreground); word-break: break-all; } +.traj-say { color: var(--foreground); opacity: 0.78; font-style: italic; font-family: var(--font-sans, sans-serif); } +.traj-clickable { cursor: pointer; } +.traj-clickable:hover { background: color-mix(in oklch, var(--primary) 7%, transparent); } +.traj-exp { color: var(--muted-foreground); opacity: 0.6; flex-shrink: 0; } +.traj-out-prev { color: var(--muted-foreground); opacity: 0.5; word-break: break-all; } +.traj-detail { + background: color-mix(in oklch, var(--foreground) 4%, transparent); + border-left: 2px solid color-mix(in oklch, var(--primary) 40%, transparent); + margin: 0.1rem 1rem 0.3rem 1.5rem; border-radius: var(--radius-sm); padding: 0.4rem 0.7rem; +} +.traj-detail-label { + display: inline-block; font-size: 0.6rem; text-transform: uppercase; letter-spacing: 0.05em; + color: var(--muted-foreground); opacity: 0.7; margin-bottom: 0.15rem; +} +.traj-detail pre { + white-space: pre-wrap; word-break: break-word; color: var(--foreground); opacity: 0.85; + margin: 0 0 0.4rem; max-height: 18rem; overflow-y: auto; line-height: 1.5; +} + +.diff-block { + background: var(--muted); border: 1px solid var(--border); border-radius: var(--radius); + padding: 0.5rem 0; max-height: 26rem; overflow: auto; + font-family: 'Geist Mono', ui-monospace, monospace; font-size: 0.72rem; line-height: 1.6; +} +.diff-line { padding: 0 1rem; white-space: pre; } +.diff-add { background: rgba(52,211,153,0.12); color: #34d399; } +.diff-del { background: rgba(248,113,113,0.12); color: #f87171; } +.diff-hunk { color: var(--primary); opacity: 0.85; } +.diff-meta { color: var(--muted-foreground); opacity: 0.6; } +.diff-ctx { color: var(--muted-foreground); } + /* ── Cat filter rows ─────────────────────────────────────────────────── */ .cat-row { cursor: pointer; border-radius: var(--radius-sm); transition: background 0.1s; } .cat-row:hover { background: var(--muted); } From 8883823d29566b2986d6cc9f0bb9a682f183698c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 26 Jun 2026 11:18:15 +0200 Subject: [PATCH 008/142] feat(sdebench): USD cost (gemini-3.5-flash prices) + per-tool-call tokens in UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - compute_cost() per-class: $1.50/1M input, $0.15 cached, $9.00 output(+reasoning); cost_usd now live in result.json + ui_export (full ~$0.31, squashed ~$0.59 — the extra intervention ~doubles $) - per model-step in/out tokens stamped onto each trajectory step (tok_in/tok_out) - UI: run-level stat boxes (Total cost / Interventions / Tokens in-out), per-task pills (interventions, $cost, in/out tokens, turns, wall), per-tool-step in→out badge --- sdebench/harness/run.py | 42 +++++++++--- sdebench/harness/ui_export.py | 27 +++++++- ui/dist/assets/index-BJAZo-4A.js | 67 ------------------- ui/dist/assets/index-BfMo5BQ9.js | 67 +++++++++++++++++++ ...{index-C3w3lMZ8.css => index-GlgNLyJV.css} | 2 +- ui/dist/index.html | 4 +- ui/src/pages/RunDetail.vue | 30 ++++++++- ui/src/style.css | 3 +- 8 files changed, 154 insertions(+), 88 deletions(-) delete mode 100644 ui/dist/assets/index-BJAZo-4A.js create mode 100644 ui/dist/assets/index-BfMo5BQ9.js rename ui/dist/assets/{index-C3w3lMZ8.css => index-GlgNLyJV.css} (61%) diff --git a/sdebench/harness/run.py b/sdebench/harness/run.py index 1fae765..2ab888c 100644 --- a/sdebench/harness/run.py +++ b/sdebench/harness/run.py @@ -18,6 +18,21 @@ REPO_ROOT = SDEBENCH.parent IMAGE = "sdebench-base" +# $ per 1M tokens, per class (update when model prices change). gemini-3.5-flash (Jun 2026): +# $1.50 input / $9.00 output, cached input 90% off ($0.15). reasoning bills as output. +PRICES = { + "google/gemini-3.5-flash": {"input": 1.50, "cache_read": 0.15, "cache_write": 1.50, "output": 9.00}, +} + + +def compute_cost(model: str, tok: dict) -> float: + p = PRICES.get(model) + if not p: + return 0.0 + return round((tok["input"] * p["input"] + tok["cache_read"] * p["cache_read"] + + tok["cache_write"] * p["cache_write"] + + (tok["output"] + tok["reasoning"]) * p["output"]) / 1_000_000, 4) + PROMPT = """\ You are a maintainer of the `{repo}` Python project. A regression was reported: @@ -98,7 +113,8 @@ def run_agent(workdir: Path, model: str, timeout: int, message: str, resume: boo # Token split kept separate (cached vs input vs output) — $ is computed later per model. tok = {"input": 0, "output": 0, "reasoning": 0, "cache_read": 0, "cache_write": 0} turns = 0 - traj = [] # structured trajectory for the UI: tool steps + assistant text + traj = [] # structured trajectory for the UI: tool steps + assistant text + seg_start = 0 # index where the current model-step's steps begin (for token stamping) for line in proc.stdout.splitlines(): line = line.strip() if not line: @@ -131,13 +147,22 @@ def run_agent(workdir: Path, model: str, timeout: int, message: str, resume: boo traj.append({"k": "say", "text": txt[:1500]}) elif t == "step_finish": tk = part.get("tokens", {}) or {} - tok["input"] += tk.get("input", 0) or 0 - tok["output"] += tk.get("output", 0) or 0 - tok["reasoning"] += tk.get("reasoning", 0) or 0 + s_in = (tk.get("input", 0) or 0) + s_out = (tk.get("output", 0) or 0) + (tk.get("reasoning", 0) or 0) + s_cache = 0 cache = tk.get("cache", {}) if isinstance(cache, dict): - tok["cache_read"] += cache.get("read", 0) or 0 + s_cache = cache.get("read", 0) or 0 + tok["cache_read"] += s_cache tok["cache_write"] += cache.get("write", 0) or 0 + tok["input"] += tk.get("input", 0) or 0 + tok["output"] += tk.get("output", 0) or 0 + tok["reasoning"] += tk.get("reasoning", 0) or 0 + # stamp this model-step's tokens onto the trajectory steps it produced + for s in traj[seg_start:]: + s["tok_in"] = s_in + s_cache + s["tok_out"] = s_out + seg_start = len(traj) return {"elapsed": elapsed, "tokens": tok, "turns": turns, "trajectory": traj} @@ -197,8 +222,6 @@ def main(): ap.add_argument("--history", choices=["full", "squashed"], default="full") ap.add_argument("--model", default="google/gemini-3.5-flash") ap.add_argument("--timeout", type=int, default=900) - ap.add_argument("--price-in", type=float, default=0.0, help="$ per 1M input tokens") - ap.add_argument("--price-out", type=float, default=0.0, help="$ per 1M output tokens") ap.add_argument("--run-id", default="r1") ap.add_argument("--max-interventions", type=int, default=5, help="cap on feedback rounds before giving up (drift guard)") @@ -241,10 +264,7 @@ def acc(m, role, prompt_text): acc(run_agent(repo, args.model, args.timeout, fb, resume=True), f"intervention-{interventions}", fb) solved = g["resolved"] - # $ is computed LATER per model; here we keep the token split. Optional convenience cost if priced: - billable_in = totals["input"] + totals["cache_read"] + totals["cache_write"] - billable_out = totals["output"] + totals["reasoning"] - cost = (billable_in * args.price_in + billable_out * args.price_out) / 1_000_000 + cost = compute_cost(args.model, {k: totals[k] for k in TOK}) result = { "task_id": task["task_id"], "history": args.history, "model": args.model, "solved": solved, "interventions": interventions, diff --git a/sdebench/harness/ui_export.py b/sdebench/harness/ui_export.py index ef14ccc..4edb56a 100644 --- a/sdebench/harness/ui_export.py +++ b/sdebench/harness/ui_export.py @@ -18,6 +18,20 @@ RUN_DIR = Path("/tmp/sdebench/run") +PRICES = { # $ per 1M tokens (gemini-3.5-flash, Jun 2026) + "google/gemini-3.5-flash": {"input": 1.50, "cache_read": 0.15, "cache_write": 1.50, "output": 9.00}, +} + + +def compute_cost(model, tok): + p = PRICES.get(model) + if not p: + return 0.0 + return round((tok.get("input", 0) * p["input"] + tok.get("cache_read", 0) * p["cache_read"] + + tok.get("cache_write", 0) * p["cache_write"] + + (tok.get("output", 0) + tok.get("reasoning", 0)) * p["output"]) / 1_000_000, 4) + + def _safe(s): return re.sub(r"[^A-Za-z0-9.+_-]", "_", s) @@ -35,6 +49,7 @@ def flatten_trace(trace): def to_query_result(t, key): tok = t.get("tokens", {}) + cost = compute_cost(t.get("model"), tok) return { "query_id": key, "query": t.get("bug_report", ""), @@ -48,8 +63,7 @@ def to_query_result(t, key): "score": None, "meta": {"interventions": t.get("interventions"), "history": t.get("history"), "tokens": tok, "wall_s": t.get("wall_s"), "turns": t.get("turns"), - "input_tok": tok.get("input"), "output_tok": tok.get("output"), - "cache_read": tok.get("cache_read")}, + "cost_usd": cost}, "raw_response": None, "category_axes": {}, } @@ -70,6 +84,10 @@ def main(): results = [to_query_result(t, key) for key, t in items] correct = sum(1 for r in results if r["correct"]) avg_interv = round(sum((r["meta"]["interventions"] or 0) for r in results) / len(results), 2) + tot_cost = round(sum(r["meta"]["cost_usd"] for r in results), 4) + avg_cost = round(tot_cost / len(results), 4) + sum_tok = {k: sum(r["meta"]["tokens"].get(k, 0) for r in results) + for k in ("input", "output", "reasoning", "cache_read", "cache_write")} summary = { "view": "agent", "dataset": "sdebench", "split": "all", "category": None, @@ -77,9 +95,12 @@ def main(): "total_queries": len(results), "correct": correct, "accuracy": correct / len(results) if results else 0.0, "ingestion_time_ms": 0.0, "ingested_docs": 0, - "description": f"sdebench — {history} history — mean interventions {avg_interv}", + "description": f"sdebench — {history} history — mean interv {avg_interv}, avg ${avg_cost}/task", "answer_llm": model, "judge_llm": "execution (FAIL_TO_PASS+PASS_TO_PASS+HIDDEN)", "avg_retrieve_time_ms": None, "avg_context_tokens": None, + # sdebench cost/speed roll-up (shown in the UI) + "sde_mean_interventions": avg_interv, "sde_total_cost_usd": tot_cost, + "sde_avg_cost_usd": avg_cost, "sde_tokens": sum_tok, "results": results, } dest = OUT / _safe(run_name) / "agent" / "all.json" diff --git a/ui/dist/assets/index-BJAZo-4A.js b/ui/dist/assets/index-BJAZo-4A.js deleted file mode 100644 index 5ba3e98..0000000 --- a/ui/dist/assets/index-BJAZo-4A.js +++ /dev/null @@ -1,67 +0,0 @@ -var bc=Object.defineProperty;var yc=(e,t,o)=>t in e?bc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[t]=o;var xn=(e,t,o)=>yc(e,typeof t!="symbol"?t+"":t,o);(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const r of document.querySelectorAll('link[rel="modulepreload"]'))n(r);new MutationObserver(r=>{for(const s of r)if(s.type==="childList")for(const i of s.addedNodes)i.tagName==="LINK"&&i.rel==="modulepreload"&&n(i)}).observe(document,{childList:!0,subtree:!0});function o(r){const s={};return r.integrity&&(s.integrity=r.integrity),r.referrerPolicy&&(s.referrerPolicy=r.referrerPolicy),r.crossOrigin==="use-credentials"?s.credentials="include":r.crossOrigin==="anonymous"?s.credentials="omit":s.credentials="same-origin",s}function n(r){if(r.ep)return;r.ep=!0;const s=o(r);fetch(r.href,s)}})();/** -* @vue/shared v3.5.30 -* (c) 2018-present Yuxi (Evan) You and Vue contributors -* @license MIT -**/function $t(e){const t=Object.create(null);for(const o of e.split(","))t[o]=1;return o=>o in t}const De=Object.freeze({}),An=Object.freeze([]),Ze=()=>{},na=()=>!1,fo=e=>e.charCodeAt(0)===111&&e.charCodeAt(1)===110&&(e.charCodeAt(2)>122||e.charCodeAt(2)<97),Po=e=>e.startsWith("onUpdate:"),Ge=Object.assign,ts=(e,t)=>{const o=e.indexOf(t);o>-1&&e.splice(o,1)},vc=Object.prototype.hasOwnProperty,Ee=(e,t)=>vc.call(e,t),ue=Array.isArray,fn=e=>po(e)==="[object Map]",oa=e=>po(e)==="[object Set]",Ns=e=>po(e)==="[object Date]",fe=e=>typeof e=="function",Be=e=>typeof e=="string",gt=e=>typeof e=="symbol",Ae=e=>e!==null&&typeof e=="object",ns=e=>(Ae(e)||fe(e))&&fe(e.then)&&fe(e.catch),ra=Object.prototype.toString,po=e=>ra.call(e),os=e=>po(e).slice(8,-1),sa=e=>po(e)==="[object Object]",rs=e=>Be(e)&&e!=="NaN"&&e[0]!=="-"&&""+parseInt(e,10)===e,Zn=$t(",key,ref,ref_for,ref_key,onVnodeBeforeMount,onVnodeMounted,onVnodeBeforeUpdate,onVnodeUpdated,onVnodeBeforeUnmount,onVnodeUnmounted"),_c=$t("bind,cloak,else-if,else,for,html,if,model,on,once,pre,show,slot,text,memo"),Xo=e=>{const t=Object.create(null);return o=>t[o]||(t[o]=e(o))},xc=/-\w/g,Ye=Xo(e=>e.replace(xc,t=>t.slice(1).toUpperCase())),wc=/\B([A-Z])/g,tn=Xo(e=>e.replace(wc,"-$1").toLowerCase()),bn=Xo(e=>e.charAt(0).toUpperCase()+e.slice(1)),cn=Xo(e=>e?`on${bn(e)}`:""),At=(e,t)=>!Object.is(e,t),Fn=(e,...t)=>{for(let o=0;o{Object.defineProperty(e,t,{configurable:!0,enumerable:!1,writable:n,value:o})},kc=e=>{const t=parseFloat(e);return isNaN(t)?e:t};let Hs;const mo=()=>Hs||(Hs=typeof globalThis<"u"?globalThis:typeof self<"u"?self:typeof window<"u"?window:typeof global<"u"?global:{});function In(e){if(ue(e)){const t={};for(let o=0;o{if(o){const n=o.split(Sc);n.length>1&&(t[n[0].trim()]=n[1].trim())}}),t}function Oe(e){let t="";if(Be(e))t=e;else if(ue(e))for(let o=0;o!!(e&&e.__v_isRef===!0),S=e=>Be(e)?e:e==null?"":ue(e)||Ae(e)&&(e.toString===ra||!fe(e.toString))?aa(e)?S(e.value):JSON.stringify(e,la,2):String(e),la=(e,t)=>aa(t)?la(e,t.value):fn(t)?{[`Map(${t.size})`]:[...t.entries()].reduce((o,[n,r],s)=>(o[ur(n,s)+" =>"]=r,o),{})}:oa(t)?{[`Set(${t.size})`]:[...t.values()].map(o=>ur(o))}:gt(t)?ur(t):Ae(t)&&!ue(t)&&!sa(t)?String(t):t,ur=(e,t="")=>{var o;return gt(e)?`Symbol(${(o=e.description)!=null?o:t})`:e};/** -* @vue/reactivity v3.5.30 -* (c) 2018-present Yuxi (Evan) You and Vue contributors -* @license MIT -**/function bt(e,...t){console.warn(`[Vue warn] ${e}`,...t)}let it;class Nc{constructor(t=!1){this.detached=t,this._active=!0,this._on=0,this.effects=[],this.cleanups=[],this._isPaused=!1,this.__v_skip=!0,this.parent=it,!t&&it&&(this.index=(it.scopes||(it.scopes=[])).push(this)-1)}get active(){return this._active}pause(){if(this._active){this._isPaused=!0;let t,o;if(this.scopes)for(t=0,o=this.scopes.length;t0&&--this._on===0&&(it=this.prevScope,this.prevScope=void 0)}stop(t){if(this._active){this._active=!1;let o,n;for(o=0,n=this.effects.length;o0)return;if(to){let t=to;for(to=void 0;t;){const o=t.next;t.next=void 0,t.flags&=-9,t=o}}let e;for(;eo;){let t=eo;for(eo=void 0;t;){const o=t.next;if(t.next=void 0,t.flags&=-9,t.flags&1)try{t.trigger()}catch(n){e||(e=n)}t=o}}if(e)throw e}function fa(e){for(let t=e.deps;t;t=t.nextDep)t.version=-1,t.prevActiveLink=t.dep.activeLink,t.dep.activeLink=t}function pa(e){let t,o=e.depsTail,n=o;for(;n;){const r=n.prevDep;n.version===-1?(n===o&&(o=r),ls(n),Uc(n)):t=n,n.dep.activeLink=n.prevActiveLink,n.prevActiveLink=void 0,n=r}e.deps=t,e.depsTail=o}function Er(e){for(let t=e.deps;t;t=t.nextDep)if(t.dep.version!==t.version||t.dep.computed&&(ma(t.dep.computed)||t.dep.version!==t.version))return!0;return!!e._dirty}function ma(e){if(e.flags&4&&!(e.flags&16)||(e.flags&=-17,e.globalVersion===io)||(e.globalVersion=io,!e.isSSR&&e.flags&128&&(!e.deps&&!e._dirty||!Er(e))))return;e.flags|=2;const t=e.dep,o=Le,n=ht;Le=e,ht=!0;try{fa(e);const r=e.fn(e._value);(t.version===0||At(r,e._value))&&(e.flags|=128,e._value=r,t.version++)}catch(r){throw t.version++,r}finally{Le=o,ht=n,pa(e),e.flags&=-3}}function ls(e,t=!1){const{dep:o,prevSub:n,nextSub:r}=e;if(n&&(n.nextSub=r,e.prevSub=void 0),r&&(r.prevSub=n,e.nextSub=void 0),o.subsHead===e&&(o.subsHead=r),o.subs===e&&(o.subs=n,!n&&o.computed)){o.computed.flags&=-5;for(let s=o.computed.deps;s;s=s.nextDep)ls(s,!0)}!t&&!--o.sc&&o.map&&o.map.delete(o.key)}function Uc(e){const{prevDep:t,nextDep:o}=e;t&&(t.nextDep=o,e.prevDep=void 0),o&&(o.prevDep=t,e.nextDep=void 0)}let ht=!0;const ha=[];function yt(){ha.push(ht),ht=!1}function vt(){const e=ha.pop();ht=e===void 0?!0:e}function Us(e){const{cleanup:t}=e;if(e.cleanup=void 0,t){const o=Le;Le=void 0;try{t()}finally{Le=o}}}let io=0;class Bc{constructor(t,o){this.sub=t,this.dep=o,this.version=o.version,this.nextDep=this.prevDep=this.nextSub=this.prevSub=this.prevActiveLink=void 0}}class cs{constructor(t){this.computed=t,this.version=0,this.activeLink=void 0,this.subs=void 0,this.map=void 0,this.key=void 0,this.sc=0,this.__v_skip=!0,this.subsHead=void 0}track(t){if(!Le||!ht||Le===this.computed)return;let o=this.activeLink;if(o===void 0||o.sub!==Le)o=this.activeLink=new Bc(Le,this),Le.deps?(o.prevDep=Le.depsTail,Le.depsTail.nextDep=o,Le.depsTail=o):Le.deps=Le.depsTail=o,ga(o);else if(o.version===-1&&(o.version=this.version,o.nextDep)){const n=o.nextDep;n.prevDep=o.prevDep,o.prevDep&&(o.prevDep.nextDep=n),o.prevDep=Le.depsTail,o.nextDep=void 0,Le.depsTail.nextDep=o,Le.depsTail=o,Le.deps===o&&(Le.deps=n)}return Le.onTrack&&Le.onTrack(Ge({effect:Le},t)),o}trigger(t){this.version++,io++,this.notify(t)}notify(t){is();try{for(let o=this.subsHead;o;o=o.nextSub)o.sub.onTrigger&&!(o.sub.flags&8)&&o.sub.onTrigger(Ge({effect:o.sub},t));for(let o=this.subs;o;o=o.prevSub)o.sub.notify()&&o.sub.dep.notify()}finally{as()}}}function ga(e){if(e.dep.sc++,e.sub.flags&4){const t=e.dep.computed;if(t&&!e.dep.subs){t.flags|=20;for(let n=t.deps;n;n=n.nextDep)ga(n)}const o=e.dep.subs;o!==e&&(e.prevSub=o,o&&(o.nextSub=e)),e.dep.subsHead===void 0&&(e.dep.subsHead=e),e.dep.subs=e}}const Ar=new WeakMap,pn=Symbol("Object iterate"),Rr=Symbol("Map keys iterate"),ao=Symbol("Array iterate");function Je(e,t,o){if(ht&&Le){let n=Ar.get(e);n||Ar.set(e,n=new Map);let r=n.get(o);r||(n.set(o,r=new cs),r.map=n,r.key=o),r.track({target:e,type:t,key:o})}}function Rt(e,t,o,n,r,s){const i=Ar.get(e);if(!i){io++;return}const a=l=>{l&&l.trigger({target:e,type:t,key:o,newValue:n,oldValue:r,oldTarget:s})};if(is(),t==="clear")i.forEach(a);else{const l=ue(e),f=l&&rs(o);if(l&&o==="length"){const c=Number(n);i.forEach((u,m)=>{(m==="length"||m===ao||!gt(m)&&m>=c)&&a(u)})}else switch((o!==void 0||i.has(void 0))&&a(i.get(o)),f&&a(i.get(ao)),t){case"add":l?f&&a(i.get("length")):(a(i.get(pn)),fn(e)&&a(i.get(Rr)));break;case"delete":l||(a(i.get(pn)),fn(e)&&a(i.get(Rr)));break;case"set":fn(e)&&a(i.get(pn));break}}as()}function wn(e){const t=ve(e);return t===e?t:(Je(t,"iterate",ao),rt(e)?t:t.map(xt))}function Zo(e){return Je(e=ve(e),"iterate",ao),e}function Et(e,t){return _t(e)?Pn(en(e)?xt(t):t):xt(t)}const Vc={__proto__:null,[Symbol.iterator](){return fr(this,Symbol.iterator,e=>Et(this,e))},concat(...e){return wn(this).concat(...e.map(t=>ue(t)?wn(t):t))},entries(){return fr(this,"entries",e=>(e[1]=Et(this,e[1]),e))},every(e,t){return Nt(this,"every",e,t,void 0,arguments)},filter(e,t){return Nt(this,"filter",e,t,o=>o.map(n=>Et(this,n)),arguments)},find(e,t){return Nt(this,"find",e,t,o=>Et(this,o),arguments)},findIndex(e,t){return Nt(this,"findIndex",e,t,void 0,arguments)},findLast(e,t){return Nt(this,"findLast",e,t,o=>Et(this,o),arguments)},findLastIndex(e,t){return Nt(this,"findLastIndex",e,t,void 0,arguments)},forEach(e,t){return Nt(this,"forEach",e,t,void 0,arguments)},includes(...e){return pr(this,"includes",e)},indexOf(...e){return pr(this,"indexOf",e)},join(e){return wn(this).join(e)},lastIndexOf(...e){return pr(this,"lastIndexOf",e)},map(e,t){return Nt(this,"map",e,t,void 0,arguments)},pop(){return Gn(this,"pop")},push(...e){return Gn(this,"push",e)},reduce(e,...t){return Bs(this,"reduce",e,t)},reduceRight(e,...t){return Bs(this,"reduceRight",e,t)},shift(){return Gn(this,"shift")},some(e,t){return Nt(this,"some",e,t,void 0,arguments)},splice(...e){return Gn(this,"splice",e)},toReversed(){return wn(this).toReversed()},toSorted(e){return wn(this).toSorted(e)},toSpliced(...e){return wn(this).toSpliced(...e)},unshift(...e){return Gn(this,"unshift",e)},values(){return fr(this,"values",e=>Et(this,e))}};function fr(e,t,o){const n=Zo(e),r=n[t]();return n!==e&&!rt(e)&&(r._next=r.next,r.next=()=>{const s=r._next();return s.done||(s.value=o(s.value)),s}),r}const Fc=Array.prototype;function Nt(e,t,o,n,r,s){const i=Zo(e),a=i!==e&&!rt(e),l=i[t];if(l!==Fc[t]){const u=l.apply(e,s);return a?xt(u):u}let f=o;i!==e&&(a?f=function(u,m){return o.call(this,Et(e,u),m,e)}:o.length>2&&(f=function(u,m){return o.call(this,u,m,e)}));const c=l.call(i,f,n);return a&&r?r(c):c}function Bs(e,t,o,n){const r=Zo(e),s=r!==e&&!rt(e);let i=o,a=!1;r!==e&&(s?(a=n.length===0,i=function(f,c,u){return a&&(a=!1,f=Et(e,f)),o.call(this,f,Et(e,c),u,e)}):o.length>3&&(i=function(f,c,u){return o.call(this,f,c,u,e)}));const l=r[t](i,...n);return a?Et(e,l):l}function pr(e,t,o){const n=ve(e);Je(n,"iterate",ao);const r=n[t](...o);return(r===-1||r===!1)&&jo(o[0])?(o[0]=ve(o[0]),n[t](...o)):r}function Gn(e,t,o=[]){yt(),is();const n=ve(e)[t].apply(e,o);return as(),vt(),n}const Gc=$t("__proto__,__v_isRef,__isVue"),ba=new Set(Object.getOwnPropertyNames(Symbol).filter(e=>e!=="arguments"&&e!=="caller").map(e=>Symbol[e]).filter(gt));function Wc(e){gt(e)||(e=String(e));const t=ve(this);return Je(t,"has",e),t.hasOwnProperty(e)}class ya{constructor(t=!1,o=!1){this._isReadonly=t,this._isShallow=o}get(t,o,n){if(o==="__v_skip")return t.__v_skip;const r=this._isReadonly,s=this._isShallow;if(o==="__v_isReactive")return!r;if(o==="__v_isReadonly")return r;if(o==="__v_isShallow")return s;if(o==="__v_raw")return n===(r?s?Ca:ka:s?wa:xa).get(t)||Object.getPrototypeOf(t)===Object.getPrototypeOf(n)?t:void 0;const i=ue(t);if(!r){let l;if(i&&(l=Vc[o]))return l;if(o==="hasOwnProperty")return Wc}const a=Reflect.get(t,o,Fe(t)?t:n);if((gt(o)?ba.has(o):Gc(o))||(r||Je(t,"get",o),s))return a;if(Fe(a)){const l=i&&rs(o)?a:a.value;return r&&Ae(l)?Ir(l):l}return Ae(a)?r?Ir(a):tr(a):a}}class va extends ya{constructor(t=!1){super(!1,t)}set(t,o,n,r){let s=t[o];const i=ue(t)&&rs(o);if(!this._isShallow){const f=_t(s);if(!rt(n)&&!_t(n)&&(s=ve(s),n=ve(n)),!i&&Fe(s)&&!Fe(n))return f?(bt(`Set operation on key "${String(o)}" failed: target is readonly.`,t[o]),!0):(s.value=n,!0)}const a=i?Number(o)e,_o=e=>Reflect.getPrototypeOf(e);function Qc(e,t,o){return function(...n){const r=this.__v_raw,s=ve(r),i=fn(s),a=e==="entries"||e===Symbol.iterator&&i,l=e==="keys"&&i,f=r[e](...n),c=o?Or:t?Pn:xt;return!t&&Je(s,"iterate",l?Rr:pn),Ge(Object.create(f),{next(){const{value:u,done:m}=f.next();return m?{value:u,done:m}:{value:a?[c(u[0]),c(u[1])]:c(u),done:m}}})}}function xo(e){return function(...t){{const o=t[0]?`on key "${t[0]}" `:"";bt(`${bn(e)} operation ${o}failed: target is readonly.`,ve(this))}return e==="delete"?!1:e==="clear"?void 0:this}}function Jc(e,t){const o={get(r){const s=this.__v_raw,i=ve(s),a=ve(r);e||(At(r,a)&&Je(i,"get",r),Je(i,"get",a));const{has:l}=_o(i),f=t?Or:e?Pn:xt;if(l.call(i,r))return f(s.get(r));if(l.call(i,a))return f(s.get(a));s!==i&&s.get(r)},get size(){const r=this.__v_raw;return!e&&Je(ve(r),"iterate",pn),r.size},has(r){const s=this.__v_raw,i=ve(s),a=ve(r);return e||(At(r,a)&&Je(i,"has",r),Je(i,"has",a)),r===a?s.has(r):s.has(r)||s.has(a)},forEach(r,s){const i=this,a=i.__v_raw,l=ve(a),f=t?Or:e?Pn:xt;return!e&&Je(l,"iterate",pn),a.forEach((c,u)=>r.call(s,f(c),f(u),i))}};return Ge(o,e?{add:xo("add"),set:xo("set"),delete:xo("delete"),clear:xo("clear")}:{add(r){const s=ve(this),i=_o(s),a=ve(r),l=!t&&!rt(r)&&!_t(r)?a:r;return i.has.call(s,l)||At(r,l)&&i.has.call(s,r)||At(a,l)&&i.has.call(s,a)||(s.add(l),Rt(s,"add",l,l)),this},set(r,s){!t&&!rt(s)&&!_t(s)&&(s=ve(s));const i=ve(this),{has:a,get:l}=_o(i);let f=a.call(i,r);f?Vs(i,a,r):(r=ve(r),f=a.call(i,r));const c=l.call(i,r);return i.set(r,s),f?At(s,c)&&Rt(i,"set",r,s,c):Rt(i,"add",r,s),this},delete(r){const s=ve(this),{has:i,get:a}=_o(s);let l=i.call(s,r);l?Vs(s,i,r):(r=ve(r),l=i.call(s,r));const f=a?a.call(s,r):void 0,c=s.delete(r);return l&&Rt(s,"delete",r,void 0,f),c},clear(){const r=ve(this),s=r.size!==0,i=fn(r)?new Map(r):new Set(r),a=r.clear();return s&&Rt(r,"clear",void 0,void 0,i),a}}),["keys","values","entries",Symbol.iterator].forEach(r=>{o[r]=Qc(r,e,t)}),o}function er(e,t){const o=Jc(e,t);return(n,r,s)=>r==="__v_isReactive"?!e:r==="__v_isReadonly"?e:r==="__v_raw"?n:Reflect.get(Ee(o,r)&&r in n?o:n,r,s)}const Yc={get:er(!1,!1)},Xc={get:er(!1,!0)},Zc={get:er(!0,!1)},eu={get:er(!0,!0)};function Vs(e,t,o){const n=ve(o);if(n!==o&&t.call(e,n)){const r=os(e);bt(`Reactive ${r} contains both the raw and reactive versions of the same object${r==="Map"?" as keys":""}, which can lead to inconsistencies. Avoid differentiating between the raw and reactive versions of an object and only use the reactive version if possible.`)}}const xa=new WeakMap,wa=new WeakMap,ka=new WeakMap,Ca=new WeakMap;function tu(e){switch(e){case"Object":case"Array":return 1;case"Map":case"Set":case"WeakMap":case"WeakSet":return 2;default:return 0}}function nu(e){return e.__v_skip||!Object.isExtensible(e)?0:tu(os(e))}function tr(e){return _t(e)?e:nr(e,!1,Kc,Yc,xa)}function Sa(e){return nr(e,!1,zc,Xc,wa)}function Ir(e){return nr(e,!0,$c,Zc,ka)}function It(e){return nr(e,!0,qc,eu,Ca)}function nr(e,t,o,n,r){if(!Ae(e))return bt(`value cannot be made ${t?"readonly":"reactive"}: ${String(e)}`),e;if(e.__v_raw&&!(t&&e.__v_isReactive))return e;const s=nu(e);if(s===0)return e;const i=r.get(e);if(i)return i;const a=new Proxy(e,s===2?n:o);return r.set(e,a),a}function en(e){return _t(e)?en(e.__v_raw):!!(e&&e.__v_isReactive)}function _t(e){return!!(e&&e.__v_isReadonly)}function rt(e){return!!(e&&e.__v_isShallow)}function jo(e){return e?!!e.__v_raw:!1}function ve(e){const t=e&&e.__v_raw;return t?ve(t):e}function ou(e){return!Ee(e,"__v_skip")&&Object.isExtensible(e)&&Mo(e,"__v_skip",!0),e}const xt=e=>Ae(e)?tr(e):e,Pn=e=>Ae(e)?Ir(e):e;function Fe(e){return e?e.__v_isRef===!0:!1}function ne(e){return Ta(e,!1)}function ru(e){return Ta(e,!0)}function Ta(e,t){return Fe(e)?e:new su(e,t)}class su{constructor(t,o){this.dep=new cs,this.__v_isRef=!0,this.__v_isShallow=!1,this._rawValue=o?t:ve(t),this._value=o?t:xt(t),this.__v_isShallow=o}get value(){return this.dep.track({target:this,type:"get",key:"value"}),this._value}set value(t){const o=this._rawValue,n=this.__v_isShallow||rt(t)||_t(t);t=n?t:ve(t),At(t,o)&&(this._rawValue=t,this._value=n?t:xt(t),this.dep.trigger({target:this,type:"set",key:"value",newValue:t,oldValue:o}))}}function Zt(e){return Fe(e)?e.value:e}function iu(e){return fe(e)?e():Zt(e)}const au={get:(e,t,o)=>t==="__v_raw"?e:Zt(Reflect.get(e,t,o)),set:(e,t,o,n)=>{const r=e[t];return Fe(r)&&!Fe(o)?(r.value=o,!0):Reflect.set(e,t,o,n)}};function Ea(e){return en(e)?e:new Proxy(e,au)}class lu{constructor(t,o,n){this.fn=t,this.setter=o,this._value=void 0,this.dep=new cs(this),this.__v_isRef=!0,this.deps=void 0,this.depsTail=void 0,this.flags=16,this.globalVersion=io-1,this.next=void 0,this.effect=this,this.__v_isReadonly=!o,this.isSSR=n}notify(){if(this.flags|=16,!(this.flags&8)&&Le!==this)return da(this,!0),!0}get value(){const t=this.dep.track({target:this,type:"get",key:"value"});return ma(this),t&&(t.version=this.dep.version),this._value}set value(t){this.setter?this.setter(t):bt("Write operation failed: computed value is readonly")}}function cu(e,t,o=!1){let n,r;fe(e)?n=e:(n=e.get,r=e.set);const s=new lu(n,r,o);return t&&!o&&(s.onTrack=t.onTrack,s.onTrigger=t.onTrigger),s}const wo={},Lo=new WeakMap;let un;function uu(e,t=!1,o=un){if(o){let n=Lo.get(o);n||Lo.set(o,n=[]),n.push(e)}else t||bt("onWatcherCleanup() was called when there was no active watcher to associate with.")}function du(e,t,o=De){const{immediate:n,deep:r,once:s,scheduler:i,augmentJob:a,call:l}=o,f=V=>{(o.onWarn||bt)("Invalid watch source: ",V,"A watch source can only be a getter/effect function, a ref, a reactive object, or an array of these types.")},c=V=>r?V:rt(V)||r===!1||r===0?Wt(V,1):Wt(V);let u,m,g,x,h=!1,k=!1;if(Fe(e)?(m=()=>e.value,h=rt(e)):en(e)?(m=()=>c(e),h=!0):ue(e)?(k=!0,h=e.some(V=>en(V)||rt(V)),m=()=>e.map(V=>{if(Fe(V))return V.value;if(en(V))return c(V);if(fe(V))return l?l(V,2):V();f(V)})):fe(e)?t?m=l?()=>l(e,2):e:m=()=>{if(g){yt();try{g()}finally{vt()}}const V=un;un=u;try{return l?l(e,3,[x]):e(x)}finally{un=V}}:(m=Ze,f(e)),t&&r){const V=m,ce=r===!0?1/0:r;m=()=>Wt(V(),ce)}const U=Hc(),P=()=>{u.stop(),U&&U.active&&ts(U.effects,u)};if(s&&t){const V=t;t=(...ce)=>{V(...ce),P()}}let w=k?new Array(e.length).fill(wo):wo;const K=V=>{if(!(!(u.flags&1)||!u.dirty&&!V))if(t){const ce=u.run();if(r||h||(k?ce.some((F,H)=>At(F,w[H])):At(ce,w))){g&&g();const F=un;un=u;try{const H=[ce,w===wo?void 0:k&&w[0]===wo?[]:w,x];w=ce,l?l(t,3,H):t(...H)}finally{un=F}}}else u.run()};return a&&a(K),u=new ca(m),u.scheduler=i?()=>i(K,!1):K,x=V=>uu(V,!1,u),g=u.onStop=()=>{const V=Lo.get(u);if(V){if(l)l(V,4);else for(const ce of V)ce();Lo.delete(u)}},u.onTrack=o.onTrack,u.onTrigger=o.onTrigger,t?n?K(!0):w=u.run():i?i(K.bind(null,!0),!0):u.run(),P.pause=u.pause.bind(u),P.resume=u.resume.bind(u),P.stop=P,P}function Wt(e,t=1/0,o){if(t<=0||!Ae(e)||e.__v_skip||(o=o||new Map,(o.get(e)||0)>=t))return e;if(o.set(e,t),t--,Fe(e))Wt(e.value,t,o);else if(ue(e))for(let n=0;n{Wt(n,t,o)});else if(sa(e)){for(const n in e)Wt(e[n],t,o);for(const n of Object.getOwnPropertySymbols(e))Object.prototype.propertyIsEnumerable.call(e,n)&&Wt(e[n],t,o)}return e}/** -* @vue/runtime-core v3.5.30 -* (c) 2018-present Yuxi (Evan) You and Vue contributors -* @license MIT -**/const mn=[];function To(e){mn.push(e)}function Eo(){mn.pop()}let mr=!1;function J(e,...t){if(mr)return;mr=!0,yt();const o=mn.length?mn[mn.length-1].component:null,n=o&&o.appContext.config.warnHandler,r=fu();if(n)Nn(n,o,11,[e+t.map(s=>{var i,a;return(a=(i=s.toString)==null?void 0:i.call(s))!=null?a:JSON.stringify(s)}).join(""),o&&o.proxy,r.map(({vnode:s})=>`at <${vo(o,s.type)}>`).join(` -`),r]);else{const s=[`[Vue warn]: ${e}`,...t];r.length&&s.push(` -`,...pu(r)),console.warn(...s)}vt(),mr=!1}function fu(){let e=mn[mn.length-1];if(!e)return[];const t=[];for(;e;){const o=t[0];o&&o.vnode===e?o.recurseCount++:t.push({vnode:e,recurseCount:0});const n=e.component&&e.component.parent;e=n&&n.vnode}return t}function pu(e){const t=[];return e.forEach((o,n)=>{t.push(...n===0?[]:[` -`],...mu(o))}),t}function mu({vnode:e,recurseCount:t}){const o=t>0?`... (${t} recursive calls)`:"",n=e.component?e.component.parent==null:!1,r=` at <${vo(e.component,e.type,n)}`,s=">"+o;return e.props?[r,...hu(e.props),s]:[r+s]}function hu(e){const t=[],o=Object.keys(e);return o.slice(0,3).forEach(n=>{t.push(...Aa(n,e[n]))}),o.length>3&&t.push(" ..."),t}function Aa(e,t,o){return Be(t)?(t=JSON.stringify(t),o?t:[`${e}=${t}`]):typeof t=="number"||typeof t=="boolean"||t==null?o?t:[`${e}=${t}`]:Fe(t)?(t=Aa(e,ve(t.value),!0),o?t:[`${e}=Ref<`,t,">"]):fe(t)?[`${e}=fn${t.name?`<${t.name}>`:""}`]:(t=ve(t),o?t:[`${e}=`,t])}const us={sp:"serverPrefetch hook",bc:"beforeCreate hook",c:"created hook",bm:"beforeMount hook",m:"mounted hook",bu:"beforeUpdate hook",u:"updated",bum:"beforeUnmount hook",um:"unmounted hook",a:"activated hook",da:"deactivated hook",ec:"errorCaptured hook",rtc:"renderTracked hook",rtg:"renderTriggered hook",0:"setup function",1:"render function",2:"watcher getter",3:"watcher callback",4:"watcher cleanup function",5:"native event handler",6:"component event handler",7:"vnode hook",8:"directive hook",9:"transition hook",10:"app errorHandler",11:"app warnHandler",12:"ref function",13:"async component loader",14:"scheduler flush",15:"component update",16:"app unmount cleanup function"};function Nn(e,t,o,n){try{return n?e(...n):e()}catch(r){ho(r,t,o)}}function Mt(e,t,o,n){if(fe(e)){const r=Nn(e,t,o,n);return r&&ns(r)&&r.catch(s=>{ho(s,t,o)}),r}if(ue(e)){const r=[];for(let s=0;s>>1,r=ot[n],s=lo(r);s=lo(o)?ot.push(e):ot.splice(yu(t),0,e),e.flags|=1,Ia()}}function Ia(){Do||(Do=Ra.then(ja))}function Pa(e){ue(e)?Rn.push(...e):Yt&&e.id===-1?Yt.splice(Tn+1,0,e):e.flags&1||(Rn.push(e),e.flags|=1),Ia()}function Fs(e,t,o=Tt+1){for(t=t||new Map;olo(o)-lo(n));if(Rn.length=0,Yt){Yt.push(...t);return}for(Yt=t,e=e||new Map,Tn=0;Tne.id==null?e.flags&2?-1:1/0:e.id;function ja(e){e=e||new Map;const t=o=>ds(e,o);try{for(Tt=0;Ttbu){const n=t.i,r=n&&xs(n.type);return ho(`Maximum recursive updates exceeded${r?` in component <${r}>`:""}. This means you have a reactive effect that is mutating its own dependencies and thus recursively triggering itself. Possible sources include component template, render function, updated hook or watcher source function.`,null,10),!0}return e.set(t,o+1),!1}let Pt=!1;const Ao=new Map;mo().__VUE_HMR_RUNTIME__={createRecord:hr(La),rerender:hr(xu),reload:hr(wu)};const yn=new Map;function vu(e){const t=e.type.__hmrId;let o=yn.get(t);o||(La(t,e.type),o=yn.get(t)),o.instances.add(e)}function _u(e){yn.get(e.type.__hmrId).instances.delete(e)}function La(e,t){return yn.has(e)?!1:(yn.set(e,{initialDef:No(t),instances:new Set}),!0)}function No(e){return xl(e)?e.__vccOpts:e}function xu(e,t){const o=yn.get(e);o&&(o.initialDef.render=t,[...o.instances].forEach(n=>{t&&(n.render=t,No(n.type).render=t),n.renderCache=[],Pt=!0,n.job.flags&8||n.update(),Pt=!1}))}function wu(e,t){const o=yn.get(e);if(!o)return;t=No(t),Gs(o.initialDef,t);const n=[...o.instances];for(let r=0;r{s.job.flags&8||(Pt=!0,s.parent.update(),Pt=!1,a.delete(s))}):s.appContext.reload?s.appContext.reload():typeof window<"u"?window.location.reload():console.warn("[HMR] Root or manually mounted instance modified. Full reload required."),s.root.ce&&s!==s.root&&s.root.ce._removeChildStyle(i)}Pa(()=>{Ao.clear()})}function Gs(e,t){Ge(e,t);for(const o in e)o!=="__file"&&!(o in t)&&delete e[o]}function hr(e){return(t,o)=>{try{return e(t,o)}catch(n){console.error(n),console.warn("[HMR] Something went wrong during Vue component hot-reload. Full reload required.")}}}let Ot,Yn=[],Pr=!1;function go(e,...t){Ot?Ot.emit(e,...t):Pr||Yn.push({event:e,args:t})}function Da(e,t){var o,n;Ot=e,Ot?(Ot.enabled=!0,Yn.forEach(({event:r,args:s})=>Ot.emit(r,...s)),Yn=[]):typeof window<"u"&&window.HTMLElement&&!((n=(o=window.navigator)==null?void 0:o.userAgent)!=null&&n.includes("jsdom"))?((t.__VUE_DEVTOOLS_HOOK_REPLAY__=t.__VUE_DEVTOOLS_HOOK_REPLAY__||[]).push(s=>{Da(s,t)}),setTimeout(()=>{Ot||(t.__VUE_DEVTOOLS_HOOK_REPLAY__=null,Pr=!0,Yn=[])},3e3)):(Pr=!0,Yn=[])}function ku(e,t){go("app:init",e,t,{Fragment:te,Text:bo,Comment:lt,Static:oo})}function Cu(e){go("app:unmount",e)}const Su=fs("component:added"),Na=fs("component:updated"),Tu=fs("component:removed"),Eu=e=>{Ot&&typeof Ot.cleanupBuffer=="function"&&!Ot.cleanupBuffer(e)&&Tu(e)};function fs(e){return t=>{go(e,t.appContext.app,t.uid,t.parent?t.parent.uid:void 0,t)}}const Au=Ha("perf:start"),Ru=Ha("perf:end");function Ha(e){return(t,o,n)=>{go(e,t.appContext.app,t.uid,t,o,n)}}function Ou(e,t,o){go("component:emit",e.appContext.app,e,t,o)}let Ke=null,Ua=null;function Ho(e){const t=Ke;return Ke=e,Ua=e&&e.type.__scopeId||null,t}function j(e,t=Ke,o){if(!t||e._n)return e;const n=(...r)=>{n._d&&Go(-1);const s=Ho(t);let i;try{i=e(...r)}finally{Ho(s),n._d&&Go(1)}return Na(t),i};return n._n=!0,n._c=!0,n._d=!0,n}function Ba(e){_c(e)&&J("Do not use built-in directive ids as custom directive id: "+e)}function En(e,t){if(Ke===null)return J("withDirectives can only be used inside render functions."),e;const o=ir(Ke),n=e.dirs||(e.dirs=[]);for(let r=0;r1)return o&&fe(t)?t.call(n&&n.proxy):t;J(`injection "${String(e)}" not found.`)}else J("inject() can only be used inside setup() or functional components.")}function Iu(){return!!(Hn()||gn)}const Pu=Symbol.for("v-scx"),Mu=()=>{{const e=ft(Pu);return e||J("Server rendering context not provided. Make sure to only call useSSRContext() conditionally in the server build."),e}};function Va(e,t){return ps(e,null,t)}function qe(e,t,o){return fe(t)||J("`watch(fn, options?)` signature has been moved to a separate API. Use `watchEffect(fn, options?)` instead. `watch` now only supports `watch(source, cb, options?) signature."),ps(e,t,o)}function ps(e,t,o=De){const{immediate:n,deep:r,flush:s,once:i}=o;t||(n!==void 0&&J('watch() "immediate" option is only respected when using the watch(source, callback, options?) signature.'),r!==void 0&&J('watch() "deep" option is only respected when using the watch(source, callback, options?) signature.'),i!==void 0&&J('watch() "once" option is only respected when using the watch(source, callback, options?) signature.'));const a=Ge({},o);a.onWarn=J;const l=t&&n||!t&&s!=="post";let f;if(uo){if(s==="sync"){const g=Mu();f=g.__watcherHandles||(g.__watcherHandles=[])}else if(!l){const g=()=>{};return g.stop=Ze,g.resume=Ze,g.pause=Ze,g}}const c=We;a.call=(g,x,h)=>Mt(g,c,x,h);let u=!1;s==="post"?a.scheduler=g=>{st(g,c&&c.suspense)}:s!=="sync"&&(u=!0,a.scheduler=(g,x)=>{x?g():or(g)}),a.augmentJob=g=>{t&&(g.flags|=4),u&&(g.flags|=2,c&&(g.id=c.uid,g.i=c))};const m=du(e,t,a);return uo&&(f?f.push(m):l&&m()),m}function ju(e,t,o){const n=this.proxy,r=Be(e)?e.includes(".")?Fa(n,e):()=>n[e]:e.bind(n,n);let s;fe(t)?s=t:(s=t.handler,o=t);const i=yo(this),a=ps(r,s.bind(n),o);return i(),a}function Fa(e,t){const o=t.split(".");return()=>{let n=e;for(let r=0;re.__isTeleport,Nu=Symbol("_leaveCb");function ms(e,t){e.shapeFlag&6&&e.component?(e.transition=t,ms(e.component.subTree,t)):e.shapeFlag&128?(e.ssContent.transition=t.clone(e.ssContent),e.ssFallback.transition=t.clone(e.ssFallback)):e.transition=t}function Ga(e,t){return fe(e)?Ge({name:e.name},t,{setup:e}):e}function Wa(e){e.ids=[e.ids[0]+e.ids[2]+++"-",0,0]}const Ws=new WeakSet;function Ks(e,t){let o;return!!((o=Object.getOwnPropertyDescriptor(e,t))&&!o.configurable)}const Uo=new WeakMap;function no(e,t,o,n,r=!1){if(ue(e)){e.forEach((h,k)=>no(h,t&&(ue(t)?t[k]:t),o,n,r));return}if(On(n)&&!r){n.shapeFlag&512&&n.type.__asyncResolved&&n.component.subTree.component&&no(e,t,o,n.component.subTree);return}const s=n.shapeFlag&4?ir(n.component):n.el,i=r?null:s,{i:a,r:l}=e;if(!a){J("Missing ref owner context. ref cannot be used on hoisted vnodes. A vnode with ref must be created inside the render function.");return}const f=t&&t.r,c=a.refs===De?a.refs={}:a.refs,u=a.setupState,m=ve(u),g=u===De?na:h=>(Ee(m,h)&&!Fe(m[h])&&J(`Template ref "${h}" used on a non-ref value. It will not work in the production build.`),Ws.has(m[h])||Ks(c,h)?!1:Ee(m,h)),x=(h,k)=>!(Ws.has(h)||k&&Ks(c,k));if(f!=null&&f!==l){if($s(t),Be(f))c[f]=null,g(f)&&(u[f]=null);else if(Fe(f)){const h=t;x(f,h.k)&&(f.value=null),h.k&&(c[h.k]=null)}}if(fe(l))Nn(l,a,12,[i,c]);else{const h=Be(l),k=Fe(l);if(h||k){const U=()=>{if(e.f){const P=h?g(l)?u[l]:c[l]:x(l)||!e.k?l.value:c[e.k];if(r)ue(P)&&ts(P,s);else if(ue(P))P.includes(s)||P.push(s);else if(h)c[l]=[s],g(l)&&(u[l]=c[l]);else{const w=[s];x(l,e.k)&&(l.value=w),e.k&&(c[e.k]=w)}}else h?(c[l]=i,g(l)&&(u[l]=i)):k?(x(l,e.k)&&(l.value=i),e.k&&(c[e.k]=i)):J("Invalid template ref type:",l,`(${typeof l})`)};if(i){const P=()=>{U(),Uo.delete(e)};P.id=-1,Uo.set(e,P),st(P,o)}else $s(e),U()}else J("Invalid template ref type:",l,`(${typeof l})`)}}function $s(e){const t=Uo.get(e);t&&(t.flags|=8,Uo.delete(e))}mo().requestIdleCallback;mo().cancelIdleCallback;const On=e=>!!e.type.__asyncLoader,hs=e=>e.type.__isKeepAlive;function Ka(e,t){za(e,"a",t)}function $a(e,t){za(e,"da",t)}function za(e,t,o=We){const n=e.__wdc||(e.__wdc=()=>{let r=o;for(;r;){if(r.isDeactivated)return;r=r.parent}return e()});if(rr(t,n,o),o){let r=o.parent;for(;r&&r.parent;)hs(r.parent.vnode)&&Hu(n,t,o,r),r=r.parent}}function Hu(e,t,o,n){const r=rr(t,e,n,!0);Mn(()=>{ts(n[t],r)},o)}function rr(e,t,o=We,n=!1){if(o){const r=o[e]||(o[e]=[]),s=t.__weh||(t.__weh=(...i)=>{yt();const a=yo(o),l=Mt(t,o,e,i);return a(),vt(),l});return n?r.unshift(s):r.push(s),s}else{const r=cn(us[e].replace(/ hook$/,""));J(`${r} is called when there is no active component instance to be associated with. Lifecycle injection APIs can only be used during execution of setup(). If you are using async setup(), make sure to register lifecycle hooks before the first await statement.`)}}const zt=e=>(t,o=We)=>{(!uo||e==="sp")&&rr(e,(...n)=>t(...n),o)},Uu=zt("bm"),at=zt("m"),Bu=zt("bu"),Vu=zt("u"),qa=zt("bum"),Mn=zt("um"),Fu=zt("sp"),Gu=zt("rtg"),Wu=zt("rtc");function Ku(e,t=We){rr("ec",e,t)}const $u="components";function Qa(e,t){return qu($u,e,!0,t)||e}const zu=Symbol.for("v-ndc");function qu(e,t,o=!0,n=!1){const r=Ke||We;if(r){const s=r.type;{const a=xs(s,!1);if(a&&(a===t||a===Ye(t)||a===bn(Ye(t))))return s}const i=zs(r[e]||s[e],t)||zs(r.appContext[e],t);return!i&&n?s:(o&&!i&&J(`Failed to resolve ${e.slice(0,-1)}: ${t} -If this is a native custom element, make sure to exclude it from component resolution via compilerOptions.isCustomElement.`),i)}else J(`resolve${bn(e.slice(0,-1))} can only be used in render() or setup().`)}function zs(e,t){return e&&(e[t]||e[Ye(t)]||e[bn(Ye(t))])}function be(e,t,o,n){let r;const s=o,i=ue(e);if(i||Be(e)){const a=i&&en(e);let l=!1,f=!1;a&&(l=!rt(e),f=_t(e),e=Zo(e)),r=new Array(e.length);for(let c=0,u=e.length;ct(a,l,void 0,s));else{const a=Object.keys(e);r=new Array(a.length);for(let l=0,f=a.length;l0;return t!=="default"&&(o.name=t),y(),Te(te,null,[L("slot",o,n)],f?-2:64)}let s=e[t];s&&s.length>1&&(J("SSR-optimized slot function detected in a non-SSR-optimized render function. You need to mark this component with $dynamic-slots in the parent template."),s=()=>[]),s&&s._c&&(s._d=!1),y();const i=s&&Ja(s(o)),a=o.key||i&&i.key,l=Te(te,{key:(a&&!gt(a)?a:`_${t}`)+(!i&&n?"_fb":"")},i||[],i&&e._===1?64:-2);return l.scopeId&&(l.slotScopeIds=[l.scopeId+"-s"]),s&&s._c&&(s._d=!0),l}function Ja(e){return e.some(t=>vn(t)?!(t.type===lt||t.type===te&&!Ja(t.children)):!0)?e:null}const Mr=e=>e?vl(e)?ir(e):Mr(e.parent):null,hn=Ge(Object.create(null),{$:e=>e,$el:e=>e.vnode.el,$data:e=>e.data,$props:e=>It(e.props),$attrs:e=>It(e.attrs),$slots:e=>It(e.slots),$refs:e=>It(e.refs),$parent:e=>Mr(e.parent),$root:e=>Mr(e.root),$host:e=>e.ce,$emit:e=>e.emit,$options:e=>Za(e),$forceUpdate:e=>e.f||(e.f=()=>{or(e.update)}),$nextTick:e=>e.n||(e.n=Oa.bind(e.proxy)),$watch:e=>ju.bind(e)}),gs=e=>e==="_"||e==="$",gr=(e,t)=>e!==De&&!e.__isScriptSetup&&Ee(e,t),Ya={get({_:e},t){if(t==="__v_skip")return!0;const{ctx:o,setupState:n,data:r,props:s,accessCache:i,type:a,appContext:l}=e;if(t==="__isVue")return!0;if(t[0]!=="$"){const m=i[t];if(m!==void 0)switch(m){case 1:return n[t];case 2:return r[t];case 4:return o[t];case 3:return s[t]}else{if(gr(n,t))return i[t]=1,n[t];if(r!==De&&Ee(r,t))return i[t]=2,r[t];if(Ee(s,t))return i[t]=3,s[t];if(o!==De&&Ee(o,t))return i[t]=4,o[t];jr&&(i[t]=0)}}const f=hn[t];let c,u;if(f)return t==="$attrs"?(Je(e.attrs,"get",""),Vo()):t==="$slots"&&Je(e,"get",t),f(e);if((c=a.__cssModules)&&(c=c[t]))return c;if(o!==De&&Ee(o,t))return i[t]=4,o[t];if(u=l.config.globalProperties,Ee(u,t))return u[t];Ke&&(!Be(t)||t.indexOf("__v")!==0)&&(r!==De&&gs(t[0])&&Ee(r,t)?J(`Property ${JSON.stringify(t)} must be accessed via $data because it starts with a reserved character ("$" or "_") and is not proxied on the render context.`):e===Ke&&J(`Property ${JSON.stringify(t)} was accessed during render but is not defined on instance.`))},set({_:e},t,o){const{data:n,setupState:r,ctx:s}=e;return gr(r,t)?(r[t]=o,!0):r.__isScriptSetup&&Ee(r,t)?(J(`Cannot mutate - - + +
diff --git a/ui/src/pages/RunDetail.vue b/ui/src/pages/RunDetail.vue index 83d98a4..1d6128a 100644 --- a/ui/src/pages/RunDetail.vue +++ b/ui/src/pages/RunDetail.vue @@ -186,9 +186,15 @@ const agentStats = computed(() => { if (!isAgent.value) return null const rs = results.value const sum = k => rs.map(r => r.meta?.[k]).filter(v => v != null).reduce((a, b) => a + b, 0) + const sumTok = k => rs.map(r => r.meta?.tokens?.[k] ?? 0).reduce((a, b) => a + b, 0) return { - tok: sum('out_tokens'), + cost: rs.map(r => r.meta?.cost_usd ?? 0).reduce((a, b) => a + b, 0), + interv: sum('interventions'), + inTok: sumTok('input') + sumTok('cache_read') + sumTok('cache_write'), + outTok: sumTok('output') + sumTok('reasoning'), turns: sum('turns'), + sde: rs.some(r => r.meta?.cost_usd != null || r.meta?.interventions != null), + tok: sum('out_tokens'), memTasks: rs.filter(r => (r.meta?.memories_injected ?? 0) > 0).length, } }) @@ -275,7 +281,17 @@ function toggleCat(axis, cat) {
Judge LLM {{ data.judge_llm }}
-
+
+
+

{{ label }}

+

{{ val }}

+
+
+
← prev {{ activeIndex + 1 }} / {{ results.length }} -