From 24098cbfc138a83277318b86113a7a5ea95ed7bd Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 31 Jul 2026 19:51:30 -0700 Subject: [PATCH] feat(evalboard): generate the rate table from pricing.py, chart in every edition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilt on top of #69, which independently landed the multi-harness overview this branch was also building. Everything #69 covers is dropped — the harness table and its colours, the multi-series chart refactor, the badge/selector changes, the All chip, whole-page ?h= scoping. What remains is the work #69 does not have. Pricing SSOT. Per-MTok rates were hand-copied from src/coder_eval/pricing.py into lib/pricing.ts, which is how 7 rates drifted (Opus 4.6/4.7/4.8 billed at 3x actual) and how a not-mirrored allowlist grew to suppress 17 models — two of which real runs use, so the board rendered "—" for their cost. scripts/gen-pricing.mjs now emits lib/pricing.generated.ts from the Python table, and the parity test asserts exact match, so the allowlist is gone and staleness is the only failure mode left. The test imports readTable, not parsePythonTable: comparing two products of the same regex cannot see a row that regex skipped, so readTable cross-checks matched rows against the ModelPricing( count and refuses to emit on a mismatch. The one deliberate gap is EXCLUDED_MODELS, which reconciles generation with #66's decision to stop statically pricing the OpenRouter open-weight models: OpenRouter routes per request, so a headline rate is wrong, and the detail view shows captured per-call cost instead. pricing.py still prices them for the Python max_usd fallback, so generating blindly would have silently reintroduced exactly what #66 removed. Guarded both ways — a stale entry throws, a reintroduced static rate fails the parity test — and the Bedrock open-weight models are pinned as NOT excluded, since those are fixed-rate and must stay priced. Analytics in every edition. The charts and tag rail sat behind isInternal, a gate dating to the initial public release rather than to anything in the block: it renders whatever runs the instance is pointed at, so a default-configured board showed only a run list. The harness selector is ungated with it — the charts name a harness per line, so gating the only control that isolates one leaves an OSS instance able to see a harness but not select it. The Harness column in the run table stays gated separately. windowHarnesses. The chip set came from listRecentHarnesses' fixed-count discovery scan, so a harness that ran inside the 30d window but not recently got a chart line with no chip to select it. getOverview now also returns every harness in the window, captured BEFORE the harness filter — invariant under selection by construction, where deriving it after the filter would collapse it to the active harness and delete every other chip on click. OverviewData.harnesses keeps its meaning (the plotted set, so "all N harnesses" still equals the lines drawn); the new test contrasts the two derivations over the same scoped input, so it fails if the chips are ever re-pointed at the filtered set. Ad-hoc runs are scoped by ?h= like the rest of the page. They carry a harness stamp like any other run and that section has no Harness column, so leaving it unfiltered showed codex uploads under a Claude Code scope with nothing on screen to explain why. module-boundaries.test.ts pins two bundle invariants previously held by comments alone, each worth a measured ~105 kB First Load JS and invisible to tsc, the suite, and a green build: lib/harness.ts stays a true leaf (client components import it for values; lib/overview.ts reaches node:fs and @azure/storage-blob), and tag-rail.tsx stays chart-free (it renders on /runs/[id] and /trends, which have no chart). Verified against the build: /runs/[id] 167 kB, /trends 119 kB. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 57 +++ evalboard/app/page.tsx | 47 ++- .../lib/__tests__/module-boundaries.test.ts | 80 +++++ evalboard/lib/__tests__/overview.test.ts | 86 ++++- .../lib/__tests__/pricing-parity.test.ts | 217 +++++++----- evalboard/lib/__tests__/pricing.test.ts | 6 +- evalboard/lib/overview.ts | 50 ++- evalboard/lib/pricing-types.ts | 12 + evalboard/lib/pricing.generated.ts | 331 ++++++++++++++++++ evalboard/lib/pricing.ts | 91 +---- evalboard/package.json | 1 + evalboard/scripts/gen-pricing.d.mts | 26 ++ evalboard/scripts/gen-pricing.mjs | 227 ++++++++++++ 13 files changed, 1044 insertions(+), 187 deletions(-) create mode 100644 evalboard/lib/__tests__/module-boundaries.test.ts create mode 100644 evalboard/lib/pricing-types.ts create mode 100644 evalboard/lib/pricing.generated.ts create mode 100644 evalboard/scripts/gen-pricing.d.mts create mode 100644 evalboard/scripts/gen-pricing.mjs diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index dd3912db..21ba3fde 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -107,3 +107,60 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule caught them. The cleanup plan explicitly deferred this as YAGNI for the one-time purge, but any future doc rename/deletion re-opens the same blind spot — caught in the 2026-07-03 open-source-docs-cleanup implementation run. + +## From 2026-07-30 evalboard pricing SSOT + multi-harness charts + +- [ ] **CI never runs the `evalboard/` test suite — the highest-value gap here.** + `grep -rn evalboard .github/workflows/ .pre-commit-config.yaml Makefile` yields + exactly one hit: a comment in `pr-checks.yml` saying `evalboard/pnpm-lock.yaml` + is *out of scope*. `pnpm verify` (`tsc --noEmit && vitest run && next build`) + exists in `evalboard/package.json` but nothing invokes it. Consequence: the + pricing drift guard that the whole generated-SSOT design rests on is never + executed. A Python dev edits a rate in `src/coder_eval/pricing.py`, never opens + `evalboard/`, and the PR goes green while every rendered USD figure keeps the + stale rate — precisely how the 7 wrong rates (Opus 4.6/4.7/4.8 at 3× actual) + accumulated in the first place. Flagged independently by two reviewers. + *Not done here because adding a required check to `pr-checks.yml` is a repo-wide + CI-policy change affecting every contributor's PR, outside a plan scoped to + `evalboard/`.* Ready to apply: a job mirroring the existing `setup-node` steps + (`pnpm install --frozen-lockfile` + `pnpm verify` with `working-directory: + evalboard`), ideally non-blocking first. Alternatively a Python-side CExxx rule + asserting `lib/pricing.generated.ts` is current, which would put the guard on + the side that actually edits rates. +- [ ] **One shared normalizer fixture for `normalizeModel` (TS) and + `_normalize_model` (Python).** Both strip the same routing/region/vendor + prefixes, but TS additionally strips a trailing `-YYYYMMDD` and Python does not + — so `claude-opus-4-6-20250514` prices in the evalboard and is unpriced in the + backend, and the frontend estimate can disagree with the backend's + authoritative Cost for dated ids. Neither side's tests can see the other, and + the generated table does not close this: generation shares the rate *values*, + not the *lookup* logic. Needs a decision on which behaviour is the reference + before a fixture can be written. Note also that plugin rates registered via + `register_pricing` (e.g. `coder_eval_uipath`) can never reach the generated + table at all, since the generator parses `pricing.py`'s literal table only. +- [ ] **No test can see `app/page.tsx`'s self-link scope threading.** `hParam`, + `base` and `buildHref` are module-private inside an async server component and + there is no test for that file. A future self-link spelled `buildHref({ tag })` + silently resets the user's harness scope with the whole suite green — now that + `?h=` scopes the charts, the tiles, the run table *and* the ad-hoc section, + dropping it is a bigger jump than it was when it only re-scoped the chart. + Guarding it means extracting `buildHref` + a `selfLinkBase` helper into a + testable module. +- [ ] **"Any chart with ≥2 series must have a hover-attribution test."** Nothing + in the suite mounted a chart tooltip before the multi-harness work, which is + how two real misattribution bugs survived a spec review: recharts silently + ignores `shared={false}` on `LineChart`, and with the default + `allowDuplicatedCategory` each `` indexes its own points with an index + into the concatenation of all series. `app/_overview/__tests__/harness-legend.test.tsx` + now covers the two overview charts (hovered-x attribution, real zeros, unknown + series keys), but that is one test file by convention, not a rule — nothing + enforces it for the next multi-series chart someone adds. +- [ ] **Nothing checks that the frontend rate table's deliberate omissions stay + deliberate.** `EXCLUDED_MODELS` in `evalboard/scripts/gen-pricing.mjs` withholds + the OpenRouter open-weight ids because they are routed per-request and shown at + captured actual cost, so a static headline rate would be confidently wrong. + That set is guarded in both directions today (a stale entry throws; a + reintroduced static rate fails `pricing-parity.test.ts`), but only *within* + `evalboard/`. The Python side has no matching notion — `pricing.py` prices them + for the `max_usd` fallback with nothing recording that the board must not — so + the rationale lives in a JS comment a Python-side reprice will never surface. diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 98b83580..079f0113 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -6,7 +6,7 @@ import { listRecentHarnesses, type TagCount, } from "@/lib/overview"; -import { parseHarnessScope } from "@/lib/harness"; +import { orderHarnesses, parseHarnessScope } from "@/lib/harness"; import { fmtDuration, fmtRunTime, fmtTimestamp } from "@/lib/format"; import { passClass } from "@/lib/pass-rate"; import { type Window } from "@/lib/reviews-types"; @@ -128,7 +128,7 @@ export default async function Page({ const [overview, listing, adhoc, harnesses] = await Promise.all([ getOverview(WINDOW, activeTag, q, harness), getRunListing(activeTag, q, limit, harness), - getAdhocRunListing(adhocLimit), + getAdhocRunListing(adhocLimit, harness), listRecentHarnesses(), ]); @@ -193,7 +193,10 @@ export default async function Page({ together. Buried in the chart card it read as a chart control while the numbers above it silently covered every harness. Same position as the selector on Path to GA, trends, and the - watchlist. Internal-only, like the analytics block below. */} + watchlist. Shown in every edition, like the analytics block it + scopes — the charts below name a harness per line, so gating the + only control that isolates one would leave an OSS instance able + to see a harness but not select it. */}

@@ -204,13 +207,23 @@ export default async function Page({ and logs.

- {isInternal && ( - - )} +
{/* The analytics block — daily success / turn-budget charts and the - colored skill/review/tag rail — is an internal-only surface (see - lib/edition.ts). The public OSS edition drops it so the front - page is just the run list. */} - {isInternal && ( + colored skill/review/tag rail — renders in EVERY edition. It used + to sit behind isInternal, a gate that dated to the initial public + release rather than to anything in the block: nothing here is + UiPath-specific, it just charts whatever runs the instance is + pointed at, so an OSS clone charts its own results and a + default-configured instance no longer shows a bare run list. + The Harness column in the run table below stays gated separately, + so on an OSS instance the chart legend can name a harness that the + table does not have a column for. */}

@@ -324,7 +342,6 @@ export default async function Page({ )}

- )}
diff --git a/evalboard/lib/__tests__/module-boundaries.test.ts b/evalboard/lib/__tests__/module-boundaries.test.ts new file mode 100644 index 00000000..27090008 --- /dev/null +++ b/evalboard/lib/__tests__/module-boundaries.test.ts @@ -0,0 +1,80 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; + +// Two bundle-correctness invariants that are otherwise enforced only by +// comments, each worth a measured ~105 kB of First Load JS. Both are violated by +// adding one import, and neither shows up in tsc, the test suite, or a passing +// `next build` — only in the bundle size, which nobody reads on a green PR. + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, "../.."); + +function readSource(rel: string): string { + return readFileSync(resolve(root, rel), "utf8"); +} + +// Matches `import … from "x"`, `import "x"`, and `export … from "x"`. +function moduleSpecifiers(src: string): string[] { + return [...src.matchAll(/(?:^|\n)\s*(?:import|export)[\s\S]*?from\s+"([^"]+)"/g)] + .map((m) => m[1]) + .concat( + [...src.matchAll(/(?:^|\n)\s*import\s+"([^"]+)"/g)].map((m) => m[1]), + ); +} + +describe("lib/harness.ts is a true leaf", () => { + // Client components ("use client" charts, the selector, the badge) all import + // this module for values, not just types. lib/overview.ts transitively reaches + // node:fs, node:path and @azure/storage-blob, so a single VALUE import from + // there would drag that whole graph into the browser bundle. Type-only + // imports erase at compile time; values do not. + // + // This is why the module's own header says it must stay dependency-free, and + // why anything that needs both harness knowledge and run data (the chart + // pivot in app/_overview/harness-series.ts) imports harness.ts rather than + // the other way round. + test("has no imports at all", () => { + const specs = moduleSpecifiers(readSource("lib/harness.ts")); + expect( + specs, + "lib/harness.ts must stay dependency-free — see the leaf-module note at its top", + ).toEqual([]); + }); +}); + +describe("the tag rail stays chart-free", () => { + // tag-rail.tsx (ChipLegend / MergedTagRail) renders on pages with no chart at + // all — /runs/[id] via run-view and /trends via trends-view — as well as on + // the overview. Pulling chart code in here measured +105 kB First Load JS on + // /runs/[id] (163 -> 268 kB) and on /trends (119 -> 225 kB). + // + // The tempting regression is to reuse the overview's swatch/legend + // primitives: app/_overview/harness-legend.tsx is a sibling that looks + // reusable, but it is chart-side and pulls the chart module's types and, via + // the charts, recharts itself. Duplicating a 6-line swatch is the cheaper + // trade. + const FORBIDDEN = ["recharts", "harness-series", "harness-legend"]; + + test("imports neither recharts nor any chart module", () => { + const specs = moduleSpecifiers(readSource("app/_overview/tag-rail.tsx")); + for (const spec of specs) { + for (const bad of FORBIDDEN) { + expect( + spec.includes(bad), + `tag-rail.tsx must not import ${spec} — it renders on chart-less pages`, + ).toBe(false); + } + } + }); + + test("its dependencies stay on a short, reviewed list", () => { + // Keeps the guard above honest: a blocklist only catches the imports we + // thought of, so any NEW dependency has to be added here deliberately — + // at which point that dependency's own graph gets re-checked. + expect( + moduleSpecifiers(readSource("app/_overview/tag-rail.tsx")).sort(), + ).toEqual(["@/lib/overview", "next/link"]); + }); +}); diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 2f8b9440..d96aaaec 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -9,7 +9,7 @@ import { type PerRun, type RunListingRow, } from "../overview"; -import { normalizeHarness } from "../harness"; +import { normalizeHarness, orderHarnesses } from "../harness"; import type { RunOverviewTask } from "../runs"; function task(overrides: Partial): RunOverviewTask { @@ -640,3 +640,87 @@ describe("projectRunRow", () => { }); }); }); + +// The switcher's chip set must be STABLE under selection. getOverview feeds it +// `windowHarnesses`, not `harnesses`: the latter is derived from the FILTERED +// points, so with `?h=codex` active it collapses to ["codex"] and every other +// chip would vanish the moment one was selected — stranding the user on a scope +// they cannot leave without hand-editing the URL. These pin the distinction. +describe("harnesses vs windowHarnesses", () => { + function run(harness: string | null, id = "2026-07-30_00-00-00"): PerRun { + return { + id, + overview: { + id, + harness, + tasks: [task({ status: "SUCCESS" })], + totalCostUsd: null, + taskDurationSeconds: null, + componentShas: [], + } as unknown as NonNullable, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + }; + } + + // The derivation getOverview applies for windowHarnesses: every non-adhoc + // run in the window with a readable overview, BEFORE the harness filter. + function windowHarnessesOf(runs: PerRun[]): string[] { + return orderHarnesses( + runs + .filter((r) => !r.adhoc && r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + } + + const window = [ + run("claude-code", "2026-07-28_00-00-00"), + run("codex", "2026-07-29_00-00-00"), + run("delegate-sdk", "2026-07-30_00-00-00"), + ]; + + test("windowHarnesses lists every harness in the window", () => { + expect(windowHarnessesOf(window)).toEqual([ + "claude-code", + "codex", + "delegate-sdk", + ]); + }); + + test("it is IDENTICAL whichever harness is selected — the anti-shift rule", () => { + // Contrast the two derivations over the SAME scoped input. The + // post-filter one collapses to the selected harness and deletes the + // other chips; the pre-filter one is invariant. Comparing the pre-filter + // result to itself would be vacuous, so apply the real harness filter + // and show only one of the two survives it. + const unscoped = windowHarnessesOf(window); + expect(unscoped).toHaveLength(3); + + for (const scope of ["claude-code", "codex", "delegate-sdk"]) { + const afterHarnessFilter = window.filter( + (r) => normalizeHarness(r.overview!.harness) === scope, + ); + + // What the chips must NOT be built from — collapses to one. + expect(windowHarnessesOf(afterHarnessFilter)).toEqual([scope]); + + // What they ARE built from: computed before that filter, so the full + // set survives and every chip stays clickable. + expect( + windowHarnessesOf(window), + `chip set shifted when scoped to ${scope}`, + ).toEqual(unscoped); + } + }); + + test("a legacy unstamped run folds into claude-code, not a phantom chip", () => { + expect(windowHarnessesOf([run(null)])).toEqual(["claude-code"]); + }); + + test("ad-hoc runs contribute no chip", () => { + const adhoc = { ...run("delegate-sdk"), adhoc: true }; + expect(windowHarnessesOf([run("codex"), adhoc])).toEqual(["codex"]); + }); +}); diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 7e5e1a67..41fe80dd 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -1,104 +1,151 @@ -import { readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { describe, expect, test } from "vitest"; -import { PRICING } from "../pricing"; +import { excludedModels, mirroredTable, readTable } from "../../scripts/gen-pricing.mjs"; +import { PRICING, resolvePricing } from "../pricing"; -// Drift guard: lib/pricing.ts is a hand-copied mirror of the authoritative -// Python table in src/coder_eval/pricing.py. It exists so the frontend's -// "estimated" USD figures agree with the backend's authoritative Cost on the -// same tokens. +// Drift guard: lib/pricing.generated.ts is generated from the authoritative +// Python table (src/coder_eval/pricing.py) by scripts/gen-pricing.mjs. Nobody +// types a rate on this side, so the only failure mode left is a STALE artifact — +// pricing.py changed and `pnpm gen:pricing` wasn't run. // -// Semantics are SUBSET, not exact-match: every model priced in lib/pricing.ts -// must exist in pricing.py with identical rates (a frontend rate that disagrees -// with the backend, or prices a model the backend doesn't, fails the build). -// The frontend is NOT required to mirror every backend model — it only needs to -// price the ones it displays, and the backend legitimately prices models the -// evalboard never renders. (Exact-match was too strict: it forced unrelated -// backend model additions into this file to keep the build green.) - -const here = dirname(fileURLToPath(import.meta.url)); -const PY_PATH = resolve(here, "../../../src/coder_eval/pricing.py"); - -// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), -const ROW_RE = - /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; - -function parsePythonTable(): Record< - string, - [number, number, number, number] -> { - const src = readFileSync(PY_PATH, "utf8"); - const out: Record = {}; - for (const m of src.matchAll(ROW_RE)) { - out[m[1]] = [ - Number(m[2]), - Number(m[3]), - Number(m[4]), - Number(m[5]), - ]; - } - return out; -} +// Semantics are EXACT-MATCH against pricing.py minus one small, explicit +// exclusion set (see EXCLUDED_MODELS in the generator). The old hand-copied +// mirror couldn't sustain exact-match — it needed a not-mirrored allowlist that +// 17 models drifted past, two of whose entries were models real runs use, +// silently rendering "—" for their cost. Generation makes exact-match free: one +// assertion subsumes orphans, missing entries, and rate drift, and the only +// models left off are the ones a static rate would be actively WRONG for. +// +// This imports readTable, NOT parsePythonTable, and the distinction is the whole +// guard: parsePythonTable on both sides would compare two products of the same +// regex, so a row that regex cannot match is missing from the artifact AND from +// the expectation — deep-equal passes while the board renders "—" for that model +// forever. readTable cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and throws if they disagree. -describe("pricing.ts ↔ pricing.py parity", () => { - const py = parsePythonTable(); +describe("pricing.generated.ts ↔ pricing.py parity", () => { + // Throws if the parser skipped a row, so this doubles as the coverage check. + const py = readTable(); test("parses a non-trivial Python table", () => { // Guard against a regex/path regression silently passing the test. expect(Object.keys(py).length).toBeGreaterThan(10); }); - test("every model in lib/pricing.ts exists in pricing.py", () => { - const orphans = Object.keys(PRICING).filter((m) => !(m in py)); + test("the generated table is current with pricing.py", () => { + const generated = Object.fromEntries( + Object.entries(PRICING).map(([model, p]) => [ + model, + [ + p.inputPerMTok, + p.outputPerMTok, + p.cacheWritePerMTok, + p.cacheReadPerMTok, + ], + ]), + ); expect( - orphans, - `priced in lib/pricing.ts but absent from pricing.py: ${orphans.join(", ")}`, - ).toEqual([]); + generated, + "lib/pricing.generated.ts is stale — run `pnpm gen:pricing`", + ).toEqual(mirroredTable(py)); }); +}); - test("shared models have identical input/output/cacheWrite/cacheRead rates", () => { - for (const [model, ts] of Object.entries(PRICING)) { - const rates = py[model]; - expect(rates, `not priced in pricing.py: ${model}`).toBeDefined(); - expect([ - ts.inputPerMTok, - ts.outputPerMTok, - ts.cacheWritePerMTok, - ts.cacheReadPerMTok, - ]).toEqual(rates); +// The one deliberate gap between the two tables. OpenRouter routes each request +// to whichever provider wins its sort, so no single headline rate is correct; +// the harness captures each call's ACTUAL cost proxy-side and the detail view +// renders it per call (provider_call_costs → ProviderCallTableSection). These +// tests pin that the estimate stays absent, because the failure mode is silent: +// a regenerated table that quietly reintroduces them puts a confidently wrong +// number beside the measured one. +describe("per-request-routed models are deliberately unpriced", () => { + const excluded = excludedModels(); + + test("the exclusion set is non-empty and still live", () => { + // mirroredTable throws on an id pricing.py no longer prices, so simply + // calling it here fails the suite on a stale exclusion. + expect(excluded.length).toBeGreaterThan(0); + expect(() => mirroredTable(readTable())).not.toThrow(); + }); + + test("none of them carry a static rate on the frontend", () => { + for (const model of excluded) { + expect( + Object.hasOwn(PRICING, model), + `${model} must not be statically priced — it is shown at captured per-call cost`, + ).toBe(false); + expect(resolvePricing(model)).toBeNull(); + } + }); + + test("Bedrock open-weight models ARE priced (they are not routed per request)", () => { + // The nearby-but-opposite case, pinned so a future widening of the + // exclusion set can't quietly take these with it: same open-weight + // families, but running at fixed Bedrock rates with no per-call capture, + // so a static rate is correct and required. + for (const model of ["deepseek.v3.2", "zai.glm-5", "moonshotai.kimi-k2.5"]) { + expect(resolvePricing(model), `${model} lost its rate`).not.toBeNull(); } }); +}); - // Python-priced models we deliberately do NOT mirror to the frontend: heavy - // frontier Claude/GPT variants the evalboard never runs, so pricing them here - // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model - // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build — - // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids - // that previously rendered "—" for cost). - const DELIBERATELY_UNMIRRORED = new Set([ - "claude-sonnet-5", - "gpt-5.4-mini", - "gpt-5.4-nano", - "gpt-5.4-pro", - "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", - // OpenRouter open-weight models: priced in pricing.py only for the Python - // max_usd static fallback. The evalboard deliberately does NOT statically - // price them — OpenRouter routes per-request, so it shows the captured - // ACTUAL per-call cost instead (see the per-call table, provider_call_costs). - "moonshotai/kimi-k3", - "z-ai/glm-5.2", - "deepseek/deepseek-v4-pro", - ]); +// Behaviour the generated table has to preserve. Deliberately NOT a second copy +// of the rate card: every expectation below is derived from pricing.py via +// `py[...]`, never a literal. The test above already proves every rate matches, +// so hardcoding one here would mean a legitimate vendor repricing breaks the +// suite even after a correct regeneration — exactly the two-places problem +// generation exists to remove. What these pin is lookup *logic* the deep-equal +// cannot see: the undated fallback, a genuine zero surviving, float precision. +describe("resolvePricing over the generated table", () => { + const py = readTable(); - test("every pricing.py model is mirrored in pricing.ts or explicitly unmirrored", () => { - const missing = Object.keys(py).filter((m) => !(m in PRICING) && !DELIBERATELY_UNMIRRORED.has(m)); - expect( - missing, - `priced in pricing.py but missing from pricing.ts — mirror it or add to DELIBERATELY_UNMIRRORED: ${missing.join(", ")}`, - ).toEqual([]); + test("a deleted dated key still prices via the undated fallback", () => { + // "claude-opus-4-6-20250514" was a dead entry (absent from pricing.py) + // and redundant: resolvePricing strips a trailing -YYYYMMDD. Deleting it + // must change no rendered figure — the dated id still resolves, and to + // exactly the undated key's rate. + for (const undated of ["claude-opus-4-6", "claude-sonnet-4-6"]) { + const dated = resolvePricing(`${undated}-20250514`); + expect(dated, `${undated}-20250514 no longer resolves`).not.toBeNull(); + expect(dated).toEqual(resolvePricing(undated)); + } + }); + + test("a model the old allowlist suppressed is now priced", () => { + // By far the most-run model in the recorded run data, yet it sat in the + // old not-mirrored allowlist — so the evalboard rendered "—" for cost on + // runs it actually executed. This is the regression that must not return. + expect(resolvePricing("gpt-5.6-terra")).not.toBeNull(); + }); + + test("a legitimately-zero rate survives as 0, not undefined", () => { + // Bedrock publishes no prompt-cache rate for the open-weight models, so + // cache-read is a real 0 — generation must not drop it as falsy. Asserted + // against pricing.py's own value, and separately that it IS zero there, + // so the test still means something if that rate ever changes. + for (const model of ["deepseek.v3.2", "zai.glm-5"]) { + expect(py[model][3]).toBe(0); + expect(resolvePricing(model)?.cacheReadPerMTok).toBe(py[model][3]); + } + }); + + test("fractional rates round-trip exactly through generation", () => { + // The generator emits numbers via string interpolation, so a precision + // loss would show up as generated !== parsed for these decimals. + for (const model of ["gpt-5-codex", "codex-mini-latest", "zai.glm-5"]) { + const p = resolvePricing(model); + expect(p).not.toBeNull(); + expect([ + p!.inputPerMTok, + p!.outputPerMTok, + p!.cacheWritePerMTok, + p!.cacheReadPerMTok, + ]).toEqual(py[model]); + // And that at least one of them really is fractional, so this test + // cannot pass vacuously if pricing.py switches to round numbers. + expect( + py[model].some((r) => !Number.isInteger(r)), + `${model} has no fractional rate left — pick another model`, + ).toBe(true); + } }); }); diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66302cf1..4125e6c7 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -36,7 +36,11 @@ describe("resolvePricing", () => { }); test("knows the current default opus id", () => { - expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75); + // Rate-agnostic on purpose: pricing-parity.test.ts already proves every + // rate equals pricing.py, so pinning a number here would break the suite + // on a legitimate vendor repricing. What matters at this seam is that the + // current default Opus id resolves at all. + expect(resolvePricing("claude-opus-4-8")).not.toBeNull(); }); test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index e02215d6..66701879 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -84,10 +84,19 @@ export interface TagCount { export interface OverviewData { runs: RunPoint[]; // one point per run, no daily aggregation - // The harnesses actually present in `runs`, in stable display order. Drives - // the chart's series list and legend, so a harness with no runs in the - // window contributes no empty line. + // The harnesses actually PLOTTED — present in `runs`, in stable display + // order. Drives the chart's series list and legend, so a harness with no + // runs in the window contributes no empty line. Derived from the windowed, + // FILTERED points, so it always equals the number of lines drawn. harnesses: string[]; + // Every harness present in the window, captured BEFORE the harness filter + // and ignoring tag/q. Feeds the switcher, which must offer a stable set: + // `harnesses` above collapses to just the active one when `?h=` is set, so + // using it for the chips would delete every other chip the moment one was + // selected. Also covers the harness that IS charted but is too old to appear + // in listRecentHarnesses()' fixed-count discovery scan — that one would + // otherwise get a line with no chip to select it. + windowHarnesses: string[]; windowStart: number; // ms — chart x-domain start windowEnd: number; // ms — chart x-domain end // The summary tiles' rollup, folded over the same runs `runs` plots so the @@ -558,11 +567,20 @@ export async function getOverview( // Ad-hoc runs never feed the daily chart or the tag rails — they're not // pipeline cadence. (Non-date-named ones are already pruned upstream by // listRunIdsInWindow; this also drops date-named runs flagged adhoc.) - const perRun = (await loadWindowData(window)).filter( + const windowRuns = (await loadWindowData(window)).filter((r) => !r.adhoc); + // Captured BEFORE the harness filter, and deliberately ignoring tag/q: this + // is what the switcher offers, so it must not depend on what is currently + // selected. Deriving it after the filter would collapse it to the active + // harness and make every other chip vanish on selection — stranding the user + // on a scope they cannot leave without editing the URL. + const windowHarnesses = orderHarnesses( + windowRuns + .filter((r) => r.overview) + .map((r) => normalizeHarness(r.overview!.harness)), + ); + const perRun = windowRuns.filter( (r) => - !r.adhoc && - (harness == null || - normalizeHarness(r.overview?.harness) === harness), + harness == null || normalizeHarness(r.overview?.harness) === harness, ); const needle = q?.trim().toLowerCase() || null; @@ -610,6 +628,7 @@ export async function getOverview( return { runs: runPoints, harnesses: orderHarnesses(seenHarnesses), + windowHarnesses, windowStart, windowEnd, totals: summarizeListing(rows), @@ -903,8 +922,21 @@ export function buildAdhocRows( // not the id, so we can't window by id and still show the most recent): the // ad-hoc set is small by construction (manual uploads only) and per-run reads // are memoized for 5 min, so a warm front page pays no extra IO. -export async function getAdhocRunListing(limit: number | null): Promise { +export async function getAdhocRunListing( + limit: number | null, + // Scoped by the same `?h=` as the rest of the page: an ad-hoc run carries a + // harness stamp like any other, and this section has no Harness column, so + // leaving it unfiltered would show codex uploads under a Claude Code scope + // with nothing on screen to explain why. + harness: string | null = null, +): Promise { const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) == null); const perRun = await mapWithConcurrency(ids, FETCH_CONCURRENCY, cachedLoadPerRun); - return buildAdhocRows(perRun, limit); + const scoped = + harness == null + ? perRun + : perRun.filter( + (r) => normalizeHarness(r.overview?.harness) === harness, + ); + return buildAdhocRows(scoped, limit); } diff --git a/evalboard/lib/pricing-types.ts b/evalboard/lib/pricing-types.ts new file mode 100644 index 00000000..120fba36 --- /dev/null +++ b/evalboard/lib/pricing-types.ts @@ -0,0 +1,12 @@ +// The per-MTok rate shape, shared by the generated table +// (lib/pricing.generated.ts) and the hand-written lookup/cost helpers in +// lib/pricing.ts. It lives in its own leaf module so the generated file can +// import it without a cycle (pricing.ts → generated → pricing.ts). +// Consumers should keep importing `Pricing` from lib/pricing.ts. + +export interface Pricing { + inputPerMTok: number; + outputPerMTok: number; + cacheWritePerMTok: number; + cacheReadPerMTok: number; +} diff --git a/evalboard/lib/pricing.generated.ts b/evalboard/lib/pricing.generated.ts new file mode 100644 index 00000000..2db1adde --- /dev/null +++ b/evalboard/lib/pricing.generated.ts @@ -0,0 +1,331 @@ +// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT. +// Source of truth: src/coder_eval/pricing.py. +// To change a rate: edit that file, then run `pnpm gen:pricing`. +// Row order mirrors pricing.py so regeneration diffs stay minimal. +// +// Not every pricing.py model appears here: the OpenRouter open-weight ids are +// withheld on purpose, because they are routed per-request and shown at their +// captured ACTUAL per-call cost instead. See EXCLUDED_MODELS in the generator. + +import type { Pricing } from "./pricing-types"; + +export const GENERATED_PRICING: Record = { + "claude-fable-5": { + inputPerMTok: 10, + outputPerMTok: 50, + cacheWritePerMTok: 12.5, + cacheReadPerMTok: 1, + }, + "claude-opus-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-8": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-7": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-6": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-5-20251101": { + inputPerMTok: 5, + outputPerMTok: 25, + cacheWritePerMTok: 6.25, + cacheReadPerMTok: 0.5, + }, + "claude-opus-4-1": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-opus-4-20250514": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-sonnet-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-6": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-5-20250929": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-sonnet-4-20250514": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-haiku-4-5": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-4-5-20251001": { + inputPerMTok: 1, + outputPerMTok: 5, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.1, + }, + "claude-haiku-3-5": { + inputPerMTok: 0.8, + outputPerMTok: 4, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.08, + }, + "claude-3-7-sonnet-20250219": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20241022": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-5-sonnet-20240620": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-opus-20240229": { + inputPerMTok: 15, + outputPerMTok: 75, + cacheWritePerMTok: 18.75, + cacheReadPerMTok: 1.5, + }, + "claude-3-sonnet-20240229": { + inputPerMTok: 3, + outputPerMTok: 15, + cacheWritePerMTok: 3.75, + cacheReadPerMTok: 0.3, + }, + "claude-3-haiku-20240307": { + inputPerMTok: 0.25, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gpt-5-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-max": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex": { + inputPerMTok: 1.25, + outputPerMTok: 10, + cacheWritePerMTok: 1.25, + cacheReadPerMTok: 0.125, + }, + "gpt-5.1-codex-mini": { + inputPerMTok: 0.25, + outputPerMTok: 2, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "codex-mini-latest": { + inputPerMTok: 1.5, + outputPerMTok: 6, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.375, + }, + "gpt-5.3-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.2-codex": { + inputPerMTok: 1.75, + outputPerMTok: 14, + cacheWritePerMTok: 1.75, + cacheReadPerMTok: 0.175, + }, + "gpt-5.4": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.5": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.5-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-pro": { + inputPerMTok: 30, + outputPerMTok: 180, + cacheWritePerMTok: 30, + cacheReadPerMTok: 3, + }, + "gpt-5.4-mini": { + inputPerMTok: 0.75, + outputPerMTok: 4.5, + cacheWritePerMTok: 0.75, + cacheReadPerMTok: 0.075, + }, + "gpt-5.4-nano": { + inputPerMTok: 0.2, + outputPerMTok: 1.25, + cacheWritePerMTok: 0.2, + cacheReadPerMTok: 0.02, + }, + "gpt-5.6-sol": { + inputPerMTok: 5, + outputPerMTok: 30, + cacheWritePerMTok: 5, + cacheReadPerMTok: 0.5, + }, + "gpt-5.6-terra": { + inputPerMTok: 2.5, + outputPerMTok: 15, + cacheWritePerMTok: 2.5, + cacheReadPerMTok: 0.25, + }, + "gpt-5.6-luna": { + inputPerMTok: 1, + outputPerMTok: 6, + cacheWritePerMTok: 1, + cacheReadPerMTok: 0.1, + }, + "gemini-3.6-flash": { + inputPerMTok: 1.5, + outputPerMTok: 7.5, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash": { + inputPerMTok: 1.5, + outputPerMTok: 9, + cacheWritePerMTok: 1.5, + cacheReadPerMTok: 0.15, + }, + "gemini-3.5-flash-lite": { + inputPerMTok: 0.3, + outputPerMTok: 2.5, + cacheWritePerMTok: 0.3, + cacheReadPerMTok: 0.03, + }, + "gemini-3.1-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-pro-preview-customtools": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "gemini-3.1-flash-lite": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3.1-flash-lite-preview": { + inputPerMTok: 0.25, + outputPerMTok: 1.5, + cacheWritePerMTok: 0.25, + cacheReadPerMTok: 0.025, + }, + "gemini-3-flash-preview": { + inputPerMTok: 0.5, + outputPerMTok: 3, + cacheWritePerMTok: 0.5, + cacheReadPerMTok: 0.05, + }, + "gemini-3-pro-preview": { + inputPerMTok: 2, + outputPerMTok: 12, + cacheWritePerMTok: 2, + cacheReadPerMTok: 0.2, + }, + "deepseek.v3.2": { + inputPerMTok: 0.74, + outputPerMTok: 2.22, + cacheWritePerMTok: 0.74, + cacheReadPerMTok: 0, + }, + "zai.glm-5": { + inputPerMTok: 1.2, + outputPerMTok: 3.84, + cacheWritePerMTok: 1.2, + cacheReadPerMTok: 0, + }, + "moonshotai.kimi-k2.5": { + inputPerMTok: 0.72, + outputPerMTok: 3.6, + cacheWritePerMTok: 0.72, + cacheReadPerMTok: 0, + }, +}; diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index ad6d0293..431e2fd5 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -1,85 +1,24 @@ -// Per-million-token prices and cost math. Ported from -// src/coder_eval/pricing.py — keep in sync when that table changes. -// Source: Anthropic / OpenAI / Google public pricing. +// Per-million-token prices and cost math. // -// This is the single source of truth for rates on the frontend: the +// Rates are NOT written here: they are generated from the authoritative Python +// table (src/coder_eval/pricing.py) into lib/pricing.generated.ts. To change a +// rate, edit that file and run `pnpm gen:pricing` — never hand-edit a rate on +// this side. lib/__tests__/pricing-parity.test.ts fails the build if the +// generated artifact drifts from pricing.py. +// +// This module is the single source of truth for rates *on the frontend*: the // cascade-aware thinking-cost simulator (lib/thinkingSim.ts) and the // per-message cost column (lib/runs.ts) both price against this table, so a -// model added or repriced here updates both at once. +// model added or repriced in pricing.py updates both at once. -export interface Pricing { - inputPerMTok: number; - outputPerMTok: number; - cacheWritePerMTok: number; - cacheReadPerMTok: number; -} +import { GENERATED_PRICING } from "./pricing.generated"; +import type { Pricing } from "./pricing-types"; -// Exported so a unit test can assert key-and-rate parity against the -// authoritative Python table (src/coder_eval/pricing.py) and fail the -// build on drift — this hand-copied mirror is otherwise guarded only by a -// comment. Not part of the consumer API; use resolvePricing() instead. -export const PRICING: Record = { - "claude-opus-4-8": p(15, 75, 18.75, 1.5), - "claude-opus-4-7": p(15, 75, 18.75, 1.5), - "claude-opus-4-6": p(15, 75, 18.75, 1.5), - "claude-opus-4-6-20250514": p(15, 75, 18.75, 1.5), - "claude-opus-4-5-20251101": p(15, 75, 18.75, 1.5), - "claude-opus-4-20250514": p(15, 75, 18.75, 1.5), - "claude-sonnet-4-6": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-6-20250514": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-5-20250929": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-20250514": p(3, 15, 3.75, 0.3), - "claude-haiku-4-5-20251001": p(0.8, 4, 1, 0.08), - "claude-3-7-sonnet-20250219": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20241022": p(3, 15, 3.75, 0.3), - "claude-3-5-sonnet-20240620": p(3, 15, 3.75, 0.3), - "claude-3-opus-20240229": p(15, 75, 18.75, 1.5), - "claude-3-sonnet-20240229": p(3, 15, 3.75, 0.3), - "claude-3-haiku-20240307": p(0.25, 1.25, 0.3, 0.03), - "gpt-5-codex": p(1.25, 10, 1.25, 0.125), - "gpt-5": p(1.25, 10, 1.25, 0.125), - "gpt-5.3-codex": p(1.75, 14, 1.75, 0.175), - "gpt-5.4": p(2.5, 15, 2.5, 0.25), - "gpt-5.5": p(5, 30, 5, 0.5), - // Google Gemini (AntigravityAgent). Gemini bills no separate cache-write - // fee (cache_write == input, effectively unused); cache_read is the cached- - // input rate. Pro's >200K-token tier is higher — this flat rate reads low - // for very-large-context runs, fine for typical eval tasks. - "gemini-3-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview": p(2, 12, 2, 0.2), - "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), - "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), - "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), - // OpenRouter open-weight models (litellm backend) are DELIBERATELY NOT priced - // here. OpenRouter routes per-request, so a static headline rate is wrong (the - // billed rate depends on the provider it landed on), and there is no per-bucket - // rate to show. Instead the harness captures each call's ACTUAL cost proxy-side - // and the detail view renders it per call (TurnRecord.provider_call_costs → - // ProviderCallTableSection); a static estimate here would only reintroduce the - // wrong number. See pricing.py (kept for the Python-side max_usd fallback). - // Bedrock open-weight models (litellm backend, eu-north-1). Mirror of pricing.py. - // These run on Bedrock (fixed rates, like Claude) with NO OpenRouter actual-cost - // capture, so static pricing is correct and required here. - // The recorded model_used arrives prefixed (e.g. "converse/zai.glm-5"), so - // resolvePricing strips the routing/region prefixes before lookup. - "deepseek.v3.2": p(0.74, 2.22, 0.74, 0), - "zai.glm-5": p(1.2, 3.84, 1.2, 0), - "moonshotai.kimi-k2.5": p(0.72, 3.6, 0.72, 0), -}; +export type { Pricing } from "./pricing-types"; -function p( - input: number, - output: number, - cacheWrite: number, - cacheRead: number, -): Pricing { - return { - inputPerMTok: input, - outputPerMTok: output, - cacheWritePerMTok: cacheWrite, - cacheReadPerMTok: cacheRead, - }; -} +// Exported so the parity test can assert the generated artifact is current +// against pricing.py. Not part of the consumer API; use resolvePricing() instead. +export const PRICING: Record = GENERATED_PRICING; // Strip the LiteLLM/Bedrock routing + region/vendor prefixes back to the bare // pricing key — mirror of src/coder_eval/pricing.py::_normalize_model, since the diff --git a/evalboard/package.json b/evalboard/package.json index 63e3d802..0b24eb65 100644 --- a/evalboard/package.json +++ b/evalboard/package.json @@ -5,6 +5,7 @@ "scripts": { "dev": "next dev -p 3030", "dev:local": "EVALBOARD_LOCAL_RUNS_DIR=../runs next dev -p 3030", + "gen:pricing": "node scripts/gen-pricing.mjs", "build": "next build", "start": "next start -p 3030", "typecheck": "tsc --noEmit", diff --git a/evalboard/scripts/gen-pricing.d.mts b/evalboard/scripts/gen-pricing.d.mts new file mode 100644 index 00000000..18a8b3dd --- /dev/null +++ b/evalboard/scripts/gen-pricing.d.mts @@ -0,0 +1,26 @@ +// Types for the plain-JS generator, so lib/__tests__/pricing-parity.test.ts can +// import its parser under `allowJs: false`. Declares only what that test +// consumes — everything else in gen-pricing.mjs is module-private, so this file +// stays a one-function surface rather than a second mirror to keep in sync. + +export declare function parsePythonTable(): Record< + string, + [number, number, number, number] +>; + +/** parsePythonTable + the coverage cross-check. Throws if a row was skipped. */ +export declare function readTable(): Record< + string, + [number, number, number, number] +>; + +/** + * The subset of pricing.py the frontend table mirrors (everything except the + * per-request-routed OpenRouter ids). Throws on a stale exclusion. + */ +export declare function mirroredTable( + table: Record, +): Record; + +/** The ids deliberately withheld from the frontend table. */ +export declare function excludedModels(): string[]; diff --git a/evalboard/scripts/gen-pricing.mjs b/evalboard/scripts/gen-pricing.mjs new file mode 100644 index 00000000..cc30078a --- /dev/null +++ b/evalboard/scripts/gen-pricing.mjs @@ -0,0 +1,227 @@ +// Generates lib/pricing.generated.ts from the authoritative Python rate table +// (src/coder_eval/pricing.py), so per-MTok rates are written in exactly one +// file in one language. Run `pnpm gen:pricing` after editing pricing.py. +// +// Also the single home of the Python-table parser: lib/__tests__/pricing-parity.test.ts +// imports parsePythonTable from here rather than re-declaring the regex, so the +// generator and its drift guard can never disagree about what pricing.py says. +// +// That shared parser is why ROW_RE's coverage has to be checked structurally +// rather than trusted: it only matches rows written as a literal +// `ModelPricing(a, b, c, d)`, and a row it fails to match is missing from BOTH +// the generated table and the parity test's expectation — so deep-equal passes +// while the board silently renders "—" for that model's cost. readTable() +// therefore cross-checks the matched-row count against the number of +// `ModelPricing(` constructions in the file and refuses to emit on a mismatch. + +import { readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Resolve both paths from the script's own location, never cwd, so the +// generator behaves identically from `evalboard/` or the repo root. +const here = dirname(fileURLToPath(import.meta.url)); +const PY_PATH = resolve(here, "../../src/coder_eval/pricing.py"); +const OUT_PATH = resolve(here, "../lib/pricing.generated.ts"); + +// Match: "model-id": ModelPricing(1.25, 10.0, 1.25, 0.125), +const ROW_RE = + /"([^"]+)":\s*ModelPricing\(\s*([\d.]+),\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)\s*\)/g; + +// Every `ModelPricing(` in the file should be one matched table row. +const CONSTRUCTOR_RE = /ModelPricing\(/g; + +// Below this, assume a regex or path regression rather than a genuinely tiny +// table — emitting an empty/near-empty table would silently unprice the board. +const MIN_ROWS = 10; + +// Models priced in pricing.py that must NOT reach the frontend table. +// +// This is the ONE exception to "generate everything", and it is not a +// convenience allowlist — the old hand-maintained not-mirrored list is exactly +// what this generator exists to delete (17 models drifted past it, two of them +// models real runs use, silently rendering "—" for their cost). An entry here +// means a static rate would be WRONG, not merely unused. +// +// OpenRouter routes each request to whichever provider wins its sort, so the +// billed rate depends on where the call landed and no single headline rate is +// correct. pricing.py keeps these solely as the Python-side `max_usd` static +// fallback; the harness captures each call's ACTUAL cost proxy-side and the +// detail view renders it per call (TurnRecord.provider_call_costs → +// ProviderCallTableSection). Mirroring the headline rate here would put a +// confidently wrong number next to the measured one. +// +// Bedrock open-weight models (deepseek.v3.2, zai.glm-5, moonshotai.kimi-k2.5) +// are deliberately NOT excluded: they run at fixed Bedrock rates like Claude, +// with no per-call capture, so static pricing is correct and required. +const EXCLUDED_MODELS = new Set([ + "moonshotai/kimi-k3", + "z-ai/glm-5.2", + "deepseek/deepseek-v4-pro", +]); + +function readSource() { + try { + return readFileSync(PY_PATH, "utf8"); + } catch (err) { + throw new Error( + `cannot read the Python pricing table at ${PY_PATH}: ${err instanceof Error ? err.message : err}`, + ); + } +} + +/** + * Parse the Python rate table into `{ model: [input, output, cacheWrite, cacheRead] }`, + * preserving pricing.py's insertion order. + * + * Uses a null-prototype object so a degenerate key (`__proto__`) becomes an own + * property instead of silently setting the prototype and dropping the row. + * @returns {Record} + */ +export function parsePythonTable() { + const src = readSource(); + /** @type {Record} */ + const out = Object.create(null); + for (const m of src.matchAll(ROW_RE)) { + const rates = [Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5])]; + if (!rates.every(Number.isFinite)) { + throw new Error( + `non-finite rate for "${m[1]}" in ${PY_PATH}: [${rates.join(", ")}]. ` + + `Emitting NaN would typecheck and render "$NaN" on the board.`, + ); + } + out[m[1]] = /** @type {[number, number, number, number]} */ (rates); + } + return out; +} + +/** + * Parse, then verify the parse actually covered the file. + * + * The drift guard MUST use this rather than parsePythonTable: comparing two + * products of the same parser cannot detect a row the parser skipped, because + * the row is missing from the artifact AND from the expectation. This is the + * only function that can tell "the mirror is faithful" apart from "the mirror + * and the expectation are equally blind". + * @returns {Record} + */ +export function readTable() { + const src = readSource(); + const table = parsePythonTable(); + const rows = Object.keys(table).length; + if (rows < MIN_ROWS) { + throw new Error( + `parsed only ${rows} rows from ${PY_PATH} (expected at least ${MIN_ROWS}). ` + + `The regex or the path has regressed — refusing to emit a partial table.`, + ); + } + const constructions = (src.match(CONSTRUCTOR_RE) ?? []).length; + if (constructions !== rows) { + throw new Error( + `${PY_PATH} has ${constructions} ModelPricing(...) constructions but ROW_RE matched ${rows} rows.\n` + + `A rate row ROW_RE cannot parse (keyword arguments, a variable, a computed expression, ` + + `scientific notation) would be dropped from the generated table AND invisible to the parity ` + + `test, silently unpricing that model. Either rewrite the row as a literal ` + + `\`"model": ModelPricing(a, b, c, d)\`, or widen ROW_RE. ` + + `(If a ModelPricing(...) legitimately lives outside the table, teach this check to skip it.)`, + ); + } + return table; +} + +/** + * The subset of pricing.py that the frontend table mirrors: everything except + * EXCLUDED_MODELS. This is what both the generator emits and the drift guard + * compares against, so the exclusion is stated once. + * + * Throws on a STALE exclusion — an id listed in EXCLUDED_MODELS that pricing.py + * no longer prices. Without this an exclusion outlives the model it was written + * for, and if that id is ever reintroduced (as a normal fixed-rate model) it + * would be silently dropped from the board, which is the very failure the + * hand-maintained allowlist kept producing. + * @param {Record} table + * @returns {Record} + */ +export function mirroredTable(table) { + const stale = [...EXCLUDED_MODELS].filter((m) => !(m in table)); + if (stale.length > 0) { + throw new Error( + `EXCLUDED_MODELS lists ${stale.join(", ")}, which ${PY_PATH} no longer prices. ` + + `Drop the stale entr${stale.length === 1 ? "y" : "ies"} — an exclusion that outlives its model ` + + `silently unprices that id if it is ever reintroduced.`, + ); + } + return Object.fromEntries( + Object.entries(table).filter(([model]) => !EXCLUDED_MODELS.has(model)), + ); +} + +/** The ids deliberately withheld from the frontend table. See EXCLUDED_MODELS. */ +export function excludedModels() { + return [...EXCLUDED_MODELS]; +} + +/** + * Render the generated TypeScript module. Every key is quoted unconditionally + * via JSON.stringify — `z-ai/glm-5.2`, `deepseek.v3.2`, and `gpt-5.1-codex-max` + * all need it, and deciding per key costs more than it saves. Rates are + * interpolated as-is, so a legitimate `0` (Bedrock open-weight cache-read) is + * emitted as `0`; parsePythonTable has already guaranteed each is finite. + * @param {Record} table + * @returns {string} + */ +function renderModule(table) { + const rows = Object.entries(table) + .map( + ([model, [input, output, cacheWrite, cacheRead]]) => + ` ${JSON.stringify(model)}: {\n` + + ` inputPerMTok: ${input},\n` + + ` outputPerMTok: ${output},\n` + + ` cacheWritePerMTok: ${cacheWrite},\n` + + ` cacheReadPerMTok: ${cacheRead},\n` + + ` },`, + ) + .join("\n"); + return ( + `// GENERATED BY scripts/gen-pricing.mjs — DO NOT EDIT.\n` + + `// Source of truth: src/coder_eval/pricing.py.\n` + + `// To change a rate: edit that file, then run \`pnpm gen:pricing\`.\n` + + `// Row order mirrors pricing.py so regeneration diffs stay minimal.\n` + + `//\n` + + `// Not every pricing.py model appears here: the OpenRouter open-weight ids are\n` + + `// withheld on purpose, because they are routed per-request and shown at their\n` + + `// captured ACTUAL per-call cost instead. See EXCLUDED_MODELS in the generator.\n` + + `\n` + + `import type { Pricing } from "./pricing-types";\n` + + `\n` + + `export const GENERATED_PRICING: Record = {\n` + + `${rows}\n` + + `};\n` + ); +} + +function main() { + let table; + try { + table = mirroredTable(readTable()); + } catch (err) { + // Every guard above throws rather than emitting, so a failed run leaves + // the previous artifact intact instead of writing a partial table. + console.error( + `gen-pricing: ${err instanceof Error ? err.message : err}`, + ); + process.exit(1); + } + writeFileSync(OUT_PATH, renderModule(table), "utf8"); + console.log( + `gen-pricing: wrote ${Object.keys(table).length} models to ${OUT_PATH}`, + ); +} + +// Only emit when run as a script; importers (the parity test) get the parser +// alone. realpathSync, not resolve: Node realpaths the main module for +// import.meta.url, so a lexical compare silently no-ops (exit 0, nothing +// written) whenever a symlink is anywhere in the invocation path. +if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +}