From d57a253d048294219b991457073fb5259a12ea8d Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:53:50 +0300 Subject: [PATCH 1/7] test(budget): require unpriced usage to fail closed --- .../src/engine/claude.unpriced.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/loop-js/src/engine/claude.unpriced.test.ts diff --git a/packages/loop-js/src/engine/claude.unpriced.test.ts b/packages/loop-js/src/engine/claude.unpriced.test.ts new file mode 100644 index 0000000..4cd9461 --- /dev/null +++ b/packages/loop-js/src/engine/claude.unpriced.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from "bun:test" +import { stepUsage } from "./claude.ts" +import { Interruption } from "./executor.ts" + +test("unpriced non-zero token usage fails closed as a budget interruption", () => { + try { + stepUsage({ input_tokens: 1 }, "some-future-model") + throw new Error("expected unpriced usage to fail closed") + } catch (err) { + expect(err).toBeInstanceOf(Interruption) + expect((err as Interruption).cause).toBe("budget") + expect((err as Interruption).detail).toContain("unpriced model") + } +}) + +test("an unpriced zero-token step is still free", () => { + expect(stepUsage({}, "some-future-model")).toEqual({ + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + usd: 0, + }) +}) From 401ac7566c7e802930f80dba545587cff2ec4a2c Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:56:37 +0300 Subject: [PATCH 2/7] fix(budget): fail closed on unpriced token usage --- packages/loop-js/src/engine/claude.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/loop-js/src/engine/claude.ts b/packages/loop-js/src/engine/claude.ts index 64f46ff..941cfd7 100644 --- a/packages/loop-js/src/engine/claude.ts +++ b/packages/loop-js/src/engine/claude.ts @@ -139,14 +139,19 @@ export function stepUsage(u: TokenUsage, model: string): StepUsage { const write5m = u.cache_creation ? (u.cache_creation.ephemeral_5m_input_tokens ?? 0) : written const p = PRICES[model] - const usd = p - ? (inputTokens * p.input + - write5m * p.input * WRITE_5M + - write1h * p.input * WRITE_1H + - cachedInputTokens * p.input * CACHE_READ + - outputTokens * p.output) / - 1e6 - : 0 // an unpriced model derives nothing; the result's `total_cost_usd` reconciles it + if (!p) { + const hasTokenUsage = inputTokens !== 0 || outputTokens !== 0 || cachedInputTokens !== 0 || written !== 0 + if (hasTokenUsage) throw new Interruption("budget", `unpriced model '${model}' produced token usage`) + return { inputTokens, outputTokens, cachedInputTokens, usd: 0 } + } + + const usd = + (inputTokens * p.input + + write5m * p.input * WRITE_5M + + write1h * p.input * WRITE_1H + + cachedInputTokens * p.input * CACHE_READ + + outputTokens * p.output) / + 1e6 return { inputTokens, outputTokens, cachedInputTokens, usd } } @@ -427,4 +432,4 @@ async function* verify(query: Query, req: VerifyRequest): PhaseStream startRound(query, req), verify: (req) => verify(query, req) } -} +} \ No newline at end of file From 07425196d4ed06cc9f65ce15d5e9c454d67434b3 Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:57:32 +0300 Subject: [PATCH 3/7] test(budget): cover unpriced streamed usage --- .../src/engine/claude.unpriced.test.ts | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/loop-js/src/engine/claude.unpriced.test.ts b/packages/loop-js/src/engine/claude.unpriced.test.ts index 4cd9461..a416c5a 100644 --- a/packages/loop-js/src/engine/claude.unpriced.test.ts +++ b/packages/loop-js/src/engine/claude.unpriced.test.ts @@ -1,10 +1,21 @@ import { expect, test } from "bun:test" -import { stepUsage } from "./claude.ts" +import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk" +import { drainSession, stepUsage } from "./claude.ts" import { Interruption } from "./executor.ts" -test("unpriced non-zero token usage fails closed as a budget interruption", () => { +async function* feed(...messages: SDKMessage[]): AsyncGenerator { + for (const message of messages) yield message +} + +test("unpriced non-zero streamed usage fails closed as a budget interruption", async () => { + const assistant = { + type: "assistant", + message: { id: "turn_1", content: [], usage: { input_tokens: 1 } }, + } as unknown as SDKMessage + const gen = drainSession(feed(assistant), "some-future-model") + try { - stepUsage({ input_tokens: 1 }, "some-future-model") + await gen.next() throw new Error("expected unpriced usage to fail closed") } catch (err) { expect(err).toBeInstanceOf(Interruption) @@ -13,11 +24,6 @@ test("unpriced non-zero token usage fails closed as a budget interruption", () = } }) -test("an unpriced zero-token step is still free", () => { - expect(stepUsage({}, "some-future-model")).toEqual({ - inputTokens: 0, - outputTokens: 0, - cachedInputTokens: 0, - usd: 0, - }) +test("stepUsage keeps zero as its best-effort value for an unpriced model", () => { + expect(stepUsage({ input_tokens: 1 }, "some-future-model").usd).toBe(0) }) From 6acaf3f5096838303d518f0b4537311e93714e33 Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:58:50 +0300 Subject: [PATCH 4/7] fix(budget): preserve stepUsage contract --- packages/loop-js/src/engine/claude.ts | 35 +++++++++++++++++---------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/packages/loop-js/src/engine/claude.ts b/packages/loop-js/src/engine/claude.ts index 941cfd7..c6e6a5d 100644 --- a/packages/loop-js/src/engine/claude.ts +++ b/packages/loop-js/src/engine/claude.ts @@ -130,6 +130,17 @@ export type TokenUsage = { cache_creation?: { ephemeral_5m_input_tokens?: number | null; ephemeral_1h_input_tokens?: number | null } | null } +function hasTokenUsage(u: TokenUsage): boolean { + return ( + (u.input_tokens ?? 0) !== 0 || + (u.output_tokens ?? 0) !== 0 || + (u.cache_creation_input_tokens ?? 0) !== 0 || + (u.cache_read_input_tokens ?? 0) !== 0 || + (u.cache_creation?.ephemeral_5m_input_tokens ?? 0) !== 0 || + (u.cache_creation?.ephemeral_1h_input_tokens ?? 0) !== 0 + ) +} + export function stepUsage(u: TokenUsage, model: string): StepUsage { const inputTokens = u.input_tokens ?? 0 const outputTokens = u.output_tokens ?? 0 @@ -139,19 +150,14 @@ export function stepUsage(u: TokenUsage, model: string): StepUsage { const write5m = u.cache_creation ? (u.cache_creation.ephemeral_5m_input_tokens ?? 0) : written const p = PRICES[model] - if (!p) { - const hasTokenUsage = inputTokens !== 0 || outputTokens !== 0 || cachedInputTokens !== 0 || written !== 0 - if (hasTokenUsage) throw new Interruption("budget", `unpriced model '${model}' produced token usage`) - return { inputTokens, outputTokens, cachedInputTokens, usd: 0 } - } - - const usd = - (inputTokens * p.input + - write5m * p.input * WRITE_5M + - write1h * p.input * WRITE_1H + - cachedInputTokens * p.input * CACHE_READ + - outputTokens * p.output) / - 1e6 + const usd = p + ? (inputTokens * p.input + + write5m * p.input * WRITE_5M + + write1h * p.input * WRITE_1H + + cachedInputTokens * p.input * CACHE_READ + + outputTokens * p.output) / + 1e6 + : 0 // an unpriced model derives nothing; drainSession fails closed before this can bypass the guard return { inputTokens, outputTokens, cachedInputTokens, usd } } @@ -177,6 +183,9 @@ export async function* drainSession( } case "assistant": { if (m.error) throw new Interruption("error", `claude: the model turn failed (${m.error})`) + if (!PRICES[model] && hasTokenUsage(m.message.usage)) { + throw new Interruption("budget", `unpriced model '${model}' produced token usage`) + } for (const b of m.message.content) { if (b.type === "text") yield { kind: "text", text: b.text } else if (b.type === "thinking") yield { kind: "reasoning", text: b.thinking } From e28abafff1ff75b644916d0e68f0d7090b2373e8 Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:03:59 +0300 Subject: [PATCH 5/7] chore: restore source newline --- packages/loop-js/src/engine/claude.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/loop-js/src/engine/claude.ts b/packages/loop-js/src/engine/claude.ts index c6e6a5d..660da42 100644 --- a/packages/loop-js/src/engine/claude.ts +++ b/packages/loop-js/src/engine/claude.ts @@ -441,4 +441,4 @@ async function* verify(query: Query, req: VerifyRequest): PhaseStream startRound(query, req), verify: (req) => verify(query, req) } -} \ No newline at end of file +} From 50e2dc1946264303edea3656804d0e1dfb56a921 Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:14:58 +0300 Subject: [PATCH 6/7] test(budget): preserve streamed content ordering --- packages/loop-js/src/engine/claude.unpriced.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/loop-js/src/engine/claude.unpriced.test.ts b/packages/loop-js/src/engine/claude.unpriced.test.ts index a416c5a..2ca0784 100644 --- a/packages/loop-js/src/engine/claude.unpriced.test.ts +++ b/packages/loop-js/src/engine/claude.unpriced.test.ts @@ -7,13 +7,14 @@ async function* feed(...messages: SDKMessage[]): AsyncGenerator { +test("unpriced non-zero streamed usage fails closed at the cost boundary", async () => { const assistant = { type: "assistant", - message: { id: "turn_1", content: [], usage: { input_tokens: 1 } }, + message: { id: "turn_1", content: [{ type: "text", text: "partial" }], usage: { input_tokens: 1 } }, } as unknown as SDKMessage const gen = drainSession(feed(assistant), "some-future-model") + expect(await gen.next()).toEqual({ done: false, value: { kind: "text", text: "partial" } }) try { await gen.next() throw new Error("expected unpriced usage to fail closed") From 7cb252ef94c56285c1ab1623c04bc887b911acf2 Mon Sep 17 00:00:00 2001 From: Mamdouh Aboammar <58908124+imMamdouhaboammar@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:16:30 +0300 Subject: [PATCH 7/7] fix(budget): fail closed at cost boundary --- packages/loop-js/src/engine/claude.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/loop-js/src/engine/claude.ts b/packages/loop-js/src/engine/claude.ts index 660da42..b336464 100644 --- a/packages/loop-js/src/engine/claude.ts +++ b/packages/loop-js/src/engine/claude.ts @@ -183,9 +183,6 @@ export async function* drainSession( } case "assistant": { if (m.error) throw new Interruption("error", `claude: the model turn failed (${m.error})`) - if (!PRICES[model] && hasTokenUsage(m.message.usage)) { - throw new Interruption("budget", `unpriced model '${model}' produced token usage`) - } for (const b of m.message.content) { if (b.type === "text") yield { kind: "text", text: b.text } else if (b.type === "thinking") yield { kind: "reasoning", text: b.thinking } @@ -195,6 +192,9 @@ export async function* drainSession( // repeating the turn's cumulative usage (contract §Mapping 1). Cost a turn once, on its id. if (m.message.id !== turn) { turn = m.message.id + if (!PRICES[model] && hasTokenUsage(m.message.usage)) { + throw new Interruption("budget", `unpriced model '${model}' produced token usage`) + } const usage = stepUsage(m.message.usage, model) derived += usage.usd yield { kind: "cost", usage }