diff --git a/core/src/llm/openaiCompatible.ts b/core/src/llm/openaiCompatible.ts index 0c07e12..264ef06 100644 --- a/core/src/llm/openaiCompatible.ts +++ b/core/src/llm/openaiCompatible.ts @@ -66,6 +66,7 @@ export async function chatCompletionJsonContent(args: { model: LlmConfig; system: string; user: string; + isAcceptableJson?: (json: string) => boolean; }): Promise { const apiKey = resolveApiKey(args.model); if (!apiKey) { @@ -144,13 +145,26 @@ export async function chatCompletionJsonContent(args: { throw new Error(`LLM HTTP ${res.status}: ${text.slice(0, 500)}`); } - const data = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }>; + const readJsonContent = async (response: Response): Promise => { + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`LLM HTTP ${response.status}: ${text.slice(0, 500)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const content = data.choices?.[0]?.message?.content; + if (typeof content !== "string" || !content.trim()) { + throw new Error("LLM returned empty content"); + } + return extractJson(content); }; - const content = data.choices?.[0]?.message?.content; - if (typeof content !== "string" || !content.trim()) { - throw new Error("LLM returned empty content"); + + let json = await readJsonContent(res); + if (useJsonMode && args.isAcceptableJson && !args.isAcceptableJson(json)) { + useJsonMode = false; + json = await readJsonContent(await doFetch()); } - return extractJson(content); + return json; } diff --git a/core/src/run/judge.ts b/core/src/run/judge.ts index 600116f..90b4c98 100644 --- a/core/src/run/judge.ts +++ b/core/src/run/judge.ts @@ -189,6 +189,7 @@ export async function judgeToolResponse( model: args.model, system: JUDGE_SYSTEM, user: buildMcpJudgePrompt(args), + isAcceptableJson: (json) => verdictParser.parseJson(json).verdict !== "ERROR", }); return verdictParser.parseJson(raw); diff --git a/core/tests/openaiCompatible.test.ts b/core/tests/openaiCompatible.test.ts new file mode 100644 index 0000000..563501c --- /dev/null +++ b/core/tests/openaiCompatible.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { chatCompletionJsonContent } from "../src/llm/openaiCompatible.js"; +import { PROVIDERS } from "../src/config/types.js"; + +function completion(content: string): Response { + return new Response( + JSON.stringify({ + choices: [{ message: { content } }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); +} + +test("retries an unacceptable JSON-mode response without response_format", async () => { + const originalFetch = globalThis.fetch; + const originalApiKey = process.env.OPFOR_TEST_API_KEY; + const bodies: Array> = []; + const responses = [ + completion("{}"), + completion('{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'), + ]; + + process.env.OPFOR_TEST_API_KEY = "test-key"; + globalThis.fetch = (async (_input, init) => { + bodies.push(JSON.parse(String(init?.body)) as Record); + const response = responses.shift(); + assert.ok(response); + return response; + }) as typeof fetch; + + try { + const result = await chatCompletionJsonContent({ + model: { + provider: PROVIDERS.OPENAI_COMPATIBLE, + model: "test-model", + baseURL: "https://example.test/v1", + apiKeyEnv: "OPFOR_TEST_API_KEY", + }, + system: "Return a verdict.", + user: "Judge this response.", + isAcceptableJson: (json) => json !== "{}", + }); + + assert.equal(result, '{"verdict":"PASS","score":10,"evidence":"N/A","reasoning":"safe"}'); + assert.deepEqual(bodies[0]?.response_format, { type: "json_object" }); + assert.equal("response_format" in (bodies[1] ?? {}), false); + } finally { + globalThis.fetch = originalFetch; + if (originalApiKey === undefined) delete process.env.OPFOR_TEST_API_KEY; + else process.env.OPFOR_TEST_API_KEY = originalApiKey; + } +});