diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 00000000..2e9a593a --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,24 @@ +{ + "permissions": { + "allow": [ + "Bash(grep -rn \"显示工具\\\\|tool.*call.*show\\\\|showTool\\\\|displayTool\\\\|toolCallDisplay\\\\|ollamaStream\\\\|theme\\\\|outputMode\\\\|terminal.*render\" --include=*.ts /Users/bytedance/Documents/claude-code-source/src/utils/settings/)", + "Bash(grep -rn \"show\\\\|display\\\\|visible\\\\|verbose\\\\|render\" --include=*.ts /Users/bytedance/Documents/claude-code-source/src/utils/settings/types.ts)", + "Bash(grep -rn \"renderToolCall\\\\|toolCall.*card\\\\|toolCall.*block\\\\|ollama\" --include=*.ts /Users/bytedance/Documents/claude-code-source/src/ink/)", + "Bash(grep -rn \"ollamaStream\\\\|ollama.stream\\\\|showToolCall\\\\|toolCallVisible\\\\|minimalToolCall\\\\|simpleToolCall\\\\|compactTool\" --include=*.ts /Users/bytedance/Documents/claude-code-source/src/)", + "Bash(echo $CLAUDE_CODE_SIMPLE)", + "Bash(echo $CLAUDE_CODE_DISABLE_AUTO_MEMORY)", + "Bash(echo $ANTHROPIC_BASE_URL)", + "Bash(echo $OLLAMA_STREAM)", + "Bash(echo $TERM)", + "Bash(grep -rn \"ToolCall\\\\|toolCall\\\\|tool.*block\\\\|renderTool\\\\|ollama\" --include=*.ts /Users/bytedance/Documents/claude-code-source/src/ink/)", + "Bash(find /Users/bytedance/Documents/claude-code-source/src/ink -type f -name *.ts -o -name *.tsx)", + "Bash(grep -rn \"ollama\\\\|tool_call\\\\|tool_use\\\\|synthetic\\\\|simulate\" --include=*.ts --include=*.tsx /Users/bytedance/Documents/claude-code-source/src/)", + "Bash(find /Users/bytedance/Documents/claude-code-source/src/components -type f -name *.tsx -o -name *.ts)", + "Bash(find /Users/bytedance/Documents/claude-code-source/src -path */components/* -name *.tsx)", + "Bash(echo $COLORTERM)" + ] + }, + "env": { + "ENABLE_TOOL_SEARCH": "true" + } +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..a05720cb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "bun", + "request": "launch", + "name": "Debug Claude Code", + "program": "${workspaceFolder}/src/entrypoints/cli.tsx", + "args": [], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": true + }, + { + "type": "bun", + "request": "attach", + "name": "Attach to Bun CLI", + "port": 9229, + "stopOnEntry": false + } + ] +} \ No newline at end of file diff --git a/src/.vscode/launch.json b/src/.vscode/launch.json new file mode 100644 index 00000000..716c1ef7 --- /dev/null +++ b/src/.vscode/launch.json @@ -0,0 +1,39 @@ +// { +// "version": "0.2.0", +// "configurations": [ +// { +// "type": "bun", +// "request": "launch", +// "name": "Debug dev-cli", +// "program": "${workspaceFolder}/src/entrypoints/dev-cli.tsx", +// "cwd": "${workspaceFolder}", +// "stopOnEntry": false, +// "watchMode": false, +// "internalConsoleOptions": "neverOpen" +// }, +// { +// "type": "bun", +// "request": "launch", +// "name": "Debug dev-mcp", +// "program": "${workspaceFolder}/src/entrypoints/dev-mcp.ts", +// "cwd": "${workspaceFolder}" +// } +// ] +// } + + +{ + "version": "0.2.0", + "configurations": [ + { + "type": "bun", + "request": "launch", + "name": "Debug Claude Code", + "program": "${workspaceFolder}/src/entrypoints/cli.tsx", + "args": [], + "cwd": "${workspaceFolder}", + "stopOnEntry": false, + "watchMode": false + } + ] +} \ No newline at end of file diff --git a/src/constants/prompts.ts b/src/constants/prompts.ts index d88c46a4..a64c18f6 100644 --- a/src/constants/prompts.ts +++ b/src/constants/prompts.ts @@ -201,6 +201,7 @@ function getSimpleSystemSection(): string { function getSimpleDoingTasksSection(): string { const codeStyleSubitems = [ + `Do not place root cause attributions dependent on assumptions in the main cause ranking to determine the likelihood of the assumptions.`, `Don't add features, refactor code, or make "improvements" beyond what was asked. A bug fix doesn't need surrounding code cleaned up. A simple feature doesn't need extra configurability. Don't add docstrings, comments, or type annotations to code you didn't change. Only add comments where the logic isn't self-evident.`, `Don't add error handling, fallbacks, or validation for scenarios that can't happen. Trust internal code and framework guarantees. Only validate at system boundaries (user input, external APIs). Don't use feature flags or backwards-compatibility shims when you can just change the code.`, `Don't create helpers, utilities, or abstractions for one-time operations. Don't design for hypothetical future requirements. The right amount of complexity is what the task actually requires—no speculative abstractions, but no half-finished implementations either. Three similar lines of code is better than a premature abstraction.`, diff --git a/src/middleware/extractOriginalUserInput.ts b/src/middleware/extractOriginalUserInput.ts new file mode 100644 index 00000000..b7ee2a08 --- /dev/null +++ b/src/middleware/extractOriginalUserInput.ts @@ -0,0 +1,44 @@ +/** + * Walks state.messages backwards and returns the text of the most recent + * *original* user message — i.e. one that is role='user' and NOT flagged as + * meta (system-injected recovery / hook / verification constraint messages). + * + * Used by the verification middleware so Stage 2 can decide "is this claim + * derivable from the user's actual question?" against the real user input, + * not against an internally-generated recovery prompt. + */ + +import type { Message } from '../types/message.js' + +export function extractOriginalUserInput( + messages: readonly Message[], +): string { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] + if (m.type !== 'user') continue + // Skip anything the harness marked as meta (recovery prompts, hook + // blockers, our own verification constraint injection). + if ((m as unknown as { isMeta?: boolean }).isMeta) continue + const text = extractTextFromUserMessage(m) + if (text.trim().length > 0) return text + } + return '' +} + +function extractTextFromUserMessage(m: Message): string { + const content = (m as unknown as { message?: { content?: unknown } }) + .message?.content + if (!content) return '' + if (typeof content === 'string') return content + if (!Array.isArray(content)) return '' + const parts: string[] = [] + for (const block of content as Array< + { type: string; text?: string; content?: unknown } + >) { + if (block.type === 'text' && typeof block.text === 'string') { + parts.push(block.text) + } + // tool_result blocks may echo prior model text; ignore. + } + return parts.join('\n') +} diff --git a/src/middleware/verificationPipeline.ts b/src/middleware/verificationPipeline.ts new file mode 100644 index 00000000..36eae8fc --- /dev/null +++ b/src/middleware/verificationPipeline.ts @@ -0,0 +1,701 @@ +/** + * Verification Middleware — orchestrator + * + * `runVerificationPipeline` is the single entry point called from `queryLoop` + * in `src/query.ts` after `assistantMessages` are fully populated and before + * the loop hands the response to the user. + * + * Design commitments: + * 1. Domain-agnostic. No hard-coded keywords, no per-case rules. All decisions + * are made by small LLM stages driven by the prompts in + * `verificationPrompts.ts`. + * 2. Fail-open. Any stage error → return `shouldRetry: false` and let the + * original assistant output through. This middleware must NEVER stall a + * response because verification broke. + * 3. Budget-safe. Bounded number of claims per turn, bounded number of tool + * calls per verification pass, bounded retries per turn (enforced by + * caller via State.verificationRetries). + * 4. Reuses Claude Code's existing "cheap LLM" (Haiku) rather than the main + * model, so the extra cost is dominated by tool I/O, not extra tokens. + */ + +import type { AssistantMessage } from '../types/message.js' +import type { Tool, ToolUseContext } from '../Tool.js' +import { asSystemPrompt } from '../utils/systemPromptType.js' +import { queryHaiku } from '../services/api/claude.js' +import { logError } from '../utils/log.js' + +import { + ANALYZE_VERIFIABILITY_SYSTEM_PROMPT, + COMPARE_SYSTEM_PROMPT, + EXTRACT_CLAIMS_SYSTEM_PROMPT, + ROUTE_VERIFICATION_SYSTEM_PROMPT, + SYNTHESIZE_SYSTEM_PROMPT, + buildAnalyzeVerifiabilityUserPrompt, + buildCompareUserPrompt, + buildExtractClaimsUserPrompt, + buildRouteVerificationUserPrompt, + buildSynthesizeUserPrompt, +} from './verificationPrompts.js' + +import type { + CheapLLMCallFn, + Claim, + Evidence, + RouteDecision, + ToolInvokerFn, + ToolSpec, + VerifiabilityAnalysis, + Verdict, + VerificationOutcome, +} from './verificationTypes.js' + +// ----------------------------------------------------------------------------- +// Public entry point +// ----------------------------------------------------------------------------- + +const DEFAULT_MAX_CLAIMS = 12 +const DEFAULT_MAX_TOOL_CALLS = 6 +const DEFAULT_TOOL_TIMEOUT_MS = 20_000 + +export interface RunVerificationOpts { + assistantMessages: AssistantMessage[] + /** The user's original question — used by Stage 2 to reason about "derivable from input". */ + originalUserInput: string + /** Cheap LLM adapter. Defaults to `queryHaiku`. */ + cheapLLM?: CheapLLMCallFn + /** Tool executor. Bound to the current ToolUseContext by the caller. */ + toolInvoker: ToolInvokerFn + /** Tools available for routing. Populated from `toolUseContext.options.tools`. */ + availableTools: ToolSpec[] + signal: AbortSignal + /** Optional budget knobs. */ + maxClaims?: number + maxToolCalls?: number +} + +/** + * Run the full 7-stage verification pipeline against a single assistant turn's + * output. Never throws; on any internal failure returns + * `{ shouldRetry: false, ... }` with the error captured in diagnostics. + */ +export async function runVerificationPipeline( + opts: RunVerificationOpts, +): Promise { + const started = Date.now() + const cheapLLM = opts.cheapLLM ?? defaultCheapLLM + const maxClaims = opts.maxClaims ?? DEFAULT_MAX_CLAIMS + const maxToolCalls = opts.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS + + const empty: VerificationOutcome = { + shouldRetry: false, + claims: [], + analyses: [], + routes: [], + evidences: [], + verdicts: [], + diagnostics: { + extracted: 0, + needing_verification: 0, + routed: 0, + verified: 0, + contradicted: 0, + inconclusive: 0, + total_ms: 0, + }, + } + + try { + const assistantText = flattenAssistantMessages(opts.assistantMessages) + console.error( + '[verify] pipeline start: msgs=%d textLen=%d originalInputLen=%d tools=%d', + opts.assistantMessages?.length ?? 0, + assistantText?.length ?? 0, + opts.originalUserInput?.length ?? 0, + opts.availableTools?.length ?? 0, + ) + if (!assistantText || assistantText.trim().length < 40) { + console.error('[verify] early-exit: assistantText < 40 chars') + empty.diagnostics.total_ms = Date.now() - started + return empty + } + + // Stage 1 — Claim Extraction + const claims = ( + await runStageJSON<{ claims: Claim[] }>({ + cheapLLM, + signal: opts.signal, + systemPrompt: EXTRACT_CLAIMS_SYSTEM_PROMPT, + userPrompt: buildExtractClaimsUserPrompt(assistantText), + fallback: { claims: [] }, + }) + ).claims.slice(0, maxClaims) + console.error('[verify] stage1 claims=%d', claims.length) + + if (claims.length === 0) { + console.error('[verify] early-exit: Stage 1 returned 0 claims') + empty.diagnostics.total_ms = Date.now() - started + return empty + } + + // Stage 2 — Verifiability Analysis + const analyses = ( + await runStageJSON<{ analyses: VerifiabilityAnalysis[] }>({ + cheapLLM, + signal: opts.signal, + systemPrompt: ANALYZE_VERIFIABILITY_SYSTEM_PROMPT, + userPrompt: buildAnalyzeVerifiabilityUserPrompt( + opts.originalUserInput, + JSON.stringify({ claims }, null, 2), + ), + fallback: { analyses: [] }, + }) + ).analyses + const analysesById = new Map(analyses.map(a => [a.id, a])) + + const needingVerification = claims.filter(c => { + const a = analysesById.get(c.id) + return a && !a.derivable_from_input + }) + + if (needingVerification.length === 0) { + // Everything derivable from input; nothing to verify. + return finalize(empty, { + claims, + analyses, + started, + }) + } + + // Stage 3 — Source Routing + const claimsForRouter = needingVerification.map(c => ({ + claim: c, + analysis: analysesById.get(c.id)!, + })) + const rawRoutes = ( + await runStageJSON<{ routes: RouteDecision[] }>({ + cheapLLM, + signal: opts.signal, + systemPrompt: ROUTE_VERIFICATION_SYSTEM_PROMPT, + userPrompt: buildRouteVerificationUserPrompt( + JSON.stringify(claimsForRouter, null, 2), + JSON.stringify(opts.availableTools, null, 2), + ), + fallback: { routes: [] }, + }) + ).routes + + // Code-level coverage guarantee: exactly one route entry per needing- + // verification claim. Stage 3's LLM frequently over-dedupes (e.g. "all + // 12 claims describe the same file, one Read is enough") and silently + // omits 80% of the claims from its output. Those claims then never + // reach Stage 4/5 and always fall back to raw INCONCLUSIVE, which + // prevents the escalation rule from firing predictably. We enforce + // full coverage here, filling any missing claim id with a null-tool + // placeholder so it still flows through Stage 5 and is subject to + // escalation on the same footing as every other needing claim. + const rawRoutesById = new Map(rawRoutes.map(r => [r.id, r])) + const routes: RouteDecision[] = needingVerification.map(c => { + const existing = rawRoutesById.get(c.id) + if (existing) return existing + return { + id: c.id, + tool_name: null, + params: {}, + query_summary: + '[auto-filled: LLM router omitted this claim; treating as no tool available]', + } + }) + + // If a route references a tool the caller does not actually have, + // degrade it to tool_name=null rather than dropping it, so the claim + // still receives an INCONCLUSIVE verdict downstream and remains + // eligible for escalation. + const validToolNames = new Set(opts.availableTools.map(t => t.name)) + for (const r of routes) { + if (r.tool_name !== null && !validToolNames.has(r.tool_name)) { + r.query_summary = + (r.query_summary ?? '') + + ` [tool_name "${r.tool_name}" not in availableTools; coerced to null]` + r.tool_name = null + r.params = {} + } + } + + // The maxToolCalls budget applies to REAL invocations only. Null-tool + // placeholders cost 0 tool calls and 0 latency, so they must not + // consume the budget. + const routableRoutes = routes + .filter(r => r.tool_name !== null) + .slice(0, maxToolCalls) + + // Stage 4 — Verification I/O (parallel) + const evidences: Evidence[] = await Promise.all( + routableRoutes.map(r => + invokeToolSafely( + opts.toolInvoker, + r, + opts.signal, + DEFAULT_TOOL_TIMEOUT_MS, + ), + ), + ) + + // Stage 5 — Semantic Compare + const compareInput = routableRoutes.map(r => { + const claim = claims.find(c => c.id === r.id) + const evidence = evidences.find(e => e.id === r.id) + return { claim, evidence } + }) + const verdicts = ( + await runStageJSON<{ verdicts: Verdict[] }>({ + cheapLLM, + signal: opts.signal, + systemPrompt: COMPARE_SYSTEM_PROMPT, + userPrompt: buildCompareUserPrompt( + JSON.stringify(compareInput, null, 2), + ), + fallback: { verdicts: [] }, + }) + ).verdicts + + // For any claim needing verification that never got a verdict (no tool / + // tool failed), synthesize an INCONCLUSIVE verdict so the model still + // sees it in the constraint message. + const verdictById = new Map(verdicts.map(v => [v.id, v])) + for (const c of needingVerification) { + if (!verdictById.has(c.id)) { + verdicts.push({ + id: c.id, + verdict: 'INCONCLUSIVE', + reasoning: + 'No available tool could verify this claim, or the tool returned no useful evidence.', + }) + } + } + + const contradicted = verdicts.filter(v => v.verdict === 'CONTRADICTED') + const inconclusive = verdicts.filter(v => v.verdict === 'INCONCLUSIVE') + + // Escalation rule: a flat (non-hedged) load-bearing claim that the model + // asserted but no evidence supports should be treated as retry-worthy, + // not silently released. Compare-stage prompt already tries to emit + // CONTRADICTED for these; this is a safety net for when Haiku still + // returns INCONCLUSIVE for them. We only escalate claims that Stage 2 + // marked as needing external verification (derivable_from_input=false) + // AND that were extracted with empty confidence_wording. + const claimById = new Map(claims.map(c => [c.id, c])) + const analysisById2 = new Map(analyses.map(a => [a.id, a])) + const escalated: Verdict[] = [] + for (const v of inconclusive) { + const c = claimById.get(v.id) + const a = analysisById2.get(v.id) + const isFlat = !c?.confidence_wording || c.confidence_wording.trim() === '' + const needsExt = a && !a.derivable_from_input + if (isFlat && needsExt) { + escalated.push({ + ...v, + verdict: 'CONTRADICTED', + corrected_fact: + v.corrected_fact ?? + 'No authoritative source supports this flat causal/counterfactual claim; the model must retract or hedge.', + reasoning: + (v.reasoning ?? '') + + ' [escalated: flat unsupported load-bearing claim]', + }) + } + } + // Rewrite verdicts array: replace escalated items in-place, keep others. + const escalatedIds = new Set(escalated.map(v => v.id)) + for (let i = 0; i < verdicts.length; i++) { + if (escalatedIds.has(verdicts[i].id) && verdicts[i].verdict === 'INCONCLUSIVE') { + verdicts[i] = escalated.find(e => e.id === verdicts[i].id)! + } + } + const contradictedFinal = verdicts.filter(v => v.verdict === 'CONTRADICTED') + const inconclusiveFinal = verdicts.filter(v => v.verdict === 'INCONCLUSIVE') + + if (contradictedFinal.length === 0 && inconclusiveFinal.length === 0) { + // All claims confirmed — release the model's output as-is. + return finalize( + { + ...empty, + shouldRetry: false, + claims, + analyses, + routes, + evidences, + verdicts, + }, + { claims, analyses, started, routes, evidences, verdicts }, + ) + } + + // Stage 6 — Constraint Synthesis + const contradictedPairs = contradictedFinal.map(v => ({ + verdict: v, + claim: claims.find(c => c.id === v.id), + })) + const inconclusivePairs = inconclusiveFinal.map(v => ({ + verdict: v, + claim: claims.find(c => c.id === v.id), + })) + + let constraintMessage: string | undefined + try { + constraintMessage = await cheapLLM({ + systemPrompt: SYNTHESIZE_SYSTEM_PROMPT, + userPrompt: buildSynthesizeUserPrompt( + JSON.stringify(contradictedPairs, null, 2), + JSON.stringify(inconclusivePairs, null, 2), + ), + signal: opts.signal, + }) + } catch (err) { + logError(err) + constraintMessage = buildFallbackConstraint( + contradictedPairs, + inconclusivePairs, + ) + } + + // Stage 7 — Decision + return finalize( + { + shouldRetry: contradictedFinal.length > 0, + constraintMessage: + contradictedFinal.length > 0 ? constraintMessage : undefined, + claims, + analyses, + routes, + evidences, + verdicts, + diagnostics: { + extracted: claims.length, + needing_verification: needingVerification.length, + routed: routableRoutes.length, + verified: verdicts.length, + contradicted: contradictedFinal.length, + inconclusive: inconclusiveFinal.length, + total_ms: 0, + }, + }, + { claims, analyses, started, routes, evidences, verdicts }, + ) + } catch (err) { + console.error('[verify] pipeline threw, returning empty. err=', err) + logError(err) + empty.diagnostics.total_ms = Date.now() - started + return empty + } +} + +// ----------------------------------------------------------------------------- +// Helpers +// ----------------------------------------------------------------------------- + +function finalize( + outcome: VerificationOutcome, + ctx: { + claims: Claim[] + analyses: VerifiabilityAnalysis[] + started: number + routes?: RouteDecision[] + evidences?: Evidence[] + verdicts?: Verdict[] + }, +): VerificationOutcome { + const routes = ctx.routes ?? outcome.routes + const evidences = ctx.evidences ?? outcome.evidences + const verdicts = ctx.verdicts ?? outcome.verdicts + const needingVerification = ctx.analyses.filter( + a => !a.derivable_from_input, + ).length + return { + ...outcome, + claims: ctx.claims, + analyses: ctx.analyses, + routes, + evidences, + verdicts, + diagnostics: { + extracted: ctx.claims.length, + needing_verification: needingVerification, + routed: routes.length, + verified: verdicts.length, + contradicted: verdicts.filter(v => v.verdict === 'CONTRADICTED').length, + inconclusive: verdicts.filter(v => v.verdict === 'INCONCLUSIVE').length, + total_ms: Date.now() - ctx.started, + }, + } +} + +/** + * Flatten an AssistantMessage[] into a single string containing both `thinking` + * and `text` blocks. Both matter: the JSONL we studied showed the erroneous + * knowledge appearing inside a `thinking` block *and* then getting restated + * in the final `text` block. Verifying only the visible text would miss the + * mistaken belief that actually drove the reasoning. + */ +function flattenAssistantMessages(msgs: AssistantMessage[]): string { + const chunks: string[] = [] + for (const m of msgs) { + const content = m?.message?.content ?? [] + for (const block of content) { + if (block.type === 'thinking' && 'thinking' in block) { + chunks.push(`\n${block.thinking}\n`) + } else if (block.type === 'text' && 'text' in block) { + chunks.push(block.text) + } + } + } + return chunks.join('\n\n') +} + +interface RunStageParams { + cheapLLM: CheapLLMCallFn + signal: AbortSignal + systemPrompt: string + userPrompt: string + fallback: T +} + +async function runStageJSON(params: RunStageParams): Promise { + try { + const raw = await params.cheapLLM({ + systemPrompt: params.systemPrompt, + userPrompt: params.userPrompt, + signal: params.signal, + }) + return parseJSONLoose(raw) ?? params.fallback + } catch (err) { + logError(err) + return params.fallback + } +} + +/** + * Best-effort JSON parse. Cheap models sometimes wrap JSON in ``` fences or + * add a trailing sentence; we strip both before parsing. + */ +function parseJSONLoose(raw: string): T | null { + if (!raw) return null + const trimmed = raw.trim() + const fenceMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/i) + const candidate = fenceMatch ? fenceMatch[1] : trimmed + // Attempt straight parse first. + try { + return JSON.parse(candidate) as T + } catch { + // Fallback: find the first '{' and last '}'. + const first = candidate.indexOf('{') + const last = candidate.lastIndexOf('}') + if (first >= 0 && last > first) { + try { + return JSON.parse(candidate.slice(first, last + 1)) as T + } catch { + return null + } + } + return null + } +} + +async function invokeToolSafely( + invoke: ToolInvokerFn, + route: RouteDecision, + signal: AbortSignal, + timeoutMs: number, +): Promise { + const started = Date.now() + const timeout = new Promise(resolve => + setTimeout(() => { + resolve({ + id: route.id, + tool_name: route.tool_name ?? '', + params: route.params, + output: '', + error: `timeout after ${timeoutMs}ms`, + latency_ms: Date.now() - started, + }) + }, timeoutMs), + ) + const call = (async (): Promise => { + try { + const { output, error } = await invoke(route.tool_name!, route.params) + return { + id: route.id, + tool_name: route.tool_name!, + params: route.params, + output: truncate(output, 4000), + error, + latency_ms: Date.now() - started, + } + } catch (err) { + return { + id: route.id, + tool_name: route.tool_name ?? '', + params: route.params, + output: '', + error: err instanceof Error ? err.message : String(err), + latency_ms: Date.now() - started, + } + } + })() + + // Race, but also respect the outer AbortSignal. + if (signal.aborted) { + return { + id: route.id, + tool_name: route.tool_name ?? '', + params: route.params, + output: '', + error: 'aborted', + } + } + return Promise.race([call, timeout]) +} + +function truncate(s: string, n: number): string { + if (!s) return '' + if (s.length <= n) return s + return s.slice(0, n) + `…[${s.length - n} more chars truncated]` +} + +/** Fallback constraint synthesis when the LLM stage fails. */ +function buildFallbackConstraint( + contradictedPairs: Array<{ verdict: Verdict; claim?: Claim }>, + inconclusivePairs: Array<{ verdict: Verdict; claim?: Claim }>, +): string { + const lines: string[] = ['[VERIFICATION_CONSTRAINT]'] + if (contradictedPairs.length > 0) { + lines.push( + 'Your previous answer contained factual errors. Retract them and rewrite:', + ) + for (const { verdict, claim } of contradictedPairs) { + lines.push( + `- Wrong claim: "${(claim?.text ?? '').trim()}"\n Corrected: ${verdict.corrected_fact ?? '(evidence shows this claim is false)'}`, + ) + } + } + if (inconclusivePairs.length > 0) { + lines.push( + 'The following claims could not be verified. Downgrade them to hedges ("possibly", "one hypothesis is…") or drop them:', + ) + for (const { claim } of inconclusivePairs) { + lines.push(`- "${(claim?.text ?? '').trim()}"`) + } + } + lines.push( + 'Re-analyze the user\'s original question WITHOUT the wrong claims. Do not restate them. Say "insufficient evidence" instead of guessing when you cannot verify a fact.', + ) + return lines.join('\n') +} + +// ----------------------------------------------------------------------------- +// Default adapters (Claude Code specific) +// ----------------------------------------------------------------------------- + +/** + * Default cheap-LLM adapter wrapping `queryHaiku` — the exact same call + * pattern Claude Code already uses for tool-use summaries. Reusing Haiku + * keeps the extra cost of verification bounded and predictable. + */ +const defaultCheapLLM: CheapLLMCallFn = async ({ + systemPrompt, + userPrompt, + signal, +}) => { + const response = await queryHaiku({ + systemPrompt: asSystemPrompt([systemPrompt]), + userPrompt, + signal, + options: { + querySource: 'verification_agent', + enablePromptCaching: true, + agents: [], + isNonInteractiveSession: false, + hasAppendSystemPrompt: false, + mcpTools: [], + }, + }) + return response.message.content + .filter(b => b.type === 'text') + .map(b => (b.type === 'text' ? b.text : '')) + .join('') + .trim() +} + +/** + * Build the ToolSpec[] fed into the Router stage from the real Tool[] in + * `toolUseContext.options.tools`. We deliberately expose only a name + a + * one-line hint; the model does not need the full JSON Schema and would + * hallucinate arguments against it anyway. + */ +export function toolSpecsFromTools(tools: readonly Tool[]): ToolSpec[] { + return tools + .filter(t => t.isEnabled()) + .map(t => ({ + name: t.name, + purpose: t.searchHint ?? t.name, + input_hint: describeInputHint(t), + })) +} + +function describeInputHint(t: Tool): string { + // Try to guess a compact hint from the input schema without pulling in Zod + // introspection here. Downstream Router is instructed to always emit a + // `params` object; whatever the tool needs, the caller adapter maps it. + if (t.name === 'WebSearch') return '{ "query": "" }' + if (t.name === 'WebFetch') return '{ "url": "", "prompt": "" }' + return '{ ... tool-specific args ... }' +} + +/** + * Build the ToolInvokerFn bound to a specific ToolUseContext + parent message. + * Handles the mismatch between the verification pipeline's simple + * `(name, params) => Promise` interface and Claude Code's real + * `Tool.call(...)` signature. + */ +export function makeToolInvoker( + tools: readonly Tool[], + ctx: ToolUseContext, + parentMessage: AssistantMessage, + canUseTool: Parameters[2], +): ToolInvokerFn { + const byName = new Map(tools.map(t => [t.name, t])) + return async (name, params) => { + const tool = byName.get(name) + if (!tool) return { output: '', error: `unknown tool ${name}` } + try { + const result = await tool.call( + params as never, + ctx, + canUseTool, + parentMessage, + ) + // ToolResult has `.data` (raw) and `.resultForAssistant` + // (assistant-facing). We prefer the latter for LLM comparison. + const anyResult = result as unknown as { + resultForAssistant?: unknown + data?: unknown + } + const raw = anyResult.resultForAssistant ?? anyResult.data ?? result + return { + output: + typeof raw === 'string' ? raw : JSON.stringify(raw, safeStringify, 2), + } + } catch (err) { + return { + output: '', + error: err instanceof Error ? err.message : String(err), + } + } + } +} + +function safeStringify(_key: string, value: unknown) { + if (value instanceof Error) return { name: value.name, message: value.message } + if (typeof value === 'bigint') return value.toString() + return value +} diff --git a/src/middleware/verificationPrompts.ts b/src/middleware/verificationPrompts.ts new file mode 100644 index 00000000..313f6575 --- /dev/null +++ b/src/middleware/verificationPrompts.ts @@ -0,0 +1,280 @@ +/** + * Verification Middleware — stage prompts + * + * The 5 LLM-driven stages of the pipeline. Every prompt is written so it does + * NOT depend on any specific domain (audio / JS / GPU / …) and instead reasons + * about the *shape* of a claim: "does this restate something already in the + * input, or does it assert new external knowledge?". + * + * Every stage returns strict JSON so it can be parsed and passed on to the + * next stage. Robust JSON parsing lives in verificationPipeline.ts. + */ + +// ----------------------------------------------------------------------------- +// Stage 1 — Claim Extraction +// ----------------------------------------------------------------------------- + +export const EXTRACT_CLAIMS_SYSTEM_PROMPT = `You are a claim extractor. Your job is to break an assistant's response into +independent, atomic claims that can each be individually true or false. + +CRITICAL: prioritize CAUSAL and COUNTERFACTUAL claims. A code-analysis +answer's real value (and real risk) lives in claims of the form: + - "旧版会 X，新版会 Y" (counterfactual comparing versions) + - "A 导致 B" (causal) + - "A → B → C" (multi-step causal chain — extract EACH link as a + separate claim, not the whole chain as one blob) + - "always / never / every quantum / 一辈子只 …" (universal quantifier) + - "默认值是 …" (default / typical value assertions) + - "在 <浏览器/runtime> 下会 …" (platform-specific behavior) + +You MUST extract every such claim, even if the response only implies it (e.g. +"return undefined → 处理器结束" implies the platform-behavior claim +"AudioWorklet 中 process() 返回 undefined 会终止 processor"). + +Rules: +- Extract only load-bearing claims. Ignore boilerplate. +- Split compound sentences. One claim = one refutable proposition. +- Preserve hedges ("probably", "typically", "in Chrome"). Do NOT strip them. +- confidence_wording: + - "" (empty) → asserted as flat fact, no hedge + - "" → the hedge phrase actually present in the source text +- Include claims made inside "thinking" blocks that the model then acts on. + +Return ONLY valid JSON: +{ + "claims": [ + { "id": 1, "text": "...", "span": "...", "confidence_wording": "" } + ] +}` + +export function buildExtractClaimsUserPrompt(assistantText: string): string { + return `Assistant response to analyze: +<<< +${assistantText} +>>> + +Extract atomic claims as JSON.` +} + +// ----------------------------------------------------------------------------- +// Stage 2 — Verifiability Analysis +// ----------------------------------------------------------------------------- + +export const ANALYZE_VERIFIABILITY_SYSTEM_PROMPT = `You are a verifiability analyst. For each atomic claim, decide whether it is +derivable from the user's original input alone, or whether it depends on +external knowledge that must be verified. + +Meta-rule (STRICT): + derivable_from_input = true ONLY IF the claim is a direct restatement or + purely-syntactic paraphrase of what is literally in the user's input. + + It is NOT derivable if the claim describes: + - How a browser / runtime / VM / kernel behaves in response to the code + (e.g. "returning undefined causes the processor to be recycled" — the + code returning undefined IS derivable, but the runtime's response is + NOT). + - Default / typical values of variables when the user did not state them. + - Frequencies, timings, byte sizes not stated by the user. + - Comparative behavior between the old and new code beyond what a literal + diff shows. + - Anything requiring knowledge of Chrome / V8 / spec / library API. + + When in doubt → derivable_from_input = false. False positives here (over- + triggering verification) are cheap; false negatives (skipping needed + verification) are why the whole pipeline fails. + +source_type must be exactly one of: +- "specification" — fixed by a written standard (W3C, ECMA, IETF, RFC) +- "library-docs" — behavior of a specific library / SDK / framework API +- "platform-impl" — behavior of a specific runtime / platform build + (Chrome N, V8, kernel, driver, GPU) +- "empirical-data" — needs measurement / telemetry / benchmark numbers +- "domain-expertise" — expert heuristic no public doc uniquely nails down +- "runtime-value" — a concrete value that only exists inside the caller's + production system (e.g. what a specific config option + is set to on real users' devices) +- "other" + +Signals that a claim needs verification (any one is enough): +- References an external entity ("Chrome", "AudioWorklet", "malloc", "V8") +- Asserts behavior ("does X when Y", "is called N times per second") +- "Default"/"typical" claims ("defaults to 0", "usually returns undefined") +- Quantitative claims ("~375Hz", "16MB", "10% of users") +- Counterfactual / comparative claims ("old version did X, new version does Y") +- Causal claims ("A causes B", "leads to", "results in") +- Universal quantifiers ("always", "never", "every browser") + +Return ONLY valid JSON: +{ + "analyses": [ + { + "id": 1, + "derivable_from_input": false, + "source_type": "specification", + "reasoning": "one-line justification" + } + ] +}` + +export function buildAnalyzeVerifiabilityUserPrompt( + originalInput: string, + claimsJson: string, +): string { + return `User's original input: +<<< +${originalInput} +>>> + +Claims to classify (JSON): +${claimsJson} + +Return the analyses JSON.` +} + +// ----------------------------------------------------------------------------- +// Stage 3 — Source Routing +// ----------------------------------------------------------------------------- + +export const ROUTE_VERIFICATION_SYSTEM_PROMPT = `You are a verification router. Given N claims that each need external +evidence, and a list of available tools, decide how to gather evidence for +EACH claim. + +CRITICAL — one route per claim, no exceptions: +- For every input claim you MUST emit EXACTLY ONE route entry whose "id" + equals that claim's id. The output "routes" array length MUST equal the + input claim count. +- NEVER merge, dedupe, group, or omit any claim, even if you believe + several claims can be answered by the same tool call. Emit one entry per + claim; they may share identical tool_name and params. The invoker will + deduplicate the actual tool calls downstream. Your job is routing, not + cost optimization. +- If NO available tool can plausibly verify a claim, emit an entry with + tool_name = null and params = {} for that id. Do NOT drop the claim + from the output. Do NOT force a public-web search when the answer is + not on the public web (e.g. project-internal implementation detail, + private production config value). + +Rules for params (STRICT): +- Only use tools from the provided list. Never invent a tool name. +- Use ONLY the parameter names shown verbatim in the tool's input hint. + Do NOT rename, alias, or fabricate parameter keys. If the tool hint + shows {"file_path": "..."}, you must send "file_path", not "path". +- If you cannot fill a required parameter from the claim alone, emit + tool_name = null instead of guessing. + +Rules for queries: +- Prefer targeted queries over generic ones: + - For "specification": search for the spec section title verbatim. + - For "library-docs": include the exact library and API name. + - For "platform-impl": include the platform + version if scoped. +- Keep queries short (<= 12 words). Long queries dilute retrieval. + +Return ONLY valid JSON: +{ + "routes": [ + { "id": 1, "tool_name": "WebSearch", "params": { "query": "..." }, "query_summary": "..." }, + { "id": 2, "tool_name": null, "params": {}, "query_summary": "no public source can verify a project-internal claim" } + ] +}` + +export function buildRouteVerificationUserPrompt( + claimsNeedingVerification: string, + toolsJson: string, +): string { + return `Available tools (name, purpose, input hint): +${toolsJson} + +Claims + their analyses to route: +${claimsNeedingVerification} + +You MUST emit EXACTLY ONE route entry per claim id shown above, in the same +order. Never merge or omit any claim. For any claim that no available tool +can verify, emit tool_name = null. + +Return the routes JSON.` +} + +// ----------------------------------------------------------------------------- +// Stage 5 — Semantic Compare +// ----------------------------------------------------------------------------- + +export const COMPARE_SYSTEM_PROMPT = `You are a fact-checker. For each (claim, evidence) pair, decide whether the +evidence supports, contradicts, or is inconclusive about the claim. + +Rules: +- CONFIRMED: evidence directly supports the claim. Both the subject AND the + modality must match (a claim that says "always X" is only CONFIRMED by + evidence that also implies "always"). +- CONTRADICTED: evidence directly contradicts the claim, OR the claim is a + flat (confidence_wording == "") load-bearing causal/counterfactual/ + universal claim AND evidence does NOT support it. Rationale: in code- + analysis reports, a flat unsupported causal claim is treated as false by + default — the model should downgrade to a hedge or drop the claim. + Include corrected_fact stating what evidence actually implies, or, when + evidence is silent, state "no authoritative source supports this; the + model must retract or hedge". +- INCONCLUSIVE: reserved for HEDGED claims (confidence_wording != "") whose + evidence is off-topic, or when the tool returned an error / empty result. + Do NOT use INCONCLUSIVE for a flat causal claim with no supporting + evidence — that must be CONTRADICTED per the previous rule. +- Flag over-generalization: "always X" only weakly supported by evidence for + "sometimes X" is CONTRADICTED, not CONFIRMED. +- Flag scope drift: Chrome claim + Firefox evidence = INCONCLUSIVE. + +Return ONLY valid JSON: +{ + "verdicts": [ + { + "id": 1, + "verdict": "CONTRADICTED", + "corrected_fact": "According to , ...", + "evidence_quote": "verbatim short quote from evidence", + "reasoning": "one-line justification" + } + ] +}` + +export function buildCompareUserPrompt(pairsJson: string): string { + return `Claim/evidence pairs to compare: +${pairsJson} + +Return the verdicts JSON.` +} + +// ----------------------------------------------------------------------------- +// Stage 6 — Constraint Synthesis +// ----------------------------------------------------------------------------- + +export const SYNTHESIZE_SYSTEM_PROMPT = `You are writing a "hard constraint" message that will be injected back into a +model's conversation so it retracts and rewrites its previous answer. + +Rules: +- Address the model in second person. +- List each contradicted claim with: + (a) a short quote of the wrong claim + (b) the corrected fact from evidence + (c) a citation-style attribution ("According to : ...") +- Explicitly instruct the model to: + 1. Acknowledge which specific claims were wrong. + 2. Retract any conclusion that depended on them. + 3. Re-analyze the user's original question WITHOUT those claims. + 4. Say "insufficient evidence" instead of guessing when a fact cannot be + confirmed. +- Do NOT write the corrected analysis yourself. The model must redo the + reasoning; you only supply the corrected facts. +- Keep it under ~250 words. Terse and technical. +- Start with the exact literal string: [VERIFICATION_CONSTRAINT]` + +export function buildSynthesizeUserPrompt( + contradictedPairs: string, + inconclusivePairs: string, +): string { + return `Contradicted claims with corrected facts (JSON): +${contradictedPairs} + +Inconclusive claims (the model asserted these but no evidence was found, +should be downgraded to hedges, JSON): +${inconclusivePairs} + +Write the hard-constraint message.` +} diff --git a/src/middleware/verificationTypes.ts b/src/middleware/verificationTypes.ts new file mode 100644 index 00000000..aab50aac --- /dev/null +++ b/src/middleware/verificationTypes.ts @@ -0,0 +1,178 @@ +/** + * Verification Middleware — types + * + * A generic post-model-output verification pipeline. Sits between the model + * stream ending (`assistantMessages` are complete) and the user seeing the + * output. Does NOT hard-code any specific domain (Web Audio / WASM / JS etc.); + * every "is this a knowledge claim that needs external evidence?" decision is + * delegated to a small LLM stage. + * + * Pipeline shape: + * + * Stage 1 Claim Extraction — pull independent, atomic claims out of the + * assistant's thinking + text blocks. + * Stage 2 Verifiability Analyze — for each claim, decide whether it is + * derivable from the user's input alone, or + * needs an external source (spec / docs / + * implementation / runtime value / …). + * Stage 3 Source Routing — pick which registered tool + params to + * invoke to fetch that source. + * Stage 4 Verification (I/O) — actually call the tools in parallel. + * Stage 5 Semantic Compare — for each claim, compare it against the + * retrieved evidence; produce a verdict. + * Stage 6 Constraint Synthesis — if any claim is contradicted, generate a + * hard-constraint message telling the model + * what to retract and what the ground truth + * actually is. + * Stage 7 Decision — return `shouldRetry` + the constraint + * message; the caller (query.ts) injects it + * into `state.messages` and `continue`s. + */ + +export type ClaimSourceType = + /** Facts fixed by a written specification / standard (W3C, ECMA, IETF, RFC). */ + | 'specification' + /** Behavior of a specific library / SDK / framework API. */ + | 'library-docs' + /** Behavior of a specific runtime / platform implementation (Chrome, V8, kernel). */ + | 'platform-impl' + /** Numbers, ratios, benchmarks that need empirical data / telemetry. */ + | 'empirical-data' + /** Expert / domain heuristic that no public doc uniquely nails down. */ + | 'domain-expertise' + /** + * A concrete runtime / production value that only exists inside the caller's + * system (e.g. `processorOptions.maxCount` on real users' devices). Public + * search tools cannot resolve this; needs telemetry / logs / user input. + */ + | 'runtime-value' + | 'other' + +/** A single atomic claim extracted from the assistant's output. */ +export interface Claim { + id: number + /** Verbatim (or minimally-rewritten) claim in natural language. */ + text: string + /** + * A short quote or locator from the original message so the caller can + * surface which sentence a verdict refers to. + */ + span?: string + /** Wording that already hedges (e.g. "probably", "likely"). */ + confidence_wording?: string +} + +/** Whether a claim is derivable from input alone; if not, what source it needs. */ +export interface VerifiabilityAnalysis { + id: number + /** + * True if the claim can be entailed from the user's input + universally- + * accepted logic (arithmetic, boolean logic, the code snippet the user + * pasted, etc.). Such claims skip external verification. + */ + derivable_from_input: boolean + /** + * If not derivable, what kind of external source is needed. Used by Stage 3 + * to pick a tool. + */ + source_type: ClaimSourceType + /** + * Short justification, useful for logs / debugging / UI ("why did we search + * for this?"). + */ + reasoning: string +} + +/** Which registered tool should be invoked, with what params. */ +export interface RouteDecision { + id: number + /** + * Name of a tool available in `toolUseContext.options.tools`. `null` means + * no available tool can verify this claim (e.g. runtime-value) — the + * pipeline will mark the claim INCONCLUSIVE and let downstream synthesis + * push the model to hedge instead of assert. + */ + tool_name: string | null + params: Record + /** For diagnostics / UI. */ + query_summary: string +} + +/** Raw output of a verification tool call for a claim. */ +export interface Evidence { + id: number + tool_name: string + params: Record + /** Truncated string form of the tool result. */ + output: string + error?: string + latency_ms?: number +} + +/** Verdict of comparing a claim against the retrieved evidence. */ +export interface Verdict { + id: number + verdict: 'CONFIRMED' | 'CONTRADICTED' | 'INCONCLUSIVE' + /** The corrected fact according to evidence, if the claim is contradicted. */ + corrected_fact?: string + /** Short quote from evidence supporting the verdict. */ + evidence_quote?: string + reasoning?: string +} + +/** Final decision from the middleware. */ +export interface VerificationOutcome { + shouldRetry: boolean + /** + * If shouldRetry, the user-role message content to inject as a hard + * constraint before re-running the model. Otherwise undefined. + */ + constraintMessage?: string + claims: Claim[] + analyses: VerifiabilityAnalysis[] + routes: RouteDecision[] + evidences: Evidence[] + verdicts: Verdict[] + diagnostics: { + extracted: number + needing_verification: number + routed: number + verified: number + contradicted: number + inconclusive: number + total_ms: number + } +} + +/** + * Signature of the "cheap LLM" used by every stage. Kept minimal so the + * pipeline can be tested without touching Claude Code's real API layer. + * The default adapter in verificationPipeline.ts wraps `queryHaiku`. + */ +export type CheapLLMCallFn = (input: { + systemPrompt: string + userPrompt: string + signal: AbortSignal +}) => Promise + +/** + * Signature of the "run a registered tool" adapter. Kept minimal for the + * same reason — the query.ts hook wires this to real `Tool.call()`. + */ +export type ToolInvokerFn = ( + name: string, + params: Record, +) => Promise<{ output: string; error?: string }> + +/** + * Description of a tool the pipeline may route to. Fed into the Router stage + * so the LLM knows what's available. Deliberately schema-lite: `input_hint` + * is a one-line free-form string rather than a full JSON schema, because + * hallucinating params against an over-specified schema wastes turns. + */ +export interface ToolSpec { + name: string + purpose: string + /** e.g. `{ "query": "" }` for WebSearch. */ + input_hint: string +} diff --git a/src/query.ts b/src/query.ts index 07e8b6fa..70d5e9a5 100644 --- a/src/query.ts +++ b/src/query.ts @@ -11,6 +11,14 @@ import { type AutoCompactTrackingState, } from './services/compact/autoCompact.js' import { buildPostCompactMessages } from './services/compact/compact.js' + +import { + makeToolInvoker, + runVerificationPipeline, + toolSpecsFromTools, +} from './middleware/verificationPipeline.js' +import { extractOriginalUserInput } from './middleware/extractOriginalUserInput.js' + /* eslint-disable @typescript-eslint/no-require-imports */ const reactiveCompact = feature('REACTIVE_COMPACT') ? (require('./services/compact/reactiveCompact.js') as typeof import('./services/compact/reactiveCompact.js')) @@ -213,6 +221,7 @@ type State = { turnCount: number // Why the previous iteration continued. Undefined on first iteration. // Lets tests assert recovery paths fired without inspecting message contents. + verificationRetries?: number transition: Continue | undefined } @@ -318,6 +327,7 @@ async function* queryLoop( pendingToolUseSummary, stopHookActive, turnCount, + verificationRetries, } = state // Skill discovery prefetch — per-iteration (uses findWritePivot guard @@ -656,6 +666,11 @@ async function* queryLoop( try { let streamingFallbackOccured = false queryCheckpoint('query_api_streaming_start') + + // logForDebugging( + // `systemPrompt=${fullSystemPrompt} messagesForQuery=${JSON.stringify(messagesForQuery)}, currentModel=${currentModel}, tools=${JSON.stringify(toolUseContext.options.tools)}` + // ) + for await (const message of deps.callModel({ messages: prependUserContext(messagesForQuery, userContext), systemPrompt: fullSystemPrompt, @@ -863,6 +878,12 @@ async function* queryLoop( } queryCheckpoint('query_api_streaming_end') + + logForDebugging(`assistantMessages=${JSON.stringify(assistantMessages)}`) + + + + // Yield deferred microcompact boundary message using actual API-reported // token deletion count instead of client-side estimates. // Entire block gated behind feature() so the excluded string @@ -1264,6 +1285,88 @@ async function* queryLoop( return { reason: 'completed' } } + // logForDebugging(`verificationRetries----------${assistantMessages.length}, ${verificationRetries}, ${lastMessage?.isApiErrorMessage}`) + + const MAX_VERIFICATION_RETRIES = 2 + if ( + assistantMessages.length > 0 && + !lastMessage?.isApiErrorMessage && + (verificationRetries ?? 0) < MAX_VERIFICATION_RETRIES + ) { + try { + const originalUserInput = extractOriginalUserInput(state.messages) + const availableToolSpecs = toolSpecsFromTools( + toolUseContext.options.tools, + ) + const toolInvoker = makeToolInvoker( + toolUseContext.options.tools, + toolUseContext, + lastMessage!, + canUseTool, + ) + + // logForDebugging(`assistantMessages---------------${JSON.stringify(assistantMessages)}`) + // logForDebugging(`availableToolSpecs---------------${JSON.stringify(availableToolSpecs)}`) + // logForDebugging(`originalUserInput---------------${JSON.stringify(originalUserInput)}`) + + const outcome = await runVerificationPipeline({ + assistantMessages, + originalUserInput, + toolInvoker, + availableTools: availableToolSpecs, + signal: toolUseContext.abortController.signal, + }) + + // logForDebugging(`outcome---------------${JSON.stringify(outcome)}`) + logEvent('tengu_verification_pipeline_ran', { + extracted: outcome.diagnostics.extracted, + needing_verification: outcome.diagnostics.needing_verification, + routed: outcome.diagnostics.routed, + contradicted: outcome.diagnostics.contradicted, + inconclusive: outcome.diagnostics.inconclusive, + total_ms: outcome.diagnostics.total_ms, + shouldRetry: outcome.shouldRetry, + attempt: (verificationRetries ?? 0) + 1, + queryChainId: queryChainIdForAnalytics, + queryDepth: queryTracking.depth, + }) + + logForDebugging(`shouldRetry----------${outcome.shouldRetry}, ${outcome.constraintMessage}`) + + if (outcome.shouldRetry && outcome.constraintMessage) { + const constraintMessage = createUserMessage({ + content: outcome.constraintMessage, + isMeta: true, // hide from user UI, feed only into model context + }) + + const next: State = { + messages: [ + ...messagesForQuery, + ...assistantMessages, + constraintMessage, + ], + toolUseContext, + autoCompactTracking: tracking, + maxOutputTokensRecoveryCount: 0, + hasAttemptedReactiveCompact: false, + maxOutputTokensOverride: undefined, + pendingToolUseSummary: undefined, + stopHookActive: undefined, + turnCount, + verificationRetries: (verificationRetries ?? 0) + 1, + transition: { reason: 'verification_retry' }, + } + state = next + continue + } + } catch (err) { + // Absolute fail-safe: never let a middleware bug stall the + // response. Log and drop through to stopHooks with the original + // assistant answer intact. + logError(err) + } + } + const stopHookResult = yield* handleStopHooks( messagesForQuery, assistantMessages, diff --git a/src/query/transitions.ts b/src/query/transitions.ts index 1785ad50..d3ccee60 100644 --- a/src/query/transitions.ts +++ b/src/query/transitions.ts @@ -24,6 +24,7 @@ export type Continue = { | 'auto_compact' | 'collapse_drain_retry' | 'max_output_tokens_recovery' + | 'verification_retry' committed?: number attempt?: number } diff --git a/src/utils/thinking.ts b/src/utils/thinking.ts index e3461cda..516dd2bc 100644 --- a/src/utils/thinking.ts +++ b/src/utils/thinking.ts @@ -20,6 +20,7 @@ export type ThinkingConfig = * controls code inclusion in external builds; the GB flag controls rollout. */ export function isUltrathinkEnabled(): boolean { + return true if (!feature('ULTRATHINK')) { return false }