From 7e39ce526a629d089a9f518b91ac735c5740b1a5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 21:49:46 -0600 Subject: [PATCH 1/5] fix(supervise): preserve runtime-selected bridge models --- src/runtime/supervise/runtime.ts | 7 +++++-- tests/runtime/bridge-executor.test.ts | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 61cb19fb..d95ea533 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -29,7 +29,7 @@ import { randomUUID } from 'node:crypto' import { request as httpRequest } from 'node:http' import { request as httpsRequest } from 'node:https' import { Readable } from 'node:stream' -import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import { estimateCost, HARNESS_NATIVE_MODEL, isModelPriced } from '@tangle-network/agent-eval' import { type AgentProfile, agentProfileSchema, @@ -1245,7 +1245,10 @@ function killWithGrace( */ function qualifyProviderModel(model: AgentProfile['model']): string | undefined { const id = model?.default - if (!id) return undefined + // Eval uses this sentinel when the profile intentionally delegates model + // selection to the configured runtime. It is provenance metadata, not a + // provider model id and must never cross the bridge wire literally. + if (!id || id === HARNESS_NATIVE_MODEL) return undefined const provider = model?.provider if (!provider || id.includes('/')) return id return `${provider}/${id}` diff --git a/tests/runtime/bridge-executor.test.ts b/tests/runtime/bridge-executor.test.ts index 49714c41..0e87ca4e 100644 --- a/tests/runtime/bridge-executor.test.ts +++ b/tests/runtime/bridge-executor.test.ts @@ -1,4 +1,5 @@ import { PassThrough, type Readable } from 'node:stream' +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { SandboxEvent } from '@tangle-network/sandbox' import { afterEach, describe, expect, it, vi } from 'vitest' import { createExecutor, type ExecutorConfig, inlineSandboxClient } from '../../src/runtime' @@ -765,4 +766,14 @@ describe('profile-selected model keeps its provider', () => { }), ).toBe('pi/tangle-router/glm-5.2') }) + + it('keeps the configured bridge model when Eval delegates model selection to runtime', async () => { + expect( + await wireModelFor({ + name: 'w', + harness: 'pi', + model: { provider: 'tangle-router', default: HARNESS_NATIVE_MODEL }, + }), + ).toBe('pi/seam-default') + }) }) From 4f43ea5a99afcf73118d8e9f337b981861778ae2 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 22:11:46 -0600 Subject: [PATCH 2/5] fix(supervise): respect runtime-selected models across executors --- src/improvement/agentic-generator.ts | 5 ++- src/mcp/local-harness.ts | 9 +++-- src/runtime/define-leaderboard.test.ts | 22 ++++++++++- src/runtime/define-leaderboard.ts | 7 +++- src/runtime/sandbox-backend.ts | 3 +- src/runtime/supervise/authoring.ts | 3 +- src/runtime/supervise/model-policy.ts | 33 ++++++++++++++++ src/runtime/supervise/runtime.ts | 38 +++++++++++-------- src/runtime/supervise/supervisor-agent.ts | 6 +-- .../supervise/worktree-cli-executor.ts | 8 ++-- tests/kernel/supervisor-agent.test.ts | 4 ++ tests/mcp/local-harness.test.ts | 17 +++++++++ tests/profile-materialization.test.ts | 17 +++++++++ tests/runtime/bridge-executor.test.ts | 35 ++++++++++++++++- tests/runtime/executor-profile-model.test.ts | 32 ++++++++++++++++ 15 files changed, 205 insertions(+), 34 deletions(-) diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index a4857850..cb5d6aed 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -59,6 +59,7 @@ import { runLocalHarness, } from '../mcp/local-harness' import { runSettledCommand } from '../mcp/worktree-harness' +import { concreteProfileModel } from '../runtime/supervise/model-policy' import type { CandidateGenerator } from './improvement-driver' import { optimizerMethod } from './optimizer-prompt' @@ -325,7 +326,7 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } if (reproducibleCostLedger) { - const model = opts.profile?.model?.default + const model = opts.profile ? concreteProfileModel(opts.profile) : undefined if (!model) { throw new Error('agenticGenerator: reproducible Codex requires profile.model.default') } @@ -702,7 +703,7 @@ function shotReceipt(input: { shot: input.shot + 1, maxShots: input.maxShots, harness: input.harness, - model: input.profile?.model?.default ?? null, + model: input.profile ? (concreteProfileModel(input.profile) ?? null) : null, reasoningEffort: input.profile?.model?.reasoningEffort ?? null, promptSha256: sha256(input.prompt), startedAt: input.startedAt.toISOString(), diff --git a/src/mcp/local-harness.ts b/src/mcp/local-harness.ts index b29df27b..98aef8cb 100644 --- a/src/mcp/local-harness.ts +++ b/src/mcp/local-harness.ts @@ -36,6 +36,7 @@ import { import { homedir, tmpdir } from 'node:os' import { basename, delimiter, dirname, isAbsolute, join, resolve, sep } from 'node:path' import type { AgentProfile, HarnessType, ReasoningEffort } from '@tangle-network/agent-interface' +import { concreteProfileModel } from '../runtime/supervise/model-policy' import { codexSensitiveEnvironmentName, collectCodexDiagnosticRedactionValues, @@ -308,8 +309,8 @@ export function harnessInvocation( } if (options.codexReproducible) { - const model = profile.model?.default - if (typeof model !== 'string' || model.trim().length === 0) { + const model = concreteProfileModel(profile) + if (!model) { throw new Error('harnessInvocation: codexReproducible requires profile.model.default') } if (profile.model?.reasoningEffort === undefined) { @@ -335,8 +336,8 @@ export function harnessInvocation( const args = buildHarnessArgs(harness, composedPrompt, options) - const model = profile.model?.default - if (typeof model === 'string' && model.length > 0) { + const model = concreteProfileModel(profile) + if (model) { args.push(...invocation.modelArgs(model)) } diff --git a/src/runtime/define-leaderboard.test.ts b/src/runtime/define-leaderboard.test.ts index 09100257..98a166b4 100644 --- a/src/runtime/define-leaderboard.test.ts +++ b/src/runtime/define-leaderboard.test.ts @@ -1,7 +1,9 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { SandboxEvent } from '@tangle-network/sandbox' +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { CreateSandboxOptions, SandboxEvent } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import { defineLeaderboard, @@ -176,7 +178,20 @@ describe('defineLeaderboard', () => { /paid-call receipt/, ) + const creates: CreateSandboxOptions[] = [] const result = await board({ + backends: { + inproc: () => { + const inner = fakeBackend() + return { + ...inner, + async create(options?: CreateSandboxOptions) { + creates.push(options ?? {}) + return inner.create(options) + }, + } + }, + }, resolveModel: (events) => { // The served model rides the backend's own usage events — here the fake // backend's llm_call stands in for the harness's terminal event. @@ -185,6 +200,11 @@ describe('defineLeaderboard', () => { }, }).run([...snappedAxis, '--cases', 'case-alpha']) expect(result.records[0]?.model).toBe('kimi-k2@2026-01-01') + expect(creates).toHaveLength(1) + expect(creates[0]?.backend?.model).toBeUndefined() + const executionProfile = creates[0]?.backend?.profile as AgentProfile + expect(executionProfile.model?.default).toBeUndefined() + expect(executionProfile.model?.default).not.toBe(HARNESS_NATIVE_MODEL) }) it('flows a structured TArtifact through parseOutput → score → records natively', async () => { diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index 8c04d331..332994f5 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -37,6 +37,7 @@ import { type AgentProfile, CODING_HARNESSES, expandProfileAxes, + HARNESS_NATIVE_MODEL, type HarnessType, harnessAxisOf, type MaximumCharge, @@ -513,6 +514,10 @@ export function defineLeaderboard( // whichever backend client runs the cell. const axis = harnessAxisOf(cellProfile) const modelId = bareModel(axis?.model ?? models[0] ?? '') + const backendModel = { + ...spec.modelBackend, + ...(modelId !== HARNESS_NATIVE_MODEL ? { model: modelId } : {}), + } return { // The naive steering directive = the no-signal retry floor: re-run the same case as // an independent attempt until one scores (>0) or the shot cap. The policy is data @@ -532,7 +537,7 @@ export function defineLeaderboard( sandboxOverrides: { backend: { type: axis.harness, - model: { ...spec.modelBackend, model: modelId }, + ...(Object.keys(backendModel).length > 0 ? { model: backendModel } : {}), }, } as never, } diff --git a/src/runtime/sandbox-backend.ts b/src/runtime/sandbox-backend.ts index 54856fbc..f6b9253b 100644 --- a/src/runtime/sandbox-backend.ts +++ b/src/runtime/sandbox-backend.ts @@ -10,6 +10,7 @@ import type { AgentProfile, HarnessType } from '@tangle-network/agent-interface' import type { CreateSandboxOptions } from '@tangle-network/sandbox' +import { profileForExecution } from './supervise/model-policy' type BackendType = NonNullable['type'] type BackendOverride = NonNullable @@ -88,7 +89,7 @@ export function buildBackendOptions( ...base, backend: { type: resolveBackendType(profile, overrideBackend), - profile, + profile: profileForExecution(profile), ...(overrideBackend?.model ? { model: overrideBackend.model } : {}), ...(overrideBackend?.server ? { server: overrideBackend.server } : {}), }, diff --git a/src/runtime/supervise/authoring.ts b/src/runtime/supervise/authoring.ts index fbd3ac3b..43f910cd 100644 --- a/src/runtime/supervise/authoring.ts +++ b/src/runtime/supervise/authoring.ts @@ -25,6 +25,7 @@ import { ValidationError } from '../../errors' import { type RouterConfig, routerChatWithUsage } from '../router-client' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' +import { concreteProfileModel } from './model-policy' import { supervisorPolicyPrompt } from './prompt-registry' import type { Agent, AgentSpec, Executor, ExecutorResult } from './types' @@ -138,7 +139,7 @@ export function authoredWorker( temperature?: number }, ): Agent { - const model = profile.model?.default ?? opts.cfg.model + const model = concreteProfileModel(profile) ?? opts.cfg.model const executorFactory: NonNullable = (spec, ctx) => { let artifact: ExecutorResult | undefined const executionId = ctx.node?.nodeId ?? `authored-router-${profile.name}` diff --git a/src/runtime/supervise/model-policy.ts b/src/runtime/supervise/model-policy.ts index 4bb705bb..80c610e9 100644 --- a/src/runtime/supervise/model-policy.ts +++ b/src/runtime/supervise/model-policy.ts @@ -4,9 +4,42 @@ * model at resolve time, so a run that names a model outside the allowed set throws before * any compute is spent — never silently swapped or silently allowed. */ +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { ConfigError } from '../../errors' +/** + * Return the model id an executor may send to a provider. + * + * Eval stamps {@link HARNESS_NATIVE_MODEL} into a profile when model selection is deliberately + * delegated to the configured runtime. That marker belongs in experiment identity and cost + * admission; it is not a provider model id. + */ +export function concreteModelId(model: string | undefined): string | undefined { + if (model === undefined) return undefined + const id = model.trim() + return id.length > 0 && id !== HARNESS_NATIVE_MODEL ? id : undefined +} + +/** Return a profile's explicitly selected provider model, if it has one. */ +export function concreteProfileModel(profile: Pick): string | undefined { + return concreteModelId(profile.model?.default) +} + +/** + * Remove only Eval's runtime-selected model marker before a profile crosses an execution boundary. + * Every other model hint remains intact, including provider, reasoning effort, and small-model + * preferences. The input profile is never mutated. + */ +export function profileForExecution(profile: AgentProfile): AgentProfile { + if (profile.model?.default !== HARNESS_NATIVE_MODEL) return profile + const { default: _runtimeSelected, ...remainingModel } = profile.model + const { model: _model, ...remainingProfile } = profile + return Object.keys(remainingModel).length > 0 + ? { ...remainingProfile, model: remainingModel } + : remainingProfile +} + /** * Throw a `ConfigError` when `allowed` is set, `model` is defined, and `model` is not a * member of `allowed`. No-op when `allowed` is unset (the unrestricted default) or when diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index d95ea533..340a649e 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -29,7 +29,7 @@ import { randomUUID } from 'node:crypto' import { request as httpRequest } from 'node:http' import { request as httpsRequest } from 'node:https' import { Readable } from 'node:stream' -import { estimateCost, HARNESS_NATIVE_MODEL, isModelPriced } from '@tangle-network/agent-eval' +import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import { type AgentProfile, agentProfileSchema, @@ -74,6 +74,7 @@ import type { import { zeroTokenUsage } from '../util' import { createInbox, type Inbox } from './inbox' import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' +import { concreteModelId, concreteProfileModel } from './model-policy' import { type ActivityLog, createActivityLog, @@ -375,7 +376,7 @@ function unmeteredSpend(ms: number): Spend { */ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerSeamKey, 'router/inline') - const model = spec.profile.model?.default ?? seam.model + const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) if (!model) { throw new ValidationError( 'routerInlineExecutor: no model — set RouterSeam.model or AgentProfile.model.default', @@ -510,7 +511,7 @@ interface RouterToolsResponse { */ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerToolsSeamKey, 'router-tools') - const model = spec.profile.model?.default ?? seam.model + const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) if (!model) { throw new ValidationError( 'routerToolsInlineExecutor: no model — set RouterToolsSeam.model or AgentProfile.model.default', @@ -785,11 +786,12 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { let artifact: ExecutorResult | undefined const executionId = ctx.node?.nodeId ?? `sandbox-run-${randomUUID()}` const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) + const profileModel = concreteProfileModel(spec.profile) const sandboxMaterialization = { effectiveProfile: spec.profile, backend: harness, - model: spec.profile.model?.default - ? ({ status: 'known', id: spec.profile.model.default } as const) + model: profileModel + ? ({ status: 'known', id: profileModel } as const) : ({ status: 'unknown', reason: 'sandbox harness selected its default model' } as const), execution: { kind: 'run', @@ -808,7 +810,7 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { binding: { executionId, harness, - model: spec.profile.model?.default ?? null, + model: profileModel ?? null, }, descriptor: { kind: 'sandbox-run', transport: 'sandbox', backend: harness }, } @@ -1244,11 +1246,8 @@ function killWithGrace( * IS the caller's declared intent rather than a gap to fill. */ function qualifyProviderModel(model: AgentProfile['model']): string | undefined { - const id = model?.default - // Eval uses this sentinel when the profile intentionally delegates model - // selection to the configured runtime. It is provenance metadata, not a - // provider model id and must never cross the bridge wire literally. - if (!id || id === HARNESS_NATIVE_MODEL) return undefined + const id = concreteModelId(model?.default) + if (!id) return undefined const provider = model?.provider if (!provider || id.includes('/')) return id return `${provider}/${id}` @@ -1272,13 +1271,20 @@ function bridgeCellModel( // " — a credential error naming a provider the caller never chose. Measured live; // the same request with `pi/tangle-router/glm-5.2` returns 200. // - // A per-cell `backend.model.model` override is left exactly as supplied: it is a caller-authored - // wire id, not a profile hint, and qualifying it would rewrite what the caller asked for. - const model = backend?.model?.model ?? qualifyProviderModel(profile.model) - if (!harness && !model) return seamModel + // A concrete per-cell `backend.model.model` override is left exactly as supplied: it is a + // caller-authored wire id, not a profile hint, and qualifying it would rewrite what the caller + // asked for. Eval's runtime-selected marker is the one exception: it selects the configured + // bridge fallback and must never cross the wire as a provider model id. + const hasBackendModel = backend?.model?.model !== undefined + const model = hasBackendModel + ? concreteModelId(backend.model?.model) + : qualifyProviderModel(profile.model) + const fallback = concreteModelId(seamModel) + if (!harness && !model) return fallback if (!harness) return model if (model) return model.startsWith(`${harness}/`) ? model : `${harness}/${model}` - return seamModel?.startsWith(`${harness}/`) ? seamModel : undefined + if (!fallback) return undefined + return fallback.startsWith(`${harness}/`) ? fallback : `${harness}/${fallback}` } export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index dab9d64f..c5066da0 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -42,6 +42,7 @@ import type { BusRecord } from './event-bus' import { bestDelivered, runFinalizer, runTree, type SupervisorFinalizer } from './finalizer' import { createInbox } from './inbox' import { attestRuntimeOwnedScopeOwner, runtimeOwnedScopeOwnerRuntime } from './materialization' +import { concreteModelId } from './model-policy' import { supervisorPolicyPrompt } from './prompt-registry' import { detachedSnapshot } from './snapshot' import type { StopRule } from './stop-rules' @@ -196,9 +197,8 @@ function resolveSupervisorSystemPrompt( * applies, exactly as when `model` is absent. */ export function resolveSupervisorModelId(profile: SupervisorProfile): string | undefined { - if (typeof profile.model === 'string') return profile.model - const fromHints = profile.model?.default - return typeof fromHints === 'string' && fromHints.length > 0 ? fromHints : undefined + if (typeof profile.model === 'string') return concreteModelId(profile.model) + return concreteModelId(profile.model?.default) } /** diff --git a/src/runtime/supervise/worktree-cli-executor.ts b/src/runtime/supervise/worktree-cli-executor.ts index 8e126a87..fff4d050 100644 --- a/src/runtime/supervise/worktree-cli-executor.ts +++ b/src/runtime/supervise/worktree-cli-executor.ts @@ -37,6 +37,7 @@ import { worktreeProfileExecutionPlan, } from '../../mcp/worktree-harness' import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' +import { concreteProfileModel } from './model-policy' import type { Executor, ExecutorResult, Spend } from './types' export type { WorktreeCommandResult, WorktreeProfileMaterializationReceipt } @@ -150,6 +151,7 @@ export function createWorktreeCliExecutor( let artifact: ExecutorResult | undefined const profilePlan = worktreeProfileExecutionPlan(options.profile, options.harness) + const profileModel = concreteProfileModel(options.profile) return attestRuntimeOwnedExecutor( { runtime: 'cli', @@ -238,8 +240,8 @@ export function createWorktreeCliExecutor( { effectiveProfile: options.profile, backend: `cli-worktree:${options.harness}`, - model: options.profile.model?.default - ? { status: 'known', id: options.profile.model.default } + model: profileModel + ? { status: 'known', id: profileModel } : { status: 'unknown', reason: `${options.harness} selected its configured default model` }, execution: { kind: 'worktree-run', id: runId }, materializer: 'agent-profile-worktree-plan', @@ -263,7 +265,7 @@ export function createWorktreeCliExecutor( repoRoot: options.repoRoot, runId, harness: options.harness, - model: options.profile.model?.default ?? null, + model: profileModel ?? null, baseRef: options.baseRef ?? 'HEAD', }, descriptor: { diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index fb4ff155..d7546cc4 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -1,3 +1,4 @@ +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' @@ -655,6 +656,9 @@ describe('resolveSupervisorProfile — a canonical AgentProfile IS a supervisor }) expect(resolveSupervisorProfile({ model: { provider: 'anthropic' } }).modelId).toBeUndefined() expect(resolveSupervisorProfile({ model: { default: '' } }).modelId).toBeUndefined() + expect( + resolveSupervisorProfile({ model: { default: HARNESS_NATIVE_MODEL } }).modelId, + ).toBeUndefined() }) it('appends prompt.instructions and resources.instructions to the system prompt, in that order', () => { diff --git a/tests/mcp/local-harness.test.ts b/tests/mcp/local-harness.test.ts index 11b63f0c..19d5fdf1 100644 --- a/tests/mcp/local-harness.test.ts +++ b/tests/mcp/local-harness.test.ts @@ -13,6 +13,7 @@ import { } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it, vi } from 'vitest' import { @@ -1175,6 +1176,14 @@ describe('harnessInvocation (the §1.5 profile-aware mapper)', () => { } }) + it('omits the model flag when Eval delegates model selection to the harness', () => { + for (const harness of ['claude-code', 'codex', 'opencode'] as const) { + const inv = harnessInvocation(harness, profileWith(undefined, HARNESS_NATIVE_MODEL), 'go') + expect(inv.args).not.toContain('-m') + expect(inv.args).not.toContain(HARNESS_NATIVE_MODEL) + } + }) + it('threads BOTH systemPrompt and model together', () => { const inv = harnessInvocation('claude-code', profileWith('SYS', 'kimi-k2.7'), 'task') expect(inv.command).toBe('claude') @@ -1307,6 +1316,14 @@ describe('harnessInvocation (the §1.5 profile-aware mapper)', () => { codexReproducible: true, }), ).toThrow(/requires profile\.model\.reasoningEffort/) + expect(() => + harnessInvocation( + 'codex', + { model: { default: HARNESS_NATIVE_MODEL, reasoningEffort: 'high' } }, + 'task', + { codexReproducible: true }, + ), + ).toThrow(/requires profile\.model\.default/) expect(() => harnessInvocation( 'claude-code', diff --git a/tests/profile-materialization.test.ts b/tests/profile-materialization.test.ts index 8f5ad479..9b838933 100644 --- a/tests/profile-materialization.test.ts +++ b/tests/profile-materialization.test.ts @@ -1,3 +1,4 @@ +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import { type AgentProfile, AGENT_PROFILE_MATERIALIZATION_AXES as CANONICAL_AXES, @@ -365,6 +366,22 @@ describe('profile materialization contracts', () => { expect(buildBackendOptions({ name: 'a' }, undefined).backend?.type).toBe('opencode') }) + it('keeps the runtime-selected model marker out of sandbox execution profiles', () => { + const profile: AgentProfile = { + name: 'runtime-selected', + model: { + default: HARNESS_NATIVE_MODEL, + provider: 'tangle-router', + reasoningEffort: 'high', + }, + } + + const executable = buildBackendOptions(profile, undefined).backend?.profile as AgentProfile + expect(executable).not.toBe(profile) + expect(executable.model).toEqual({ provider: 'tangle-router', reasoningEffort: 'high' }) + expect(profile.model?.default).toBe(HARNESS_NATIVE_MODEL) + }) + it('refuses a declared harness the sandbox cannot run', () => { // Falling through to opencode would run a gemini profile on a different harness and // report success, so the mismatch has to surface as a failure. diff --git a/tests/runtime/bridge-executor.test.ts b/tests/runtime/bridge-executor.test.ts index 0e87ca4e..fa174712 100644 --- a/tests/runtime/bridge-executor.test.ts +++ b/tests/runtime/bridge-executor.test.ts @@ -718,7 +718,10 @@ describe('profile-selected model keeps its provider', () => { // pi fell back to its own default provider and died with "No API key found for opencode" — a // credential error naming a provider nobody chose. Measured live against a real cli-bridge: // `pi/tangle-router/glm-5.2` returns 200, `pi/glm-5.2` does not. - async function wireModelFor(profile: Record): Promise { + async function wireModelFor( + profile: Record, + seamModel = 'pi/seam-default', + ): Promise { const seen: Array> = [] bridgeHttpHandler = (payload) => { seen.push(payload) @@ -728,7 +731,7 @@ describe('profile-selected model keeps its provider', () => { backend: 'bridge', bridgeUrl: 'http://bridge.test', bridgeBearer: 'secret', - model: 'pi/seam-default', + model: seamModel, })({ profile, harness: null } as unknown as AgentSpec, { signal: new AbortController().signal, seams: {}, @@ -776,4 +779,32 @@ describe('profile-selected model keeps its provider', () => { }), ).toBe('pi/seam-default') }) + + it('qualifies a bare configured bridge model with the selected harness', async () => { + expect( + await wireModelFor( + { + name: 'w', + harness: 'pi', + model: { provider: 'tangle-router', default: HARNESS_NATIVE_MODEL }, + }, + 'seam-default', + ), + ).toBe('pi/seam-default') + }) + + it('keeps the configured model when a real per-create override delegates selection', async () => { + const seen: Array> = [] + bridgeHttpHandler = (payload) => { + seen.push(payload) + return sse('ok', 1, 2) + } + + await runOnce(bridgeClient('seam-default'), 'go', { + type: 'pi', + model: { model: HARNESS_NATIVE_MODEL }, + }) + + expect(seen[0]?.model).toBe('pi/seam-default') + }) }) diff --git a/tests/runtime/executor-profile-model.test.ts b/tests/runtime/executor-profile-model.test.ts index a634b9ff..bab45c17 100644 --- a/tests/runtime/executor-profile-model.test.ts +++ b/tests/runtime/executor-profile-model.test.ts @@ -1,5 +1,6 @@ import { createServer, type Server } from 'node:http' import type { AddressInfo } from 'node:net' +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' import { type AgentSpec, createExecutor } from '../../src/runtime' @@ -80,4 +81,35 @@ describe('router executor model precedence', () => { expect(request?.model).toBe('profile-selected-model') }) + + it.each(['router', 'router-tools'] as const)( + 'uses the %s configured model when Eval delegates model selection', + async (backend) => { + let request: Record | undefined + const routerBaseUrl = await startRouter((body) => { + request = body + }) + const factory = createExecutor({ + backend, + routerBaseUrl, + routerKey: 'key', + model: 'backend-fallback-model', + ...(backend === 'router-tools' ? { tools: [], executeToolCall: async () => '' } : {}), + }) + const executor = factory( + { + profile: { + name: 'runtime-selected-model', + model: { default: HARNESS_NATIVE_MODEL }, + }, + harness: null, + }, + { signal: new AbortController().signal, seams: {} }, + ) + + await executor.execute('do the task', new AbortController().signal) + + expect(request?.model).toBe('backend-fallback-model') + }, + ) }) From da4e817556341a62d6a020b7e3bee921c790e4b8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 22:35:43 -0600 Subject: [PATCH 3/5] fix(supervise): normalize delegated models at execution --- src/runtime/define-leaderboard.test.ts | 32 ++++++++++++++++ src/runtime/define-leaderboard.ts | 4 +- src/runtime/environment-provider.test.ts | 48 ++++++++++++++++++++++++ src/runtime/environment-provider.ts | 7 +++- src/runtime/supervise/model-policy.ts | 12 ++++-- src/runtime/supervise/runtime.ts | 32 ++++++++++++---- tests/runtime/bridge-executor.test.ts | 4 +- 7 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/runtime/define-leaderboard.test.ts b/src/runtime/define-leaderboard.test.ts index 98a166b4..e9dce037 100644 --- a/src/runtime/define-leaderboard.test.ts +++ b/src/runtime/define-leaderboard.test.ts @@ -207,6 +207,38 @@ describe('defineLeaderboard', () => { expect(executionProfile.model?.default).not.toBe(HARNESS_NATIVE_MODEL) }) + it('sends the runtime-selected marker only to cli-bridge cells', async () => { + const creates: CreateSandboxOptions[] = [] + const inner = fakeBackend() + const result = await board({ + backends: { + 'cli-bridge': () => ({ + ...inner, + async create(options?: CreateSandboxOptions) { + creates.push(options ?? {}) + return inner.create(options) + }, + }), + }, + resolveModel: () => 'kimi-k2@2026-01-01', + }).run([ + '--backend', + 'cli-bridge', + '--harnesses', + 'claude-code', + '--models', + 'moonshot/kimi-k2@2026-01-01', + '--cases', + 'case-alpha', + ]) + + expect(result.records[0]?.model).toBe('kimi-k2@2026-01-01') + expect(creates).toHaveLength(1) + expect(creates[0]?.backend?.model?.model).toBe(HARNESS_NATIVE_MODEL) + const executionProfile = creates[0]?.backend?.profile as AgentProfile + expect(executionProfile.model?.default).toBeUndefined() + }) + it('flows a structured TArtifact through parseOutput → score → records natively', async () => { interface Structured { answer: string diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index 332994f5..e20d1de6 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -516,7 +516,9 @@ export function defineLeaderboard( const modelId = bareModel(axis?.model ?? models[0] ?? '') const backendModel = { ...spec.modelBackend, - ...(modelId !== HARNESS_NATIVE_MODEL ? { model: modelId } : {}), + ...(modelId !== HARNESS_NATIVE_MODEL || backendName === 'cli-bridge' + ? { model: modelId } + : {}), } return { // The naive steering directive = the no-signal retry floor: re-run the same case as diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index 69121554..b50a647d 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -1,3 +1,4 @@ +import { HARNESS_NATIVE_MODEL } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import type { BackendType, @@ -1085,6 +1086,53 @@ describe('environment provider adapters', () => { expect(executor.resultArtifact().out).toMatchObject({ content: 'from-package' }) }) + it('keeps the runtime-selected model marker out of provider.create', async () => { + let createdProfile: AgentProfile | string | undefined + let taskProfile: AgentProfile | undefined + const provider: AgentEnvironmentProvider = { + name: 'runtime-model-provider', + capabilities: () => fakeCapabilities(), + async create(input) { + createdProfile = input.profile + return fakeEnvironment({ + stream: async function* (): AsyncIterable { + yield { type: 'result', data: { finalText: 'provider-selected-model' } } + }, + }) + }, + } + const factory = createExecutor({ + backend: 'provider', + provider, + profileForCreate: (profile) => ({ ...profile, description: 'create-only transform' }), + taskToTurn: (task, profile) => { + taskProfile = profile + return { prompt: String(task) } + }, + }) + const profile: AgentProfile = { + name: 'runtime-model-worker', + model: { + provider: 'tangle-router', + default: ` ${HARNESS_NATIVE_MODEL} `, + reasoningEffort: 'high', + }, + } + const spec: AgentSpec = { profile, harness: null } + const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } + const executor = factory(spec, ctx) + + await collect(executor.execute('task', ctx.signal) as AsyncIterable) + + expect(createdProfile).toMatchObject({ + name: 'runtime-model-worker', + description: 'create-only transform', + model: { provider: 'tangle-router', reasoningEffort: 'high' }, + }) + expect((createdProfile as AgentProfile).model?.default).toBeUndefined() + expect(taskProfile).toBe(profile) + }) + it('resolves a named provider through the runtime registry', async () => { let created: unknown const provider: AgentEnvironmentProvider = { diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 4a0dc686..1ab08772 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -309,6 +309,10 @@ export interface ProviderExecutorOptions { runtime?: Runtime destroyOnSettle?: boolean requireTerminalEvent?: boolean + /** Transform only the profile sent to `provider.create`. The original profile + * remains the input to `taskToTurn`, so execution-only normalization cannot + * rewrite the caller's task mapping. */ + profileForCreate?: (profile: AgentProfile) => AgentProfile taskToTurn?: (task: unknown, specProfile: AgentProfile) => AgentTurnInput } @@ -387,9 +391,10 @@ async function* streamProviderExecutor( ): AsyncIterable { const started = Date.now() const linked = mergeAbortSignals(args.signal, args.controller.signal) + const createProfile = args.options.profileForCreate?.(args.profile) ?? args.profile const environment = await args.provider.create({ ...(args.options.defaults ?? {}), - profile: args.profile, + profile: createProfile, signal: linked, }) args.onEnvironment(environment) diff --git a/src/runtime/supervise/model-policy.ts b/src/runtime/supervise/model-policy.ts index 80c610e9..370376f0 100644 --- a/src/runtime/supervise/model-policy.ts +++ b/src/runtime/supervise/model-policy.ts @@ -18,7 +18,12 @@ import { ConfigError } from '../../errors' export function concreteModelId(model: string | undefined): string | undefined { if (model === undefined) return undefined const id = model.trim() - return id.length > 0 && id !== HARNESS_NATIVE_MODEL ? id : undefined + return id.length > 0 && !isHarnessNativeModel(id) ? id : undefined +} + +/** Whether a model value delegates selection to the chosen execution system. */ +export function isHarnessNativeModel(model: string | undefined): boolean { + return model?.trim() === HARNESS_NATIVE_MODEL } /** Return a profile's explicitly selected provider model, if it has one. */ @@ -32,8 +37,9 @@ export function concreteProfileModel(profile: Pick): stri * preferences. The input profile is never mutated. */ export function profileForExecution(profile: AgentProfile): AgentProfile { - if (profile.model?.default !== HARNESS_NATIVE_MODEL) return profile - const { default: _runtimeSelected, ...remainingModel } = profile.model + const model = profile.model + if (!isHarnessNativeModel(model?.default) || model === undefined) return profile + const { default: _runtimeSelected, ...remainingModel } = model const { model: _model, ...remainingProfile } = profile return Object.keys(remainingModel).length > 0 ? { ...remainingProfile, model: remainingModel } diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 340a649e..cf5ea6fa 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -74,7 +74,12 @@ import type { import { zeroTokenUsage } from '../util' import { createInbox, type Inbox } from './inbox' import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' -import { concreteModelId, concreteProfileModel } from './model-policy' +import { + concreteModelId, + concreteProfileModel, + isHarnessNativeModel, + profileForExecution, +} from './model-policy' import { type ActivityLog, createActivityLog, @@ -1273,11 +1278,18 @@ function bridgeCellModel( // // A concrete per-cell `backend.model.model` override is left exactly as supplied: it is a // caller-authored wire id, not a profile hint, and qualifying it would rewrite what the caller - // asked for. Eval's runtime-selected marker is the one exception: it selects the configured - // bridge fallback and must never cross the wire as a provider model id. - const hasBackendModel = backend?.model?.model !== undefined + // asked for. Eval's runtime-selected marker is the one exception: it selects the harness's + // configured model and must never cross the wire as a provider model id. + const backendModel = backend?.model?.model + const hasBackendModel = backendModel !== undefined + // cli-bridge already treats a bare harness id (`pi`, `codex`, …) as "use that + // harness's configured model". Translate Eval's marker into that existing + // wire form rather than leaking `default` or substituting an unrelated fallback. + if (hasBackendModel && isHarnessNativeModel(backendModel)) { + return harness ?? concreteModelId(seamModel) + } const model = hasBackendModel - ? concreteModelId(backend.model?.model) + ? concreteModelId(backendModel) : qualifyProviderModel(profile.model) const fallback = concreteModelId(seamModel) if (!harness && !model) return fallback @@ -2695,7 +2707,7 @@ export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { }) } case 'provider': { - const { provider, registry, taskToTurn, ...decisionData } = config + const { provider, registry, profileForCreate, taskToTurn, ...decisionData } = config const snapshot = detachedSnapshot(decisionData, 'createExecutor provider config') // A registry is a live service. Resolve its mutable name mapping exactly once at intake and // retain the resulting provider instance, never the registry lookup for later execution. @@ -2703,6 +2715,7 @@ export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { return Object.freeze({ ...snapshot, provider: resolvedProvider, + ...(profileForCreate === undefined ? {} : { profileForCreate }), ...(taskToTurn === undefined ? {} : { taskToTurn }), }) } @@ -2868,7 +2881,12 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory runtime: providerSeam.runtime ?? (provider.name as Runtime), } } - return providerAsExecutor(provider, providerSeam)(spec, seamed) + const profileForCreate = providerSeam.profileForCreate + return providerAsExecutor(provider, { + ...providerSeam, + profileForCreate: (profile) => + profileForExecution(profileForCreate?.(profile) ?? profile), + })(spec, seamed) } case 'sandbox': { // The sandbox executor requires a concrete harness; a spec-level harness diff --git a/tests/runtime/bridge-executor.test.ts b/tests/runtime/bridge-executor.test.ts index fa174712..3ef485a7 100644 --- a/tests/runtime/bridge-executor.test.ts +++ b/tests/runtime/bridge-executor.test.ts @@ -793,7 +793,7 @@ describe('profile-selected model keeps its provider', () => { ).toBe('pi/seam-default') }) - it('keeps the configured model when a real per-create override delegates selection', async () => { + it('uses the harness configured model when a real per-create override delegates selection', async () => { const seen: Array> = [] bridgeHttpHandler = (payload) => { seen.push(payload) @@ -805,6 +805,6 @@ describe('profile-selected model keeps its provider', () => { model: { model: HARNESS_NATIVE_MODEL }, }) - expect(seen[0]?.model).toBe('pi/seam-default') + expect(seen[0]?.model).toBe('pi') }) }) From de1985b8ef4f1db64e4f1e5b0a3a4e26dba0ffa3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 22:43:32 -0600 Subject: [PATCH 4/5] fix(eval): normalize delegated model markers --- src/runtime/define-leaderboard.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/define-leaderboard.ts b/src/runtime/define-leaderboard.ts index e20d1de6..78063f71 100644 --- a/src/runtime/define-leaderboard.ts +++ b/src/runtime/define-leaderboard.ts @@ -37,7 +37,6 @@ import { type AgentProfile, CODING_HARNESSES, expandProfileAxes, - HARNESS_NATIVE_MODEL, type HarnessType, harnessAxisOf, type MaximumCharge, @@ -55,6 +54,7 @@ import { leaderboard, renderLeaderboardMarkdown } from './benchmark-report' import { loopDispatch } from './loop-dispatch' import { resolveSandboxClient } from './resolve-sandbox-client' import { type SteeringDecision, steeringDriver } from './steering-drivers' +import { isHarnessNativeModel } from './supervise/model-policy' import type { LoopResult, SandboxClient } from './types' /** Structured per-case verdict a `score` function may return (a bare number is @@ -516,7 +516,7 @@ export function defineLeaderboard( const modelId = bareModel(axis?.model ?? models[0] ?? '') const backendModel = { ...spec.modelBackend, - ...(modelId !== HARNESS_NATIVE_MODEL || backendName === 'cli-bridge' + ...(!isHarnessNativeModel(modelId) || backendName === 'cli-bridge' ? { model: modelId } : {}), } From ac9a3eccb224024e742ca3bd52d05443c8efe032 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 2 Aug 2026 22:44:17 -0600 Subject: [PATCH 5/5] chore(release): 0.123.1 --- docs/api/primitive-catalog.md | 2 +- docs/api/runtime.md | 22 +++++++++++++++++++ docs/api/runtime/environment-provider.md | 20 +++++++++++++++++ docs/canonical-api.md | 2 +- package.json | 2 +- .../fixtures/agent-improvement-proposal.json | 10 ++++----- .../agent-profile-improvement-proposal.json | 6 ++--- 7 files changed, 53 insertions(+), 11 deletions(-) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 58052dfd..c57f71e2 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.123.0` and `@tangle-network/agent-eval@0.142.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.123.1` and `@tangle-network/agent-eval@0.142.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 4b3f8722..314c8de3 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -12274,6 +12274,28 @@ Generic environment provider executor config. External packages implement [`ProviderExecutorOptions`](runtime/environment-provider.md#providerexecutoroptions).[`requireTerminalEvent`](runtime/environment-provider.md#requireterminalevent-1) +##### profileForCreate? + +> `optional` **profileForCreate?**: (`profile`) => `AgentProfile` + +Transform only the profile sent to `provider.create`. The original profile +remains the input to `taskToTurn`, so execution-only normalization cannot +rewrite the caller's task mapping. + +###### Parameters + +###### profile + +`AgentProfile` + +###### Returns + +`AgentProfile` + +###### Inherited from + +[`ProviderExecutorOptions`](runtime/environment-provider.md#providerexecutoroptions).[`profileForCreate`](runtime/environment-provider.md#profileforcreate) + ##### taskToTurn? > `optional` **taskToTurn?**: (`task`, `specProfile`) => `AgentTurnInput` diff --git a/docs/api/runtime/environment-provider.md b/docs/api/runtime/environment-provider.md index d0525fba..2dee68d2 100644 --- a/docs/api/runtime/environment-provider.md +++ b/docs/api/runtime/environment-provider.md @@ -284,6 +284,26 @@ Options for running a provider as a supervise-mode executor. **`Experimental`** +##### profileForCreate? + +> `optional` **profileForCreate?**: (`profile`) => `AgentProfile` + +**`Experimental`** + +Transform only the profile sent to `provider.create`. The original profile +remains the input to `taskToTurn`, so execution-only normalization cannot +rewrite the caller's task mapping. + +###### Parameters + +###### profile + +`AgentProfile` + +###### Returns + +`AgentProfile` + ##### taskToTurn? > `optional` **taskToTurn?**: (`task`, `specProfile`) => `AgentTurnInput` diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 001cd83a..37fe92f3 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.123.0.** +> **Version 0.123.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.142.2 <0.143.0`. > `sandbox` must satisfy `>=0.17.2 <0.18.0`. diff --git a/package.json b/package.json index 2357b07f..7fba3565 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.123.0", + "version": "0.123.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index c60b85d0..ed72b64c 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:0f4c0fe5e1339dd5f387766056e7797cbd9064cab15ba4c3edc922569a2685b4", + "digest": "sha256:fb18b750e1c943753d05266412a3240b548e47b3a7b049c6b49baf4387d14578", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.123.0" + "runtimeVersion": "0.123.1" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:ac7412b9a1e554d1b89aaaa80fcea7901cf5d258e8ae8c3777f70701c8b86d6f", - "runId": "agent-runtime-0.123.0-proposal-fixture", + "recordDigest": "sha256:9ff787d43525daee437bd6dd1518ef0d119ff0e3d91c5907068b29bac1a31d0a", + "runId": "agent-runtime-0.123.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.123.0-proposal-fixture" + "runId": "agent-runtime-0.123.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 75ac9326..bab9660b 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:58231ee3d872020cce4c8f1dc8179b11e598c5daf1566c643677eb5dc231e453", + "digest": "sha256:af06351a86081b1db9bdd82aa7c01aade535163410bb5d769e0ce5fcf67fabd1", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.123.0" + "runtimeVersion": "0.123.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:f3bbd6c1b07feacc0679a0c38b9c1fbf2eea5fa8c35ddaf950a4960a2e7e3d0f", + "recordDigest": "sha256:b2cdacb049f574cded93ebeddfe3f90dbce5f8b9c16abd59b6e310d74eb07e08", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }