Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion packages/agent-core/src/agent/turn/kosong-llm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,19 @@ export function downgradeUnsupportedMedia(
messages: readonly Message[],
capability: ModelCapability | undefined,
): Message[] {
if (capability === undefined || isUnknownCapability(capability)) return [...messages];
// Leave history alone when the host omitted a matrix (undefined) or only
// knows the model is uncatalogued (UNKNOWN_CAPABILITY). UNKNOWN means "we
// do not know", not "text-only" — a multimodal custom model can still use
// that marker (SingleModelProvider default, KimiForCodingProvider).
//
// Explicit matrices still strip: ProviderManager merges declared
// capabilities with the catalog into a plain ModelCapability object, so a
// text-only custom model configured as capabilities = ["thinking",
// "tool_use"] gets image_in: false without the UNKNOWN marker and is
// protected against image_url 400s that brick later turns (#2669).
if (capability === undefined || isUnknownCapability(capability)) {
return [...messages];
}
const dropImage = !capability.image_in;
const dropVideo = !capability.video_in;
const dropAudio = !capability.audio_in;
Expand Down
32 changes: 30 additions & 2 deletions packages/agent-core/test/agent/kosong-llm.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -440,12 +440,40 @@ describe('downgradeUnsupportedMedia', () => {
expect(out[0]?.content).toEqual([imagePart, videoPart]);
});

it('does not downgrade for UNKNOWN_CAPABILITY or an undefined capability', () => {
it('does not downgrade when capability is undefined (host omitted matrix)', () => {
const input = [mediaMessage([videoPart])];
expect(downgradeUnsupportedMedia(input, UNKNOWN_CAPABILITY)[0]?.content).toEqual([videoPart]);
expect(downgradeUnsupportedMedia(input, undefined)[0]?.content).toEqual([videoPart]);
});

it('preserves media for UNKNOWN_CAPABILITY (uncatalogued ≠ text-only)', () => {
// UNKNOWN is the SingleModelProvider / unresolved default: the model may
// still be multimodal. Explicit text-only matrices are covered below.
const input = [mediaMessage([imagePart, videoPart, audioPart])];
expect(downgradeUnsupportedMedia(input, UNKNOWN_CAPABILITY)[0]?.content).toEqual([
imagePart,
videoPart,
audioPart,
]);
});

it('strips images when an explicit matrix has no image_in (#2669)', () => {
// ProviderManager resolves declared capabilities into a plain object
// (not the UNKNOWN marker), e.g. capabilities = ["thinking", "tool_use"].
const capability: ModelCapability = {
image_in: false,
video_in: false,
audio_in: false,
thinking: true,
tool_use: true,
max_context_tokens: 128_000,
};
const input = [mediaMessage([{ type: 'text', text: 'see this' }, imagePart])];
expect(downgradeUnsupportedMedia(input, capability)[0]?.content).toEqual([
{ type: 'text', text: 'see this' },
{ type: 'text', text: '[image omitted: current model has no image input]' },
]);
});

it('returns a new array and never mutates the caller input', () => {
const capability = makeCapability(1000); // all media dropped
const message = mediaMessage([videoPart]);
Expand Down