Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added .pr-assets/36517-test-results.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
9 changes: 6 additions & 3 deletions packages/llm/src/cache-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,17 @@ const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]
messages.findLastIndex((m) => m.role === role)

// Mark the last text part of `messages[index]`. If no text part exists, mark
// the last content part regardless of type — that's the breakpoint position
// in tool-result-only messages too.
// the last non-reasoning content part — Bedrock rejects cache markers that
// sit immediately after a reasoning block, and tool-result-only messages still
// need a breakpoint on their last result part.
const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
if (index < 0 || index >= messages.length) return messages
const target = messages[index]!
if (target.content.length === 0) return messages
const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
const lastSafeIndex = target.content.findLastIndex((part) => part.type !== "reasoning")
const markAt = lastTextIndex >= 0 ? lastTextIndex : lastSafeIndex
if (markAt < 0) return messages
const existing = target.content[markAt]!
if ("cache" in existing && existing.cache) return messages
const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
Expand Down
2 changes: 2 additions & 0 deletions packages/llm/src/protocols/bedrock-converse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,8 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
continue
}
if (part.type === "reasoning") {
// Never attach cachePoint after reasoning — Bedrock returns
// ValidationException: "Cache point cannot be inserted after reasoning block."
content.push({
reasoningContent: {
reasoningText: { text: part.text, signature: reasoningSignature(part) },
Expand Down
49 changes: 49 additions & 0 deletions packages/llm/test/cache-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,55 @@ describe("applyCachePolicy", () => {
}),
)

it.effect("'latest-assistant' anchors on text before trailing reasoning, not the reasoning block", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: bedrockModel,
messages: [
Message.user("u1"),
Message.assistant([
{ type: "text", text: "visible" },
{ type: "reasoning", text: "hidden", encrypted: "sig" },
]),
],
cache: { messages: "latest-assistant" },
}),
)

const body = prepared.body as { messages: Array<{ role: string; content: Array<Record<string, unknown>> }> }
const assistant = body.messages.find((message) => message.role === "assistant")
expect(assistant?.content).toEqual([
{ text: "visible" },
{ cachePoint: { type: "default" } },
{
reasoningContent: {
reasoningText: { text: "hidden", signature: "sig" },
},
},
])
expect(assistant?.content.at(-1)).not.toHaveProperty("cachePoint")
}),
)

it.effect("skips cache markers for reasoning-only assistant messages", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: bedrockModel,
messages: [
Message.user("u1"),
Message.assistant([{ type: "reasoning", text: "only thinking", encrypted: "sig" }]),
],
cache: { messages: "latest-assistant" },
}),
)

const flat = JSON.stringify(prepared.body)
expect(flat).not.toContain("cachePoint")
}),
)

test("returns the same request reference when policy is a no-op (pure function)", () => {
const request = LLM.request({
model: anthropicModel,
Expand Down
36 changes: 36 additions & 0 deletions packages/llm/test/provider/bedrock-converse.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,42 @@ describe("Bedrock Converse route", () => {
}),
)

it.effect("does not place cachePoint after a trailing reasoning block (#36517)", () =>
Effect.gen(function* () {
const cache = new CacheHint({ type: "ephemeral" })
const prepared = yield* LLMClient.prepare(
LLM.request({
id: "req_cache_reasoning",
model,
cache: "none",
messages: [
Message.user("hi"),
Message.assistant([
{ type: "text", text: "reply", cache },
{ type: "reasoning", text: "thoughts", encrypted: "sig" },
]),
],
generation: { maxTokens: 16, temperature: 0 },
}),
)

const assistant = (prepared.body as { messages: Array<{ role: string; content: unknown[] }> }).messages.find(
(message) => message.role === "assistant",
)
expect(assistant?.content).toEqual([
{ text: "reply" },
{ cachePoint: { type: "default" } },
{
reasoningContent: {
reasoningText: { text: "thoughts", signature: "sig" },
},
},
])
// Final block must not be a cachePoint (Bedrock ValidationException).
expect(assistant?.content.at(-1)).not.toHaveProperty("cachePoint")
}),
)

it.effect("does not emit cachePoint when no cache hint is set", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(baseRequest)
Expand Down
29 changes: 29 additions & 0 deletions packages/opencode/src/provider/transform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,33 @@ function normalizeMessages(
return msgs
}

function isBedrockModel(model: Provider.Model) {
return model.providerID.includes("bedrock") || model.api.npm === "@ai-sdk/amazon-bedrock"
}

// Bedrock rejects cachePoint immediately after a reasoning block. The AI SDK
// emits message-level bedrock.cachePoint after every content part, so when an
// assistant message ends in reasoning we must anchor the breakpoint on the
// last non-reasoning part instead (or skip if the message is reasoning-only).
function applyBedrockCacheOptions(msg: ModelMessage, providerOptions: Record<string, unknown>): "applied" | "skipped" {
if (msg.role !== "assistant" || !Array.isArray(msg.content) || msg.content.length === 0) return "skipped"
const last = msg.content[msg.content.length - 1]
if (!last || typeof last !== "object" || last.type !== "reasoning") return "skipped"

for (let i = msg.content.length - 1; i >= 0; i--) {
const part = msg.content[i]
if (!part || typeof part !== "object" || part.type === "reasoning") continue
// Same shape as the content-level path below; cast keeps mergeDeep assignable
// to SharedV3ProviderOptions without widening the whole message transform.
;(part as { providerOptions?: Record<string, unknown> }).providerOptions = mergeDeep(
(part as { providerOptions?: Record<string, unknown> }).providerOptions ?? {},
providerOptions,
)
return "applied"
}
return "applied"
}

function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage[] {
const system = msgs.filter((msg) => msg.role === "system").slice(0, 2)
const final = msgs.filter((msg) => msg.role !== "system").slice(-2)
Expand Down Expand Up @@ -365,6 +392,8 @@ function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage
}
}

if (isBedrockModel(model) && applyBedrockCacheOptions(msg, providerOptions) === "applied") continue

msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions)
}

Expand Down
86 changes: 86 additions & 0 deletions packages/opencode/test/provider/transform.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2821,6 +2821,92 @@ describe("ProviderTransform.message - bedrock caching with non-bedrock providerI
})
})

describe("ProviderTransform.message - bedrock cachePoint after reasoning (#36517)", () => {
const bedrockModel = {
id: "amazon-bedrock/anthropic.claude-sonnet-4-20250514-v1:0",
providerID: "amazon-bedrock",
api: {
id: "anthropic.claude-sonnet-4-20250514-v1:0",
url: "https://bedrock-runtime.us-east-1.amazonaws.com",
npm: "@ai-sdk/amazon-bedrock",
},
name: "Claude Sonnet 4",
capabilities: {},
options: {},
headers: {},
} as any

test("anchors cachePoint on last non-reasoning part when assistant ends with reasoning", () => {
const msgs = [
{ role: "user", content: "Hello" },
{
role: "assistant",
content: [
{ type: "text", text: "Visible reply" },
{
type: "reasoning",
text: "internal thoughts",
providerOptions: { bedrock: { signature: "sig" } },
},
],
},
] as any[]

const result = ProviderTransform.message(msgs, bedrockModel, {}) as any[]
const assistant = result.find((msg) => msg.role === "assistant")
expect(assistant?.providerOptions?.bedrock?.cachePoint).toBeUndefined()
expect(assistant?.content[0].providerOptions?.bedrock).toEqual({
cachePoint: { type: "default" },
})
expect(assistant?.content[1].providerOptions?.bedrock?.cachePoint).toBeUndefined()
})

test("skips cachePoint when assistant content is reasoning-only", () => {
const msgs = [
{ role: "user", content: "Hello" },
{
role: "assistant",
content: [
{
type: "reasoning",
text: "only thinking",
providerOptions: { bedrock: { signature: "sig" } },
},
],
},
] as any[]

const result = ProviderTransform.message(msgs, bedrockModel, {}) as any[]
const assistant = result.find((msg) => msg.role === "assistant")
expect(assistant?.providerOptions?.bedrock?.cachePoint).toBeUndefined()
expect(assistant?.content[0].providerOptions?.bedrock?.cachePoint).toBeUndefined()
})

test("keeps message-level cachePoint when assistant does not end with reasoning", () => {
const msgs = [
{ role: "user", content: "Hello" },
{
role: "assistant",
content: [
{
type: "reasoning",
text: "internal thoughts",
providerOptions: { bedrock: { signature: "sig" } },
},
{ type: "text", text: "Visible reply" },
],
},
] as any[]

const result = ProviderTransform.message(msgs, bedrockModel, {}) as any[]
const assistant = result.find((msg) => msg.role === "assistant")
expect(assistant?.providerOptions?.bedrock).toEqual({
cachePoint: { type: "default" },
})
expect(assistant?.content[1].providerOptions?.bedrock?.cachePoint).toBeUndefined()
})
})

describe("ProviderTransform.message - cache control on gateway", () => {
const createModel = (overrides: Partial<any> = {}) =>
({
Expand Down
Loading