diff --git a/src/CodexAcpServer.ts b/src/CodexAcpServer.ts index adc4e079..6f7837e0 100644 --- a/src/CodexAcpServer.ts +++ b/src/CodexAcpServer.ts @@ -1,5 +1,8 @@ import * as acp from "@agentclientprotocol/sdk"; import {RequestError, type SessionId, type SessionModeState} from "@agentclientprotocol/sdk"; +import {readFile} from "node:fs/promises"; +import {extname} from "node:path"; +import {fileURLToPath, pathToFileURL} from "node:url"; import {CodexEventHandler} from "./CodexEventHandler"; import {CodexApprovalHandler} from "./CodexApprovalHandler"; import {CodexElicitationHandler} from "./CodexElicitationHandler"; @@ -1114,13 +1117,11 @@ export class CodexAcpServer { } } - private createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): UpdateSessionEvent[] { + private async createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): Promise { const updates: UpdateSessionEvent[] = []; - const messageId = item.id; for (const input of item.content) { - const blocks = this.userInputToContentBlocks(input); - for (const block of blocks) { - updates.push(createUserMessageChunk(block, messageId)); + for (const block of await this.userInputToContentBlocks(input)) { + updates.push(createUserMessageChunk(block, item.id)); } } return updates; @@ -1173,20 +1174,44 @@ export class CodexAcpServer { }; } - private userInputToContentBlocks(input: UserInput): acp.ContentBlock[] { + private async userInputToContentBlocks(input: UserInput): Promise { switch (input.type) { - case "text": - return input.text.length > 0 ? [{ type: "text", text: input.text }] : []; - case "image": + case "text": { + const text = input.text.trimStart(); + const requestMarker = "\n## My request for Codex:\n"; + const requestMarkerIndex = text.indexOf(requestMarker); + const visibleText = text.startsWith("# Files mentioned by the user:\n") && requestMarkerIndex !== -1 + ? text.slice(requestMarkerIndex + requestMarker.length) + : input.text; + return visibleText.length > 0 ? [{ type: "text", text: visibleText }] : []; + } + case "image": { + const match = /^data:(image\/[^;]+);base64,(.+)$/.exec(input.url); + const mimeType = match?.[1]; + const data = match?.[2]; + if (mimeType && data) { + return [{ type: "image", mimeType, data }]; + } return [{ type: "text", text: this.formatUriAsLink("image", input.url) }]; + } case "localImage": { - const uri = input.path.startsWith("file://") ? input.path : `file://${input.path}`; + const path = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path; + const uri = pathToFileURL(path).href; + const extension = extname(path).slice(1).toLowerCase(); + const data = extension ? await readFile(path).catch(() => null) : null; + if (data) { + const mimeType = `image/${extension === "jpg" ? "jpeg" : extension}`; + return [{ type: "image", data: data.toString("base64"), mimeType, uri }]; + } return [{ type: "text", text: this.formatUriAsLink(null, uri) }]; } case "skill": return [{ type: "text", text: `skill:${input.name} (${input.path})` }]; + case "mention": { + const uri = input.path.startsWith("file://") ? input.path : pathToFileURL(input.path).href; + return [{ type: "resource_link", name: input.name, uri }]; + } } - return []; } private formatUriAsLink(name: string | null, uri: string): string { diff --git a/src/__tests__/CodexACPAgent/data/load-session-history.json b/src/__tests__/CodexACPAgent/data/load-session-history.json index bc5b3f96..3663efe2 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-history.json +++ b/src/__tests__/CodexACPAgent/data/load-session-history.json @@ -108,6 +108,58 @@ } ] } +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "mimeType": "image/png", + "data": "dGVzdCBpbWFnZQ==" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "data": "dGVzdCBpbWFnZQ==", + "mimeType": "image/png", + "uri": "file:///tmp/codex-acp-load-session-image.png" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "resource_link", + "name": "notes.txt", + "uri": "file:///test/project/notes.txt" + } + } + } + ] +} { "method": "sessionUpdate", "args": [ @@ -357,4 +409,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/__tests__/CodexACPAgent/load-session.test.ts b/src/__tests__/CodexACPAgent/load-session.test.ts index f8aea520..a476c90c 100644 --- a/src/__tests__/CodexACPAgent/load-session.test.ts +++ b/src/__tests__/CodexACPAgent/load-session.test.ts @@ -3,11 +3,16 @@ import type * as acp from "@agentclientprotocol/sdk"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { createCodexMockTestFixture, createTestModel } from "../acp-test-utils"; import type { Model, Thread } from "../../app-server/v2"; describe("CodexACPAgent - loadSession", () => { it("should replay history during loadSession", async () => { + const localImageDirectory = await mkdtemp(join(tmpdir(), "codex-acp-load-session-")); + const localImagePath = join(localImageDirectory, "image.png"); + await writeFile(localImagePath, "test image"); + const fixture = createCodexMockTestFixture(); const codexAcpAgent = fixture.getCodexAcpAgent(); const codexAcpClient = fixture.getCodexAcpClient(); @@ -82,8 +87,15 @@ describe("CodexACPAgent - loadSession", () => { id: "item-user-1", clientId: null, content: [ - { type: "text", text: "Hi", text_elements: [] }, + { + type: "text", + text: `\n# Files mentioned by the user:\n\n## image.png: ${localImagePath}\n\n## My request for Codex:\nHi`, + text_elements: [], + }, { type: "image", url: "https://example.com/image.png" }, + { type: "image", url: "data:image/png;base64,dGVzdCBpbWFnZQ==" }, + { type: "localImage", path: localImagePath }, + { type: "mention", name: "notes.txt", path: "/test/project/notes.txt" }, ], }, { @@ -204,9 +216,14 @@ describe("CodexACPAgent - loadSession", () => { threadId: thread.id, includeTurns: true, }); - await expect(fixture.getAcpConnectionDump([])).toMatchFileSnapshot( + const replay = fixture.getAcpConnectionDump([]).replaceAll( + pathToFileURL(localImagePath).href, + "file:///tmp/codex-acp-load-session-image.png", + ); + await expect(`${replay}\n`).toMatchFileSnapshot( "data/load-session-history.json" ); + await rm(localImageDirectory, { recursive: true }); }); it("should not recover session mcp servers during loadSession when request omits them", async () => {