diff --git a/packages/slack/src/message.ts b/packages/slack/src/message.ts index 0a278d1a..f7c389f6 100644 --- a/packages/slack/src/message.ts +++ b/packages/slack/src/message.ts @@ -396,6 +396,25 @@ export interface MessageMetadata { * channel is the channel the message was sent in. */ channel: ConversationsInfoResponse["channel"]; + + /** + * imageBlocks is a list of image blocks in the message (from block kit). + * These are different from file attachments - they're inline images with URLs. + */ + imageBlocks: Array<{ + image_url: string; + alt_text: string; + title?: string; + result: + | { + type: "downloaded"; + content: Buffer; + } + | { + type: "error"; + error: Error; + }; + }>; } export interface ExtractMessagesMetadataOptions { @@ -469,6 +488,14 @@ export const extractMessagesMetadata = async < | { type: "not_supported" }; }> = []; + // Collect image blocks (inline images from block kit) + const imageBlockUrls: Array<{ + url: string; + alt_text: string; + title?: string; + messageIndex: number; + }> = []; + // First pass: collect all IDs and files from all messages for (let i = 0; i < messages.length; i++) { const message = messages[i]; @@ -497,6 +524,24 @@ export const extractMessagesMetadata = async < break; } } + + // Extract image blocks + for (const block of message.blocks) { + if (block.type === "image" && "image_url" in block) { + const imageBlock = block as { + type: "image"; + image_url: string; + alt_text: string; + title?: { text: string }; + }; + imageBlockUrls.push({ + url: imageBlock.image_url, + alt_text: imageBlock.alt_text, + title: imageBlock.title?.text, + messageIndex: i, + }); + } + } } // Collect user IDs from message authors @@ -643,6 +688,58 @@ export const extractMessagesMetadata = async < } } + // Fetch image blocks (no auth needed for external URLs) + const imageBlockResults: Map< + number, // messageIndex + MessageMetadata["imageBlocks"] + > = new Map(); + + for (const entry of imageBlockUrls) { + const { url, alt_text, title, messageIndex } = entry; + + // Ensure messageIndex array exists + if (!imageBlockResults.has(messageIndex)) { + imageBlockResults.set(messageIndex, []); + } + const messageImageBlocks = imageBlockResults.get(messageIndex)!; + + // Download the image (external URL, no auth needed) + promises.push( + (async () => { + try { + const response = await fetch(url, { + redirect: "follow", + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(text); + } + const content = await response.arrayBuffer(); + messageImageBlocks.push({ + image_url: url, + alt_text, + title, + result: { + type: "downloaded", + content: Buffer.from(content), + }, + }); + } catch (err) { + messageImageBlocks.push({ + image_url: url, + alt_text, + title, + result: { + type: "error", + error: err as Error, + }, + }); + } + })() + ); + } + // Wait for all promises to resolve await Promise.all([ ...promises, @@ -750,6 +847,9 @@ export const extractMessagesMetadata = async < } } + // Get image blocks + const imageBlocks = imageBlockResults.get(i) ?? []; + // Get user info const user = message.user ? users[message.user] : undefined; @@ -774,6 +874,7 @@ export const extractMessagesMetadata = async < metadata: { mentions, files, + imageBlocks, user, createdAt, channel, diff --git a/packages/slack/src/tools.ts b/packages/slack/src/tools.ts index 28f6e43f..140b5b9c 100644 --- a/packages/slack/src/tools.ts +++ b/packages/slack/src/tools.ts @@ -299,10 +299,22 @@ IMPORTANT: This MUST be text, not an emoji.`), } : file.result, })), + imageBlocks: msg.metadata.imageBlocks.map((imageBlock) => ({ + image_url: imageBlock.image_url, + alt_text: imageBlock.alt_text, + title: imageBlock.title, + result: + imageBlock.result.type === "downloaded" + ? { + type: "downloaded", + base64: imageBlock.result.content.toString("base64"), + } + : imageBlock.result, + })), }; }, toModelOutput(output) { - const { message, files } = output; + const { message, files, imageBlocks } = output; const parts: Extract< LanguageModelV2ToolResultOutput, { type: "content" } @@ -321,6 +333,21 @@ IMPORTANT: This MUST be text, not an emoji.`), }); } } + // Add image blocks (inline images from block kit) + for (const imageBlock of imageBlocks) { + if (imageBlock.result.type === "downloaded") { + parts.push({ + type: "media", + data: imageBlock.result.base64, + mediaType: "image/png", // Default to PNG for external images + }); + } else { + parts.push({ + type: "text", + text: `The message contains an image block (${imageBlock.alt_text}), but it could not be downloaded: ${JSON.stringify(imageBlock.result)}`, + }); + } + } return { type: "content", value: [