From ae3fd3c5a05bd44d8cd8642523d0de6462ade01b Mon Sep 17 00:00:00 2001 From: sedanah-m Date: Tue, 11 Aug 2026 09:37:23 -0700 Subject: [PATCH 1/6] Implement Image-Generation --- .../src/features/image-generation/index.tsx | 144 ++++++++++++++++- .../src/features/image-generation/service.ts | 147 +++++++++++++++++- 2 files changed, 286 insertions(+), 5 deletions(-) diff --git a/ai/ai-samples/src/features/image-generation/index.tsx b/ai/ai-samples/src/features/image-generation/index.tsx index da08d1226..20d4c8537 100644 --- a/ai/ai-samples/src/features/image-generation/index.tsx +++ b/ai/ai-samples/src/features/image-generation/index.tsx @@ -1,9 +1,145 @@ -import React from 'react'; +import { useState, useRef, useEffect } from 'react'; +import { ChatSession, ImageConfigAspectRatio } from 'firebase/ai'; +import { startImageChat, sendImageChatMessage } from './service'; + +interface ChatMessage { + role: 'user' | 'model'; + text: string; + images?: { mimeType: string; base64: string }[]; +} + +export default function ImageGenerationView() { + const [messages, setMessages] = useState([]); + const [prompt, setPrompt] = useState('Generate an image of the Eiffel Tower with fireworks in the background.'); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const fileInputRef = useRef(null); + + const chatSessionRef = useRef(null); + useEffect(() => { + try { + chatSessionRef.current = startImageChat(ImageConfigAspectRatio.SQUARE_1x1); + setError(null); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : 'Failed to initialize chat session. Please check your Firebase configuration.'; + setError(message); + } + }, []); + + const handleGenerate = async () => { + const cleanedPrompt = prompt.trim(); + if (!cleanedPrompt || !chatSessionRef.current) return; + const files = Array.from(fileInputRef.current?.files ?? []); + const referenceFile = files.length > 0 ? files[0] : undefined; + + // Note: Inline Base64 conversion is meant for lightweight files. TODO: finish this comment + + setLoading(true); + setError(null); + setPrompt(''); + + try { + setMessages((prev) => [...prev, { role: 'user', text: cleanedPrompt }]); + const result = await sendImageChatMessage(chatSessionRef.current, cleanedPrompt, referenceFile); + setMessages((prev) => [...prev, { + role: 'model', + text: result.text, + images: result.images + }]); + if (fileInputRef.current) fileInputRef.current.value = ''; + + } catch (err: unknown) { + console.error(err); + if (err instanceof Error) { + setError(err.message); + } else { + setError('An error occurred during image generation.'); + } + } finally { + setLoading(false); + } + }; + + const handleResetChat = () => { + try { + chatSessionRef.current = startImageChat(ImageConfigAspectRatio.SQUARE_1x1); + setMessages([]); + setError(null); + setPrompt(''); + if (fileInputRef.current) fileInputRef.current.value = ''; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : 'Failed to reset chat session.'; + setError(message); + } + }; -export default function ImageGenerationFeature() { return ( -
-

image-generation

+
+
+

Image Generation & Iterative Editing

+ +
+ +

+ Ask Gemini to generate an image, or upload a reference file and ask the model to edit it iteratively. +

+ + {error && ( +
+ {error} +
+ )} + +
+ {messages.map((msg, idx) => ( +
+
+ {msg.text &&

{msg.text}

} + + {/* Native browser rendering of the extracted Base64 data */} + {msg.images && msg.images.map((img, i) => ( + Generated by Gemini + ))} +
+
+ ))} + {loading &&
Generating content...
} +
+ +
+ +
+ setPrompt(e.target.value)} + placeholder="e.g., Make the sky purple and add a spaceship..." + style={{ flex: 1, padding: '10px', borderRadius: '4px', border: '1px solid #ccc' }} + disabled={loading} + /> + +
+
); } \ No newline at end of file diff --git a/ai/ai-samples/src/features/image-generation/service.ts b/ai/ai-samples/src/features/image-generation/service.ts index 27c3a6599..9d253e133 100644 --- a/ai/ai-samples/src/features/image-generation/service.ts +++ b/ai/ai-samples/src/features/image-generation/service.ts @@ -1 +1,146 @@ -// Service for image-generation \ No newline at end of file +import { ChatSession, ImageConfigAspectRatio, ImageConfigImageSize, Part, ResponseModality } from 'firebase/ai'; +import { getAiModel } from '../../services/firebaseAIService'; + +export interface ImageGenerationResult { + text: string; + images: { mimeType: string; base64: string }[]; +} + +/** + * Helper: Safely extracts text and image Base64 data from a response parts array. + * Iterates over all parts and capturing everything. + */ +function extractTextAndImages(parts: Part[] = []): ImageGenerationResult { + let extractedText = ''; + const extractedImages: { mimeType: string; base64: string }[] = []; + + for (const part of parts) { + if (part.text) { + extractedText += part.text + '\n'; + } + if (part.inlineData) { + extractedImages.push({ + mimeType: part.inlineData.mimeType, + base64: part.inlineData.data + }); + } + } + + return { + text: extractedText.trim(), + images: extractedImages + }; +} + +/** + * Helper: Converts a standard browser File object into a Firebase AI SDK Part. + * Uses the native browser FileReader API to extract the Base64 string. + */ +export async function fileToGenerativePart(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => { + if (typeof reader.result === 'string') { + resolve({ + inlineData: { + data: reader.result.split(',')[1], + mimeType: file.type + } + }); + } else { + reject(new Error("Failed to parse file data as Base64.")); + } + }; + reader.onerror = reject; + reader.readAsDataURL(file); + }); +} + +/** + * Concept 1 & 5: Generate Images (Text-Only Input) + Image Configuration + * Demonstrates unary text-to-image generation, injecting custom aspect ratio and size parameters. + */ +export async function generateImage( + prompt: string, + aspectRatio: ImageConfigAspectRatio = ImageConfigAspectRatio.SQUARE_1x1, + imageSize: ImageConfigImageSize = ImageConfigImageSize.SIZE_1K +): Promise { + const model = getAiModel('gemini-3.1-flash-lite-image', { + generationConfig: { + responseModalities: [ResponseModality.IMAGE], + imageConfig: { aspectRatio, imageSize } + } + }); + + const result = await model.generateContent(prompt); + const parts = result.response.candidates?.[0]?.content?.parts ?? []; + return extractTextAndImages(parts); +} + +/** + * Concept 2: Generate Interleaved Images and Text + * Demonstrates instructing the model to return both text blocks and images in a single unary response. + */ +export async function generateInterleavedContent(prompt: string): Promise { + const model = getAiModel('gemini-3.1-flash-lite-image', { + generationConfig: { + responseModalities: [ResponseModality.TEXT, ResponseModality.IMAGE] // Both modalities active + } + }); + + const result = await model.generateContent(prompt); + const parts = result.response.candidates?.[0]?.content?.parts ?? []; + return extractTextAndImages(parts); +} + +/** + * Concept 3: Edit Images (Text-and-Image Input) + * Demonstrates unary multimodal prompting where you pass a reference image and a text instruction. + */ +export async function editSingleImage(prompt: string, file: File): Promise { + const imagePart = await fileToGenerativePart(file); + const model = getAiModel('gemini-3.1-flash-lite-image', { + generationConfig: { + responseModalities: [ResponseModality.IMAGE] + } + }); + + const result = await model.generateContent([prompt, imagePart]); + const parts = result.response.candidates?.[0]?.content?.parts ?? []; + return extractTextAndImages(parts); +} + +/** + * Concept 4: Iterate and Edit Images Using Multi-Turn Chat + * Initializes a stateful chat session specifically for iterative visual editing. + */ +export function startImageChat(aspectRatio: ImageConfigAspectRatio = ImageConfigAspectRatio.SQUARE_1x1): ChatSession { + const model = getAiModel('gemini-3.1-flash-lite-image', { + generationConfig: { + responseModalities: [ResponseModality.TEXT, ResponseModality.IMAGE], + imageConfig: { aspectRatio } + } + }); + + return model.startChat({ history: [] }); +} + +/** + * Sends a message to the active image chat session. + * For the initial turn, you can pass a reference file. Follow-up turns can omit the file + * and rely purely on the ChatSession history. + */ +export async function sendImageChatMessage( + chat: ChatSession, + prompt: string, + file?: File +): Promise { + const messagePayload: (string | Part) [] = [prompt]; + if (file) { + messagePayload.push(await fileToGenerativePart(file)); + } + + const result = await chat.sendMessage(messagePayload); + const parts = result.response.candidates?.[0]?.content?.parts ?? []; + return extractTextAndImages(parts); +} \ No newline at end of file From ab3ff3e48a5d3a51d9cdd6abb99209ea0fb9600f Mon Sep 17 00:00:00 2001 From: sedanah-m Date: Tue, 11 Aug 2026 15:52:44 -0700 Subject: [PATCH 2/6] fix:fixing the bug where interleaved text and image contexts were flattened and lost; fix: missing `await` typo from official documentation snippets. Add: comment about memory accumulation --- .../src/features/image-generation/index.tsx | 73 +++++++++++-------- .../src/features/image-generation/service.ts | 47 ++++++------ 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/ai/ai-samples/src/features/image-generation/index.tsx b/ai/ai-samples/src/features/image-generation/index.tsx index 20d4c8537..3fc4970a0 100644 --- a/ai/ai-samples/src/features/image-generation/index.tsx +++ b/ai/ai-samples/src/features/image-generation/index.tsx @@ -1,11 +1,12 @@ import { useState, useRef, useEffect } from 'react'; import { ChatSession, ImageConfigAspectRatio } from 'firebase/ai'; import { startImageChat, sendImageChatMessage } from './service'; +import { ImageGenerationSegment } from './service'; interface ChatMessage { role: 'user' | 'model'; - text: string; - images?: { mimeType: string; base64: string }[]; + text?: string; + segments?: ImageGenerationSegment[]; } export default function ImageGenerationView() { @@ -15,7 +16,7 @@ export default function ImageGenerationView() { const [error, setError] = useState(null); const fileInputRef = useRef(null); - + const chatSessionRef = useRef(null); useEffect(() => { try { @@ -33,8 +34,6 @@ export default function ImageGenerationView() { const files = Array.from(fileInputRef.current?.files ?? []); const referenceFile = files.length > 0 ? files[0] : undefined; - // Note: Inline Base64 conversion is meant for lightweight files. TODO: finish this comment - setLoading(true); setError(null); setPrompt(''); @@ -42,11 +41,11 @@ export default function ImageGenerationView() { try { setMessages((prev) => [...prev, { role: 'user', text: cleanedPrompt }]); const result = await sendImageChatMessage(chatSessionRef.current, cleanedPrompt, referenceFile); - setMessages((prev) => [...prev, { - role: 'model', - text: result.text, - images: result.images + setMessages((prev) => [...prev, { + role: 'model', + segments: result.segments }]); + if (fileInputRef.current) fileInputRef.current.value = ''; } catch (err: unknown) { @@ -59,7 +58,7 @@ export default function ImageGenerationView() { } finally { setLoading(false); } - }; + } const handleResetChat = () => { try { @@ -82,7 +81,6 @@ export default function ImageGenerationView() { Reset Session
-

Ask Gemini to generate an image, or upload a reference file and ask the model to edit it iteratively.

@@ -98,16 +96,31 @@ export default function ImageGenerationView() {
{msg.text &&

{msg.text}

} - - {/* Native browser rendering of the extracted Base64 data */} - {msg.images && msg.images.map((img, i) => ( - Generated by Gemini - ))} + {/* Native browser rendering of the ordered segments */} + {msg.segments && msg.segments.map((segment, i) => { + + if (segment.type === 'text') { + return ( +

+ {segment.text} +

+ ); + } + + if (segment.type === 'image') { + return ( + Generated by Gemini + ); + } + + return null; + })} +
))} @@ -115,24 +128,24 @@ export default function ImageGenerationView() {
-
- setPrompt(e.target.value)} - placeholder="e.g., Make the sky purple and add a spaceship..." + setPrompt(e.target.value)} + placeholder="e.g., Make the sky purple and add a spaceship..." style={{ flex: 1, padding: '10px', borderRadius: '4px', border: '1px solid #ccc' }} disabled={loading} /> -