diff --git a/.changeset/message-queue.md b/.changeset/message-queue.md new file mode 100644 index 000000000..6f91ebae0 --- /dev/null +++ b/.changeset/message-queue.md @@ -0,0 +1,14 @@ +--- +'@tanstack/ai-client': minor +'@tanstack/ai-react': minor +'@tanstack/ai-solid': minor +'@tanstack/ai-vue': minor +'@tanstack/ai-svelte': minor +'@tanstack/ai-preact': minor +--- + +Messages sent while a stream is already in flight are now queued by default and automatically sent once the in-flight stream settles, instead of being silently dropped. **This is a behavior change.** + +The behavior is configurable via a new `queue` option, which accepts `whenBusy: 'queue' | 'drop' | 'interrupt'`, `drain: 'fifo' | 'batch'`, `maxSize`, and `onOverflow`, or a custom strategy function for full control. + +Queued messages are exposed on the hook as `queue` and can be cancelled before they send via `cancelQueued(id)`. `sendMessage` also accepts a per-call `{ whenBusy }` override. diff --git a/docs/chat/streaming.md b/docs/chat/streaming.md index db32def3c..a4159ac50 100644 --- a/docs/chat/streaming.md +++ b/docs/chat/streaming.md @@ -203,6 +203,54 @@ export async function POST(request: Request) { } ``` +## Queueing Messages + +By default, calling `sendMessage` while a stream is already in flight **queues** the message instead of dropping it — it sends automatically once the current run settles. Configure this with the `queue` option, which accepts a `QueueConfig` object, a plain shorthand string, or a strategy function: + +```tsx group=queueing-messages +import { useChat, fetchServerSentEvents } from "@tanstack/ai-react"; + +const { messages, queue, sendMessage, cancelQueued, isLoading } = useChat({ + connection: fetchServerSentEvents("/api/chat"), + queue: { whenBusy: "queue", drain: "fifo", maxSize: 5 }, +}); +``` + +- **`whenBusy`** — what happens to a send that arrives mid-stream: + - `"queue"` (default) — hold the message; it sends once the stream settles. + - `"drop"` — ignore the send. + - `"interrupt"` — abort the current stream (like calling `stop()`) and send the new message immediately. +- **`drain`** — how queued items leave the queue: `"fifo"` (default) sends them one at a time in order; `"batch"` merges everything currently queued into a single send once the stream settles (string contents joined with `\n`, multimodal content concatenated in order). +- **`maxSize`** — caps how many messages can be queued. +- **`onOverflow`** — `"reject"` (default) ignores a send once `maxSize` is reached; `"drop-oldest"` evicts the oldest queued item to make room. + +You can also pass a plain `WhenBusy` string (e.g. `queue: "interrupt"`) as shorthand for `{ whenBusy: "interrupt" }` — this applies for any `WhenBusy` value (`"queue"`, `"drop"`, or `"interrupt"`), not just `"interrupt"` — or a `QueueStrategy` function for full control over the per-send decision (the drain order stays FIFO for the function form). + +`useChat` exposes the pending queue as `queue` so you can render it distinctly from `messages`, along with `cancelQueued(id)` to cancel an item before it sends: + +```tsx group=queueing-messages +function PendingQueue() { + return ( + <> + {queue.map((q) => ( +
+ {typeof q.content === "string" ? q.content : "[attachment]"} + +
+ ))} + + ); +} +``` + +Override the configured policy for a single send with the second argument to `sendMessage`: + +```tsx group=queueing-messages +sendMessage("Never mind, do this instead", { whenBusy: "interrupt" }); +``` + +> **Note:** This is a default-behavior change — messages sent while streaming used to be silently dropped. They are now queued unless you opt into `whenBusy: "drop"` or `"interrupt"`. + ## Best Practices 1. **Handle loading states** - Use `isLoading` to show loading indicators @@ -210,6 +258,7 @@ export async function POST(request: Request) { 3. **Cancel on unmount** - Clean up streams when components unmount 4. **Optimize rendering** - Batch updates if needed for performance 5. **Show progress** - Display partial content as it streams +6. **Render queued messages distinctly** - Use `queue` to show pending sends separately from `messages` ## Next Steps diff --git a/docs/config.json b/docs/config.json index c7b16c418..1625517ae 100644 --- a/docs/config.json +++ b/docs/config.json @@ -157,7 +157,7 @@ "label": "Streaming", "to": "chat/streaming", "addedAt": "2026-04-15", - "updatedAt": "2026-07-03" + "updatedAt": "2026-07-06" }, { "label": "Connection Adapters", diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index e7fbc6a0a..090a8d9b4 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -10,6 +10,7 @@ import { Guitar, Home, Image, + Layers, Menu, LayoutGrid, Mic, @@ -231,6 +232,19 @@ export default function Header() { Persistent Chats + setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-2', + }} + > + + Queueing Strategies + + setIsOpen(false)} diff --git a/examples/ts-react-chat/src/routeTree.gen.ts b/examples/ts-react-chat/src/routeTree.gen.ts index 69a7764a2..a66c65d4c 100644 --- a/examples/ts-react-chat/src/routeTree.gen.ts +++ b/examples/ts-react-chat/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as ThreadsRouteImport } from './routes/threads' import { Route as ServerFnChatRouteImport } from './routes/server-fn-chat' import { Route as SandboxesRouteImport } from './routes/sandboxes' import { Route as RealtimeRouteImport } from './routes/realtime' +import { Route as QueueingRouteImport } from './routes/queueing' import { Route as McpDemoRouteImport } from './routes/mcp-demo' import { Route as McpAppsRouteImport } from './routes/mcp-apps' import { Route as Issue176ToolResultRouteImport } from './routes/issue-176-tool-result' @@ -74,6 +75,11 @@ const RealtimeRoute = RealtimeRouteImport.update({ path: '/realtime', getParentRoute: () => rootRouteImport, } as any) +const QueueingRoute = QueueingRouteImport.update({ + id: '/queueing', + path: '/queueing', + getParentRoute: () => rootRouteImport, +} as any) const McpDemoRoute = McpDemoRouteImport.update({ id: '/mcp-demo', path: '/mcp-demo', @@ -287,6 +293,7 @@ export interface FileRoutesByFullPath { '/issue-176-tool-result': typeof Issue176ToolResultRoute '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute + '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute @@ -333,6 +340,7 @@ export interface FileRoutesByTo { '/issue-176-tool-result': typeof Issue176ToolResultRoute '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute + '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute @@ -380,6 +388,7 @@ export interface FileRoutesById { '/issue-176-tool-result': typeof Issue176ToolResultRoute '/mcp-apps': typeof McpAppsRoute '/mcp-demo': typeof McpDemoRoute + '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute @@ -428,6 +437,7 @@ export interface FileRouteTypes { | '/issue-176-tool-result' | '/mcp-apps' | '/mcp-demo' + | '/queueing' | '/realtime' | '/sandboxes' | '/server-fn-chat' @@ -474,6 +484,7 @@ export interface FileRouteTypes { | '/issue-176-tool-result' | '/mcp-apps' | '/mcp-demo' + | '/queueing' | '/realtime' | '/sandboxes' | '/server-fn-chat' @@ -520,6 +531,7 @@ export interface FileRouteTypes { | '/issue-176-tool-result' | '/mcp-apps' | '/mcp-demo' + | '/queueing' | '/realtime' | '/sandboxes' | '/server-fn-chat' @@ -567,6 +579,7 @@ export interface RootRouteChildren { Issue176ToolResultRoute: typeof Issue176ToolResultRoute McpAppsRoute: typeof McpAppsRoute McpDemoRoute: typeof McpDemoRoute + QueueingRoute: typeof QueueingRoute RealtimeRoute: typeof RealtimeRoute SandboxesRoute: typeof SandboxesRoute ServerFnChatRoute: typeof ServerFnChatRoute @@ -635,6 +648,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof RealtimeRouteImport parentRoute: typeof rootRouteImport } + '/queueing': { + id: '/queueing' + path: '/queueing' + fullPath: '/queueing' + preLoaderRoute: typeof QueueingRouteImport + parentRoute: typeof rootRouteImport + } '/mcp-demo': { id: '/mcp-demo' path: '/mcp-demo' @@ -927,6 +947,7 @@ const rootRouteChildren: RootRouteChildren = { Issue176ToolResultRoute: Issue176ToolResultRoute, McpAppsRoute: McpAppsRoute, McpDemoRoute: McpDemoRoute, + QueueingRoute: QueueingRoute, RealtimeRoute: RealtimeRoute, SandboxesRoute: SandboxesRoute, ServerFnChatRoute: ServerFnChatRoute, diff --git a/examples/ts-react-chat/src/routes/queueing.tsx b/examples/ts-react-chat/src/routes/queueing.tsx new file mode 100644 index 000000000..de6a340ba --- /dev/null +++ b/examples/ts-react-chat/src/routes/queueing.tsx @@ -0,0 +1,288 @@ +import { useMemo, useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { Send, Square, X, Zap } from 'lucide-react' +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' +import type { QueueConfig, QueuedMessage, UIMessage } from '@tanstack/ai-react' +import { DEFAULT_MODEL_OPTION, MODEL_OPTIONS } from '@/lib/model-selection' +import type { ModelOption } from '@/lib/model-selection' + +/** + * Showcase route for the client-side message queue. + * + * Three chats run side by side against the same endpoint, each hardwired to a + * different `queue` strategy. A shared composer broadcasts the same message to + * all three at once, so you can send a message, then send another WHILE the + * first is still streaming, and watch each strategy diverge: + * + * - queue/fifo → the second message is held and auto-sent after the first + * settles; it shows up as a cancellable pending item. + * - interrupt → the in-flight stream is aborted and the new message sends + * immediately (the queue never fills). + * - queue/batch → multiple mid-stream messages are held, then merged into a + * single send when the stream settles. + */ + +interface StrategyMeta { + title: string + blurb: string + config: QueueConfig +} + +const FIFO: StrategyMeta = { + title: 'queue · fifo', + blurb: + 'Messages sent while streaming are held and auto-sent one at a time, in order. Pending messages are cancellable until they drain.', + config: { whenBusy: 'queue', drain: 'fifo' }, +} +const INTERRUPT: StrategyMeta = { + title: 'interrupt', + blurb: + 'A message sent while streaming aborts the in-flight response and sends immediately. The queue never fills.', + config: { whenBusy: 'interrupt' }, +} +const BATCH: StrategyMeta = { + title: 'queue · batch', + blurb: + 'Messages sent while streaming are held, then merged into a single send (joined by newlines) when the stream settles.', + config: { whenBusy: 'queue', drain: 'batch' }, +} + +function textOf(message: UIMessage): string { + return message.parts + .filter((part) => part.type === 'text') + .map((part) => (part.type === 'text' ? part.content : '')) + .join('') +} + +interface PanelChat { + messages: Array + isLoading: boolean + queue: Array + cancelQueued: (id: string) => void + stop: () => void + error: Error | undefined +} + +function StrategyPanel({ + strategy, + chat, +}: { + strategy: StrategyMeta + chat: PanelChat +}) { + const { messages, isLoading, queue, cancelQueued, stop, error } = chat + const visible = messages.filter( + (message) => textOf(message).trim().length > 0, + ) + + return ( +
+
+
+ + {strategy.title} + + {isLoading && ( + + )} +
+

+ {strategy.blurb} +

+
+ +
+ {visible.length === 0 && ( +

No messages yet.

+ )} + {visible.map((message) => ( +
+ + {message.role === 'assistant' ? 'AI' : 'You'}: + + {textOf(message)} +
+ ))} + {isLoading && ( +

⏳ streaming…

+ )} +
+ + {error && ( +
+ {error.message} +
+ )} + +
+

+ Queue ({queue.length}) +

+ {queue.length === 0 ? ( +

empty

+ ) : ( +
    + {queue.map((item) => ( +
  • + + {typeof item.content === 'string' + ? item.content + : '[multimodal]'} + + +
  • + ))} +
+ )} +
+
+ ) +} + +function QueueingPage() { + const [selectedModel, setSelectedModel] = + useState(DEFAULT_MODEL_OPTION) + const [input, setInput] = useState('') + + const body = useMemo( + () => ({ provider: selectedModel.provider, model: selectedModel.model }), + [selectedModel.provider, selectedModel.model], + ) + + // One chat per strategy, called explicitly (fixed count, fixed order) so the + // rules of hooks hold. The shared composer broadcasts the same text to all + // three via their sendMessage functions. + const fifo = useChat({ + connection: fetchServerSentEvents('/api/tanchat'), + body, + queue: FIFO.config, + }) + const interrupt = useChat({ + connection: fetchServerSentEvents('/api/tanchat'), + body, + queue: INTERRUPT.config, + }) + const batch = useChat({ + connection: fetchServerSentEvents('/api/tanchat'), + body, + queue: BATCH.config, + }) + + const panels = [ + { meta: FIFO, chat: fifo }, + { meta: INTERRUPT, chat: interrupt }, + { meta: BATCH, chat: batch }, + ] + + const broadcast = () => { + const text = input.trim() + if (!text) return + void fifo.sendMessage(text) + void interrupt.sendMessage(text) + void batch.sendMessage(text) + setInput('') + } + + return ( +
+
+

+ Queueing Strategies +

+

+ Send a message, then send another while it is still streaming{' '} + — each panel reacts differently based on its queue{' '} + config. +

+
+ + +
+
+ +
+ {panels.map((panel) => ( + + ))} +
+ +
+
+