diff --git a/.changeset/quiet-runs-stall.md b/.changeset/quiet-runs-stall.md new file mode 100644 index 000000000..5643bf8e2 --- /dev/null +++ b/.changeset/quiet-runs-stall.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-sandbox-cloudflare': minor +--- + +Add the app-wide `stallTimeoutMs` option for both coordinator modes, with authenticated `/_bridge` and `/tool-exec` callbacks refreshing run activity on arrival and completion. diff --git a/docs/config.json b/docs/config.json index dfe4cd277..52875dd39 100644 --- a/docs/config.json +++ b/docs/config.json @@ -699,7 +699,8 @@ { "label": "Reaping & Retention", "to": "sandbox/reaping", - "addedAt": "2026-08-04" + "addedAt": "2026-08-04", + "updatedAt": "2026-08-20" }, { "label": "Provisioning", @@ -717,7 +718,7 @@ "label": "Cloudflare (Edge)", "to": "sandbox/cloudflare", "addedAt": "2026-06-29", - "updatedAt": "2026-08-04" + "updatedAt": "2026-08-20" } ] }, diff --git a/docs/sandbox/cloudflare.md b/docs/sandbox/cloudflare.md index bbd360153..425427241 100644 --- a/docs/sandbox/cloudflare.md +++ b/docs/sandbox/cloudflare.md @@ -126,6 +126,42 @@ migrated in place on first read — nothing to run, but note that `GET /runs/:id` and the WebSocket terminal `status` frame now carry the converged status strings and field names. +### Stall watchdog + +`createCloudflareSandboxAgent` sets one stall policy for the whole app. The +`stallTimeoutMs` field applies to every run in both `do-drives` (the default) +and `colocated` mode: + +| Value | Behavior | +| --- | --- | +| Omitted | Treat the run as stalled after `300000` ms (five minutes) without persisted activity | +| Positive safe integer | Use that many milliseconds as the stall threshold | +| `false` | Disable stall detection | + +```ts +import { grokBuildText } from '@tanstack/ai-grok-build' +import { createCloudflareSandboxAgent } from '@tanstack/ai-sandbox-cloudflare/agent' + +export const agent = createCloudflareSandboxAgent({ + adapter: () => grokBuildText('grok-build'), + stallTimeoutMs: 10 * 60_000, +}) +``` + +The alarm checks every 30 seconds, or every `stallTimeoutMs` when that is +shorter, so detection can lag the configured threshold by up to one check +interval. Persisted run events refresh +`updatedAt`; authenticated `/_bridge` and `/tool-exec` callbacks also refresh it +on arrival and completion. Unknown-run and unauthorized requests do not. + +Choose a timeout longer than the longest legitimate quiet period. Native +operations that emit no event and make no callback are invisible to the +watchdog. An authenticated callback is protected while it remains in flight. +A callback that never completes can leave an orphaned `running` record. +Disabling the watchdog with `false` can do the same. When a stall is detected, the watchdog marks +the run log `failed`; it may not kill the underlying agent process or container. +Use provider lifecycle controls and [reaping](./reaping) for resource cleanup. + ### Three layers, three homes Keep these separate — each has a different home on Cloudflare, and conflating diff --git a/docs/sandbox/reaping.md b/docs/sandbox/reaping.md index ed1c3256f..d29fa384a 100644 --- a/docs/sandbox/reaping.md +++ b/docs/sandbox/reaping.md @@ -371,12 +371,24 @@ export class RunReaper { } ``` -One thing this is **not** interchangeable with: the coordinator from -`@tanstack/ai-sandbox-cloudflare` ships a *stall watchdog* — an alarm that -fails run records whose log has gone quiet for too long. That is log hygiene, -not reaping: it never probes a journal for the exit sentinel and never -reclaims a sandbox. On Cloudflare you still schedule `sweepDetachedRuns`, and -a DO alarm like the one above is the natural place for it. +### Cloudflare's stall watchdog is not a reaper + +`createCloudflareSandboxAgent({ stallTimeoutMs })` sets an app-wide policy for +both `do-drives` and `colocated` modes. Omit it for the `300000` ms default, pass +a positive safe-integer millisecond value to customize it, or pass `false` to +disable it. Its alarm checks about every 30 seconds. + +Persisted events and authenticated `/_bridge` or `/tool-exec` callback arrival +and completion refresh the run record's `updatedAt`; unknown-run and +unauthorized requests do not. Silent native operations therefore need a timeout +longer than their longest expected quiet period. In-flight authenticated +callbacks are protected. A callback that never completes can leave an orphaned +`running` record. Disabling the watchdog with `false` can do the same. + +The watchdog atomically marks a stale log `failed`, but may not kill the +underlying process and never probes a journal or reclaims a sandbox. You must +still schedule `sweepDetachedRuns`; see the +[Cloudflare stall-watchdog details](./cloudflare#stall-watchdog). ## `pruneJournals`: bounding the journal directory diff --git a/packages/ai-sandbox-cloudflare/src/chat-coordinator.ts b/packages/ai-sandbox-cloudflare/src/chat-coordinator.ts index e4c15048e..1cb1ccb61 100644 --- a/packages/ai-sandbox-cloudflare/src/chat-coordinator.ts +++ b/packages/ai-sandbox-cloudflare/src/chat-coordinator.ts @@ -30,6 +30,7 @@ import { withSandbox, } from '@tanstack/ai-sandbox' import { SandboxCoordinator, resolveBridgeOrigin } from './coordinator' +import { runWithCallbackActivity } from './coordinator-callbacks' import { timingSafeBearerEqualWeb } from './web-crypto' import type { StartRunInput } from './coordinator' import type { @@ -233,21 +234,29 @@ export abstract class ChatSandboxCoordinator< ) { return new Response('unauthorized', { status: 401 }) } - let message: unknown - try { - message = await request.json() - } catch { - // A malformed body must still produce a valid JSON-RPC error so the agent's - // MCP client can react, rather than an opaque DO 500 that can wedge the run. - return this.jsonResponse({ - jsonrpc: '2.0', - id: null, - error: { code: -32700, message: 'Parse error' }, - }) - } - const reply = await handleBridgeJsonRpc(bridge.core, message) - // A notification (no id) yields null → MCP expects an empty 202 ack. - if (reply === null) return new Response(null, { status: 202 }) - return this.jsonResponse(reply) + return runWithCallbackActivity( + this, + runId, + (id) => this.log.touch(id), + async () => { + let message: unknown + try { + message = await request.json() + } catch { + // A malformed body must still produce a valid JSON-RPC error so the + // agent's MCP client can react, rather than an opaque DO 500 that can + // wedge the run. + return this.jsonResponse({ + jsonrpc: '2.0', + id: null, + error: { code: -32700, message: 'Parse error' }, + }) + } + const reply = await handleBridgeJsonRpc(bridge.core, message) + // A notification (no id) yields null → MCP expects an empty 202 ack. + if (reply === null) return new Response(null, { status: 202 }) + return this.jsonResponse(reply) + }, + ) } } diff --git a/packages/ai-sandbox-cloudflare/src/container-coordinator.ts b/packages/ai-sandbox-cloudflare/src/container-coordinator.ts index 4a58b7e82..7385ce6d6 100644 --- a/packages/ai-sandbox-cloudflare/src/container-coordinator.ts +++ b/packages/ai-sandbox-cloudflare/src/container-coordinator.ts @@ -34,6 +34,7 @@ import { } from '@tanstack/ai-sandbox' import { getSandbox } from '@cloudflare/sandbox' import { SandboxCoordinator, resolveBridgeOrigin } from './coordinator' +import { runWithCallbackActivity } from './coordinator-callbacks' import { timingSafeBearerEqualWeb } from './web-crypto' import type { StartRunInput } from './coordinator' import type { ContainerRunRequest, HarnessId } from './protocol' @@ -409,29 +410,41 @@ export abstract class ContainerSandboxCoordinator< ) { return new Response('unauthorized', { status: 401 }) } - let payload: unknown - try { - payload = await request.json() - } catch { - return this.jsonResponse({ error: 'body must be valid JSON' }, 400) - } - if (!isToolExecRequest(payload)) { - return this.jsonResponse({ error: 'body must be { name, args }' }, 400) - } - try { - const result = await executeHostTool( - state.hostTools, - payload.name, - payload.args, - { - ...(state.context !== undefined ? { context: state.context } : {}), - signal: state.abort.signal, - }, - ) - return this.jsonResponse({ result }) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - return this.jsonResponse({ error: message }, 500) - } + return runWithCallbackActivity( + this, + runId, + (id) => this.log.touch(id), + async () => { + let payload: unknown + try { + payload = await request.json() + } catch { + return this.jsonResponse({ error: 'body must be valid JSON' }, 400) + } + if (!isToolExecRequest(payload)) { + return this.jsonResponse( + { error: 'body must be { name, args }' }, + 400, + ) + } + try { + const result = await executeHostTool( + state.hostTools, + payload.name, + payload.args, + { + ...(state.context !== undefined + ? { context: state.context } + : {}), + signal: state.abort.signal, + }, + ) + return this.jsonResponse({ result }) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + return this.jsonResponse({ error: message }, 500) + } + }, + ) } } diff --git a/packages/ai-sandbox-cloudflare/src/coordinator-callbacks.ts b/packages/ai-sandbox-cloudflare/src/coordinator-callbacks.ts new file mode 100644 index 000000000..11dc73e45 --- /dev/null +++ b/packages/ai-sandbox-cloudflare/src/coordinator-callbacks.ts @@ -0,0 +1,58 @@ +type TouchRun = (runId: string) => Promise + +const inFlightCallbacks = new WeakMap>() + +function increment(owner: object, runId: string): void { + let runs = inFlightCallbacks.get(owner) + if (!runs) { + runs = new Map() + inFlightCallbacks.set(owner, runs) + } + runs.set(runId, (runs.get(runId) ?? 0) + 1) +} + +function decrement(owner: object, runId: string): void { + const runs = inFlightCallbacks.get(owner) + if (!runs) return + const count = runs.get(runId) ?? 0 + if (count > 1) { + runs.set(runId, count - 1) + return + } + runs.delete(runId) + if (runs.size === 0) inFlightCallbacks.delete(owner) +} + +export function hasInFlightCallback(owner: object, runId: string): boolean { + return (inFlightCallbacks.get(owner)?.get(runId) ?? 0) > 0 +} + +export async function runWithCallbackActivity( + owner: object, + runId: string, + touch: TouchRun, + operation: () => Promise, +): Promise { + increment(owner, runId) + try { + // Failure here proves liveness could not be persisted, so do not execute the + // callback operation. + await touch(runId) + try { + return await operation() + } finally { + // This is best-effort bookkeeping after an operation has produced its + // result or error. It must never replace that original outcome. + try { + await touch(runId) + } catch (error) { + console.error( + `[sandbox-coordinator] completion activity touch failed for run ${runId}:`, + error, + ) + } + } + } finally { + decrement(owner, runId) + } +} diff --git a/packages/ai-sandbox-cloudflare/src/coordinator.ts b/packages/ai-sandbox-cloudflare/src/coordinator.ts index 9f4b4707a..f48e6f7e0 100644 --- a/packages/ai-sandbox-cloudflare/src/coordinator.ts +++ b/packages/ai-sandbox-cloudflare/src/coordinator.ts @@ -31,21 +31,30 @@ import { EventType, isTerminalRunStatus } from '@tanstack/ai' // migrated on read; see './run-log'). import { RunController } from '@tanstack/ai-sandbox' import { runLogStore, runLogStream } from './durability' +import { hasInFlightCallback } from './coordinator-callbacks' import { DurableObjectRunEventLog } from './run-log-do' import type { ModelMessage, StreamChunk } from '@tanstack/ai' import type { RunLogRecord } from './run-log' -/** Re-arm window for the liveness watchdog while a run is in flight (ms). */ +/** Upper bound on the watchdog check interval while a run is in flight (ms). */ const WATCHDOG_MS = 30_000 -/** - * How long a non-terminal run may go without ANY new event before the watchdog - * presumes the orchestrator driving it is dead (eviction that lost the - * `waitUntil` promise, an uncaught fault, a hung container) and fails the run so - * tailing clients stop waiting forever. Generous so a legitimately slow agent - * step (a long tool call that emits no chunks) is not killed prematurely. - */ -const WATCHDOG_STALL_MS = 5 * 60_000 +/** Default permitted period without persisted run activity (ms). */ +const DEFAULT_STALL_TIMEOUT_MS = 5 * 60_000 + +/** @internal Shared validation for direct subclasses and the eager factory path. */ +export function normalizeStallTimeoutMs( + stallTimeoutMs: number | false | undefined, +): number | false { + if (stallTimeoutMs === undefined) return DEFAULT_STALL_TIMEOUT_MS + if (stallTimeoutMs === false) return false + if (!Number.isSafeInteger(stallTimeoutMs) || stallTimeoutMs <= 0) { + throw new TypeError( + 'stallTimeoutMs must be a positive safe integer or false', + ) + } + return stallTimeoutMs +} /** What the Worker hands the coordinator to start a run. */ export interface StartRunInput { @@ -106,9 +115,15 @@ export abstract class SandboxCoordinator< * running — double-delivering events and racing the persisted cursor. */ private readonly pumping = new WeakSet() + private readonly stallTimeoutMs: number | false - constructor(ctx: DurableObjectState, env: TEnv) { + constructor( + ctx: DurableObjectState, + env: TEnv, + stallTimeoutMs?: number | false, + ) { super(ctx, env) + this.stallTimeoutMs = normalizeStallTimeoutMs(stallTimeoutMs) this.log = new DurableObjectRunEventLog(ctx.storage) this.controller = new RunController({ runs: runLogStore(this.log), @@ -154,7 +169,12 @@ export abstract class SandboxCoordinator< async startRun(input: StartRunInput): Promise<{ runId: string }> { const existing = await this.log.get(input.runId) - if (existing) return { runId: input.runId } // idempotent re-trigger + if (existing) { + if (!isTerminalRunStatus(existing.status)) await this.armWatchdog() + return { runId: input.runId } + } + + await this.armWatchdog() // Open the run BEFORE building the stream. `pipeToRunLog`'s never-rejects // guarantee only covers failures AFTER the stream is handed to it — a throw @@ -189,7 +209,6 @@ export abstract class SandboxCoordinator< // running the settle hook. const settle = (): void => this.onRunSettled(input.runId) this.ctx.waitUntil(done.then(settle, settle)) - await this.ctx.storage.setAlarm(Date.now() + WATCHDOG_MS) return { runId: input.runId } } @@ -320,41 +339,58 @@ export abstract class SandboxCoordinator< // =========================================================================== override async alarm(): Promise { + // A previously configured alarm may still be delivered once after watchdogs + // are disabled. It must self-extinguish before reading storage or entering a + // catch path that could re-arm it. Deliberately do not call deleteAlarm(). + if (this.stallTimeoutMs === false) return + try { // Through the log (not a raw `rec:` list) so legacy records are migrated // on the way out — the storage layout is the log's private concern. const runs = await this.log.list() - const now = Date.now() + const cutoff = Date.now() - this.stallTimeoutMs let active = false for (const record of runs) { if (isTerminalRunStatus(record.status)) continue - if (now - record.updatedAt > WATCHDOG_STALL_MS) { - // No progress for too long — the driver is presumed dead. Fail the run - // so tailing clients stop waiting forever (the whole point of the - // watchdog; without this a stuck run sits at `running` indefinitely). - await this.failStalledRun(record.runId) - } else { + if ( + hasInFlightCallback(this, record.runId) || + record.updatedAt >= cutoff + ) { active = true + continue } + + // Re-check and terminalize atomically against the same strict cutoff. + // A callback touch or normal completion racing this alarm wins cleanly. + const finished = await this.failStalledRun(record.runId, cutoff) + if (finished) this.onRunSettled(record.runId) + else active = true } - if (active) await this.ctx.storage.setAlarm(Date.now() + WATCHDOG_MS) + if (active) await this.armWatchdog() } catch (error) { // Never let the watchdog die silently: a transient storage error must not // permanently disable liveness detection. Re-arm and try again next tick. console.error('[sandbox-coordinator] watchdog alarm failed:', error) - await this.ctx.storage.setAlarm(Date.now() + WATCHDOG_MS) + await this.armWatchdog() } } - /** Mark a stalled (orchestrator-presumed-dead) run as a terminal error. */ - private async failStalledRun(runId: string): Promise { - const message = 'run watchdog: no progress; orchestrator presumed dead' - try { - await this.log.append(runId, { type: EventType.RUN_ERROR, message }) - } catch { - // The run may have just reached terminal concurrently; finish is idempotent. + /** Arm without allowing a new run to postpone an earlier pending check. */ + private async armWatchdog(): Promise { + if (this.stallTimeoutMs === false) return + const next = Date.now() + Math.min(WATCHDOG_MS, this.stallTimeoutMs) + const pending = await this.ctx.storage.getAlarm() + if (pending === null || pending > next) { + await this.ctx.storage.setAlarm(next) } - await this.log.finish(runId, 'failed', { message }) - this.onRunSettled(runId) + } + + /** Mark a strictly stale run as a terminal error if it still qualifies. */ + private failStalledRun(runId: string, cutoff: number): Promise { + const message = 'run watchdog: no progress; orchestrator presumed dead' + return this.log.finishIfStale(runId, cutoff, { + type: EventType.RUN_ERROR, + message, + }) } } diff --git a/packages/ai-sandbox-cloudflare/src/factory.ts b/packages/ai-sandbox-cloudflare/src/factory.ts index 443f633fe..8b934bbbf 100644 --- a/packages/ai-sandbox-cloudflare/src/factory.ts +++ b/packages/ai-sandbox-cloudflare/src/factory.ts @@ -45,7 +45,7 @@ import { cloudflareSandbox } from './provider' import { ChatSandboxCoordinator } from './chat-coordinator' import { ContainerSandboxCoordinator } from './container-coordinator' import { createSandboxAgentWorker } from './worker' -import { resolvePreviewHost } from './coordinator' +import { normalizeStallTimeoutMs, resolvePreviewHost } from './coordinator' import type { ChatCoordinatorEnv, ChatRunConfig } from './chat-coordinator' import type { ContainerCoordinatorEnv, @@ -83,6 +83,11 @@ export interface SandboxAgentEnv interface BaseAgentConfig { /** chat()-provided server tools, resolved per run (DO-drives: bridged over MCP). */ tools?: (input: StartRunInput, env: TEnv) => Array + /** + * Fail a run after this many milliseconds without persisted activity. Omitted + * defaults to five minutes; `false` disables the watchdog. + */ + stallTimeoutMs?: number | false } /** DO-drives config: the DO runs `chat()` with the given adapter. */ @@ -188,11 +193,16 @@ function resolveCoordinator( export function createCloudflareSandboxAgent< TEnv extends SandboxAgentEnv = SandboxAgentEnv, >(config: CloudflareSandboxAgentConfig): CloudflareSandboxAgent { + const stallTimeoutMs = normalizeStallTimeoutMs(config.stallTimeoutMs) const worker = createSandboxAgentWorker(resolveCoordinator) if (config.mode === 'colocated') { const colocated = config class ConfiguredContainerCoordinator extends ContainerSandboxCoordinator { + constructor(ctx: DurableObjectState, env: TEnv) { + super(ctx, env, stallTimeoutMs) + } + protected override config(input: StartRunInput): ContainerRunConfig { return { hostTools: colocated.tools?.(input, this.env) ?? [], @@ -207,6 +217,10 @@ export function createCloudflareSandboxAgent< const doDrives = config class ConfiguredChatCoordinator extends ChatSandboxCoordinator { + constructor(ctx: DurableObjectState, env: TEnv) { + super(ctx, env, stallTimeoutMs) + } + protected override config(input: StartRunInput): ChatRunConfig { const tools = doDrives.tools?.(input, this.env) return { diff --git a/packages/ai-sandbox-cloudflare/src/run-log-do.ts b/packages/ai-sandbox-cloudflare/src/run-log-do.ts index 9151317b3..758d1c305 100644 --- a/packages/ai-sandbox-cloudflare/src/run-log-do.ts +++ b/packages/ai-sandbox-cloudflare/src/run-log-do.ts @@ -149,15 +149,79 @@ export class DurableObjectRunEventLog implements RunEventLog { this.wake(runId) } + async touch(runId: string): Promise { + await this.storage.transaction(async (txn) => { + const stored = await txn.get(recKey(runId)) + if (!stored) return + const { record, migrated } = migrateStoredRunRecord(stored) + if (isTerminalRunStatus(record.status)) { + if (migrated) await txn.put(recKey(runId), record) + return + } + await txn.put(recKey(runId), { ...record, updatedAt: Date.now() }) + }) + // Activity without a new event or status transition gives a tailing reader + // nothing to observe, so intentionally do not wake it. + } + + async finishIfStale( + runId: string, + cutoff: number, + chunk: Extract, + ): Promise { + const finished = await this.storage.transaction(async (txn) => { + const stored = await txn.get(recKey(runId)) + if (!stored) return false + const { record, migrated } = migrateStoredRunRecord(stored) + if (isTerminalRunStatus(record.status) || record.updatedAt >= cutoff) { + if (migrated) await txn.put(recKey(runId), record) + return false + } + + const now = Date.now() + const seq = record.lastSeq + 1 + const next: RunLogRecord = { + ...record, + status: 'failed', + lastSeq: seq, + error: { + message: chunk.message, + ...(chunk.code !== undefined ? { code: chunk.code } : {}), + }, + finishedAt: now, + updatedAt: now, + } + // Read the current activity clock and commit the terminal event + record + // in this one transaction. Keep wake-ups outside: transaction callbacks + // may be retried and must contain no external work. + await txn.put(evtKey(runId, seq), chunk) + await txn.put(recKey(runId), next) + return true + }) + if (finished) this.wake(runId) + return finished + } + async update(runId: string, patch: RunRecordPatch): Promise { - const record = await this.getRecord(runId) - if (!record) return // unknown runId is a no-op - const next: RunLogRecord = { ...record, ...patch, updatedAt: Date.now() } - await this.storage.put(recKey(runId), next) + const updated = await this.storage.transaction(async (txn) => { + const stored = await txn.get(recKey(runId)) + if (!stored) return false + const { record, migrated } = migrateStoredRunRecord(stored) + if (isTerminalRunStatus(record.status)) { + if (migrated) await txn.put(recKey(runId), record) + return false + } + await txn.put(recKey(runId), { + ...record, + ...patch, + updatedAt: Date.now(), + }) + return true + }) // A patch may terminalize the shared status field (core's driver writes its - // terminal status through `RunStore.update`) — parked readers must see it - // now, not a TAIL_POLL_MS later. - this.wake(runId) + // terminal status through `RunStore.update`) — wake only after that update + // commits. A late update racing a terminal writer is an atomic no-op. + if (updated) this.wake(runId) } async get(runId: string): Promise { diff --git a/packages/ai-sandbox-cloudflare/src/run-log.ts b/packages/ai-sandbox-cloudflare/src/run-log.ts index a326c6c90..9447deefc 100644 --- a/packages/ai-sandbox-cloudflare/src/run-log.ts +++ b/packages/ai-sandbox-cloudflare/src/run-log.ts @@ -97,7 +97,8 @@ export interface RunEventLogReadOptions { * - `append` assigns the next `seq` (0, 1, 2, …) and returns it. * - `read` yields the backlog after `fromSeq` in order, then live-tails new * events, and RETURNS once the run is terminal and the cursor has caught up. - * - All methods reject for an unknown `runId` except `get`, which resolves null. + * - Unknown-run behavior is method-specific: `append`/`finish`/`read` reject, + * `get` resolves null, and `update` is a no-op. */ export interface RunEventLog { /** @@ -120,9 +121,9 @@ export interface RunEventLog { error?: RunError, ) => Promise /** - * Patch the record's mutable fields ({@link RunRecordPatch}). Unknown `runId` - * is a NO-OP (never a throw, never a create) — core's `RunStore.update` - * invariant, which `runLogStore` maps onto this method. + * Patch the record's mutable fields ({@link RunRecordPatch}). Unknown and + * already-terminal runs are NO-OPs (never a throw, never a create), preserving + * the first terminal status against late driver updates. * * MUST wake blocked readers, exactly like `append`/`finish`: the record and * the event log share one status field here, so a driver that terminalizes @@ -299,9 +300,49 @@ export class InMemoryRunEventLog implements RunEventLog { return Promise.resolve() } + touch(runId: string): Promise { + const state = this.runs.get(runId) + if (!state || isTerminalRunStatus(state.record.status)) { + return Promise.resolve() + } + state.record.updatedAt = this.now() + return Promise.resolve() + } + + finishIfStale( + runId: string, + cutoff: number, + chunk: Extract, + ): Promise { + const state = this.runs.get(runId) + if ( + !state || + isTerminalRunStatus(state.record.status) || + state.record.updatedAt >= cutoff + ) { + return Promise.resolve(false) + } + + const now = this.now() + const seq = state.record.lastSeq + 1 + state.chunks.push(chunk) + state.record.lastSeq = seq + state.record.status = 'failed' + state.record.error = { + message: chunk.message, + ...(chunk.code !== undefined ? { code: chunk.code } : {}), + } + state.record.finishedAt = now + state.record.updatedAt = now + this.wake(state) + return Promise.resolve(true) + } + update(runId: string, patch: RunRecordPatch): Promise { const state = this.runs.get(runId) - if (!state) return Promise.resolve() // unknown runId is a no-op + if (!state || isTerminalRunStatus(state.record.status)) { + return Promise.resolve() + } state.record = { ...state.record, ...patch, updatedAt: this.now() } // A patch may terminalize the shared status field (core's driver writes its // terminal status through `RunStore.update`) — parked readers must see it. diff --git a/packages/ai-sandbox-cloudflare/tests/cloudflare-workers.ts b/packages/ai-sandbox-cloudflare/tests/cloudflare-workers.ts new file mode 100644 index 000000000..f24b774ac --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/cloudflare-workers.ts @@ -0,0 +1,9 @@ +export class DurableObject { + protected readonly ctx: DurableObjectState + protected readonly env: TEnv + + constructor(ctx: DurableObjectState, env: TEnv) { + this.ctx = ctx + this.env = env + } +} diff --git a/packages/ai-sandbox-cloudflare/tests/coordinator-callbacks.test.ts b/packages/ai-sandbox-cloudflare/tests/coordinator-callbacks.test.ts new file mode 100644 index 000000000..c9fe98fee --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/coordinator-callbacks.test.ts @@ -0,0 +1,240 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { defineWorkspace } from '@tanstack/ai-sandbox' +import { ChatSandboxCoordinator } from '../src/chat-coordinator' +import { ContainerSandboxCoordinator } from '../src/container-coordinator' +import { + hasInFlightCallback, + runWithCallbackActivity, +} from '../src/coordinator-callbacks' +import { deferred, fakeCoordinatorState, seedRunningRecord } from './fixtures' +import type { AnyTextAdapter } from '@tanstack/ai' +import type { + SandboxDefinition, + ToolBridgeProvisioner, +} from '@tanstack/ai-sandbox' + +const mocks = vi.hoisted(() => ({ + operation: vi.fn(), + provisioned: undefined as Promise<{ token: string }> | undefined, +})) +vi.mock('@cloudflare/sandbox', () => ({ getSandbox: vi.fn() })) +type MockChatOptions = { middleware: [{ setup(context: never): void }] } +vi.mock('@tanstack/ai', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + chat: vi.fn((options: MockChatOptions) => { + let provisioner: ToolBridgeProvisioner | undefined + options.middleware[0].setup({ + provide: (_capability: unknown, value: ToolBridgeProvisioner) => + (provisioner = value), + } as never) + if (!provisioner) throw new Error('bridge provisioner was not provided') + mocks.provisioned = provisioner.provision([], { provider: 'test' }) + return (async function* () {})() + }), + } +}) +vi.mock('@tanstack/ai-sandbox', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + createToolBridgeCore: () => ({}), + handleBridgeJsonRpc: (...args: Array) => mocks.operation(...args), + executeHostTool: (...args: Array) => mocks.operation(...args), + withSandbox: () => ({ name: 'test-sandbox' }), + } +}) +const workspace = defineWorkspace({ source: { type: 'none' } }) +const input = { runId: 'run-1', threadId: 'thread-1', messages: [] } +const uuid = '00000000-0000-4000-8000-000000000000' +const containerToken = `${uuid}${uuid.replaceAll('-', '')}` +type Mode = 'do-drives' | 'colocated' +class TestChatCoordinator extends ChatSandboxCoordinator { + constructor(state: DurableObjectState) { + super(state, { PUBLIC_HOSTNAME: 'example.com' }, 100) + } + async activate(): Promise { + this.buildRunStream(input) + const provisioned = await mocks.provisioned + if (!provisioned) throw new Error('bridge was not provisioned') + return provisioned.token + } + protected config() { + return { adapter: {} as AnyTextAdapter, sandbox: {} as SandboxDefinition } + } +} +class TestContainerCoordinator extends ContainerSandboxCoordinator { + constructor(state: DurableObjectState) { + super(state, { Sandbox: {} as never, PUBLIC_HOSTNAME: 'example.com' }, 100) + } + activate(): Promise { + this.buildRunStream(input) + return Promise.resolve(containerToken) + } + protected config() { + return { + hostTools: [], + workspace, + harness: 'claude-code' as const, + model: 'test-model', + } + } +} +async function setup(mode: Mode) { + const { state, storage } = fakeCoordinatorState() + seedRunningRecord(storage, { updatedAt: 100 }) + const coordinator = + mode === 'do-drives' + ? new TestChatCoordinator(state) + : new TestContainerCoordinator(state) + return { coordinator, token: await coordinator.activate() } +} +function callbackRequest(mode: Mode, token: string, runId = input.runId) { + const path = mode === 'do-drives' ? '_bridge' : 'tool-exec' + return new Request(`https://example.com/${path}/${runId}`, { + method: 'POST', + headers: { authorization: `Bearer ${token}` }, + body: JSON.stringify( + mode === 'do-drives' + ? { jsonrpc: '2.0', id: 1, method: 'tools/list' } + : { name: 'test-tool', args: {} }, + ), + }) +} +afterEach(() => { + mocks.operation.mockReset() + mocks.provisioned = undefined + vi.restoreAllMocks() +}) +describe('runWithCallbackActivity', () => { + it('reference-counts overlaps while isolating runs and owners', async () => { + const owner = {} + const otherOwner = {} + const first = deferred() + const last = deferred() + const isolated = deferred() + const allEntered = deferred() + const activity: Array = [] + let entered = 0 + const start = ( + callbackOwner: object, + runId: string, + gate: { promise: Promise }, + label?: string, + ) => + runWithCallbackActivity( + callbackOwner, + runId, + async () => { + if (label) activity.push(`${label}:touch`) + }, + async () => { + if (label) activity.push(`${label}:operation`) + entered += 1 + if (entered === 4) allEntered.resolve() + return gate.promise + }, + ) + const firstPending = start(owner, 'run-1', first, 'first') + const lastPending = start(owner, 'run-1', last, 'last') + const isolatedPending = [ + start(owner, 'run-2', isolated), + start(otherOwner, 'run-1', isolated), + ] + await allEntered.promise + const initial = 'first:touch,last:touch,first:operation,last:operation' + expect(activity.join()).toBe(initial) + expect(hasInFlightCallback(owner, 'run-1')).toBe(true) + expect(hasInFlightCallback(owner, 'run-2')).toBe(true) + expect(hasInFlightCallback(otherOwner, 'run-1')).toBe(true) + first.resolve() + await firstPending + expect(activity.join()).toBe(`${initial},first:touch`) + expect(hasInFlightCallback(owner, 'run-1')).toBe(true) + isolated.resolve() + await Promise.all(isolatedPending) + expect(hasInFlightCallback(owner, 'run-2')).toBe(false) + expect(hasInFlightCallback(otherOwner, 'run-1')).toBe(false) + expect(hasInFlightCallback(owner, 'run-1')).toBe(true) + last.resolve() + await lastPending + expect(activity.join()).toBe(`${initial},first:touch,last:touch`) + expect(hasInFlightCallback(owner, 'run-1')).toBe(false) + }) + it('propagates an arrival-touch failure without entering the operation', async () => { + const owner = {} + const activity: Array = [] + await expect( + runWithCallbackActivity( + owner, + 'run-1', + () => Promise.reject(new Error('arrival failed')), + async () => activity.push('operation'), + ), + ).rejects.toThrow('arrival failed') + expect(activity).toEqual([]) + expect(hasInFlightCallback(owner, 'run-1')).toBe(false) + }) + it('preserves resolve and reject outcomes when completion touch fails', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const owner = {} + const original = new Error('operation failed') + for (const outcome of ['resolve', 'reject'] as const) { + const activity: Array = [] + let touches = 0 + const pending = runWithCallbackActivity( + owner, + 'run-1', + async () => { + activity.push('touch') + touches += 1 + if (touches === 2) throw new Error('completion failed') + }, + async () => { + activity.push('operation') + if (outcome === 'reject') throw original + return 'ok' + }, + ) + if (outcome === 'resolve') await expect(pending).resolves.toBe('ok') + else await expect(pending).rejects.toBe(original) + expect(activity).toEqual(['touch', 'operation', 'touch']) + expect(hasInFlightCallback(owner, 'run-1')).toBe(false) + } + }) +}) +describe.each([ + ['do-drives', { jsonrpc: '2.0', id: 1, result: 'ok' }], + ['colocated', 'ok'], +] as const)('%s callback endpoint', (mode, operationResult) => { + it('authenticates callbacks and records arrival and completion activity', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(100) + vi.spyOn(crypto, 'randomUUID').mockReturnValue(uuid) + const { coordinator, token } = await setup(mode) + now.mockReturnValue(200) + const [unknown, unauthorized] = await Promise.all([ + coordinator.fetch(callbackRequest(mode, token, 'unknown')), + coordinator.fetch(callbackRequest(mode, 'wrong-token')), + ]) + expect([unknown.status, unauthorized.status]).toEqual([404, 401]) + expect((await coordinator.status(input.runId))?.updatedAt).toBe(100) + const entered = deferred() + const operation = deferred() + mocks.operation.mockImplementation(() => { + entered.resolve() + return operation.promise + }) + const responsePending = coordinator.fetch(callbackRequest(mode, token)) + await entered.promise + expect((await coordinator.status(input.runId))?.updatedAt).toBe(200) + now.mockReturnValue(300) + operation.resolve(operationResult) + const response = await responsePending + expect(response.status).toBe(200) + expect(await response.json()).toEqual( + mode === 'do-drives' ? operationResult : { result: operationResult }, + ) + expect((await coordinator.status(input.runId))?.updatedAt).toBe(300) + }) +}) diff --git a/packages/ai-sandbox-cloudflare/tests/coordinator-watchdog.test.ts b/packages/ai-sandbox-cloudflare/tests/coordinator-watchdog.test.ts new file mode 100644 index 000000000..28a410fca --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/coordinator-watchdog.test.ts @@ -0,0 +1,105 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { SandboxCoordinator } from '../src/coordinator' +import { runWithCallbackActivity } from '../src/coordinator-callbacks' +import { deferred, fakeCoordinatorState } from './fixtures' +import type { StartRunInput } from '../src/coordinator' +import type { StreamChunk } from '@tanstack/ai' + +class TestCoordinator extends SandboxCoordinator> { + readonly settledRuns = new Set() + + constructor(state: DurableObjectState, stallTimeoutMs?: number | false) { + super(state, {}, stallTimeoutMs) + } + open(input: { runId: string; threadId: string }): Promise { + return this.log.open(input) + } + touch(runId: string): Promise { + return this.log.touch(runId) + } + protected buildRunStream(_input: StartRunInput): AsyncIterable { + return (async function* () {})() + } + protected override onRunSettled(runId: string): void { + this.settledRuns.add(runId) + } +} + +describe('SandboxCoordinator watchdog', () => { + afterEach(() => vi.restoreAllMocks()) + + it('enforces the strict lifecycle around callback activity', async () => { + const startedAt = 100 + const stallTimeoutMs = 300 + const now = vi.spyOn(Date, 'now').mockReturnValue(startedAt) + const fixture = fakeCoordinatorState() + const coordinator = new TestCoordinator(fixture.state, stallTimeoutMs) + await coordinator.open({ runId: 'run-1', threadId: 'thread-1' }) + + now.mockReturnValue(startedAt + stallTimeoutMs) + await fixture.invokeAlarm(coordinator) + expect((await coordinator.status('run-1'))?.status).toBe('running') + + const entered = deferred() + const operation = deferred() + const callback = runWithCallbackActivity( + coordinator, + 'run-1', + (runId) => coordinator.touch(runId), + async () => { + entered.resolve() + return operation.promise + }, + ) + await entered.promise + + const staleDuringCallback = startedAt + 2 * stallTimeoutMs + 1 + now.mockReturnValue(staleDuringCallback) + await fixture.invokeAlarm(coordinator) + expect((await coordinator.status('run-1'))?.status).toBe('running') + expect(fixture.storage.alarm).toBeGreaterThan(staleDuringCallback) + + operation.resolve() + await callback + now.mockReturnValue(staleDuringCallback + stallTimeoutMs + 1) + await fixture.invokeAlarm(coordinator) + + expect(await coordinator.status('run-1')).toMatchObject({ + status: 'failed', + error: { + message: 'run watchdog: no progress; orchestrator presumed dead', + }, + }) + expect(coordinator.settledRuns.has('run-1')).toBe(true) + expect(fixture.storage.alarm).toBeNull() + }) + + it('schedules the next check within a sub-30s stall timeout', async () => { + const startedAt = 100 + vi.spyOn(Date, 'now').mockReturnValue(startedAt) + const fixture = fakeCoordinatorState() + const coordinator = new TestCoordinator(fixture.state, 1_000) + await coordinator.open({ runId: 'run-1', threadId: 'thread-1' }) + + await fixture.invokeAlarm(coordinator) + + expect(fixture.storage.alarm).toBe(startedAt + 1_000) + }) + + it('fails closed when arming cannot persist an alarm', async () => { + const fixture = fakeCoordinatorState() + const coordinator = new TestCoordinator(fixture.state) + vi.spyOn(fixture.storage, 'setAlarm').mockRejectedValueOnce( + new Error('setAlarm failed'), + ) + + await expect( + coordinator.startRun({ + runId: 'run-1', + threadId: 'thread-1', + messages: [], + }), + ).rejects.toThrow('setAlarm failed') + expect(await coordinator.status('run-1')).toBeNull() + }) +}) diff --git a/packages/ai-sandbox-cloudflare/tests/factory-watchdog.test.ts b/packages/ai-sandbox-cloudflare/tests/factory-watchdog.test.ts new file mode 100644 index 000000000..41349f293 --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/factory-watchdog.test.ts @@ -0,0 +1,108 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { defineWorkspace } from '@tanstack/ai-sandbox' +import { createCloudflareSandboxAgent } from '../src/factory' +import { fakeCoordinatorState, seedRunningRecord } from './fixtures' +import type { AnyTextAdapter } from '@tanstack/ai' +import type { + CloudflareSandboxAgentConfig, + SandboxAgentEnv, +} from '../src/factory' + +vi.mock('@cloudflare/sandbox', () => ({ + Sandbox: class {}, + getSandbox: vi.fn(), +})) + +const workspace = defineWorkspace({ source: { type: 'none' } }) +const adapter = {} as unknown as AnyTextAdapter + +type Mode = 'do-drives' | 'colocated' + +function configFor( + mode: Mode, + stallTimeoutMs?: number | false, +): CloudflareSandboxAgentConfig { + const policy = stallTimeoutMs === undefined ? {} : { stallTimeoutMs } + return mode === 'colocated' + ? { + mode, + harness: 'claude-code', + model: 'test-model', + workspace, + ...policy, + } + : { mode, adapter: () => adapter, ...policy } +} + +async function statusAfterAlarm( + mode: Mode, + stallTimeoutMs: number | false | undefined, + staleAge: number, +): Promise { + const now = 2_000_000 + vi.spyOn(Date, 'now').mockReturnValue(now) + const agent = createCloudflareSandboxAgent(configFor(mode, stallTimeoutMs)) + const fixture = fakeCoordinatorState() + seedRunningRecord(fixture.storage, { updatedAt: now - staleAge }) + const coordinator = new agent.Coordinator( + fixture.state, + {} as SandboxAgentEnv, + ) + fixture.storage.alarm = now - 1 + await fixture.invokeAlarm(coordinator) + expect(fixture.storage.alarm).toBeNull() + return (await coordinator.status('run-1'))?.status +} + +describe('createCloudflareSandboxAgent watchdog config', () => { + afterEach(() => vi.restoreAllMocks()) + + it.each<{ + name: string + mode: Mode + stallTimeoutMs: number | false | undefined + staleAge: number + expectedStatus: 'failed' | 'running' + }>([ + { + name: 'default do-drives', + mode: 'do-drives', + stallTimeoutMs: undefined, + staleAge: 300_001, + expectedStatus: 'failed', + }, + { + name: 'custom do-drives', + mode: 'do-drives', + stallTimeoutMs: 1_000, + staleAge: 1_001, + expectedStatus: 'failed', + }, + { + name: 'custom colocated', + mode: 'colocated', + stallTimeoutMs: 1_000, + staleAge: 1_001, + expectedStatus: 'failed', + }, + { + name: 'disabled colocated', + mode: 'colocated', + stallTimeoutMs: false, + staleAge: 1_000_000, + expectedStatus: 'running', + }, + ])('$name', async ({ mode, stallTimeoutMs, staleAge, expectedStatus }) => { + expect(await statusAfterAlarm(mode, stallTimeoutMs, staleAge)).toBe( + expectedStatus, + ) + }) + + it('rejects invalid timeout values eagerly', () => { + for (const invalid of [0, 1.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => + createCloudflareSandboxAgent(configFor('do-drives', invalid)), + ).toThrow(TypeError) + } + }) +}) diff --git a/packages/ai-sandbox-cloudflare/tests/fixtures.ts b/packages/ai-sandbox-cloudflare/tests/fixtures.ts new file mode 100644 index 000000000..aea70b91d --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/fixtures.ts @@ -0,0 +1,111 @@ +import type { RunLogRecord } from '../src/run-log' + +export class FakeDurableStorage { + readonly data = new Map() + alarm: number | null = null + private transactionTail = Promise.resolve() + + get(key: string): Promise { + return Promise.resolve(this.data.get(key) as T | undefined) + } + + put(key: string, value: unknown): Promise { + this.data.set(key, value) + return Promise.resolve() + } + + list(options?: { + prefix?: string + start?: string + }): Promise> { + let entries = [...this.data.entries()].sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ) + if (options?.prefix !== undefined) { + entries = entries.filter(([key]) => key.startsWith(options.prefix!)) + } + if (options?.start !== undefined) { + entries = entries.filter(([key]) => key >= options.start!) + } + return Promise.resolve(new Map(entries) as Map) + } + + transaction( + closure: (txn: { + get: (key: string) => Promise + put: (key: string, value: unknown) => Promise + }) => Promise, + ): Promise { + const result = this.transactionTail.then(() => + closure({ + get: (key: string) => + Promise.resolve(this.data.get(key) as V | undefined), + put: (key, value) => { + this.data.set(key, value) + return Promise.resolve() + }, + }), + ) + this.transactionTail = result.then( + () => undefined, + () => undefined, + ) + return result + } + + getAlarm(): Promise { + return Promise.resolve(this.alarm) + } + + setAlarm(timestamp: number | Date): Promise { + this.alarm = timestamp instanceof Date ? timestamp.getTime() : timestamp + return Promise.resolve() + } +} + +export function fakeDurableStorage(): FakeDurableStorage & + DurableObjectStorage { + return new FakeDurableStorage() as unknown as FakeDurableStorage & + DurableObjectStorage +} + +export function fakeCoordinatorState() { + const storage = fakeDurableStorage() + const state = { + storage, + waitUntil(_promise: Promise) {}, + acceptWebSocket() {}, + } as unknown as DurableObjectState + return { + storage, + state, + async invokeAlarm(handler: { alarm: () => Promise }) { + storage.alarm = null + await handler.alarm() + }, + } +} + +export function seedRunningRecord( + storage: FakeDurableStorage, + options: { runId?: string; updatedAt: number; threadId?: string }, +): void { + const runId = options.runId ?? 'run-1' + const record: RunLogRecord = { + runId, + threadId: options.threadId ?? 'thread-1', + status: 'running', + startedAt: options.updatedAt, + updatedAt: options.updatedAt, + lastSeq: -1, + } + storage.data.set(`rec:${runId}`, record) +} + +export function deferred() { + let resolve = (_value: T): void => {} + const promise = new Promise((res) => { + resolve = res + }) + return { promise, resolve } +} diff --git a/packages/ai-sandbox-cloudflare/tests/run-log-do.test.ts b/packages/ai-sandbox-cloudflare/tests/run-log-do.test.ts index 488720086..50c75d55f 100644 --- a/packages/ai-sandbox-cloudflare/tests/run-log-do.test.ts +++ b/packages/ai-sandbox-cloudflare/tests/run-log-do.test.ts @@ -1,58 +1,9 @@ -/** - * Behavioral tests for {@link DurableObjectRunEventLog} against a Map-backed - * `DurableObjectStorage` stub (no Workers runtime). Re-runs the core run-log - * contract — gap-free seq, replay-then-tail, fromSeq resume, terminal rejection, - * unknown-runId handling — plus the durable-specific eviction/re-poll path. - */ import { describe, expect, it } from 'vitest' import { DurableObjectRunEventLog } from '../src/run-log-do' +import { fakeDurableStorage } from './fixtures' import type { StreamChunk } from '@tanstack/ai' -/** A minimal in-memory `DurableObjectStorage`: a sorted-key Map. */ -function fakeStorage(): DurableObjectStorage { - const map = new Map() - const sortedEntries = (): Array<[string, unknown]> => - [...map.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - const storage = { - get(key: string): Promise { - return Promise.resolve(map.get(key) as T | undefined) - }, - put(key: string, value: unknown): Promise { - map.set(key, value) - return Promise.resolve() - }, - delete(key: string): Promise { - return Promise.resolve(map.delete(key)) - }, - list(options?: { - prefix?: string - start?: string - }): Promise> { - const out = new Map() - for (const [key, value] of sortedEntries()) { - if (options?.prefix !== undefined && !key.startsWith(options.prefix)) { - continue - } - if (options?.start !== undefined && key < options.start) continue - out.set(key, value as T) - } - return Promise.resolve(out) - }, - async transaction( - closure: (txn: { - put: (k: string, v: unknown) => Promise - }) => Promise, - ): Promise { - return closure({ - put: (k, v) => { - map.set(k, v) - return Promise.resolve() - }, - }) - }, - } - return storage as unknown as DurableObjectStorage -} +const fakeStorage = fakeDurableStorage const chunk = (n: number): StreamChunk => ({ type: 'TEXT', text: `c${n}` }) as unknown as StreamChunk @@ -81,22 +32,9 @@ describe('DurableObjectRunEventLog', () => { for await (const event of log.read('r1', { fromSeq: 0 })) { seen.push(event.seq) } - // fromSeq is EXCLUSIVE: seq 0 is skipped, 1 and 2 replayed. expect(seen).toEqual([1, 2]) }) - it('replays from the start when fromSeq is omitted', async () => { - const log = new DurableObjectRunEventLog(fakeStorage()) - await log.open({ runId: 'r1', threadId: 't1' }) - await log.append('r1', chunk(0)) - await log.append('r1', chunk(1)) - await log.finish('r1', 'completed') - - const seen: Array = [] - for await (const event of log.read('r1')) seen.push(event.seq) - expect(seen).toEqual([0, 1]) - }) - it('live-tails: a reader that joins mid-run sees backlog + new events', async () => { const log = new DurableObjectRunEventLog(fakeStorage()) await log.open({ runId: 'r1', threadId: 't1' }) @@ -106,7 +44,6 @@ describe('DurableObjectRunEventLog', () => { const reading = (async () => { for await (const event of log.read('r1')) seen.push(event.seq) })() - // Append more, then finish, after the reader is tailing. await log.append('r1', chunk(1)) await log.append('r1', chunk(2)) await log.finish('r1', 'completed') @@ -114,14 +51,7 @@ describe('DurableObjectRunEventLog', () => { expect(seen).toEqual([0, 1, 2]) }) - it('rejects append after terminal', async () => { - const log = new DurableObjectRunEventLog(fakeStorage()) - await log.open({ runId: 'r1', threadId: 't1' }) - await log.finish('r1', 'completed') - await expect(log.append('r1', chunk(0))).rejects.toThrow(/terminal/) - }) - - it('finish is idempotent and keeps the first terminal status', async () => { + it('finish is idempotent, immutable, and rejects later appends', async () => { const log = new DurableObjectRunEventLog(fakeStorage()) await log.open({ runId: 'r1', threadId: 't1' }) await log.finish('r1', 'failed', { message: 'boom' }) @@ -129,13 +59,13 @@ describe('DurableObjectRunEventLog', () => { const record = await log.get('r1') expect(record?.status).toBe('failed') expect(record?.error?.message).toBe('boom') + await expect(log.append('r1', chunk(0))).rejects.toThrow(/terminal/) }) it('open is idempotent', async () => { const log = new DurableObjectRunEventLog(fakeStorage()) const a = await log.open({ runId: 'r1', threadId: 't1' }) await log.append('r1', chunk(0)) - // The second call's threadId is ignored: the existing record wins. const b = await log.open({ runId: 'r1', threadId: 't2' }) expect(b.lastSeq).toBe(a.lastSeq + 1) expect(b.threadId).toBe('t1') @@ -152,9 +82,6 @@ describe('DurableObjectRunEventLog', () => { it('a reader whose in-memory waiter was lost still progresses (eviction poll)', async () => { const storage = fakeStorage() - // Two independent log instances over the SAME storage simulate eviction: the - // writer's appends never wake the reader's waiter set, so the reader can only - // make progress via the TAIL_POLL_MS fallback re-read. const reader = new DurableObjectRunEventLog(storage) const writer = new DurableObjectRunEventLog(storage) await writer.open({ runId: 'r1', threadId: 't1' }) @@ -172,8 +99,6 @@ describe('DurableObjectRunEventLog', () => { it('migrates a legacy terminal record on read and writes it back', async () => { const storage = fakeStorage() - // A record persisted by the pre-convergence layout: legacy status - // vocabulary, createdAt/updatedAt, no threadId. await storage.put('rec:legacy', { runId: 'legacy', status: 'done', @@ -190,18 +115,14 @@ describe('DurableObjectRunEventLog', () => { expect(record?.startedAt).toBe(100) expect(record?.finishedAt).toBe(200) expect(record?.updatedAt).toBe(200) - // No thread was stored — the self-referential backfill, never a fake one. expect(record?.threadId).toBe('legacy') - // Write-back: the stored value is now the converged layout, so the - // conversion is paid exactly once. const stored = await storage.get<{ status: string; startedAt?: number }>( 'rec:legacy', ) expect(stored?.status).toBe('completed') expect(stored?.startedAt).toBe(100) - // The migrated run replays like any other. const seen: Array = [] for await (const event of log.read('legacy')) seen.push(event.seq) expect(seen).toEqual([0]) @@ -227,17 +148,14 @@ describe('DurableObjectRunEventLog', () => { }) const log = new DurableObjectRunEventLog(storage) - // `error` maps to `failed` and stays terminal: appends still reject. await expect(log.append('legacy', chunk(9))).rejects.toThrow(/terminal/) expect((await log.get('legacy'))?.status).toBe('failed') - // A running record migrates without gaining finishedAt and keeps its cursor. const live = await log.get('live') expect(live?.status).toBe('running') expect(live?.finishedAt).toBeUndefined() expect(await log.append('live', chunk(3))).toBe(3) - // `list` (the watchdog's view) also observes only the converged layout. const statuses = (await log.list()).map((r) => r.status).sort() expect(statuses).toEqual(['failed', 'running']) }) diff --git a/packages/ai-sandbox-cloudflare/tests/run-log-watchdog.test.ts b/packages/ai-sandbox-cloudflare/tests/run-log-watchdog.test.ts new file mode 100644 index 000000000..fb80ebba6 --- /dev/null +++ b/packages/ai-sandbox-cloudflare/tests/run-log-watchdog.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { EventType } from '@tanstack/ai' +import { DurableObjectRunEventLog } from '../src/run-log-do' +import { InMemoryRunEventLog } from '../src/run-log' +import { fakeDurableStorage } from './fixtures' +import type { RunEventLog } from '../src/run-log' +import type { StreamChunk } from '@tanstack/ai' + +type AssertRunEventLog = T +type ThirdPartyRunEventLog = AssertRunEventLog< + Omit +> +type WatchdogLog = ThirdPartyRunEventLog & + Pick + +const implementations: Array<[string, () => WatchdogLog]> = [ + ['in-memory', () => new InMemoryRunEventLog()], + ['durable', () => new DurableObjectRunEventLog(fakeDurableStorage())], +] + +const errorChunk = ( + message: string, + code?: string, +): Extract => ({ + type: EventType.RUN_ERROR, + message, + ...(code === undefined ? {} : { code }), +}) + +async function collect(log: WatchdogLog) { + const events = [] + for await (const event of log.read('run-1')) events.push(event) + return events +} + +describe('watchdog run-log contract', () => { + afterEach(() => vi.restoreAllMocks()) + + describe.each(implementations)('%s', (_name, createLog) => { + it('enforces touch and strict stale-finish semantics', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(100) + const log = createLog() + const errors = ['first', 'second'].map((message, index) => + errorChunk(message, String(index)), + ) + + await log.open({ runId: 'run-1', threadId: 'thread-1' }) + await log.touch('unknown') + const before = await log.get('run-1') + if (!before) throw new Error('expected run') + + now.mockReturnValue(200) + await log.touch('run-1') + expect(await log.get('run-1')).toEqual({ ...before, updatedAt: 200 }) + expect(await log.finishIfStale('run-1', 200, errors[0]!)).toBe(false) + + now.mockReturnValue(300) + const winners = await Promise.all( + errors.map((error) => log.finishIfStale('run-1', 201, error)), + ) + expect(winners.filter(Boolean)).toHaveLength(1) + + const events = await collect(log) + expect(events).toHaveLength(1) + const [event] = events + expect(event?.seq).toBe(0) + expect(event?.chunk.type).toBe(EventType.RUN_ERROR) + if (event?.chunk.type !== EventType.RUN_ERROR) { + throw new Error('expected terminal error') + } + const terminal = await log.get('run-1') + expect(terminal).toMatchObject({ + status: 'failed', + lastSeq: 0, + error: { message: event.chunk.message, code: event.chunk.code }, + finishedAt: 300, + updatedAt: 300, + }) + + now.mockReturnValue(400) + await log.touch('run-1') + expect(await log.get('run-1')).toEqual(terminal) + await log.update('run-1', { status: 'completed', finishedAt: 400 }) + expect(await log.get('run-1')).toEqual(terminal) + }) + }) +}) diff --git a/packages/ai-sandbox-cloudflare/tests/run-log.test.ts b/packages/ai-sandbox-cloudflare/tests/run-log.test.ts index e3d840166..a73ed5ffe 100644 --- a/packages/ai-sandbox-cloudflare/tests/run-log.test.ts +++ b/packages/ai-sandbox-cloudflare/tests/run-log.test.ts @@ -48,61 +48,34 @@ describe('InMemoryRunEventLog', () => { expect(events.map((e) => e.seq)).toEqual([2, 3]) }) - it('live-tails: a blocked reader wakes on append and on finish', async () => { + it('replays backlog, then blocks for append and finish', async () => { const log = new InMemoryRunEventLog() await log.open({ runId: 'r1', threadId: 't1' }) - - const seen: Array = [] - const reader = (async () => { - for await (const e of log.read('r1')) seen.push(e.seq) - })() - - // Reader is blocked (no events yet). Append over a few microtask turns. await log.append('r1', chunk('a')) - await new Promise((r) => setTimeout(r, 0)) - await log.append('r1', chunk('b')) - await new Promise((r) => setTimeout(r, 0)) - await log.finish('r1', 'completed') + const reader = log.read('r1')[Symbol.asyncIterator]() - await reader - expect(seen).toEqual([0, 1]) - }) - - it('a reader that joins mid-run gets backlog + live tail, resumably', async () => { - const log = new InMemoryRunEventLog() - await log.open({ runId: 'r1', threadId: 't1' }) - await log.append('r1', chunk('a')) // seq 0 — before the reader joins - - const seen: Array = [] - const reader = (async () => { - for await (const e of log.read('r1', { fromSeq: -1 })) seen.push(e.seq) - })() - - await new Promise((r) => setTimeout(r, 0)) - await log.append('r1', chunk('b')) // seq 1 — live + const backlog = await reader.next() + expect(backlog.value?.seq).toBe(0) + const next = reader.next() + await log.append('r1', chunk('b')) + const appended = await next + expect(appended.value?.seq).toBe(1) + const done = reader.next() await log.finish('r1', 'completed') - await reader - - expect(seen).toEqual([0, 1]) + expect((await done).done).toBe(true) }) it('stops tailing when the read signal aborts (client disconnect)', async () => { const log = new InMemoryRunEventLog() await log.open({ runId: 'r1', threadId: 't1' }) await log.append('r1', chunk('a')) - const ac = new AbortController() - const seen: Array = [] - const reader = (async () => { - for await (const e of log.read('r1', { signal: ac.signal })) { - seen.push(e.seq) - } - })() + const reader = log.read('r1', { signal: ac.signal })[Symbol.asyncIterator]() - await new Promise((r) => setTimeout(r, 0)) - ac.abort() // run never finishes; reader must still return - await reader - expect(seen).toEqual([0]) + expect((await reader.next()).value?.seq).toBe(0) + const done = reader.next() + ac.abort() + expect((await done).done).toBe(true) }) it('open is idempotent and rejects appends after terminal', async () => { diff --git a/packages/ai-sandbox-cloudflare/vite.config.ts b/packages/ai-sandbox-cloudflare/vite.config.ts index ae6bfdf2f..b45f89b70 100644 --- a/packages/ai-sandbox-cloudflare/vite.config.ts +++ b/packages/ai-sandbox-cloudflare/vite.config.ts @@ -1,3 +1,4 @@ +import { fileURLToPath } from 'node:url' import { defineConfig, mergeConfig } from 'vitest/config' import { tanstackViteConfig } from '@tanstack/vite-config' import packageJson from './package.json' @@ -11,6 +12,11 @@ const config = defineConfig({ globals: true, environment: 'node', include: ['tests/**/*.test.ts'], + alias: { + 'cloudflare:workers': fileURLToPath( + new URL('./tests/cloudflare-workers.ts', import.meta.url), + ), + }, coverage: { provider: 'v8', reporter: ['text', 'json', 'html', 'lcov'], diff --git a/scripts/lovable-gateway.models.json b/scripts/lovable-gateway.models.json index 8a193376e..79c6babbc 100644 --- a/scripts/lovable-gateway.models.json +++ b/scripts/lovable-gateway.models.json @@ -12,15 +12,8 @@ "max_tokens": 65536, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -85,15 +78,8 @@ "max_tokens": 65536, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "video" - ], - "output": [ - "text", - "image" - ] + "input": ["text", "image", "video"], + "output": ["text", "image"] }, "pricing": { "input": { @@ -150,15 +136,8 @@ "max_tokens": 65536, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -222,12 +201,8 @@ "context_window": 8192, "max_tokens": 16384, "modalities": { - "input": [ - "text" - ], - "output": [ - "audio" - ] + "input": ["text"], + "output": ["audio"] }, "pricing": { "input": { @@ -265,12 +240,8 @@ "max_tokens": 16384, "knowledge": "2025-01", "modalities": { - "input": [ - "text" - ], - "output": [ - "audio" - ] + "input": ["text"], + "output": ["audio"] }, "pricing": { "input": { @@ -308,15 +279,8 @@ "max_tokens": 65536, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -406,12 +370,8 @@ "max_tokens": 16384, "knowledge": "2025-01", "modalities": { - "input": [ - "text" - ], - "output": [ - "audio" - ] + "input": ["text"], + "output": ["audio"] }, "pricing": { "input": { @@ -449,15 +409,8 @@ "max_tokens": 65000, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -522,15 +475,8 @@ "max_tokens": 32768, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "video" - ], - "output": [ - "text", - "image" - ] + "input": ["text", "image", "video"], + "output": ["text", "image"] }, "pricing": { "input": { @@ -595,15 +541,8 @@ "max_tokens": 32768, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "video" - ], - "output": [ - "text", - "image" - ] + "input": ["text", "image", "video"], + "output": ["text", "image"] }, "pricing": { "input": { @@ -668,15 +607,8 @@ "max_tokens": 65000, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -740,15 +672,8 @@ "context_window": 65536, "max_tokens": 4096, "modalities": { - "input": [ - "text", - "image", - "video" - ], - "output": [ - "text", - "image" - ] + "input": ["text", "image", "video"], + "output": ["text", "image"] }, "pricing": { "input": { @@ -813,12 +738,8 @@ "max_tokens": 16384, "knowledge": "2025-01", "modalities": { - "input": [ - "text" - ], - "output": [ - "audio" - ] + "input": ["text"], + "output": ["audio"] }, "pricing": { "input": { @@ -856,15 +777,8 @@ "max_tokens": 64000, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -954,15 +868,8 @@ "max_tokens": 64000, "knowledge": "2025-01", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -1027,15 +934,8 @@ "max_tokens": 64000, "knowledge": "2026-03", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -1100,12 +1000,8 @@ "max_tokens": 0, "knowledge": "2025-05", "modalities": { - "input": [ - "text" - ], - "output": [ - "text" - ] + "input": ["text"], + "output": ["text"] }, "pricing": { "input": { @@ -1133,15 +1029,8 @@ "max_tokens": 0, "knowledge": "2025-11", "modalities": { - "input": [ - "text", - "image", - "audio", - "video" - ], - "output": [ - "text" - ] + "input": ["text", "image", "audio", "video"], + "output": ["text"] }, "pricing": { "input": { @@ -1192,13 +1081,8 @@ "context_window": 0, "max_tokens": 0, "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "video" - ] + "input": ["text", "image"], + "output": ["video"] }, "pricing": { "video_duration": { @@ -1226,13 +1110,8 @@ "context_window": 0, "max_tokens": 0, "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "video" - ] + "input": ["text", "image"], + "output": ["video"] }, "pricing": { "video_duration": { @@ -1260,13 +1139,8 @@ "context_window": 0, "max_tokens": 0, "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "video" - ] + "input": ["text", "image"], + "output": ["video"] }, "pricing": { "video_duration": { @@ -1291,13 +1165,8 @@ "context_window": 16000, "max_tokens": 2000, "modalities": { - "input": [ - "text", - "audio" - ], - "output": [ - "text" - ] + "input": ["text", "audio"], + "output": ["text"] }, "pricing": { "input": { @@ -1342,12 +1211,8 @@ "context_window": 2000, "max_tokens": 0, "modalities": { - "input": [ - "text" - ], - "output": [ - "audio" - ] + "input": ["text"], + "output": ["audio"] }, "pricing": { "input": { @@ -1384,13 +1249,8 @@ "context_window": 16000, "max_tokens": 2000, "modalities": { - "input": [ - "text", - "audio" - ], - "output": [ - "text" - ] + "input": ["text", "audio"], + "output": ["text"] }, "pricing": { "input": { @@ -1436,13 +1296,8 @@ "max_tokens": 128000, "knowledge": "2024-09-30", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1526,13 +1381,8 @@ "max_tokens": 128000, "knowledge": "2024-05-30", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1616,13 +1466,8 @@ "max_tokens": 128000, "knowledge": "2024-05-30", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1671,13 +1516,8 @@ "max_tokens": 128000, "knowledge": "2024-10", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1761,13 +1601,8 @@ "max_tokens": 128000, "knowledge": "2025-08-31", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1866,13 +1701,8 @@ "max_tokens": 128000, "knowledge": "2025-08-31", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -1956,13 +1786,8 @@ "max_tokens": 128000, "knowledge": "2025-08-31", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2011,13 +1836,8 @@ "max_tokens": 128000, "knowledge": "2025-08-31", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2081,13 +1901,8 @@ "max_tokens": 128000, "knowledge": "2025-12-01", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2186,13 +2001,8 @@ "max_tokens": 128000, "knowledge": "2025-12-01", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2256,13 +2066,8 @@ "max_tokens": 128000, "knowledge": "2026-02-16", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2361,13 +2166,8 @@ "max_tokens": 128000, "knowledge": "2026-02-16", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2466,13 +2266,8 @@ "max_tokens": 128000, "knowledge": "2026-02-16", "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "text" - ] + "input": ["text", "image"], + "output": ["text"] }, "pricing": { "input": { @@ -2570,13 +2365,8 @@ "context_window": 0, "max_tokens": 0, "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "image" - ] + "input": ["text", "image"], + "output": ["image"] }, "pricing": { "input": { @@ -2621,13 +2411,8 @@ "context_window": 0, "max_tokens": 0, "modalities": { - "input": [ - "text", - "image" - ], - "output": [ - "image" - ] + "input": ["text", "image"], + "output": ["image"] }, "pricing": { "input": { @@ -2672,12 +2457,8 @@ "context_window": 8191, "max_tokens": 0, "modalities": { - "input": [ - "text" - ], - "output": [ - "text" - ] + "input": ["text"], + "output": ["text"] }, "pricing": { "input": { @@ -2704,12 +2485,8 @@ "context_window": 8191, "max_tokens": 0, "modalities": { - "input": [ - "text" - ], - "output": [ - "text" - ] + "input": ["text"], + "output": ["text"] }, "pricing": { "input": { diff --git a/testing/e2e/tests/workerd-cloudflare-watchdog.spec.ts b/testing/e2e/tests/workerd-cloudflare-watchdog.spec.ts new file mode 100644 index 000000000..40d365d01 --- /dev/null +++ b/testing/e2e/tests/workerd-cloudflare-watchdog.spec.ts @@ -0,0 +1,124 @@ +/** + * Exempt from the aimock policy: the harness overrides `buildRunStream` with a + * local never-resolving stream and never reaches an LLM provider's HTTP layer, + * so there is nothing to mock. + */ +import { fileURLToPath } from 'node:url' +import { expect, test } from '@playwright/test' +import { Miniflare } from 'miniflare' +import { build } from 'vite' + +const AGENT_DIST = fileURLToPath( + new URL( + '../../../packages/ai-sandbox-cloudflare/dist/esm/agent.js', + import.meta.url, + ), +) +const RUN_ID = 'watchdog-e2e' +const STALL_MS = 250 +const POLL_TIMEOUT_MS = 15_000 +const WATCHDOG_ERROR = 'run watchdog: no progress; orchestrator presumed dead' + +const harness = ` +import { SandboxCoordinator } from './agent.js' +const RUN_ID = '${RUN_ID}' +export class TestCoordinator extends SandboxCoordinator { + constructor(ctx, env) { + super(ctx, env, Number(env.STALL_TIMEOUT_MS)) + } + buildRunStream() { + return (async function* () { await new Promise(() => {}) })() + } + async handleRoute(_request, parts) { + if (parts[0] === 'start') { + const result = await this.startRun({ + runId: RUN_ID, threadId: 'thread-e2e', messages: [], + }) + return this.jsonResponse(result) + } + if (parts[0] === 'state') { + const record = await this.status(RUN_ID) + const events = [] + if (record && record.status !== 'running') { + for await (const event of this.log.read(RUN_ID)) events.push(event.chunk) + } + return this.jsonResponse({ record, events }) + } + return new Response('not found', { status: 404 }) + } +} +export default { + fetch(request, env) { + const id = env.COORDINATOR.idFromName('singleton') + return env.COORDINATOR.get(id).fetch(request) + }, +} +` + +async function buildWorkerModules() { + const result = await build({ + configFile: false, + logLevel: 'silent', + build: { + write: false, + rollupOptions: { + input: { agent: AGENT_DIST }, + external: ['cloudflare:workers'], + preserveEntrySignatures: 'strict', + output: { + format: 'es', + entryFileNames: '[name].js', + chunkFileNames: '[name].js', + }, + }, + }, + }) + if (Array.isArray(result) || !('output' in result)) { + throw new Error('expected one Vite bundle') + } + return [ + { type: 'ESModule' as const, path: 'worker.js', contents: harness }, + ...result.output.flatMap((output) => + output.type === 'chunk' + ? [ + { + type: 'ESModule' as const, + path: output.fileName, + contents: output.code, + }, + ] + : [], + ), + ] +} + +test('a real alarm fails a stalled Cloudflare coordinator run', async () => { + test.setTimeout(60_000) + const mf = new Miniflare({ + modules: await buildWorkerModules(), + compatibilityDate: '2025-01-01', + compatibilityFlags: ['nodejs_compat'], + bindings: { STALL_TIMEOUT_MS: String(STALL_MS) }, + durableObjects: { COORDINATOR: 'TestCoordinator' }, + }) + const dispatch = (route: string) => + mf.dispatchFetch(`http://localhost/${route}`) + const state = async () => (await dispatch('state')).json() + + try { + expect(await (await dispatch('start')).json()).toEqual({ runId: RUN_ID }) + await expect + .poll(async () => (await state()).record?.status, { + timeout: POLL_TIMEOUT_MS, + }) + .toBe('failed') + + const final = await state() + expect(final.record.error).toEqual({ message: WATCHDOG_ERROR }) + expect(final.events).toEqual([ + { type: 'RUN_ERROR', message: WATCHDOG_ERROR }, + ]) + } finally { + await mf.dispose() + } +})