fix(execution): offload buffered event values at a budget-aware threshold

The replay buffer keeps 1000 events per execution inside a 64 MiB budget, so a
full ring only fits if events stay modest. Values were offloaded to object
storage at 8 MiB, far above that, so a run emitting large block outputs
exhausted its budget within a few dozen events and stayed pinned there for the
rest of its life.

Give buffered events their own offload threshold, sized against the observed
distribution of per-event payload sizes so ordinary runs stay fully inline and
only the outlier tail is offloaded. The shared LARGE_VALUE_THRESHOLD_BYTES
stays at 8 MiB: executor snapshots, provider attachment limits and inline
UserFile base64 all derive from it and are bounded per payload rather than
accumulated across a thousand of them.

Offloading costs fidelity — a ref renders as a truncated preview, and for
arrays and objects as a shape summary rather than contents — which is why the
threshold tracks the measured distribution instead of the stricter bound the
budget arithmetic alone would suggest.

Author: waleedlatif1

apps/sim/lib/execution/event-buffer.test.ts

@@ -368,6 +368,28 @@ describe('execution event buffer', () => {
     expect(persistedEntries).toEqual([])
   })
 
+  /**
+   * Buffered events accumulate against a 64 MiB per-execution budget across a
+   * 1000-event ring, so a value that a one-off payload would happily inline
+   * must still be offloaded here.
+   */
+  it('offloads values far below the shared large-value threshold', async () => {
+    mockRedis.incrby.mockResolvedValue(100)
+    // Comfortably under LARGE_VALUE_THRESHOLD_BYTES (8 MiB), over the event one.
+    const payload = 'x'.repeat(512 * 1024)
+
+    const writer = createExecutionEventWriter('exec-1', {
+      workspaceId: 'ws-1',
+      workflowId: 'wf-1',
+    })
+    await writer.write(makeEvent(payload))
+    await writer.flush()
+
+    expect(persistedEntries).toHaveLength(1)
+    const persistedBytes = Buffer.byteLength(JSON.stringify(persistedEntries[0]), 'utf8')
+    expect(persistedBytes).toBeLessThan(256 * 1024)
+  })
+
   it('preserves requested UserFile base64 when buffering terminal events', async () => {
     mockRedis.incrby.mockResolvedValue(100)
     const base64 = Buffer.from('hello').toString('base64')

apps/sim/lib/execution/event-buffer.ts

@@ -3,7 +3,10 @@ import { toError } from '@sim/utils/errors'
 import { randomInt } from '@sim/utils/random'
 import { env } from '@/lib/core/config/env'
 import { getRedisClient } from '@/lib/core/config/redis'
-import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref'
+import {
+  EXECUTION_EVENT_VALUE_THRESHOLD_BYTES,
+  LARGE_VALUE_THRESHOLD_BYTES,
+} from '@/lib/execution/payloads/large-value-ref'
 import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
 import type { LargeValueStoreContext } from '@/lib/execution/payloads/store'
 import {
@@ -301,6 +304,9 @@ async function compactEventForBuffer(
     executionId: context.executionId ?? event.executionId,
     requireDurable: context.requireDurablePayloads,
     preserveRoot: true,
+    // Buffered events accumulate against a per-execution byte budget, so they
+    // offload far earlier than a one-off payload needs to.
+    thresholdBytes: EXECUTION_EVENT_VALUE_THRESHOLD_BYTES,
   }
 
   let compactedData = await compactExecutionPayload(event.data, {

apps/sim/lib/execution/payloads/large-value-ref.ts

@@ -1,6 +1,31 @@
 export const LARGE_VALUE_REF_MARKER = '__simLargeValueRef'
 
 export const LARGE_VALUE_THRESHOLD_BYTES = 8 * 1024 * 1024
+
+/**
+ * Offload threshold for values inlined into buffered execution events.
+ *
+ * The replay buffer keeps `EVENT_LIMIT` (1000) events per execution inside a
+ * `MAX_EXECUTION_REDIS_BYTES` (64 MiB) budget. Values were offloaded at 8 MiB,
+ * so a run emitting large block outputs exhausted that budget within a few
+ * dozen events and stayed pinned at its ceiling for the rest of its life.
+ *
+ * Sized against the observed distribution of per-event payload sizes rather
+ * than the budget arithmetic alone: it sits well above what a typical run
+ * emits and well below the outlier tail, so ordinary runs stay fully inline
+ * and only the shape that exhausts the budget is offloaded.
+ *
+ * Offloading is not free: a ref renders in the terminal as a truncated string
+ * preview, or for arrays and objects as a shape summary rather than their
+ * contents. That cost is why this tracks the measured distribution instead of
+ * the stricter bound the budget arithmetic alone would suggest.
+ *
+ * Deliberately separate from {@link LARGE_VALUE_THRESHOLD_BYTES}, which stays
+ * at 8 MiB for executor snapshots and inline UserFile base64. Those are bounded
+ * per payload rather than accumulated across a thousand of them, and the base64
+ * limit is derived from it arithmetically.
+ */
+export const EXECUTION_EVENT_VALUE_THRESHOLD_BYTES = 256 * 1024
 export const LARGE_VALUE_REF_VERSION = 1
 
 export const LARGE_VALUE_KINDS = ['array', 'object', 'string', 'json'] as const