improvement(resolver): lazy resolution for underlying fields greater than 10MB

Author: icecrasher321

apps/docs/content/docs/en/blocks/function.mdx

@@ -185,6 +185,14 @@ plt.show()
 
 ## Best Practices
 
+### Large Inputs and Payload Limits
+
+Function blocks receive their code, parameters, resolved references, and previous block context in an internal execution request. Sim can safely reference oversized workflow outputs, such as large `loop.results` or `parallel.results`, when you select a smaller nested field like `<loop.results[0][0].id>`.
+
+Avoid passing a full large object into a Function block when you only need one field. For example, prefer `<api.data.customerId>` over `<api.data>` when the API response is large. If the complete function request body is still larger than the platform limit, execution can fail before your code starts.
+
+For large generated data, write the result to a file or table with `outputPath`, `outputSandboxPath`, or `outputTable` instead of returning the entire payload inline.
+
 - **Keep functions focused**: Write functions that do one thing well to improve maintainability and debugging
 - **Handle errors gracefully**: Use try/catch blocks to handle potential errors and provide meaningful error messages
 - **Test edge cases**: Ensure your code handles unusual inputs, null values, and boundary conditions correctly

apps/docs/content/docs/en/execution/api-deployment.mdx

@@ -215,6 +215,25 @@ while (true) {
   </Tab>
 </Tabs>
 
+#### Oversized outputs
+
+Workflow execution responses are capped by platform request and response limits. When an internal output, log field, streamed field, or async status payload contains a value that is too large to inline, Sim may replace that nested value with a versioned reference:
+
+```json
+{
+  "__simLargeValueRef": true,
+  "version": 1,
+  "id": "lv_abc123DEF456",
+  "kind": "array",
+  "size": 12582912,
+  "key": "execution-values/workspace/workflow/execution/lv_abc123DEF456.json",
+  "executionId": "exec_xyz",
+  "preview": { "length": 25000 }
+}
+```
+
+The `version` field is part of the external API contract. Treat the reference as an opaque placeholder for a value that could not be safely embedded in the response. `id`, `key`, and `executionId` are not fetch URLs; use `selectedOutputs` to request a smaller nested field, reduce the data passed between blocks, or return the data from a Response block when your workflow intentionally owns the HTTP response body.
+
 ### Asynchronous
 
 For long-running workflows, async mode returns a job ID immediately so you don't need to hold the connection open. Add the `X-Execution-Mode: async` header to your request. The API returns HTTP 202 with a job ID and status URL. Poll the status URL until the job completes.

apps/sim/app/api/workflows/[id]/execute/route.ts

@@ -36,6 +36,7 @@ import {
   registerManualExecutionAborter,
   unregisterManualExecutionAborter,
 } from '@/lib/execution/manual-cancellation'
+import { compactBlockLogs, compactExecutionPayload } from '@/lib/execution/payloads/serializer'
 import { preprocessExecution } from '@/lib/execution/preprocessing'
 import { LoggingSession } from '@/lib/logs/execution/logging-session'
 import {
@@ -65,7 +66,7 @@ import type {
   IterationContext,
   SerializableExecutionState,
 } from '@/executor/execution/types'
-import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types'
+import type { BlockLog, NormalizedBlockOutput, StreamingExecution } from '@/executor/types'
 import { getExecutionErrorStatus, hasExecutionResult } from '@/executor/utils/errors'
 import { Serializer } from '@/serializer'
 import { CORE_TRIGGER_TYPES, type CoreTriggerType } from '@/stores/logs/filters/types'
@@ -75,6 +76,20 @@ const logger = createLogger('WorkflowExecuteAPI')
 export const runtime = 'nodejs'
 export const dynamic = 'force-dynamic'
 
+async function compactRoutePayload<T>(
+  value: T,
+  context: {
+    workspaceId?: string
+    workflowId?: string
+    executionId?: string
+    userId?: string
+    preserveUserFileBase64?: boolean
+    preserveRoot?: boolean
+  }
+): Promise<T> {
+  return compactExecutionPayload(value, { ...context, requireDurable: true })
+}
+
 function resolveOutputIds(
   selectedOutputs: string[] | undefined,
   blocks: Record<string, any>
@@ -719,6 +734,14 @@ async function handleExecutePost(
         })
 
         await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession })
+        const compactResultOutput = await compactRoutePayload(result.output, {
+          workspaceId,
+          workflowId,
+          executionId,
+          userId: actorUserId,
+          preserveUserFileBase64: true,
+          preserveRoot: true,
+        })
 
         if (
           result.status === 'cancelled' &&
@@ -734,7 +757,7 @@ async function handleExecutePost(
           return NextResponse.json(
             {
               success: false,
-              output: result.output,
+              output: compactResultOutput,
               error: timeoutErrorMessage,
               metadata: result.metadata
                 ? {
@@ -756,16 +779,23 @@ async function handleExecutePost(
             })) as NormalizedBlockOutput)
           : result.output
 
-        const resultWithBase64 = { ...result, output: outputWithBase64 }
-
-        if (auth.authType !== AuthType.INTERNAL_JWT && workflowHasResponseBlock(resultWithBase64)) {
-          return createHttpResponseFromBlock(resultWithBase64)
+        if (auth.authType !== AuthType.INTERNAL_JWT && workflowHasResponseBlock(result)) {
+          return createHttpResponseFromBlock({ ...result, output: outputWithBase64 })
         }
 
+        const compactOutput = await compactRoutePayload(outputWithBase64, {
+          workspaceId,
+          workflowId,
+          executionId,
+          userId: actorUserId,
+          preserveUserFileBase64: true,
+          preserveRoot: true,
+        })
+
         const filteredResult = {
           success: result.success,
           executionId,
-          output: outputWithBase64,
+          output: compactOutput,
           error: result.error,
           metadata: result.metadata
             ? {
@@ -784,11 +814,21 @@ async function handleExecutePost(
 
         const executionResult = hasExecutionResult(error) ? error.executionResult : undefined
         const status = getExecutionErrorStatus(error)
+        const compactErrorOutput = executionResult?.output
+          ? await compactRoutePayload(executionResult.output, {
+              workspaceId,
+              workflowId,
+              executionId,
+              userId: actorUserId,
+              preserveUserFileBase64: true,
+              preserveRoot: true,
+            })
+          : undefined
 
         return NextResponse.json(
           {
             success: false,
-            output: executionResult?.output,
+            output: compactErrorOutput,
             error: executionResult?.error || errorMessage || 'Execution failed',
             metadata: executionResult?.metadata
               ? {
@@ -838,6 +878,9 @@ async function handleExecutePost(
           timeoutMs: preprocessResult.executionTimeout?.sync,
         },
         executionId,
+        workspaceId,
+        workflowId,
+        userId: actorUserId,
         executeFn: async ({ onStream, onBlockComplete, abortSignal }) =>
           executeWorkflow(
             streamWorkflow,
@@ -872,7 +915,12 @@ async function handleExecutePost(
     let isStreamClosed = false
     let isManualAbortRegistered = false
 
-    const eventWriter = createExecutionEventWriter(executionId)
+    const eventWriter = createExecutionEventWriter(executionId, {
+      workspaceId,
+      workflowId,
+      userId: actorUserId,
+      preserveUserFileBase64: includeFileBase64,
+    })
     const metaInitialized = await initializeExecutionStreamMeta(executionId, {
       userId: actorUserId,
       workflowId,
@@ -898,16 +946,18 @@ async function handleExecutePost(
           terminalStatus?: TerminalExecutionStreamStatus
         ) => {
           const isBuffered = event.type !== 'stream:chunk' && event.type !== 'stream:done'
+          let eventToSend = event
           if (isBuffered) {
             const entry = terminalStatus
               ? await eventWriter.writeTerminal(event, terminalStatus)
               : await eventWriter.write(event)
-            event.eventId = entry.eventId
+            eventToSend = entry.event
+            eventToSend.eventId = entry.eventId
             terminalEventPublished ||= Boolean(terminalStatus)
           }
           if (!isStreamClosed) {
             try {
-              controller.enqueue(encodeSSEEvent(event))
+              controller.enqueue(encodeSSEEvent(eventToSend))
             } catch {
               isStreamClosed = true
             }
@@ -971,7 +1021,26 @@ async function handleExecutePost(
             iterationContext?: IterationContext,
             childWorkflowContext?: ChildWorkflowContext
           ) => {
-            const hasError = callbackData.output?.error
+            const compactCallbackData = {
+              ...callbackData,
+              input: await compactRoutePayload(callbackData.input, {
+                workspaceId,
+                workflowId,
+                executionId,
+                userId: actorUserId,
+                preserveUserFileBase64: includeFileBase64,
+                preserveRoot: true,
+              }),
+              output: await compactRoutePayload(callbackData.output, {
+                workspaceId,
+                workflowId,
+                executionId,
+                userId: actorUserId,
+                preserveUserFileBase64: includeFileBase64,
+                preserveRoot: true,
+              }),
+            }
+            const hasError = compactCallbackData.output?.error
             const childWorkflowData = childWorkflowContext
               ? {
                   childWorkflowBlockId: childWorkflowContext.parentBlockId,
@@ -988,7 +1057,7 @@ async function handleExecutePost(
                 blockId,
                 blockName,
                 blockType,
-                error: callbackData.output.error,
+                error: compactCallbackData.output.error,
               })
               await sendEvent({
                 type: 'block:error',
@@ -999,12 +1068,12 @@ async function handleExecutePost(
                   blockId,
                   blockName,
                   blockType,
-                  input: callbackData.input,
-                  error: callbackData.output.error,
-                  durationMs: callbackData.executionTime || 0,
-                  startedAt: callbackData.startedAt,
-                  executionOrder: callbackData.executionOrder,
-                  endedAt: callbackData.endedAt,
+                  input: compactCallbackData.input,
+                  error: compactCallbackData.output.error,
+                  durationMs: compactCallbackData.executionTime || 0,
+                  startedAt: compactCallbackData.startedAt,
+                  executionOrder: compactCallbackData.executionOrder,
+                  endedAt: compactCallbackData.endedAt,
                   ...(iterationContext && {
                     iterationCurrent: iterationContext.iterationCurrent,
                     iterationTotal: iterationContext.iterationTotal,
@@ -1033,12 +1102,12 @@ async function handleExecutePost(
                   blockId,
                   blockName,
                   blockType,
-                  input: callbackData.input,
-                  output: callbackData.output,
-                  durationMs: callbackData.executionTime || 0,
-                  startedAt: callbackData.startedAt,
-                  executionOrder: callbackData.executionOrder,
-                  endedAt: callbackData.endedAt,
+                  input: compactCallbackData.input,
+                  output: compactCallbackData.output,
+                  durationMs: compactCallbackData.executionTime || 0,
+                  startedAt: compactCallbackData.startedAt,
+                  executionOrder: compactCallbackData.executionOrder,
+                  endedAt: compactCallbackData.endedAt,
                   ...(iterationContext && {
                     iterationCurrent: iterationContext.iterationCurrent,
                     iterationTotal: iterationContext.iterationTotal,
@@ -1171,6 +1240,13 @@ async function handleExecutePost(
           })
 
           await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession })
+          const compactTerminalLogs = await compactBlockLogs(result.logs, {
+            workspaceId,
+            workflowId,
+            executionId,
+            userId: actorUserId,
+            requireDurable: true,
+          })
 
           if (result.status === 'cancelled') {
             if (timeoutController.isTimedOut() && timeoutController.timeoutMs) {
@@ -1191,7 +1267,7 @@ async function handleExecutePost(
                   data: {
                     error: timeoutErrorMessage,
                     duration: result.metadata?.duration || 0,
-                    finalBlockLogs: result.logs,
+                    finalBlockLogs: compactTerminalLogs,
                   },
                 },
                 'error'
@@ -1208,7 +1284,7 @@ async function handleExecutePost(
                   workflowId,
                   data: {
                     duration: result.metadata?.duration || 0,
-                    finalBlockLogs: result.logs,
+                    finalBlockLogs: compactTerminalLogs,
                   },
                 },
                 'cancelled'
@@ -1224,6 +1300,21 @@ async function handleExecutePost(
                 maxBytes: base64MaxBytes,
               })
             : result.output
+          const compactSseOutput = await compactRoutePayload(sseOutput, {
+            workspaceId,
+            workflowId,
+            executionId,
+            userId: actorUserId,
+            preserveUserFileBase64: true,
+            preserveRoot: true,
+          })
+          const compactFinalBlockLogs = await compactBlockLogs(result.logs, {
+            workspaceId,
+            workflowId,
+            executionId,
+            userId: actorUserId,
+            requireDurable: true,
+          })
 
           if (result.status === 'paused') {
             finalMetaStatus = 'complete'
@@ -1234,11 +1325,11 @@ async function handleExecutePost(
                 executionId,
                 workflowId,
                 data: {
-                  output: sseOutput,
+                  output: compactSseOutput,
                   duration: result.metadata?.duration || 0,
                   startTime: result.metadata?.startTime || startTime.toISOString(),
                   endTime: result.metadata?.endTime || new Date().toISOString(),
-                  finalBlockLogs: result.logs,
+                  finalBlockLogs: compactFinalBlockLogs,
                 },
               },
               'complete'
@@ -1253,11 +1344,11 @@ async function handleExecutePost(
                 workflowId,
                 data: {
                   success: result.success,
-                  output: sseOutput,
+                  output: compactSseOutput,
                   duration: result.metadata?.duration || 0,
                   startTime: result.metadata?.startTime || startTime.toISOString(),
                   endTime: result.metadata?.endTime || new Date().toISOString(),
-                  finalBlockLogs: result.logs,
+                  finalBlockLogs: compactFinalBlockLogs,
                 },
               },
               'complete'
@@ -1274,6 +1365,22 @@ async function handleExecutePost(
           reqLogger.error(`SSE execution failed: ${errorMessage}`, { isTimeout })
 
           const executionResult = hasExecutionResult(error) ? error.executionResult : undefined
+          let compactErrorLogs: BlockLog[] | undefined
+          try {
+            compactErrorLogs = executionResult?.logs
+              ? await compactBlockLogs(executionResult.logs, {
+                  workspaceId,
+                  workflowId,
+                  executionId,
+                  userId: actorUserId,
+                  requireDurable: true,
+                })
+              : undefined
+          } catch (compactionError) {
+            reqLogger.warn('Failed to compact SSE error logs, omitting oversized error details', {
+              error: toError(compactionError).message,
+            })
+          }
 
           finalMetaStatus = 'error'
           await sendEvent(
@@ -1285,7 +1392,7 @@ async function handleExecutePost(
               data: {
                 error: executionResult?.error || errorMessage,
                 duration: executionResult?.metadata?.duration || 0,
-                finalBlockLogs: executionResult?.logs,
+                finalBlockLogs: compactErrorLogs,
               },
             },
             'error'