Performance: avoid serial Azure Blob payload operations for fan-out messages

[berndverst]

1. What is the issue?

The Azure Blob payload interceptor awaits externalization and resolution one field at a time when a response, history chunk, work item, or entity batch contains multiple externalized payloads.

2. Likely priority

Medium. It primarily affects fan-out/fan-in workflows with several payloads over the configured threshold and large history replay batches.

3. Impact on performance

Independent Azure Storage uploads and downloads become strictly additive in latency. A message containing N externalized fields waits for approximately N network round trips, including each request's retry behavior, before it can proceed.

4. Details and code reference

Sequential resolution path:

durabletask-dotnet/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs

Lines 108 to 193 in 883211a

	/// <inheritdoc/>
	protected override async Task ResolveResponsePayloadsAsync<TResponse>(TResponse response, CancellationToken cancellation)
	{
	// Sidecar -> client/worker
	switch (response)
	{
	case P.GetInstanceResponse r when r.OrchestrationState is { } s:
	s.Input = await this.MaybeResolveAsync(s.Input, cancellation);
	s.Output = await this.MaybeResolveAsync(s.Output, cancellation);
	s.CustomStatus = await this.MaybeResolveAsync(s.CustomStatus, cancellation);
	break;
	case P.HistoryChunk c when c.Events != null:
	foreach (P.HistoryEvent e in c.Events)
	{
	await this.ResolveEventPayloadsAsync(e, cancellation);
	}
	break;
	case P.QueryInstancesResponse r:
	foreach (P.OrchestrationState s in r.OrchestrationState)
	{
	s.Input = await this.MaybeResolveAsync(s.Input, cancellation);
	s.Output = await this.MaybeResolveAsync(s.Output, cancellation);
	s.CustomStatus = await this.MaybeResolveAsync(s.CustomStatus, cancellation);
	}
	break;
	case P.GetEntityResponse r when r.Entity is { } em:
	em.SerializedState = await this.MaybeResolveAsync(em.SerializedState, cancellation);
	break;
	case P.QueryEntitiesResponse r:
	foreach (P.EntityMetadata em in r.Entities)
	{
	em.SerializedState = await this.MaybeResolveAsync(em.SerializedState, cancellation);
	}
	break;
	case P.WorkItem wi:
	// Resolve activity input
	if (wi.ActivityRequest is { } ar)
	{
	ar.Input = await this.MaybeResolveAsync(ar.Input, cancellation);
	}
	// Resolve orchestration input embedded in ExecutionStarted event and external events
	if (wi.OrchestratorRequest is { } or)
	{
	foreach (P.HistoryEvent? e in or.PastEvents)
	{
	await this.ResolveEventPayloadsAsync(e, cancellation);
	}
	foreach (P.HistoryEvent? e in or.NewEvents)
	{
	await this.ResolveEventPayloadsAsync(e, cancellation);
	}
	}
	// Resolve entity V1 batch request (OperationRequest inputs and entity state)
	if (wi.EntityRequest is { } er1)
	{
	er1.EntityState = await this.MaybeResolveAsync(er1.EntityState, cancellation);
	if (er1.Operations != null)
	{
	foreach (P.OperationRequest op in er1.Operations)
	{
	op.Input = await this.MaybeResolveAsync(op.Input, cancellation);
	}
	}
	}
	// Resolve entity V2 request (history-based operation requests and entity state)
	if (wi.EntityRequestV2 is { } er2)
	{
	er2.EntityState = await this.MaybeResolveAsync(er2.EntityState, cancellation);
	if (er2.OperationRequests != null)
	{
	foreach (P.HistoryEvent opEvt in er2.OperationRequests)
	{
	await this.ResolveEventPayloadsAsync(opEvt, cancellation);
	}
	}
	}
	break;
	}

This resolves history, work-item, and query fields sequentially.

Sequential externalization path:

durabletask-dotnet/src/Extensions/AzureBlobPayloads/Interceptors/AzureBlobPayloadsSideCarInterceptor.cs

Lines 196 to 283 in 883211a

	async Task ExternalizeOrchestratorResponseAsync(P.OrchestratorResponse r, CancellationToken cancellation)
	{
	r.CustomStatus = await this.MaybeExternalizeAsync(r.CustomStatus, cancellation);
	foreach (P.OrchestratorAction a in r.Actions)
	{
	if (a.CompleteOrchestration is { } complete)
	{
	complete.Result = await this.MaybeExternalizeAsync(complete.Result, cancellation);
	complete.Details = await this.MaybeExternalizeAsync(complete.Details, cancellation);
	}
	if (a.TerminateOrchestration is { } term)
	{
	term.Reason = await this.MaybeExternalizeAsync(term.Reason, cancellation);
	}
	if (a.ScheduleTask is { } schedule)
	{
	schedule.Input = await this.MaybeExternalizeAsync(schedule.Input, cancellation);
	}
	if (a.CreateSubOrchestration is { } sub)
	{
	sub.Input = await this.MaybeExternalizeAsync(sub.Input, cancellation);
	}
	if (a.SendEvent is { } sendEvt)
	{
	sendEvt.Data = await this.MaybeExternalizeAsync(sendEvt.Data, cancellation);
	}
	if (a.SendEntityMessage is { } entityMsg)
	{
	if (entityMsg.EntityOperationSignaled is { } sig)
	{
	sig.Input = await this.MaybeExternalizeAsync(sig.Input, cancellation);
	}
	if (entityMsg.EntityOperationCalled is { } called)
	{
	called.Input = await this.MaybeExternalizeAsync(called.Input, cancellation);
	}
	}
	}
	}
	async Task ExternalizeEntityBatchResultAsync(P.EntityBatchResult r, CancellationToken cancellation)
	{
	r.EntityState = await this.MaybeExternalizeAsync(r.EntityState, cancellation);
	if (r.Results != null)
	{
	foreach (P.OperationResult result in r.Results)
	{
	if (result.Success is { } success)
	{
	success.Result = await this.MaybeExternalizeAsync(success.Result, cancellation);
	}
	}
	}
	if (r.Actions != null)
	{
	foreach (P.OperationAction action in r.Actions)
	{
	if (action.SendSignal is { } sendSig)
	{
	sendSig.Input = await this.MaybeExternalizeAsync(sendSig.Input, cancellation);
	}
	if (action.StartNewOrchestration is { } start)
	{
	start.Input = await this.MaybeExternalizeAsync(start.Input, cancellation);
	}
	}
	}
	}
	async Task ExternalizeEntityBatchRequestAsync(P.EntityBatchRequest r, CancellationToken cancellation)
	{
	r.EntityState = await this.MaybeExternalizeAsync(r.EntityState, cancellation);
	if (r.Operations != null)
	{
	foreach (P.OperationRequest op in r.Operations)
	{
	op.Input = await this.MaybeExternalizeAsync(op.Input, cancellation);
	}
	}
	}

This awaits each action or operation in sequence.

5. Guidance for how to fix

Collect independent upload or download operations and execute them with bounded concurrency, assigning results back to the same fields after completion. Do not use unbounded Task.WhenAll for arbitrary fan-out: set a conservative limit that respects Azure Storage throttling and preserves cancellation and first-failure behavior. Keep event/action ordering unchanged so transport conversion remains replay-safe.