From 9b0bf33d3e9d0509674c8914254b1bfb43cd3b6c Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 14:35:29 -0400 Subject: [PATCH 01/13] feat(exstore): add to client and worker options. --- .../client/WorkflowClientOptions.java | 40 ++++++++++++++++--- .../ExternalStorageMessageConverter.java | 17 ++++++-- .../internal/worker/SingleWorkerOptions.java | 25 +++++++++++- .../main/java/io/temporal/worker/Worker.java | 10 +++++ .../temporal/worker/WorkerFactoryOptions.java | 35 ++++++++++++++++ 5 files changed, 117 insertions(+), 10 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientOptions.java b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientOptions.java index e10defba51..7251b52d89 100644 --- a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientOptions.java +++ b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientOptions.java @@ -7,6 +7,7 @@ import io.temporal.common.converter.DataConverter; import io.temporal.common.converter.GlobalDataConverter; import io.temporal.common.interceptors.WorkflowClientInterceptor; +import io.temporal.payload.storage.ExternalStorageOptions; import java.lang.management.ManagementFactory; import java.time.Duration; import java.util.Arrays; @@ -52,6 +53,7 @@ public static final class Builder { private QueryRejectCondition queryRejectCondition; private WorkflowClientPlugin[] plugins; private Duration workerHeartbeatInterval; + private ExternalStorageOptions externalStorage; private Builder() {} @@ -68,6 +70,7 @@ private Builder(WorkflowClientOptions options) { queryRejectCondition = options.queryRejectCondition; plugins = options.plugins; workerHeartbeatInterval = options.workerHeartbeatInterval; + externalStorage = options.externalStorage; } public Builder setNamespace(String namespace) { @@ -170,6 +173,17 @@ public Builder setWorkerHeartbeatInterval(Duration workerHeartbeatInterval) { return this; } + /** + * Configures offloading of large payloads to external storage for workflows and activities + * created through this client and its workers. When null (the default), external storage is + * disabled. + */ + @Experimental + public Builder setExternalStorage(ExternalStorageOptions externalStorage) { + this.externalStorage = externalStorage; + return this; + } + public WorkflowClientOptions build() { return new WorkflowClientOptions( namespace, @@ -180,7 +194,8 @@ public WorkflowClientOptions build() { contextPropagators, queryRejectCondition, plugins == null ? EMPTY_PLUGINS : plugins, - resolveHeartbeatInterval(workerHeartbeatInterval)); + resolveHeartbeatInterval(workerHeartbeatInterval), + externalStorage); } /** @@ -207,7 +222,8 @@ public WorkflowClientOptions validateAndBuildWithDefaults() { ? QueryRejectCondition.QUERY_REJECT_CONDITION_UNSPECIFIED : queryRejectCondition, plugins == null ? EMPTY_PLUGINS : plugins, - resolveHeartbeatInterval(workerHeartbeatInterval)); + resolveHeartbeatInterval(workerHeartbeatInterval), + externalStorage); } private static Duration resolveHeartbeatInterval(Duration raw) { @@ -250,6 +266,8 @@ private static Duration resolveHeartbeatInterval(Duration raw) { private final Duration workerHeartbeatInterval; + private final ExternalStorageOptions externalStorage; + private WorkflowClientOptions( String namespace, DataConverter dataConverter, @@ -259,7 +277,8 @@ private WorkflowClientOptions( List contextPropagators, QueryRejectCondition queryRejectCondition, WorkflowClientPlugin[] plugins, - Duration workerHeartbeatInterval) { + Duration workerHeartbeatInterval, + ExternalStorageOptions externalStorage) { this.namespace = namespace; this.dataConverter = dataConverter; this.interceptors = interceptors; @@ -269,6 +288,7 @@ private WorkflowClientOptions( this.queryRejectCondition = queryRejectCondition; this.plugins = plugins; this.workerHeartbeatInterval = workerHeartbeatInterval; + this.externalStorage = externalStorage; } /** @@ -335,6 +355,12 @@ public Duration getWorkerHeartbeatInterval() { return workerHeartbeatInterval; } + /** External storage configuration, or null when external storage is disabled. */ + @Experimental + public ExternalStorageOptions getExternalStorage() { + return externalStorage; + } + @Override public String toString() { return "WorkflowClientOptions{" @@ -359,6 +385,8 @@ public String toString() { + Arrays.toString(plugins) + ", workerHeartbeatInterval=" + workerHeartbeatInterval + + ", externalStorage=" + + externalStorage + '}'; } @@ -376,7 +404,8 @@ public boolean equals(Object o) { && queryRejectCondition == that.queryRejectCondition && Arrays.equals(plugins, that.plugins) && com.google.common.base.Objects.equal( - workerHeartbeatInterval, that.workerHeartbeatInterval); + workerHeartbeatInterval, that.workerHeartbeatInterval) + && com.google.common.base.Objects.equal(externalStorage, that.externalStorage); } @Override @@ -390,6 +419,7 @@ public int hashCode() { contextPropagators, queryRejectCondition, Arrays.hashCode(plugins), - workerHeartbeatInterval); + workerHeartbeatInterval, + externalStorage); } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java index 726505b7be..f28f33abea 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java @@ -4,6 +4,7 @@ import io.temporal.internal.payload.visitor.PayloadVisitor; import io.temporal.internal.payload.visitor.PayloadVisitorOptions; import io.temporal.internal.payload.visitor.PayloadVisitors; +import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriverTargetInfo; import java.util.concurrent.CompletableFuture; import javax.annotation.Nullable; @@ -14,17 +15,27 @@ * *

Search attributes stay inline because the server indexes and validates their payload values. */ -final class ExternalStorageMessageConverter { +public final class ExternalStorageMessageConverter { private final ExternalStoragePayloadConverter payloadConverter; private final int payloadVisitConcurrency; + /** + * Builds a message converter from user-facing options. {@code payloadVisitConcurrency} bounds the + * number of concurrent payload-list visits within a single message walk (at least {@code 1}). + */ + public static ExternalStorageMessageConverter create( + ExternalStorageOptions options, int payloadVisitConcurrency) { + return new ExternalStorageMessageConverter( + ExternalStoragePayloadConverter.fromOptions(options), payloadVisitConcurrency); + } + ExternalStorageMessageConverter( ExternalStoragePayloadConverter payloadConverter, int payloadVisitConcurrency) { this.payloadConverter = payloadConverter; this.payloadVisitConcurrency = payloadVisitConcurrency; } - CompletableFuture store( + public CompletableFuture store( T message, @Nullable StorageDriverTargetInfo target) { PayloadVisitorOptions options = PayloadVisitorOptions.newBuilder( @@ -36,7 +47,7 @@ CompletableFuture store( return PayloadVisitors.visit(message, options); } - CompletableFuture retrieve(T message) { + public CompletableFuture retrieve(T message) { PayloadVisitorOptions options = PayloadVisitorOptions.newBuilder( (PayloadVisitor) (context, payloads) -> payloadConverter.retrieve(payloads)) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/SingleWorkerOptions.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/SingleWorkerOptions.java index 8e0288566e..53a5d4b2b8 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/SingleWorkerOptions.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/SingleWorkerOptions.java @@ -7,10 +7,12 @@ import io.temporal.common.converter.DataConverter; import io.temporal.common.converter.GlobalDataConverter; import io.temporal.common.interceptors.WorkerInterceptor; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.worker.PreferredVersionProvider; import io.temporal.worker.WorkerDeploymentOptions; import java.time.Duration; import java.util.List; +import javax.annotation.Nullable; public final class SingleWorkerOptions { @@ -45,6 +47,7 @@ public static final class Builder { private boolean allowActivityHeartbeatDuringShutdown; private String workerControlTaskQueue; private PreferredVersionProvider preferredVersionProvider; + private ExternalStorageMessageConverter externalStorageMessageConverter; private Builder() {} @@ -73,6 +76,7 @@ private Builder(SingleWorkerOptions options) { this.allowActivityHeartbeatDuringShutdown = options.getAllowActivityHeartbeatDuringShutdown(); this.workerControlTaskQueue = options.getWorkerControlTaskQueue(); this.preferredVersionProvider = options.getPreferredVersionProvider(); + this.externalStorageMessageConverter = options.getExternalStorageMessageConverter(); } public Builder setIdentity(String identity) { @@ -185,6 +189,13 @@ public Builder setPreferredVersionProvider(PreferredVersionProvider preferredVer return this; } + /** The message converter that offloads/restores external-storage payloads, or null. */ + public Builder setExternalStorageMessageConverter( + ExternalStorageMessageConverter externalStorageMessageConverter) { + this.externalStorageMessageConverter = externalStorageMessageConverter; + return this; + } + public SingleWorkerOptions build() { PollerOptions pollerOptions = this.pollerOptions; if (pollerOptions == null) { @@ -227,7 +238,8 @@ public SingleWorkerOptions build() { this.workerInstanceKey, this.allowActivityHeartbeatDuringShutdown, this.workerControlTaskQueue, - this.preferredVersionProvider); + this.preferredVersionProvider, + this.externalStorageMessageConverter); } } @@ -252,6 +264,7 @@ public SingleWorkerOptions build() { private final boolean allowActivityHeartbeatDuringShutdown; private final String workerControlTaskQueue; private final PreferredVersionProvider preferredVersionProvider; + private final ExternalStorageMessageConverter externalStorageMessageConverter; private SingleWorkerOptions( String identity, @@ -274,7 +287,8 @@ private SingleWorkerOptions( String workerInstanceKey, boolean allowActivityHeartbeatDuringShutdown, String workerControlTaskQueue, - PreferredVersionProvider preferredVersionProvider) { + PreferredVersionProvider preferredVersionProvider, + ExternalStorageMessageConverter externalStorageMessageConverter) { this.identity = identity; this.binaryChecksum = binaryChecksum; this.buildId = buildId; @@ -296,6 +310,7 @@ private SingleWorkerOptions( this.allowActivityHeartbeatDuringShutdown = allowActivityHeartbeatDuringShutdown; this.workerControlTaskQueue = workerControlTaskQueue; this.preferredVersionProvider = preferredVersionProvider; + this.externalStorageMessageConverter = externalStorageMessageConverter; } public String getIdentity() { @@ -393,6 +408,12 @@ public PreferredVersionProvider getPreferredVersionProvider() { return preferredVersionProvider; } + /** The external-storage message converter for this worker, or null when disabled. */ + @Nullable + public ExternalStorageMessageConverter getExternalStorageMessageConverter() { + return externalStorageMessageConverter; + } + public WorkerVersioningOptions getWorkerVersioningOptions() { return new WorkerVersioningOptions( this.getBuildId(), this.isUsingBuildIdForVersioning(), this.getDeploymentOptions()); diff --git a/temporal-sdk/src/main/java/io/temporal/worker/Worker.java b/temporal-sdk/src/main/java/io/temporal/worker/Worker.java index b755134448..8929924455 100644 --- a/temporal-sdk/src/main/java/io/temporal/worker/Worker.java +++ b/temporal-sdk/src/main/java/io/temporal/worker/Worker.java @@ -22,10 +22,12 @@ import io.temporal.common.converter.DataConverter; import io.temporal.common.converter.EncodedValues; import io.temporal.failure.TemporalFailure; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.sync.WorkflowInternal; import io.temporal.internal.sync.WorkflowThreadExecutor; import io.temporal.internal.worker.*; import io.temporal.internal.worker.TaskCounter; +import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.serviceclient.MetricsTag; import io.temporal.serviceclient.Version; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -1079,8 +1081,16 @@ private static SingleWorkerOptions.Builder toSingleWorkerOptions( identity = options.getIdentity(); } + ExternalStorageOptions externalStorage = clientOptions.getExternalStorage(); + ExternalStorageMessageConverter externalStorageMessageConverter = + externalStorage == null + ? null + : ExternalStorageMessageConverter.create( + externalStorage, factoryOptions.getMaxConcurrentExternalStorageVisits()); + return SingleWorkerOptions.newBuilder() .setDataConverter(clientOptions.getDataConverter()) + .setExternalStorageMessageConverter(externalStorageMessageConverter) .setIdentity(identity) .setBuildId(buildId) .setUseBuildIdForVersioning(options.isUsingBuildIdForVersioning()) diff --git a/temporal-sdk/src/main/java/io/temporal/worker/WorkerFactoryOptions.java b/temporal-sdk/src/main/java/io/temporal/worker/WorkerFactoryOptions.java index dae83334de..092414e3b5 100644 --- a/temporal-sdk/src/main/java/io/temporal/worker/WorkerFactoryOptions.java +++ b/temporal-sdk/src/main/java/io/temporal/worker/WorkerFactoryOptions.java @@ -25,6 +25,7 @@ public static WorkerFactoryOptions getDefaultInstance() { private static final int DEFAULT_WORKFLOW_CACHE_SIZE = 600; private static final int DEFAULT_MAX_WORKFLOW_THREAD_COUNT = 600; private static final Duration DEFAULT_SHUTDOWN_CHECK_INTERVAL = Duration.ofMillis(250); + private static final int DEFAULT_MAX_CONCURRENT_EXTERNAL_STORAGE_VISITS = 3; private static final WorkerFactoryOptions DEFAULT_INSTANCE; @@ -43,6 +44,7 @@ public static class Builder { private boolean usingVirtualWorkflowThreads; private ExecutorService overrideLocalActivityTaskExecutor; private Duration shutdownCheckInterval; + private int maxConcurrentExternalStorageVisits; private Builder() {} @@ -60,6 +62,7 @@ private Builder(WorkerFactoryOptions options) { this.usingVirtualWorkflowThreads = options.usingVirtualWorkflowThreads; this.overrideLocalActivityTaskExecutor = options.overrideLocalActivityTaskExecutor; this.shutdownCheckInterval = options.shutdownCheckInterval; + this.maxConcurrentExternalStorageVisits = options.maxConcurrentExternalStorageVisits; } /** @@ -173,6 +176,19 @@ public Builder setShutdownCheckInterval(Duration shutdownCheckInterval) { return this; } + /** + * Maximum number of concurrent payload-list visits performed while offloading or restoring + * external-storage payloads for a single worker task. Only applies when external storage is + * configured on the client. Must be at least 1. + * + *

Default is 3. + */ + @Experimental + public Builder setMaxConcurrentExternalStorageVisits(int maxConcurrentExternalStorageVisits) { + this.maxConcurrentExternalStorageVisits = maxConcurrentExternalStorageVisits; + return this; + } + public WorkerFactoryOptions build() { return new WorkerFactoryOptions( workflowCacheSize, @@ -184,6 +200,7 @@ public WorkerFactoryOptions build() { usingVirtualWorkflowThreads, overrideLocalActivityTaskExecutor, shutdownCheckInterval, + maxConcurrentExternalStorageVisits, false); } @@ -209,6 +226,7 @@ public WorkerFactoryOptions validateAndBuildWithDefaults() { usingVirtualWorkflowThreads, overrideLocalActivityTaskExecutor, shutdownCheckInterval, + maxConcurrentExternalStorageVisits, true); } } @@ -222,6 +240,7 @@ public WorkerFactoryOptions validateAndBuildWithDefaults() { private final boolean usingVirtualWorkflowThreads; private final ExecutorService overrideLocalActivityTaskExecutor; private final Duration shutdownCheckInterval; + private final int maxConcurrentExternalStorageVisits; private WorkerFactoryOptions( int workflowCacheSize, @@ -233,6 +252,7 @@ private WorkerFactoryOptions( boolean usingVirtualWorkflowThreads, ExecutorService overrideLocalActivityTaskExecutor, Duration shutdownCheckInterval, + int maxConcurrentExternalStorageVisits, boolean validate) { if (validate) { Preconditions.checkState(workflowCacheSize >= 0, "negative workflowCacheSize"); @@ -262,6 +282,11 @@ private WorkerFactoryOptions( } else { shutdownCheckInterval = DEFAULT_SHUTDOWN_CHECK_INTERVAL; } + Preconditions.checkState( + maxConcurrentExternalStorageVisits >= 0, "negative maxConcurrentExternalStorageVisits"); + if (maxConcurrentExternalStorageVisits <= 0) { + maxConcurrentExternalStorageVisits = DEFAULT_MAX_CONCURRENT_EXTERNAL_STORAGE_VISITS; + } } this.workflowCacheSize = workflowCacheSize; this.maxWorkflowThreadCount = maxWorkflowThreadCount; @@ -273,6 +298,7 @@ private WorkerFactoryOptions( this.usingVirtualWorkflowThreads = usingVirtualWorkflowThreads; this.overrideLocalActivityTaskExecutor = overrideLocalActivityTaskExecutor; this.shutdownCheckInterval = shutdownCheckInterval; + this.maxConcurrentExternalStorageVisits = maxConcurrentExternalStorageVisits; } public int getWorkflowCacheSize() { @@ -330,6 +356,15 @@ public Duration getShutdownCheckInterval() { return shutdownCheckInterval; } + /** + * Returns the maximum number of concurrent payload-list visits performed while offloading or + * restoring external-storage payloads for a single worker task. + */ + @Experimental + public int getMaxConcurrentExternalStorageVisits() { + return maxConcurrentExternalStorageVisits; + } + /** * @deprecated not used anymore by JavaSDK, this value doesn't have any effect */ From 61b6960dc0790113307dd793954f40a565e6338c Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 16:27:01 -0400 Subject: [PATCH 02/13] feat(extstore): integrate external storage into client and worker pipelines. --- .../client/WorkflowClientInternalImpl.java | 15 +- .../PayloadAndFailureDataConverter.java | 7 + .../ExternalStorageGenericWorkflowClient.java | 320 ++++++++++++++++++ .../ExternalStorageMessageConverter.java | 24 ++ .../replay/ReplayWorkflowTaskHandler.java | 7 +- .../ServiceWorkflowHistoryIterator.java | 19 +- .../internal/worker/ActivityWorker.java | 88 +++-- .../internal/worker/WorkflowWorker.java | 38 ++- .../ExternalStorageReferenceGuardTest.java | 48 +++ .../storage/ExternalStoragePipelineTest.java | 142 ++++++++ 10 files changed, 681 insertions(+), 27 deletions(-) create mode 100644 temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java create mode 100644 temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java diff --git a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java index e856ca4b46..71129abd0b 100644 --- a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java @@ -18,12 +18,15 @@ import io.temporal.internal.WorkflowThreadMarker; import io.temporal.internal.client.*; import io.temporal.internal.client.NexusStartWorkflowResponse; +import io.temporal.internal.client.external.ExternalStorageGenericWorkflowClient; import io.temporal.internal.client.external.GenericWorkflowClient; import io.temporal.internal.client.external.GenericWorkflowClientImpl; import io.temporal.internal.client.external.ManualActivityCompletionClientFactory; import io.temporal.internal.common.PluginUtils; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.sync.StubMarker; import io.temporal.internal.worker.HeartbeatManager; +import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.serviceclient.MetricsTag; import io.temporal.serviceclient.WorkflowServiceStubs; import io.temporal.serviceclient.WorkflowServiceStubsPlugin; @@ -106,7 +109,17 @@ public static WorkflowClient newInstance( .getOptions() .getMetricsScope() .tagged(MetricsTag.defaultTags(options.getNamespace())); - this.genericClient = new GenericWorkflowClientImpl(workflowServiceStubs, metricsScope); + GenericWorkflowClient genericClient = + new GenericWorkflowClientImpl(workflowServiceStubs, metricsScope); + ExternalStorageOptions externalStorage = options.getExternalStorage(); + if (externalStorage != null) { + genericClient = + new ExternalStorageGenericWorkflowClient( + genericClient, + ExternalStorageMessageConverter.create(externalStorage, 1), + options.getNamespace()); + } + this.genericClient = genericClient; this.interceptors = options.getInterceptors(); this.workflowClientCallsInvoker = initializeClientInvoker(); this.manualActivityCompletionClientFactory = diff --git a/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java b/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java index 935fd8462c..09460c46dd 100644 --- a/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java +++ b/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java @@ -71,6 +71,13 @@ public T fromPayload(Payload payload, Class valueClass, Type valueType) return (T) new RawValue(payload); } + if (payload.getExternalPayloadsCount() > 0) { + throw new DataConverterException( + "[TMPRL-1105] Encountered an external-storage reference payload but external storage is not " + + "configured. Configure WorkflowClientOptions.Builder.setExternalStorage(...) with a " + + "driver able to retrieve it."); + } + try { String encoding = payload.getMetadataOrThrow(EncodingKeys.METADATA_ENCODING_KEY).toString(UTF_8); diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java new file mode 100644 index 0000000000..f2a5df1b56 --- /dev/null +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java @@ -0,0 +1,320 @@ +package io.temporal.internal.client.external; + +import com.google.common.base.Strings; +import io.grpc.Deadline; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.workflowservice.v1.*; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.payload.storage.StorageDriverTargetInfo; +import io.temporal.payload.storage.StorageDriverWorkflowInfo; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Decorates a {@link GenericWorkflowClient} to offload outbound request payloads to external + * storage and restore inbound response payloads. + * + *

Only constructed when external storage is configured, so the converter is never null. + */ +public final class ExternalStorageGenericWorkflowClient implements GenericWorkflowClient { + private final GenericWorkflowClient next; + private final ExternalStorageMessageConverter externalStorage; + private final String namespace; + + public ExternalStorageGenericWorkflowClient( + GenericWorkflowClient next, + ExternalStorageMessageConverter externalStorage, + String namespace) { + this.next = next; + this.externalStorage = externalStorage; + this.namespace = namespace; + } + + @Nullable + private StorageDriverTargetInfo workflowTarget(String workflowId, String runId, String type) { + return new StorageDriverWorkflowInfo( + namespace, + Strings.emptyToNull(workflowId), + Strings.emptyToNull(runId), + Strings.emptyToNull(type)); + } + + @Nullable + private StorageDriverTargetInfo workflowTarget(WorkflowExecution execution, String type) { + return workflowTarget(execution.getWorkflowId(), execution.getRunId(), type); + } + + @Override + public StartWorkflowExecutionResponse start(StartWorkflowExecutionRequest request) { + return next.start( + externalStorage.storeBlocking( + request, + workflowTarget(request.getWorkflowId(), null, request.getWorkflowType().getName()))); + } + + @Override + public SignalWorkflowExecutionResponse signal(SignalWorkflowExecutionRequest request) { + return next.signal( + externalStorage.storeBlocking( + request, workflowTarget(request.getWorkflowExecution(), null))); + } + + @Override + public SignalWithStartWorkflowExecutionResponse signalWithStart( + SignalWithStartWorkflowExecutionRequest request) { + return next.signalWithStart( + externalStorage.storeBlocking( + request, + workflowTarget(request.getWorkflowId(), null, request.getWorkflowType().getName()))); + } + + @Override + public void requestCancel(RequestCancelWorkflowExecutionRequest parameters) { + next.requestCancel(parameters); + } + + @Override + public QueryWorkflowResponse query(QueryWorkflowRequest queryParameters) { + QueryWorkflowRequest stored = + externalStorage.storeBlocking( + queryParameters, workflowTarget(queryParameters.getExecution(), null)); + return externalStorage.retrieveBlocking(next.query(stored)); + } + + @Override + public UpdateWorkflowExecutionResponse update( + @Nonnull UpdateWorkflowExecutionRequest updateParameters, @Nonnull Deadline deadline) { + UpdateWorkflowExecutionRequest stored = + externalStorage.storeBlocking( + updateParameters, workflowTarget(updateParameters.getWorkflowExecution(), null)); + return externalStorage.retrieveBlocking(next.update(stored, deadline)); + } + + @Override + public CompletableFuture pollUpdateAsync( + @Nonnull PollWorkflowExecutionUpdateRequest request, @Nonnull Deadline deadline) { + return next.pollUpdateAsync(request, deadline).thenCompose(externalStorage::retrieve); + } + + @Override + public void terminate(TerminateWorkflowExecutionRequest request) { + next.terminate( + externalStorage.storeBlocking( + request, workflowTarget(request.getWorkflowExecution(), null))); + } + + @Override + public GetWorkflowExecutionHistoryResponse longPollHistory( + @Nonnull GetWorkflowExecutionHistoryRequest request, @Nonnull Deadline deadline) { + return externalStorage.retrieveBlocking(next.longPollHistory(request, deadline)); + } + + @Override + public CompletableFuture longPollHistoryAsync( + @Nonnull GetWorkflowExecutionHistoryRequest request, @Nonnull Deadline deadline) { + return next.longPollHistoryAsync(request, deadline).thenCompose(externalStorage::retrieve); + } + + @Override + public GetWorkflowExecutionHistoryResponse getWorkflowExecutionHistory( + @Nonnull GetWorkflowExecutionHistoryRequest request) { + return externalStorage.retrieveBlocking(next.getWorkflowExecutionHistory(request)); + } + + @Override + public CompletableFuture getWorkflowExecutionHistoryAsync( + @Nonnull GetWorkflowExecutionHistoryRequest request) { + return next.getWorkflowExecutionHistoryAsync(request).thenCompose(externalStorage::retrieve); + } + + @Override + public ListWorkflowExecutionsResponse listWorkflowExecutions( + ListWorkflowExecutionsRequest listRequest) { + return next.listWorkflowExecutions(listRequest); + } + + @Override + public CompletableFuture listWorkflowExecutionsAsync( + ListWorkflowExecutionsRequest listRequest) { + return next.listWorkflowExecutionsAsync(listRequest); + } + + @Override + public CountWorkflowExecutionsResponse countWorkflowExecutions( + CountWorkflowExecutionsRequest request) { + return next.countWorkflowExecutions(request); + } + + @Override + public CreateScheduleResponse createSchedule(CreateScheduleRequest request) { + return next.createSchedule(request); + } + + @Override + public CompletableFuture listSchedulesAsync(ListSchedulesRequest request) { + return next.listSchedulesAsync(request); + } + + @Override + public UpdateScheduleResponse updateSchedule(UpdateScheduleRequest request) { + return next.updateSchedule(request); + } + + @Override + public PatchScheduleResponse patchSchedule(PatchScheduleRequest request) { + return next.patchSchedule(request); + } + + @Override + public DeleteScheduleResponse deleteSchedule(DeleteScheduleRequest request) { + return next.deleteSchedule(request); + } + + @Override + public DescribeScheduleResponse describeSchedule(DescribeScheduleRequest request) { + return next.describeSchedule(request); + } + + @Override + public DescribeWorkflowExecutionResponse describeWorkflowExecution( + DescribeWorkflowExecutionRequest request) { + return next.describeWorkflowExecution(request); + } + + @Override + public StartNexusOperationExecutionResponse startNexusOperationExecution( + @Nonnull StartNexusOperationExecutionRequest request) { + return next.startNexusOperationExecution(request); + } + + @Override + public DescribeNexusOperationExecutionResponse describeNexusOperationExecution( + @Nonnull DescribeNexusOperationExecutionRequest request) { + return next.describeNexusOperationExecution(request); + } + + @Override + public PollNexusOperationExecutionResponse pollNexusOperationExecution( + @Nonnull PollNexusOperationExecutionRequest request, @Nonnull Deadline deadline) { + return next.pollNexusOperationExecution(request, deadline); + } + + @Override + public CompletableFuture pollNexusOperationExecutionAsync( + @Nonnull PollNexusOperationExecutionRequest request, @Nonnull Deadline deadline) { + return next.pollNexusOperationExecutionAsync(request, deadline); + } + + @Override + public CompletableFuture listNexusOperationExecutionsAsync( + @Nonnull ListNexusOperationExecutionsRequest request) { + return next.listNexusOperationExecutionsAsync(request); + } + + @Override + public CountNexusOperationExecutionsResponse countNexusOperationExecutions( + @Nonnull CountNexusOperationExecutionsRequest request) { + return next.countNexusOperationExecutions(request); + } + + @Override + public RequestCancelNexusOperationExecutionResponse requestCancelNexusOperationExecution( + @Nonnull RequestCancelNexusOperationExecutionRequest request) { + return next.requestCancelNexusOperationExecution(request); + } + + @Override + public TerminateNexusOperationExecutionResponse terminateNexusOperationExecution( + @Nonnull TerminateNexusOperationExecutionRequest request) { + return next.terminateNexusOperationExecution(request); + } + + @Override + public DeleteNexusOperationExecutionResponse deleteNexusOperationExecution( + @Nonnull DeleteNexusOperationExecutionRequest request) { + return next.deleteNexusOperationExecution(request); + } + + @Override + @SuppressWarnings("deprecation") + public UpdateWorkerBuildIdCompatibilityResponse updateWorkerBuildIdCompatability( + UpdateWorkerBuildIdCompatibilityRequest request) { + return next.updateWorkerBuildIdCompatability(request); + } + + @Override + public ExecuteMultiOperationResponse executeMultiOperation( + ExecuteMultiOperationRequest request, @Nonnull Deadline deadline) { + ExecuteMultiOperationRequest stored = externalStorage.storeBlocking(request, null); + return externalStorage.retrieveBlocking(next.executeMultiOperation(stored, deadline)); + } + + @Override + public StartActivityExecutionResponse startActivity(StartActivityExecutionRequest request) { + return next.startActivity(request); + } + + @Override + public PollActivityExecutionResponse pollActivity(PollActivityExecutionRequest request) { + return next.pollActivity(request); + } + + @Override + public PollActivityExecutionResponse pollActivity( + PollActivityExecutionRequest request, @Nonnull Deadline deadline) { + return next.pollActivity(request, deadline); + } + + @Override + public CompletableFuture pollActivityAsync( + PollActivityExecutionRequest request, @Nonnull Deadline deadline) { + return next.pollActivityAsync(request, deadline); + } + + @Override + public DescribeActivityExecutionResponse describeActivity( + DescribeActivityExecutionRequest request) { + return next.describeActivity(request); + } + + @Override + public void cancelActivity(RequestCancelActivityExecutionRequest request) { + next.cancelActivity(request); + } + + @Override + public void terminateActivity(TerminateActivityExecutionRequest request) { + next.terminateActivity(request); + } + + @Override + public ListActivityExecutionsResponse listActivities(ListActivityExecutionsRequest request) { + return next.listActivities(request); + } + + @Override + public CompletableFuture listActivitiesAsync( + ListActivityExecutionsRequest request) { + return next.listActivitiesAsync(request); + } + + @Override + public CountActivityExecutionsResponse countActivities(CountActivityExecutionsRequest request) { + return next.countActivities(request); + } + + @Override + @SuppressWarnings("deprecation") + public GetWorkerBuildIdCompatibilityResponse getWorkerBuildIdCompatability( + GetWorkerBuildIdCompatibilityRequest req) { + return next.getWorkerBuildIdCompatability(req); + } + + @Override + @SuppressWarnings("deprecation") + public GetWorkerTaskReachabilityResponse GetWorkerTaskReachability( + GetWorkerTaskReachabilityRequest req) { + return next.GetWorkerTaskReachability(req); + } +} diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java index f28f33abea..fde415fb66 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java @@ -1,5 +1,6 @@ package io.temporal.internal.payload.storage; +import com.google.common.base.Throwables; import com.google.protobuf.Message; import io.temporal.internal.payload.visitor.PayloadVisitor; import io.temporal.internal.payload.visitor.PayloadVisitorOptions; @@ -7,6 +8,7 @@ import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriverTargetInfo; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import javax.annotation.Nullable; /** @@ -56,4 +58,26 @@ public CompletableFuture retrieve(T message) { .build(); return PayloadVisitors.visit(message, options); } + + /** Blocking variant of {@link #store}, for synchronous worker call sites. */ + public T storeBlocking(T message, @Nullable StorageDriverTargetInfo target) { + return join(store(message, target)); + } + + /** Blocking variant of {@link #retrieve}, for synchronous worker call sites. */ + public T retrieveBlocking(T message) { + return join(retrieve(message)); + } + + private static T join(CompletableFuture future) { + try { + return future.join(); + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause != null) { + Throwables.throwIfUnchecked(cause); + } + throw e; + } + } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java index f5b7cb0d29..6190ad9e9d 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java @@ -94,7 +94,12 @@ private Result handleWorkflowTaskWithQuery( logWorkflowTaskToBeProcessed(workflowTask, createdNew); ServiceWorkflowHistoryIterator historyIterator = - new ServiceWorkflowHistoryIterator(service, namespace, workflowTask, metricsScope); + new ServiceWorkflowHistoryIterator( + service, + namespace, + workflowTask, + metricsScope, + options.getExternalStorageMessageConverter()); boolean finalCommand; Result result; diff --git a/temporal-sdk/src/main/java/io/temporal/internal/replay/ServiceWorkflowHistoryIterator.java b/temporal-sdk/src/main/java/io/temporal/internal/replay/ServiceWorkflowHistoryIterator.java index 229b66186e..3aac3c9edb 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/replay/ServiceWorkflowHistoryIterator.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/replay/ServiceWorkflowHistoryIterator.java @@ -12,12 +12,14 @@ import io.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest; import io.temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse; import io.temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponseOrBuilder; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.serviceclient.RpcRetryOptions; import io.temporal.serviceclient.WorkflowServiceStubs; import java.time.Duration; import java.util.Iterator; import java.util.NoSuchElementException; +import javax.annotation.Nullable; /** Supports iteration over history while loading new pages through calls to the service. */ class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator { @@ -29,6 +31,7 @@ class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator { private final Scope metricsScope; private final PollWorkflowTaskQueueResponseOrBuilder task; private final GrpcRetryer grpcRetryer; + private final @Nullable ExternalStorageMessageConverter externalStorage; private Deadline deadline; private Iterator current; ByteString nextPageToken; @@ -38,10 +41,20 @@ class ServiceWorkflowHistoryIterator implements WorkflowHistoryIterator { String namespace, PollWorkflowTaskQueueResponseOrBuilder task, Scope metricsScope) { + this(service, namespace, task, metricsScope, null); + } + + ServiceWorkflowHistoryIterator( + WorkflowServiceStubs service, + String namespace, + PollWorkflowTaskQueueResponseOrBuilder task, + Scope metricsScope, + @Nullable ExternalStorageMessageConverter externalStorage) { this.service = service; this.namespace = namespace; this.task = task; this.metricsScope = metricsScope; + this.externalStorage = externalStorage; // TODO Refactor WorkflowHistoryIteratorTest or WorkflowHistoryIterator to remove this check. // `service == null` shouldn't be allowed as it's needed for a normal functioning of this // class. @@ -64,7 +77,11 @@ public boolean hasNext() { // true. GetWorkflowExecutionHistoryResponse response = queryWorkflowExecutionHistory(); - current = response.getHistory().getEventsList().iterator(); + History history = response.getHistory(); + if (externalStorage != null) { + history = externalStorage.retrieveBlocking(history); + } + current = history.getEventsList().iterator(); nextPageToken = response.getNextPageToken(); // Server can return an empty page, but a valid nextPageToken that contains // more events. diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java index ff528d46b3..324fe98c8c 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java @@ -3,6 +3,7 @@ import static io.temporal.serviceclient.MetricsTag.METRICS_TAGS_CALL_OPTIONS_KEY; import com.google.protobuf.ByteString; +import com.google.protobuf.Message; import com.uber.m3.tally.Scope; import com.uber.m3.tally.Stopwatch; import com.uber.m3.util.Duration; @@ -13,8 +14,11 @@ import io.temporal.internal.activity.ActivityPollResponseToInfo; import io.temporal.internal.common.ProtobufTimeUtils; import io.temporal.internal.logging.LoggerTag; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.internal.worker.ActivityTaskHandler.Result; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.MetricsTag; import io.temporal.serviceclient.WorkflowServiceStubs; import io.temporal.serviceclient.rpcretry.DefaultStubServiceOperationRpcRetryOptions; @@ -27,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -331,6 +336,7 @@ public void handle(ActivityTask task) throws Exception { } private ActivityTaskHandler.Result handleActivity(ActivityTask task, Scope metricsScope) { + task = retrieveInboundPayloads(task); PollActivityTaskQueueResponseOrBuilder pollResponse = task.getResponse(); ByteString taskToken = pollResponse.getTaskToken(); metricsScope @@ -354,7 +360,7 @@ private ActivityTaskHandler.Result handleActivity(ActivityTask task, Scope metri } try { - sendReply(taskToken, result, metricsScope); + sendReply(taskToken, result, metricsScope, activityStorageTarget(pollResponse)); } catch (Exception e) { logExceptionDuringResultReporting(e, pollResponse, result); // TODO this class doesn't report activity success and failure metrics now, instead it's @@ -392,16 +398,21 @@ public Throwable wrapFailure(ActivityTask t, Throwable failure) { // TODO: Suppress warning until the SDK supports deployment @SuppressWarnings("deprecation") private void sendReply( - ByteString taskToken, ActivityTaskHandler.Result response, Scope metricsScope) { + ByteString taskToken, + ActivityTaskHandler.Result response, + Scope metricsScope, + @Nullable StorageDriverTargetInfo storageTarget) { RespondActivityTaskCompletedRequest taskCompleted = response.getTaskCompleted(); if (taskCompleted != null) { RespondActivityTaskCompletedRequest request = - taskCompleted.toBuilder() - .setTaskToken(taskToken) - .setIdentity(options.getIdentity()) - .setNamespace(namespace) - .setWorkerVersion(options.workerVersionStamp()) - .build(); + storeOutboundPayloads( + taskCompleted.toBuilder() + .setTaskToken(taskToken) + .setIdentity(options.getIdentity()) + .setNamespace(namespace) + .setWorkerVersion(options.workerVersionStamp()) + .build(), + storageTarget); grpcRetryer.retry( () -> @@ -414,12 +425,14 @@ private void sendReply( Result.TaskFailedResult taskFailed = response.getTaskFailed(); if (taskFailed != null) { RespondActivityTaskFailedRequest request = - taskFailed.getTaskFailedRequest().toBuilder() - .setTaskToken(taskToken) - .setIdentity(options.getIdentity()) - .setNamespace(namespace) - .setWorkerVersion(options.workerVersionStamp()) - .build(); + storeOutboundPayloads( + taskFailed.getTaskFailedRequest().toBuilder() + .setTaskToken(taskToken) + .setIdentity(options.getIdentity()) + .setNamespace(namespace) + .setWorkerVersion(options.workerVersionStamp()) + .build(), + storageTarget); grpcRetryer.retry( () -> @@ -432,12 +445,14 @@ private void sendReply( RespondActivityTaskCanceledRequest taskCanceled = response.getTaskCanceled(); if (taskCanceled != null) { RespondActivityTaskCanceledRequest request = - taskCanceled.toBuilder() - .setTaskToken(taskToken) - .setIdentity(options.getIdentity()) - .setNamespace(namespace) - .setWorkerVersion(options.workerVersionStamp()) - .build(); + storeOutboundPayloads( + taskCanceled.toBuilder() + .setTaskToken(taskToken) + .setIdentity(options.getIdentity()) + .setNamespace(namespace) + .setWorkerVersion(options.workerVersionStamp()) + .build(), + storageTarget); grpcRetryer.retry( () -> @@ -452,6 +467,39 @@ private void sendReply( // Manual activity completion } + private ActivityTask retrieveInboundPayloads(ActivityTask task) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + if (converter == null) { + return task; + } + PollActivityTaskQueueResponseOrBuilder response = task.getResponse(); + PollActivityTaskQueueResponse built = + response instanceof PollActivityTaskQueueResponse + ? (PollActivityTaskQueueResponse) response + : ((PollActivityTaskQueueResponse.Builder) response).build(); + PollActivityTaskQueueResponse retrieved = converter.retrieveBlocking(built); + return new ActivityTask(retrieved, task.getPermit(), task.getCompletionCallback()); + } + + private T storeOutboundPayloads( + T request, @Nullable StorageDriverTargetInfo target) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + return converter == null ? request : converter.storeBlocking(request, target); + } + + @Nullable + private StorageDriverTargetInfo activityStorageTarget( + PollActivityTaskQueueResponseOrBuilder pollResponse) { + if (options.getExternalStorageMessageConverter() == null) { + return null; + } + return new StorageDriverActivityInfo( + namespace, + pollResponse.getActivityId(), + pollResponse.getWorkflowExecution().getRunId(), + pollResponse.getActivityType().getName()); + } + private void logExceptionDuringResultReporting( Exception e, PollActivityTaskQueueResponseOrBuilder pollResponse, diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java index 3eed1099d3..78f7bebba3 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java @@ -18,9 +18,12 @@ import io.temporal.api.workflowservice.v1.*; import io.temporal.failure.ApplicationFailure; import io.temporal.internal.logging.LoggerTag; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcMessageTooLargeException; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.payload.context.WorkflowSerializationContext; +import io.temporal.payload.storage.StorageDriverTargetInfo; +import io.temporal.payload.storage.StorageDriverWorkflowInfo; import io.temporal.serviceclient.MetricsTag; import io.temporal.serviceclient.RpcRetryOptions; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -381,6 +384,28 @@ public String toString() { options.getIdentity(), namespace, taskQueue); } + private PollWorkflowTaskQueueResponse retrieveInboundPayloads( + PollWorkflowTaskQueueResponse response) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + return converter == null ? response : converter.retrieveBlocking(response); + } + + private RespondWorkflowTaskCompletedRequest storeOutboundPayloads( + RespondWorkflowTaskCompletedRequest request, @Nullable StorageDriverTargetInfo target) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + return converter == null ? request : converter.storeBlocking(request, target); + } + + @Nullable + private StorageDriverTargetInfo workflowStorageTarget( + WorkflowExecution execution, String workflowType) { + if (options.getExternalStorageMessageConverter() == null) { + return null; + } + return new StorageDriverWorkflowInfo( + namespace, execution.getWorkflowId(), execution.getRunId(), workflowType); + } + private class TaskHandlerImpl implements PollTaskExecutor.TaskHandler { final WorkflowTaskHandler handler; @@ -439,7 +464,8 @@ public void handle(WorkflowTask task) throws Exception { Optional nextWFTResponse = Optional.of(workflowTaskResponse); do { - PollWorkflowTaskQueueResponse currentTask = nextWFTResponse.get(); + PollWorkflowTaskQueueResponse currentTask = + retrieveInboundPayloads(nextWFTResponse.get()); nextWFTResponse = Optional.empty(); boolean iterationFailed = false; try { @@ -489,7 +515,8 @@ public void handle(WorkflowTask task) throws Exception { currentTask.getTaskToken(), requestBuilder, result.getRequestRetryOptions(), - workflowTypeScope); + workflowTypeScope, + workflowStorageTarget(workflowExecution, workflowType)); // If we were processing a speculative WFT the server may instruct us that the // task was dropped by resting out event ID. long resetEventId = response.getResetHistoryEventId(); @@ -651,7 +678,8 @@ private RespondWorkflowTaskCompletedResponse sendTaskCompleted( ByteString taskToken, RespondWorkflowTaskCompletedRequest.Builder taskCompleted, RpcRetryOptions retryOptions, - Scope workflowTypeMetricsScope) { + Scope workflowTypeMetricsScope, + @Nullable StorageDriverTargetInfo storageTarget) { GrpcRetryer.GrpcRetryerOptions grpcRetryOptions = new GrpcRetryer.GrpcRetryerOptions( RpcRetryOptions.newBuilder().buildWithDefaultsFrom(retryOptions), null); @@ -674,12 +702,14 @@ private RespondWorkflowTaskCompletedResponse sendTaskCompleted( taskCompleted.setBinaryChecksum(options.getBuildId()); } + RespondWorkflowTaskCompletedRequest request = + storeOutboundPayloads(taskCompleted.build(), storageTarget); return grpcRetryer.retryWithResult( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope) - .respondWorkflowTaskCompleted(taskCompleted.build()), + .respondWorkflowTaskCompleted(request), grpcRetryOptions); } diff --git a/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java b/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java new file mode 100644 index 0000000000..ea14e30396 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java @@ -0,0 +1,48 @@ +package io.temporal.common.converter; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.google.protobuf.ByteString; +import io.temporal.api.common.v1.Payload; +import org.junit.Test; + +/** + * When external storage is not configured, an inbound reference payload reaching value + * deserialization must fail with the clear {@code [TMPRL-1105]} error instead of an opaque decoding + * failure. + */ +public class ExternalStorageReferenceGuardTest { + + private final DataConverter dataConverter = DefaultDataConverter.newDefaultInstance(); + + @Test + public void referencePayloadWithoutConfiguredStorageThrows() { + Payload reference = + Payload.newBuilder() + .putMetadata( + EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/protobuf")) + .setData(ByteString.copyFromUtf8("{}")) + .addExternalPayloads( + Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(1024).build()) + .build(); + + DataConverterException e = + assertThrows( + DataConverterException.class, + () -> dataConverter.fromPayload(reference, String.class, String.class)); + assertTrue(e.getMessage(), e.getMessage().contains("[TMPRL-1105]")); + } + + @Test + public void rawValueBypassesTheGuard() { + Payload reference = + Payload.newBuilder() + .addExternalPayloads( + Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(1024).build()) + .build(); + + RawValue raw = dataConverter.fromPayload(reference, RawValue.class, RawValue.class); + assertTrue(raw.getPayload().getExternalPayloadsCount() > 0); + } +} diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java new file mode 100644 index 0000000000..ffa73a1a74 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -0,0 +1,142 @@ +package io.temporal.internal.payload.storage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.temporal.activity.ActivityInterface; +import io.temporal.activity.ActivityMethod; +import io.temporal.activity.ActivityOptions; +import io.temporal.api.common.v1.Payload; +import io.temporal.client.WorkflowClientOptions; +import io.temporal.payload.storage.ExternalStorageOptions; +import io.temporal.payload.storage.StorageDriver; +import io.temporal.payload.storage.StorageDriverClaim; +import io.temporal.payload.storage.StorageDriverRetrieveContext; +import io.temporal.payload.storage.StorageDriverStoreContext; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Rule; +import org.junit.Test; + +/** + * End-to-end round trip through the worker and client pipelines: with external storage configured + * and a zero threshold (offload everything), a workflow that passes payloads through an activity + * must still complete correctly, and the driver must have actually stored and restored payloads. + */ +public class ExternalStoragePipelineTest { + + private static final InMemoryDriver DRIVER = new InMemoryDriver("test"); + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + SDKTestWorkflowRule.newBuilder() + .setWorkflowTypes(EchoWorkflowImpl.class) + .setActivityImplementations(new EchoActivityImpl()) + .setWorkflowClientOptions( + WorkflowClientOptions.newBuilder() + .setExternalStorage( + ExternalStorageOptions.newBuilder() + .setDriver(DRIVER) + .setPayloadSizeThreshold(0) + .build()) + .build()) + .build(); + + @Test + public void payloadsRoundTripThroughStorage() { + EchoWorkflow workflow = testWorkflowRule.newWorkflowStub(EchoWorkflow.class); + + String result = workflow.run("hello"); + + assertEquals("echo: hello", result); + assertTrue("expected the driver to have stored payloads", DRIVER.stores.get() > 0); + assertTrue("expected the driver to have restored payloads", DRIVER.retrieves.get() > 0); + } + + @WorkflowInterface + public interface EchoWorkflow { + @WorkflowMethod + String run(String input); + } + + @ActivityInterface + public interface EchoActivity { + @ActivityMethod + String echo(String input); + } + + public static class EchoWorkflowImpl implements EchoWorkflow { + private final EchoActivity activity = + Workflow.newActivityStub( + EchoActivity.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()); + + @Override + public String run(String input) { + return activity.echo(input); + } + } + + public static class EchoActivityImpl implements EchoActivity { + @Override + public String echo(String input) { + return "echo: " + input; + } + } + + private static final class InMemoryDriver implements StorageDriver { + private final String name; + private final Map objects = new ConcurrentHashMap<>(); + final AtomicInteger stores = new AtomicInteger(); + final AtomicInteger retrieves = new AtomicInteger(); + private final AtomicInteger counter = new AtomicInteger(); + + InMemoryDriver(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getType() { + return "test.inmemory"; + } + + @Override + public CompletableFuture> store( + StorageDriverStoreContext context, List payloads) { + List claims = new ArrayList<>(payloads.size()); + for (Payload payload : payloads) { + stores.incrementAndGet(); + String key = name + "-" + counter.incrementAndGet(); + objects.put(key, payload); + claims.add(new StorageDriverClaim(Collections.singletonMap("key", key))); + } + return CompletableFuture.completedFuture(claims); + } + + @Override + public CompletableFuture> retrieve( + StorageDriverRetrieveContext context, List claims) { + List payloads = new ArrayList<>(claims.size()); + for (StorageDriverClaim claim : claims) { + retrieves.incrementAndGet(); + payloads.add(objects.get(claim.getClaimData().get("key"))); + } + return CompletableFuture.completedFuture(payloads); + } + } +} From 802f0d0d1daa7acb5f8d51db1cb11aaece6ad3cf Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 17:06:24 -0400 Subject: [PATCH 03/13] feat(extstore): offload activity completion and heartbeat payloads --- .../client/WorkflowClientInternalImpl.java | 13 +-- .../ActivityExecutionContextFactoryImpl.java | 11 ++- .../ActivityExecutionContextImpl.java | 8 +- .../activity/HeartbeatContextImpl.java | 20 ++++- .../internal/client/ActivityClientHelper.java | 31 +++++-- ...ManualActivityCompletionClientFactory.java | 16 ++-- ...alActivityCompletionClientFactoryImpl.java | 12 ++- .../ManualActivityCompletionClientImpl.java | 84 +++++++++++++------ .../internal/worker/SyncActivityWorker.java | 3 +- .../activity/HeartbeatContextImplTest.java | 4 +- .../storage/ExternalStoragePipelineTest.java | 2 + .../TestActivityEnvironmentInternal.java | 3 +- 12 files changed, 148 insertions(+), 59 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java index 71129abd0b..fabf75615f 100644 --- a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java @@ -109,15 +109,15 @@ public static WorkflowClient newInstance( .getOptions() .getMetricsScope() .tagged(MetricsTag.defaultTags(options.getNamespace())); + ExternalStorageOptions externalStorage = options.getExternalStorage(); + ExternalStorageMessageConverter externalStorageConverter = + externalStorage == null ? null : ExternalStorageMessageConverter.create(externalStorage, 1); GenericWorkflowClient genericClient = new GenericWorkflowClientImpl(workflowServiceStubs, metricsScope); - ExternalStorageOptions externalStorage = options.getExternalStorage(); - if (externalStorage != null) { + if (externalStorageConverter != null) { genericClient = new ExternalStorageGenericWorkflowClient( - genericClient, - ExternalStorageMessageConverter.create(externalStorage, 1), - options.getNamespace()); + genericClient, externalStorageConverter, options.getNamespace()); } this.genericClient = genericClient; this.interceptors = options.getInterceptors(); @@ -127,7 +127,8 @@ public static WorkflowClient newInstance( workflowServiceStubs, options.getNamespace(), options.getIdentity(), - options.getDataConverter()); + options.getDataConverter(), + externalStorageConverter); java.time.Duration heartbeatInterval = options.getWorkerHeartbeatInterval(); if (!heartbeatInterval.isNegative()) { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextFactoryImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextFactoryImpl.java index 4acc1d17dd..a173207d4f 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextFactoryImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextFactoryImpl.java @@ -4,6 +4,7 @@ import io.temporal.client.WorkflowClient; import io.temporal.common.converter.DataConverter; import io.temporal.internal.client.external.ManualActivityCompletionClientFactory; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; @@ -21,6 +22,7 @@ public class ActivityExecutionContextFactoryImpl implements ActivityExecutionCon private final DataConverter dataConverter; private final ScheduledExecutorService heartbeatExecutor; private final ManualActivityCompletionClientFactory manualCompletionClientFactory; + private final ExternalStorageMessageConverter externalStorage; private final ConcurrentMap activeContexts = new ConcurrentHashMap<>(); @@ -31,7 +33,8 @@ public ActivityExecutionContextFactoryImpl( Duration maxHeartbeatThrottleInterval, Duration defaultHeartbeatThrottleInterval, DataConverter dataConverter, - ScheduledExecutorService heartbeatExecutor) { + ScheduledExecutorService heartbeatExecutor, + ExternalStorageMessageConverter externalStorage) { this.client = Objects.requireNonNull(client); this.identity = identity; this.namespace = Objects.requireNonNull(namespace); @@ -40,9 +43,10 @@ public ActivityExecutionContextFactoryImpl( Objects.requireNonNull(defaultHeartbeatThrottleInterval); this.dataConverter = Objects.requireNonNull(dataConverter); this.heartbeatExecutor = Objects.requireNonNull(heartbeatExecutor); + this.externalStorage = externalStorage; this.manualCompletionClientFactory = ManualActivityCompletionClientFactory.newFactory( - client.getWorkflowServiceStubs(), namespace, identity, dataConverter); + client.getWorkflowServiceStubs(), namespace, identity, dataConverter, externalStorage); } @Override @@ -63,7 +67,8 @@ public InternalActivityExecutionContext createContext( identity, maxHeartbeatThrottleInterval, defaultHeartbeatThrottleInterval, - () -> cleanupContext(info.getTaskToken(), false)); + () -> cleanupContext(info.getTaskToken(), false), + externalStorage); activeContexts.put(taskToken, context); return context; } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java index 40fe45c326..4cb9e66108 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java @@ -10,6 +10,7 @@ import io.temporal.common.CancellationToken; import io.temporal.common.converter.DataConverter; import io.temporal.internal.client.external.ManualActivityCompletionClientFactory; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; import io.temporal.workflow.Functions; import java.lang.reflect.Type; @@ -18,6 +19,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; /** @@ -55,7 +57,8 @@ class ActivityExecutionContextImpl implements InternalActivityExecutionContext { String identity, Duration maxHeartbeatThrottleInterval, Duration defaultHeartbeatThrottleInterval, - Functions.Proc closeCallback) { + Functions.Proc closeCallback, + @Nullable ExternalStorageMessageConverter externalStorage) { this.client = client; this.activity = activity; this.metricsScope = metricsScope; @@ -73,7 +76,8 @@ class ActivityExecutionContextImpl implements InternalActivityExecutionContext { metricsScope, identity, maxHeartbeatThrottleInterval, - defaultHeartbeatThrottleInterval); + defaultHeartbeatThrottleInterval, + externalStorage); } /** diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java index 91da94ab0a..07ddfafe23 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java @@ -14,7 +14,10 @@ import io.temporal.failure.TimeoutFailure; import io.temporal.internal.client.ActivityClientHelper; import io.temporal.internal.concurrent.structured.CancelSource; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.WorkflowServiceStubs; import java.lang.reflect.Type; import java.time.Duration; @@ -24,6 +27,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +62,7 @@ static long getLocalHeartbeatTimeoutBufferMillis() { private final long heartbeatIntervalMillis; private final DataConverter dataConverter; private final DataConverter dataConverterWithActivityContext; + private final @Nullable ExternalStorageMessageConverter externalStorage; private final Scope metricsScope; private final Optional prevAttemptHeartbeatDetails; @@ -89,7 +94,8 @@ public HeartbeatContextImpl( Scope metricsScope, String identity, Duration maxHeartbeatThrottleInterval, - Duration defaultHeartbeatThrottleInterval) { + Duration defaultHeartbeatThrottleInterval, + @Nullable ExternalStorageMessageConverter externalStorage) { this( service, namespace, @@ -100,6 +106,7 @@ public HeartbeatContextImpl( identity, maxHeartbeatThrottleInterval, defaultHeartbeatThrottleInterval, + externalStorage, getLocalHeartbeatTimeoutBufferMillis()); } @@ -113,10 +120,12 @@ public HeartbeatContextImpl( String identity, Duration maxHeartbeatThrottleInterval, Duration defaultHeartbeatThrottleInterval, + @Nullable ExternalStorageMessageConverter externalStorage, long localHeartbeatTimeoutBufferMillis) { this.service = service; this.metricsScope = metricsScope; this.dataConverter = dataConverter; + this.externalStorage = externalStorage; this.dataConverterWithActivityContext = dataConverter.withContext( new ActivitySerializationContext( @@ -330,6 +339,11 @@ private void checkHeartbeatTimeoutDeadlineLocked() { } } + private StorageDriverTargetInfo activityStorageTarget() { + return new StorageDriverActivityInfo( + namespace, info.getActivityId(), info.getWorkflowRunId(), info.getActivityType()); + } + private void sendHeartbeatRequest(Object details) { try { RecordActivityTaskHeartbeatResponse status = @@ -339,7 +353,9 @@ private void sendHeartbeatRequest(Object details) { identity, info.getTaskToken(), dataConverterWithActivityContext.toPayloads(details), - metricsScope); + metricsScope, + externalStorage, + activityStorageTarget()); if (status.getCancelRequested()) { requestCancelLocked(); } else if (status.getActivityReset()) { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/ActivityClientHelper.java b/temporal-sdk/src/main/java/io/temporal/internal/client/ActivityClientHelper.java index eb3e98107c..9cfd527191 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/ActivityClientHelper.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/ActivityClientHelper.java @@ -12,9 +12,12 @@ import io.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatByIdResponse; import io.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest; import io.temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatResponse; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.WorkflowServiceStubs; import java.util.Optional; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** * Contains methods that could but didn't become a part of the main {@link @@ -30,17 +33,23 @@ public static RecordActivityTaskHeartbeatResponse sendHeartbeatRequest( String identity, byte[] taskToken, Optional payloads, - Scope metricsScope) { - RecordActivityTaskHeartbeatRequest.Builder request = + Scope metricsScope, + @Nullable ExternalStorageMessageConverter externalStorage, + @Nullable StorageDriverTargetInfo storageTarget) { + RecordActivityTaskHeartbeatRequest.Builder builder = RecordActivityTaskHeartbeatRequest.newBuilder() .setTaskToken(ByteString.copyFrom(taskToken)) .setNamespace(namespace) .setIdentity(identity); - payloads.ifPresent(request::setDetails); + payloads.ifPresent(builder::setDetails); + RecordActivityTaskHeartbeatRequest request = builder.build(); + if (externalStorage != null) { + request = externalStorage.storeBlocking(request, storageTarget); + } return service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .recordActivityTaskHeartbeat(request.build()); + .recordActivityTaskHeartbeat(request); } public static RecordActivityTaskHeartbeatByIdResponse recordActivityTaskHeartbeatById( @@ -50,19 +59,25 @@ public static RecordActivityTaskHeartbeatByIdResponse recordActivityTaskHeartbea WorkflowExecution execution, @Nonnull String activityId, Optional payloads, - Scope metricsScope) { + Scope metricsScope, + @Nullable ExternalStorageMessageConverter externalStorage, + @Nullable StorageDriverTargetInfo storageTarget) { Preconditions.checkNotNull(activityId, "Either activity id or task token are required"); - RecordActivityTaskHeartbeatByIdRequest.Builder request = + RecordActivityTaskHeartbeatByIdRequest.Builder builder = RecordActivityTaskHeartbeatByIdRequest.newBuilder() .setRunId(execution.getRunId()) .setWorkflowId(execution.getWorkflowId()) .setActivityId(activityId) .setNamespace(namespace) .setIdentity(identity); - payloads.ifPresent(request::setDetails); + payloads.ifPresent(builder::setDetails); + RecordActivityTaskHeartbeatByIdRequest request = builder.build(); + if (externalStorage != null) { + request = externalStorage.storeBlocking(request, storageTarget); + } return service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .recordActivityTaskHeartbeatById(request.build()); + .recordActivityTaskHeartbeatById(request); } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java index 74eb0a5e7d..9d5f35e777 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java @@ -4,6 +4,7 @@ import io.temporal.activity.ManualActivityCompletionClient; import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.common.converter.DataConverter; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; import io.temporal.serviceclient.WorkflowServiceStubs; import javax.annotation.Nonnull; @@ -11,17 +12,22 @@ public interface ManualActivityCompletionClientFactory { - /** - * Create a {@link ManualActivityCompletionClientFactory} that emits simple {@link - * ManualActivityCompletionClientImpl} implementations - */ static ManualActivityCompletionClientFactory newFactory( @Nonnull WorkflowServiceStubs service, @Nonnull String namespace, @Nonnull String identity, @Nonnull DataConverter dataConverter) { + return newFactory(service, namespace, identity, dataConverter, null); + } + + static ManualActivityCompletionClientFactory newFactory( + @Nonnull WorkflowServiceStubs service, + @Nonnull String namespace, + @Nonnull String identity, + @Nonnull DataConverter dataConverter, + @Nullable ExternalStorageMessageConverter externalStorage) { return new ManualActivityCompletionClientFactoryImpl( - service, namespace, identity, dataConverter); + service, namespace, identity, dataConverter, externalStorage); } ManualActivityCompletionClient getClient(@Nonnull byte[] taskToken, @Nonnull Scope metricsScope); diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java index 6c8237401e..761970b753 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java @@ -5,6 +5,7 @@ import io.temporal.activity.ManualActivityCompletionClient; import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.common.converter.DataConverter; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; import io.temporal.serviceclient.WorkflowServiceStubs; import java.util.Objects; @@ -16,16 +17,19 @@ class ManualActivityCompletionClientFactoryImpl implements ManualActivityComplet private final DataConverter dataConverter; private final String namespace; private final String identity; + private final @Nullable ExternalStorageMessageConverter externalStorage; ManualActivityCompletionClientFactoryImpl( @Nonnull WorkflowServiceStubs service, @Nonnull String namespace, @Nonnull String identity, - @Nonnull DataConverter dataConverter) { + @Nonnull DataConverter dataConverter, + @Nullable ExternalStorageMessageConverter externalStorage) { this.service = Objects.requireNonNull(service); this.namespace = Objects.requireNonNull(namespace); this.identity = Objects.requireNonNull(identity); this.dataConverter = Objects.requireNonNull(dataConverter); + this.externalStorage = externalStorage; } @Override @@ -51,7 +55,8 @@ public ManualActivityCompletionClient getClient( taskToken, null, null, - activitySerializationContext); + activitySerializationContext, + externalStorage); } @Override @@ -80,6 +85,7 @@ public ManualActivityCompletionClient getClient( null, execution, activityId, - activitySerializationContext); + activitySerializationContext, + externalStorage); } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java index 0e68b107b5..7b863b9862 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java @@ -4,6 +4,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.ByteString; +import com.google.protobuf.Message; import com.uber.m3.tally.Scope; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -16,8 +17,11 @@ import io.temporal.failure.CanceledFailure; import io.temporal.internal.client.ActivityClientHelper; import io.temporal.internal.common.OptionsUtils; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.RpcRetryOptions; import io.temporal.serviceclient.WorkflowServiceStubs; import java.util.Optional; @@ -41,6 +45,7 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie private final byte[] taskToken; private final GrpcRetryer grpcRetryer; private final GrpcRetryer.GrpcRetryerOptions replyGrpcRetryerOptions; + private final @Nullable ExternalStorageMessageConverter externalStorage; ManualActivityCompletionClientImpl( @Nonnull WorkflowServiceStubs service, @@ -51,8 +56,10 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie @Nullable byte[] taskToken, @Nullable WorkflowExecution execution, @Nullable String activityId, - @Nullable ActivitySerializationContext context) { + @Nullable ActivitySerializationContext context, + @Nullable ExternalStorageMessageConverter externalStorage) { this.service = service; + this.externalStorage = externalStorage; this.dataConverterWithActivityExecutionContext = context != null ? dataConverter.withContext(context) : dataConverter; this.namespace = namespace; @@ -75,23 +82,35 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie this.activityId = activityId; } + private T storeOutbound(T request) { + return externalStorage == null + ? request + : externalStorage.storeBlocking(request, activityTarget()); + } + + private StorageDriverTargetInfo activityTarget() { + return new StorageDriverActivityInfo( + namespace, activityId, execution != null ? execution.getRunId() : null, null); + } + @Override public void complete(@Nullable Object result) { Optional payloads = dataConverterWithActivityExecutionContext.toPayloads(result); if (taskToken != null) { - RespondActivityTaskCompletedRequest.Builder request = + RespondActivityTaskCompletedRequest.Builder builder = RespondActivityTaskCompletedRequest.newBuilder() .setNamespace(namespace) .setIdentity(identity) .setTaskToken(ByteString.copyFrom(taskToken)); - payloads.ifPresent(request::setResult); + payloads.ifPresent(builder::setResult); + RespondActivityTaskCompletedRequest request = storeOutbound(builder.build()); try { grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .respondActivityTaskCompleted(request.build()), + .respondActivityTaskCompleted(request), replyGrpcRetryerOptions); } catch (Exception e) { processException(e); @@ -100,20 +119,21 @@ public void complete(@Nullable Object result) { if (activityId == null) { throw new IllegalArgumentException("Either activity id or task token are required"); } - RespondActivityTaskCompletedByIdRequest.Builder request = + RespondActivityTaskCompletedByIdRequest.Builder builder = RespondActivityTaskCompletedByIdRequest.newBuilder() .setActivityId(activityId) .setNamespace(namespace) .setWorkflowId(execution.getWorkflowId()) .setRunId(execution.getRunId()); - payloads.ifPresent(request::setResult); + payloads.ifPresent(builder::setResult); + RespondActivityTaskCompletedByIdRequest request = storeOutbound(builder.build()); try { grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .respondActivityTaskCompletedById(request.build()), + .respondActivityTaskCompletedById(request), replyGrpcRetryerOptions); } catch (Exception e) { processException(e); @@ -127,11 +147,13 @@ public void fail(@Nonnull Throwable exception) { // When converting failures reason is class name, details are serialized exception. if (taskToken != null) { RespondActivityTaskFailedRequest request = - RespondActivityTaskFailedRequest.newBuilder() - .setFailure(dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) - .setNamespace(namespace) - .setTaskToken(ByteString.copyFrom(taskToken)) - .build(); + storeOutbound( + RespondActivityTaskFailedRequest.newBuilder() + .setFailure( + dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) + .setNamespace(namespace) + .setTaskToken(ByteString.copyFrom(taskToken)) + .build()); try { grpcRetryer.retry( () -> @@ -153,13 +175,15 @@ public void fail(@Nonnull Throwable exception) { throw new IllegalArgumentException("Either activity id or task token are required"); } RespondActivityTaskFailedByIdRequest request = - RespondActivityTaskFailedByIdRequest.newBuilder() - .setFailure(dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) - .setNamespace(namespace) - .setWorkflowId(execution.getWorkflowId()) - .setRunId(execution.getRunId()) - .setActivityId(activityId) - .build(); + storeOutbound( + RespondActivityTaskFailedByIdRequest.newBuilder() + .setFailure( + dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) + .setNamespace(namespace) + .setWorkflowId(execution.getWorkflowId()) + .setRunId(execution.getRunId()) + .setActivityId(activityId) + .build()); try { grpcRetryer.retry( () -> @@ -185,7 +209,9 @@ public void recordHeartbeat(@Nullable Object details) throws CanceledFailure { identity, taskToken, dataConverterWithActivityExecutionContext.toPayloads(details), - metricsScope); + metricsScope, + externalStorage, + activityTarget()); if (status.getCancelRequested()) { throw new ActivityCanceledException(); } else if (status.getActivityReset()) { @@ -202,7 +228,9 @@ public void recordHeartbeat(@Nullable Object details) throws CanceledFailure { execution, activityId, dataConverterWithActivityExecutionContext.toPayloads(details), - metricsScope); + metricsScope, + externalStorage, + activityTarget()); if (status.getCancelRequested()) { throw new ActivityCanceledException(); } else if (status.getActivityReset()) { @@ -221,18 +249,19 @@ public void reportCancellation(@Nullable Object details) { Optional convertedDetails = dataConverterWithActivityExecutionContext.toPayloads(details); if (taskToken != null) { - RespondActivityTaskCanceledRequest.Builder request = + RespondActivityTaskCanceledRequest.Builder builder = RespondActivityTaskCanceledRequest.newBuilder() .setNamespace(namespace) .setTaskToken(ByteString.copyFrom(taskToken)); - convertedDetails.ifPresent(request::setDetails); + convertedDetails.ifPresent(builder::setDetails); + RespondActivityTaskCanceledRequest request = storeOutbound(builder.build()); try { grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .respondActivityTaskCanceled(request.build()), + .respondActivityTaskCanceled(request), replyGrpcRetryerOptions); } catch (Exception e) { // There is nothing that can be done at this point. @@ -243,20 +272,21 @@ public void reportCancellation(@Nullable Object details) { if (activityId == null) { throw new IllegalArgumentException("Either activity id or task token are required"); } - RespondActivityTaskCanceledByIdRequest.Builder request = + RespondActivityTaskCanceledByIdRequest.Builder builder = RespondActivityTaskCanceledByIdRequest.newBuilder() .setNamespace(namespace) .setWorkflowId(execution.getWorkflowId()) .setRunId(OptionsUtils.safeGet(execution.getRunId())) .setActivityId(activityId); - convertedDetails.ifPresent(request::setDetails); + convertedDetails.ifPresent(builder::setDetails); + RespondActivityTaskCanceledByIdRequest request = storeOutbound(builder.build()); try { grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .respondActivityTaskCanceledById(request.build()), + .respondActivityTaskCanceledById(request), replyGrpcRetryerOptions); } catch (Exception e) { // There is nothing that can be done at this point. diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/SyncActivityWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/SyncActivityWorker.java index 94d2f5dee3..15de1ce397 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/SyncActivityWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/SyncActivityWorker.java @@ -59,7 +59,8 @@ public SyncActivityWorker( options.getMaxHeartbeatThrottleInterval(), options.getDefaultHeartbeatThrottleInterval(), options.getDataConverter(), - heartbeatExecutor); + heartbeatExecutor, + options.getExternalStorageMessageConverter()); this.taskHandler = new ActivityTaskHandlerImpl( namespace, diff --git a/temporal-sdk/src/test/java/io/temporal/internal/activity/HeartbeatContextImplTest.java b/temporal-sdk/src/test/java/io/temporal/internal/activity/HeartbeatContextImplTest.java index 1379aed154..35f01926ed 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/activity/HeartbeatContextImplTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/activity/HeartbeatContextImplTest.java @@ -329,7 +329,8 @@ public void factoryCancelByTaskTokenCompletesCancellationToken() { Duration.ofSeconds(60), Duration.ofSeconds(30), GlobalDataConverter.get(), - heartbeatExecutor); + heartbeatExecutor, + null); ActivityInfoInternal info = activityInfoWithHeartbeatTimeout(Duration.ofSeconds(10)); InternalActivityExecutionContext context = @@ -363,6 +364,7 @@ private HeartbeatContextImpl createHeartbeatContext( "test-identity", maxHeartbeatThrottleInterval, defaultHeartbeatThrottleInterval, + null, TEST_BUFFER_MILLIS); } diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java index ffa73a1a74..8936090ed7 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import io.temporal.activity.Activity; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; import io.temporal.activity.ActivityOptions; @@ -90,6 +91,7 @@ public String run(String input) { public static class EchoActivityImpl implements EchoActivity { @Override public String echo(String input) { + Activity.getExecutionContext().heartbeat("heartbeat: " + input); return "echo: " + input; } } diff --git a/temporal-testing/src/main/java/io/temporal/testing/TestActivityEnvironmentInternal.java b/temporal-testing/src/main/java/io/temporal/testing/TestActivityEnvironmentInternal.java index 1d6ebb92de..360ac63867 100644 --- a/temporal-testing/src/main/java/io/temporal/testing/TestActivityEnvironmentInternal.java +++ b/temporal-testing/src/main/java/io/temporal/testing/TestActivityEnvironmentInternal.java @@ -109,7 +109,8 @@ public TestActivityEnvironmentInternal(@Nullable TestEnvironmentOptions options) WorkerOptions.getDefaultInstance().getMaxHeartbeatThrottleInterval(), WorkerOptions.getDefaultInstance().getDefaultHeartbeatThrottleInterval(), testEnvironmentOptions.getWorkflowClientOptions().getDataConverter(), - heartbeatExecutor); + heartbeatExecutor, + null); activityTaskHandler = new ActivityTaskHandlerImpl( testEnvironmentOptions.getWorkflowClientOptions().getNamespace(), From b366b7a3ccb532c5e203eb3ee893599a3b6ea851 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 17:17:31 -0400 Subject: [PATCH 04/13] feat(extstore): offload nexus worker task payloads --- .../temporal/internal/worker/NexusWorker.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java index 33416a807b..7355f07a8a 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java @@ -4,6 +4,7 @@ import static io.temporal.serviceclient.MetricsTag.TASK_FAILURE_TYPE; import com.google.protobuf.ByteString; +import com.google.protobuf.Message; import com.uber.m3.tally.Scope; import com.uber.m3.tally.Stopwatch; import com.uber.m3.util.Duration; @@ -17,6 +18,7 @@ import io.temporal.internal.common.NexusUtil; import io.temporal.internal.common.ProtobufTimeUtils; import io.temporal.internal.logging.LoggerTag; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.serviceclient.MetricsTag; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -304,6 +306,7 @@ private String getNexusTaskOperation(PollNexusTaskQueueResponseOrBuilder pollRes @Override public void handle(NexusTask task) { + task = retrieveInboundPayloads(task); PollNexusTaskQueueResponseOrBuilder pollResponse = task.getResponse(); // Extract service and operation from the request and set them as MDC and metrics // scope tags. If the request does not have a service or operation, do not set the tags. @@ -485,12 +488,13 @@ private void sendReply( taskResponse = getResponseForOldServer(taskResponse); } RespondNexusTaskCompletedRequest request = - RespondNexusTaskCompletedRequest.newBuilder() - .setTaskToken(taskToken) - .setIdentity(options.getIdentity()) - .setNamespace(namespace) - .setResponse(taskResponse) - .build(); + storeOutbound( + RespondNexusTaskCompletedRequest.newBuilder() + .setTaskToken(taskToken) + .setIdentity(options.getIdentity()) + .setNamespace(namespace) + .setResponse(taskResponse) + .build()); grpcRetryer.retry( () -> @@ -512,17 +516,37 @@ private void sendReply( } else { request.setError(NexusUtil.handlerErrorToNexusError(handlerException, dataConverter)); } + RespondNexusTaskFailedRequest failedRequest = storeOutbound(request.build()); grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, metricsScope) - .respondNexusTaskFailed(request.build()), + .respondNexusTaskFailed(failedRequest), replyGrpcRetryerOptions); } else { throw new IllegalArgumentException("[BUG] Either response or failure must be set"); } } } + + private NexusTask retrieveInboundPayloads(NexusTask task) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + if (converter == null) { + return task; + } + PollNexusTaskQueueResponseOrBuilder response = task.getResponse(); + PollNexusTaskQueueResponse built = + response instanceof PollNexusTaskQueueResponse + ? (PollNexusTaskQueueResponse) response + : ((PollNexusTaskQueueResponse.Builder) response).build(); + PollNexusTaskQueueResponse retrieved = converter.retrieveBlocking(built); + return new NexusTask(retrieved, task.getPermit(), task.getCompletionCallback()); + } + + private T storeOutbound(T request) { + ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); + return converter == null ? request : converter.storeBlocking(request, null); + } } } From 1077c1581a278363475d8e7468da8a71161d2876 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 17:37:09 -0400 Subject: [PATCH 05/13] feat(extstore): offload remaining client RPCs and WFT query/failure payloads --- .../ExternalStorageGenericWorkflowClient.java | 39 ++++++++++--------- .../internal/worker/WorkflowWorker.java | 34 +++++++++++----- .../storage/ExternalStoragePipelineTest.java | 14 ++++++- 3 files changed, 57 insertions(+), 30 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java index f2a5df1b56..8ef56b88de 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java @@ -131,13 +131,13 @@ public CompletableFuture getWorkflowExecuti @Override public ListWorkflowExecutionsResponse listWorkflowExecutions( ListWorkflowExecutionsRequest listRequest) { - return next.listWorkflowExecutions(listRequest); + return externalStorage.retrieveBlocking(next.listWorkflowExecutions(listRequest)); } @Override public CompletableFuture listWorkflowExecutionsAsync( ListWorkflowExecutionsRequest listRequest) { - return next.listWorkflowExecutionsAsync(listRequest); + return next.listWorkflowExecutionsAsync(listRequest).thenCompose(externalStorage::retrieve); } @Override @@ -148,17 +148,17 @@ public CountWorkflowExecutionsResponse countWorkflowExecutions( @Override public CreateScheduleResponse createSchedule(CreateScheduleRequest request) { - return next.createSchedule(request); + return next.createSchedule(externalStorage.storeBlocking(request, null)); } @Override public CompletableFuture listSchedulesAsync(ListSchedulesRequest request) { - return next.listSchedulesAsync(request); + return next.listSchedulesAsync(request).thenCompose(externalStorage::retrieve); } @Override public UpdateScheduleResponse updateSchedule(UpdateScheduleRequest request) { - return next.updateSchedule(request); + return next.updateSchedule(externalStorage.storeBlocking(request, null)); } @Override @@ -173,43 +173,44 @@ public DeleteScheduleResponse deleteSchedule(DeleteScheduleRequest request) { @Override public DescribeScheduleResponse describeSchedule(DescribeScheduleRequest request) { - return next.describeSchedule(request); + return externalStorage.retrieveBlocking(next.describeSchedule(request)); } @Override public DescribeWorkflowExecutionResponse describeWorkflowExecution( DescribeWorkflowExecutionRequest request) { - return next.describeWorkflowExecution(request); + return externalStorage.retrieveBlocking(next.describeWorkflowExecution(request)); } @Override public StartNexusOperationExecutionResponse startNexusOperationExecution( @Nonnull StartNexusOperationExecutionRequest request) { - return next.startNexusOperationExecution(request); + return next.startNexusOperationExecution(externalStorage.storeBlocking(request, null)); } @Override public DescribeNexusOperationExecutionResponse describeNexusOperationExecution( @Nonnull DescribeNexusOperationExecutionRequest request) { - return next.describeNexusOperationExecution(request); + return externalStorage.retrieveBlocking(next.describeNexusOperationExecution(request)); } @Override public PollNexusOperationExecutionResponse pollNexusOperationExecution( @Nonnull PollNexusOperationExecutionRequest request, @Nonnull Deadline deadline) { - return next.pollNexusOperationExecution(request, deadline); + return externalStorage.retrieveBlocking(next.pollNexusOperationExecution(request, deadline)); } @Override public CompletableFuture pollNexusOperationExecutionAsync( @Nonnull PollNexusOperationExecutionRequest request, @Nonnull Deadline deadline) { - return next.pollNexusOperationExecutionAsync(request, deadline); + return next.pollNexusOperationExecutionAsync(request, deadline) + .thenCompose(externalStorage::retrieve); } @Override public CompletableFuture listNexusOperationExecutionsAsync( @Nonnull ListNexusOperationExecutionsRequest request) { - return next.listNexusOperationExecutionsAsync(request); + return next.listNexusOperationExecutionsAsync(request).thenCompose(externalStorage::retrieve); } @Override @@ -252,30 +253,30 @@ public ExecuteMultiOperationResponse executeMultiOperation( @Override public StartActivityExecutionResponse startActivity(StartActivityExecutionRequest request) { - return next.startActivity(request); + return next.startActivity(externalStorage.storeBlocking(request, null)); } @Override public PollActivityExecutionResponse pollActivity(PollActivityExecutionRequest request) { - return next.pollActivity(request); + return externalStorage.retrieveBlocking(next.pollActivity(request)); } @Override public PollActivityExecutionResponse pollActivity( PollActivityExecutionRequest request, @Nonnull Deadline deadline) { - return next.pollActivity(request, deadline); + return externalStorage.retrieveBlocking(next.pollActivity(request, deadline)); } @Override public CompletableFuture pollActivityAsync( PollActivityExecutionRequest request, @Nonnull Deadline deadline) { - return next.pollActivityAsync(request, deadline); + return next.pollActivityAsync(request, deadline).thenCompose(externalStorage::retrieve); } @Override public DescribeActivityExecutionResponse describeActivity( DescribeActivityExecutionRequest request) { - return next.describeActivity(request); + return externalStorage.retrieveBlocking(next.describeActivity(request)); } @Override @@ -290,13 +291,13 @@ public void terminateActivity(TerminateActivityExecutionRequest request) { @Override public ListActivityExecutionsResponse listActivities(ListActivityExecutionsRequest request) { - return next.listActivities(request); + return externalStorage.retrieveBlocking(next.listActivities(request)); } @Override public CompletableFuture listActivitiesAsync( ListActivityExecutionsRequest request) { - return next.listActivitiesAsync(request); + return next.listActivitiesAsync(request).thenCompose(externalStorage::retrieve); } @Override diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java index 78f7bebba3..a8e4c6b953 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java @@ -390,8 +390,8 @@ private PollWorkflowTaskQueueResponse retrieveInboundPayloads( return converter == null ? response : converter.retrieveBlocking(response); } - private RespondWorkflowTaskCompletedRequest storeOutboundPayloads( - RespondWorkflowTaskCompletedRequest request, @Nullable StorageDriverTargetInfo target) { + private T storeOutboundPayloads( + T request, @Nullable StorageDriverTargetInfo target) { ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); return converter == null ? request : converter.storeBlocking(request, target); } @@ -479,7 +479,10 @@ public void handle(WorkflowTask task) throws Exception { if (queryCompleted != null) { try { sendDirectQueryCompletedResponse( - currentTask.getTaskToken(), queryCompleted.toBuilder(), workflowTypeScope); + currentTask.getTaskToken(), + queryCompleted.toBuilder(), + workflowTypeScope, + workflowStorageTarget(workflowExecution, workflowType)); } catch (StatusRuntimeException e) { GrpcMessageTooLargeException tooLargeException = GrpcMessageTooLargeException.tryWrap(e); @@ -499,7 +502,10 @@ public void handle(WorkflowTask task) throws Exception { .setErrorMessage(failure.getMessage()) .setFailure(failure); sendDirectQueryCompletedResponse( - currentTask.getTaskToken(), queryFailedBuilder, workflowTypeScope); + currentTask.getTaskToken(), + queryFailedBuilder, + workflowTypeScope, + workflowStorageTarget(workflowExecution, workflowType)); } } else { try { @@ -536,7 +542,8 @@ public void handle(WorkflowTask task) throws Exception { currentTask.getTaskToken(), taskFailed.toBuilder(), result.getRequestRetryOptions(), - workflowTypeScope); + workflowTypeScope, + workflowStorageTarget(workflowExecution, workflowType)); } // Apply post-completion metrics only if runnable present and the above succeeded @@ -573,7 +580,8 @@ public void handle(WorkflowTask task) throws Exception { currentTask.getTaskToken(), taskFailedBuilder, result.getRequestRetryOptions(), - workflowTypeScope); + workflowTypeScope, + workflowStorageTarget(workflowExecution, workflowType)); } } } catch (Exception e) { @@ -718,7 +726,8 @@ private void sendTaskFailed( ByteString taskToken, RespondWorkflowTaskFailedRequest.Builder taskFailed, RpcRetryOptions retryOptions, - Scope workflowTypeMetricsScope) { + Scope workflowTypeMetricsScope, + @Nullable StorageDriverTargetInfo storageTarget) { GrpcRetryer.GrpcRetryerOptions grpcRetryOptions = new GrpcRetryer.GrpcRetryerOptions( RpcRetryOptions.newBuilder().buildWithDefaultsFrom(retryOptions), null); @@ -732,25 +741,30 @@ private void sendTaskFailed( taskFailed.setWorkerVersion(options.workerVersionStamp()); } + RespondWorkflowTaskFailedRequest request = + storeOutboundPayloads(taskFailed.build(), storageTarget); grpcRetryer.retry( () -> service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope) - .respondWorkflowTaskFailed(taskFailed.build()), + .respondWorkflowTaskFailed(request), grpcRetryOptions); } private void sendDirectQueryCompletedResponse( ByteString taskToken, RespondQueryTaskCompletedRequest.Builder queryCompleted, - Scope workflowTypeMetricsScope) { + Scope workflowTypeMetricsScope, + @Nullable StorageDriverTargetInfo storageTarget) { queryCompleted.setTaskToken(taskToken).setNamespace(namespace); + RespondQueryTaskCompletedRequest request = + storeOutboundPayloads(queryCompleted.build(), storageTarget); // Do not retry query response service .blockingStub() .withOption(METRICS_TAGS_CALL_OPTIONS_KEY, workflowTypeMetricsScope) - .respondQueryTaskCompleted(queryCompleted.build()); + .respondQueryTaskCompleted(request); } private void logExceptionDuringResultReporting( diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java index 8936090ed7..98fbef3634 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -15,6 +15,7 @@ import io.temporal.payload.storage.StorageDriverRetrieveContext; import io.temporal.payload.storage.StorageDriverStoreContext; import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.QueryMethod; import io.temporal.workflow.Workflow; import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; @@ -60,6 +61,7 @@ public void payloadsRoundTripThroughStorage() { String result = workflow.run("hello"); assertEquals("echo: hello", result); + assertEquals("echo: hello", workflow.lastResult()); assertTrue("expected the driver to have stored payloads", DRIVER.stores.get() > 0); assertTrue("expected the driver to have restored payloads", DRIVER.retrieves.get() > 0); } @@ -68,6 +70,9 @@ public void payloadsRoundTripThroughStorage() { public interface EchoWorkflow { @WorkflowMethod String run(String input); + + @QueryMethod + String lastResult(); } @ActivityInterface @@ -81,10 +86,17 @@ public static class EchoWorkflowImpl implements EchoWorkflow { Workflow.newActivityStub( EchoActivity.class, ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()); + private String lastResult = ""; @Override public String run(String input) { - return activity.echo(input); + lastResult = activity.echo(input); + return lastResult; + } + + @Override + public String lastResult() { + return lastResult; } } From 347cd020bf58e611dbb2bae73e7027bb49d888c2 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 17:57:28 -0400 Subject: [PATCH 06/13] add clientWithoutExternalStorageReadingAReferenceFails test --- .../storage/ExternalStoragePipelineTest.java | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java index 98fbef3634..7d221cc60c 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -1,6 +1,7 @@ package io.temporal.internal.payload.storage; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import io.temporal.activity.Activity; @@ -8,7 +9,10 @@ import io.temporal.activity.ActivityMethod; import io.temporal.activity.ActivityOptions; import io.temporal.api.common.v1.Payload; +import io.temporal.client.WorkflowClient; import io.temporal.client.WorkflowClientOptions; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriver; import io.temporal.payload.storage.StorageDriverClaim; @@ -37,7 +41,7 @@ */ public class ExternalStoragePipelineTest { - private static final InMemoryDriver DRIVER = new InMemoryDriver("test"); + private final InMemoryDriver driver = new InMemoryDriver("test"); @Rule public SDKTestWorkflowRule testWorkflowRule = @@ -48,7 +52,7 @@ public class ExternalStoragePipelineTest { WorkflowClientOptions.newBuilder() .setExternalStorage( ExternalStorageOptions.newBuilder() - .setDriver(DRIVER) + .setDriver(driver) .setPayloadSizeThreshold(0) .build()) .build()) @@ -62,8 +66,43 @@ public void payloadsRoundTripThroughStorage() { assertEquals("echo: hello", result); assertEquals("echo: hello", workflow.lastResult()); - assertTrue("expected the driver to have stored payloads", DRIVER.stores.get() > 0); - assertTrue("expected the driver to have restored payloads", DRIVER.retrieves.get() > 0); + assertTrue("expected the driver to have stored payloads", driver.stores.get() > 0); + assertTrue("expected the driver to have restored payloads", driver.retrieves.get() > 0); + } + + @Test + public void clientWithoutExternalStorageReadingAReferenceFails() { + String workflowId = "extstore-not-configured-" + testWorkflowRule.getTaskQueue(); + EchoWorkflow workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + EchoWorkflow.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId(workflowId) + .build()); + workflow.run("hello"); + + WorkflowClient withoutStorage = + WorkflowClient.newInstance( + testWorkflowRule.getWorkflowServiceStubs(), + WorkflowClientOptions.newBuilder() + .setNamespace(testWorkflowRule.getTestEnvironment().getNamespace()) + .build()); + WorkflowStub stub = withoutStorage.newUntypedWorkflowStub(workflowId); + + Exception e = assertThrows(Exception.class, () -> stub.getResult(String.class)); + assertTrue(e.toString(), chainContains(e, "[TMPRL-1105]")); + } + + private static boolean chainContains(Throwable t, String needle) { + for (Throwable c = t; c != null; c = c.getCause()) { + if (c.getMessage() != null && c.getMessage().contains(needle)) { + return true; + } + } + return false; } @WorkflowInterface From e2324dcd80bafcc3ae0be05bd6ac32ae87efb76e Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 29 Jul 2026 20:53:15 -0400 Subject: [PATCH 07/13] fix(extstore): resolve external-storage references during offline replay --- .../replay/ReplayWorkflowTaskHandler.java | 5 +++ .../internal/worker/WorkflowWorker.java | 9 +--- .../storage/ExternalStoragePipelineTest.java | 45 +++++++++++++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java index 6190ad9e9d..56ca5188cf 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java @@ -23,6 +23,7 @@ import io.temporal.common.converter.DataConverter; import io.temporal.internal.common.ProtobufTimeUtils; import io.temporal.internal.common.WorkflowExecutionUtils; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.worker.*; import io.temporal.payload.context.WorkflowSerializationContext; import io.temporal.serviceclient.MetricsTag; @@ -77,6 +78,10 @@ public WorkflowTaskHandler.Result handleWorkflowTask(PollWorkflowTaskQueueRespon String workflowType = workflowTask.getWorkflowType().getName(); Scope metricsScope = options.getMetricsScope().tagged(ImmutableMap.of(MetricsTag.WORKFLOW_TYPE, workflowType)); + ExternalStorageMessageConverter externalStorage = options.getExternalStorageMessageConverter(); + if (externalStorage != null) { + workflowTask = externalStorage.retrieveBlocking(workflowTask); + } return handleWorkflowTaskWithQuery(workflowTask.toBuilder(), metricsScope); } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java index a8e4c6b953..f6fd848feb 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/WorkflowWorker.java @@ -384,12 +384,6 @@ public String toString() { options.getIdentity(), namespace, taskQueue); } - private PollWorkflowTaskQueueResponse retrieveInboundPayloads( - PollWorkflowTaskQueueResponse response) { - ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); - return converter == null ? response : converter.retrieveBlocking(response); - } - private T storeOutboundPayloads( T request, @Nullable StorageDriverTargetInfo target) { ExternalStorageMessageConverter converter = options.getExternalStorageMessageConverter(); @@ -464,8 +458,7 @@ public void handle(WorkflowTask task) throws Exception { Optional nextWFTResponse = Optional.of(workflowTaskResponse); do { - PollWorkflowTaskQueueResponse currentTask = - retrieveInboundPayloads(nextWFTResponse.get()); + PollWorkflowTaskQueueResponse currentTask = nextWFTResponse.get(); nextWFTResponse = Optional.empty(); boolean iterationFailed = false; try { diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java index 7d221cc60c..943dce190f 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -13,11 +13,15 @@ import io.temporal.client.WorkflowClientOptions; import io.temporal.client.WorkflowOptions; import io.temporal.client.WorkflowStub; +import io.temporal.common.WorkflowExecutionHistory; import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriver; import io.temporal.payload.storage.StorageDriverClaim; import io.temporal.payload.storage.StorageDriverRetrieveContext; import io.temporal.payload.storage.StorageDriverStoreContext; +import io.temporal.testing.TestEnvironmentOptions; +import io.temporal.testing.TestWorkflowEnvironment; +import io.temporal.testing.WorkflowReplayer; import io.temporal.testing.internal.SDKTestWorkflowRule; import io.temporal.workflow.QueryMethod; import io.temporal.workflow.Workflow; @@ -96,6 +100,47 @@ public void clientWithoutExternalStorageReadingAReferenceFails() { assertTrue(e.toString(), chainContains(e, "[TMPRL-1105]")); } + @Test + public void offlineReplayerResolvesReferences() throws Exception { + String workflowId = "extstore-replay-" + testWorkflowRule.getTaskQueue(); + EchoWorkflow workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + EchoWorkflow.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId(workflowId) + .build()); + workflow.run("hello"); + + WorkflowClient withoutStorage = + WorkflowClient.newInstance( + testWorkflowRule.getWorkflowServiceStubs(), + WorkflowClientOptions.newBuilder() + .setNamespace(testWorkflowRule.getTestEnvironment().getNamespace()) + .build()); + WorkflowExecutionHistory rawHistory = withoutStorage.fetchHistory(workflowId); + + TestWorkflowEnvironment replayEnv = + TestWorkflowEnvironment.newInstance( + TestEnvironmentOptions.newBuilder() + .setWorkflowClientOptions( + WorkflowClientOptions.newBuilder() + .setExternalStorage( + ExternalStorageOptions.newBuilder() + .setDriver(driver) + .setPayloadSizeThreshold(0) + .build()) + .build()) + .build()); + try { + WorkflowReplayer.replayWorkflowExecution(rawHistory, replayEnv, EchoWorkflowImpl.class); + } finally { + replayEnv.close(); + } + } + private static boolean chainContains(Throwable t, String needle) { for (Throwable c = t; c != null; c = c.getCause()) { if (c.getMessage() != null && c.getMessage().contains(needle)) { From 0d499a61386a19d018eeec42338cd22b267214b0 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 12:59:35 -0400 Subject: [PATCH 08/13] fix(extstore): release Nexus slots on retrieval failure --- .../temporal/internal/worker/NexusWorker.java | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java index 7355f07a8a..594386df3c 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/NexusWorker.java @@ -306,29 +306,30 @@ private String getNexusTaskOperation(PollNexusTaskQueueResponseOrBuilder pollRes @Override public void handle(NexusTask task) { - task = retrieveInboundPayloads(task); - PollNexusTaskQueueResponseOrBuilder pollResponse = task.getResponse(); - // Extract service and operation from the request and set them as MDC and metrics - // scope tags. If the request does not have a service or operation, do not set the tags. - // If we don't know how to handle the task, we will fail the task further down the line. - Scope metricsScope = workerMetricsScope; - String service = getNexusTaskService(pollResponse); - if (!service.isEmpty()) { - MDC.put(LoggerTag.NEXUS_SERVICE, service); - metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_SERVICE, service)); - } - String operation = getNexusTaskOperation(pollResponse); - if (!operation.isEmpty()) { - MDC.put(LoggerTag.NEXUS_OPERATION, operation); - metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_OPERATION, operation)); - } - slotSupplier.markSlotUsed( - new NexusSlotInfo( - service, operation, taskQueue, options.getIdentity(), options.getBuildId()), - task.getPermit()); - boolean taskFailed = false; try { + task = retrieveInboundPayloads(task); + PollNexusTaskQueueResponseOrBuilder pollResponse = task.getResponse(); + // Extract service and operation from the request and set them as MDC and metrics + // scope tags. If the request does not have a service or operation, do not set the tags. + // If we don't know how to handle the task, we will fail the task further down the line. + Scope metricsScope = workerMetricsScope; + String service = getNexusTaskService(pollResponse); + if (!service.isEmpty()) { + MDC.put(LoggerTag.NEXUS_SERVICE, service); + metricsScope = metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_SERVICE, service)); + } + String operation = getNexusTaskOperation(pollResponse); + if (!operation.isEmpty()) { + MDC.put(LoggerTag.NEXUS_OPERATION, operation); + metricsScope = + metricsScope.tagged(ImmutableMap.of(MetricsTag.NEXUS_OPERATION, operation)); + } + slotSupplier.markSlotUsed( + new NexusSlotInfo( + service, operation, taskQueue, options.getIdentity(), options.getBuildId()), + task.getPermit()); + taskFailed = handleNexusTask(task, metricsScope); } catch (Throwable e) { taskFailed = true; From eb9443446022e3fc19195fd9fc489b9efb336dc0 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 13:27:08 -0400 Subject: [PATCH 09/13] fix(extstore): preserve missing storage failures --- .../PayloadAndFailureDataConverter.java | 6 ++-- .../activity/ActivityTaskExecutors.java | 6 ++++ .../internal/nexus/NexusTaskHandlerImpl.java | 11 +++++++ ...ExternalStorageNotConfiguredException.java | 32 +++++++++++++++++++ .../replay/ReplayWorkflowTaskHandler.java | 7 ++++ .../internal/sync/SignalDispatcher.java | 6 ++++ .../sync/WorkflowExecutionHandler.java | 6 ++++ .../ExternalStorageReferenceGuardTest.java | 6 ++-- 8 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageNotConfiguredException.java diff --git a/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java b/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java index 09460c46dd..adede8bf53 100644 --- a/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java +++ b/temporal-sdk/src/main/java/io/temporal/common/converter/PayloadAndFailureDataConverter.java @@ -8,6 +8,7 @@ import io.temporal.api.common.v1.Payloads; import io.temporal.api.failure.v1.Failure; import io.temporal.failure.DefaultFailureConverter; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.payload.context.SerializationContext; import java.lang.reflect.Type; import java.util.*; @@ -72,10 +73,7 @@ public T fromPayload(Payload payload, Class valueClass, Type valueType) } if (payload.getExternalPayloadsCount() > 0) { - throw new DataConverterException( - "[TMPRL-1105] Encountered an external-storage reference payload but external storage is not " - + "configured. Configure WorkflowClientOptions.Builder.setExternalStorage(...) with a " - + "driver able to retrieve it."); + throw new ExternalStorageNotConfiguredException(); } try { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java index 7789bd2716..cf94800017 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityTaskExecutors.java @@ -17,6 +17,7 @@ import io.temporal.common.interceptors.Header; import io.temporal.common.interceptors.WorkerInterceptor; import io.temporal.internal.common.FailureUtils; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.internal.worker.ActivityTaskHandler; import io.temporal.payload.context.ActivitySerializationContext; import io.temporal.serviceclient.CheckedExceptionWrapper; @@ -95,6 +96,11 @@ public ActivityTaskHandler.Result execute(ActivityInfoInternal info, Scope metri return this.constructSuccessfulResultValue(info, result, dataConverterWithActivityContext); } catch (Throwable e) { Throwable ex = CheckedExceptionWrapper.unwrap(e); + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(ex); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } boolean local = info.isLocal(); if (ex instanceof ActivityCanceledException) { log.info( diff --git a/temporal-sdk/src/main/java/io/temporal/internal/nexus/NexusTaskHandlerImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/nexus/NexusTaskHandlerImpl.java index adaecca330..91a50733ca 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/nexus/NexusTaskHandlerImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/nexus/NexusTaskHandlerImpl.java @@ -22,6 +22,7 @@ import io.temporal.internal.common.InternalUtils; import io.temporal.internal.common.LinkConverter; import io.temporal.internal.common.NexusUtil; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.internal.worker.NexusTask; import io.temporal.internal.worker.NexusTaskHandler; import io.temporal.internal.worker.ShutdownManager; @@ -136,8 +137,18 @@ public Result handle(NexusTask task, Scope metricsScope) throws TimeoutException (Throwable) null); } } catch (HandlerException e) { + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(e); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } return new Result(e); } catch (Throwable e) { + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(e); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } return new Result( new HandlerException(HandlerException.ErrorType.INTERNAL, "internal handler error", e)); } finally { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageNotConfiguredException.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageNotConfiguredException.java new file mode 100644 index 0000000000..bbd7afc18b --- /dev/null +++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageNotConfiguredException.java @@ -0,0 +1,32 @@ +package io.temporal.internal.payload.storage; + +import io.temporal.common.converter.DataConverterException; +import javax.annotation.Nullable; + +/** + * Signals that an external-storage reference reached a data converter without storage configured. + */ +public final class ExternalStorageNotConfiguredException extends DataConverterException { + public ExternalStorageNotConfiguredException() { + super( + "[TMPRL-1105] Encountered an external-storage reference payload but external storage is not " + + "configured. Configure WorkflowClientOptions.Builder.setExternalStorage(...) with a " + + "driver able to retrieve it."); + } + + /** Returns the missing-storage failure from a wrapper chain, if present. */ + @Nullable + public static ExternalStorageNotConfiguredException find(Throwable failure) { + while (failure != null) { + if (failure instanceof ExternalStorageNotConfiguredException) { + return (ExternalStorageNotConfiguredException) failure; + } + Throwable cause = failure.getCause(); + if (cause == failure) { + break; + } + failure = cause; + } + return null; + } +} diff --git a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java index 56ca5188cf..1719ef7e4d 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/replay/ReplayWorkflowTaskHandler.java @@ -24,6 +24,7 @@ import io.temporal.internal.common.ProtobufTimeUtils; import io.temporal.internal.common.WorkflowExecutionUtils; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.internal.worker.*; import io.temporal.payload.context.WorkflowSerializationContext; import io.temporal.serviceclient.MetricsTag; @@ -161,6 +162,12 @@ private Result handleWorkflowTaskWithQuery( } } + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(e); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } + if (directQuery) { return createDirectQueryResult(workflowTask, null, e); } else { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/sync/SignalDispatcher.java b/temporal-sdk/src/main/java/io/temporal/internal/sync/SignalDispatcher.java index 9bc6bdfa7b..0e96d2f817 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/sync/SignalDispatcher.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/sync/SignalDispatcher.java @@ -11,6 +11,7 @@ import io.temporal.common.interceptors.Header; import io.temporal.common.interceptors.WorkflowInboundCallsInterceptor; import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptor; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.worker.MetricsType; import io.temporal.workflow.DynamicSignalHandler; import io.temporal.workflow.HandlerUnfinishedPolicy; @@ -87,6 +88,11 @@ public void handleSignal( dataConverterWithWorkflowContext.fromPayloads( input, handler.getArgTypes(), handler.getGenericArgTypes()); } catch (DataConverterException e) { + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(e); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } logSerializationException(signalName, eventId, e); return; } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/sync/WorkflowExecutionHandler.java b/temporal-sdk/src/main/java/io/temporal/internal/sync/WorkflowExecutionHandler.java index afeb70b14b..c7d8325658 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/sync/WorkflowExecutionHandler.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/sync/WorkflowExecutionHandler.java @@ -8,6 +8,7 @@ import io.temporal.common.interceptors.Header; import io.temporal.failure.CanceledFailure; import io.temporal.failure.TemporalFailure; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import io.temporal.internal.replay.ReplayWorkflowContext; import io.temporal.internal.statemachines.UnsupportedContinueAsNewRequest; import io.temporal.internal.worker.WorkflowExecutionException; @@ -131,6 +132,11 @@ private void applyWorkflowFailurePolicyAndRethrow(Throwable e) { throw (DestroyWorkflowThreadError) e; } Throwable exception = unwrap(e); + ExternalStorageNotConfiguredException externalStorageFailure = + ExternalStorageNotConfiguredException.find(exception); + if (externalStorageFailure != null) { + throw externalStorageFailure; + } Class[] failTypes = implementationOptions.getFailWorkflowExceptionTypes(); if (exception instanceof TemporalFailure) { diff --git a/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java b/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java index ea14e30396..df9076a341 100644 --- a/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java +++ b/temporal-sdk/src/test/java/io/temporal/common/converter/ExternalStorageReferenceGuardTest.java @@ -5,6 +5,7 @@ import com.google.protobuf.ByteString; import io.temporal.api.common.v1.Payload; +import io.temporal.internal.payload.storage.ExternalStorageNotConfiguredException; import org.junit.Test; /** @@ -27,11 +28,12 @@ public void referencePayloadWithoutConfiguredStorageThrows() { Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(1024).build()) .build(); - DataConverterException e = + ExternalStorageNotConfiguredException e = assertThrows( - DataConverterException.class, + ExternalStorageNotConfiguredException.class, () -> dataConverter.fromPayload(reference, String.class, String.class)); assertTrue(e.getMessage(), e.getMessage().contains("[TMPRL-1105]")); + assertTrue(ExternalStorageNotConfiguredException.find(new RuntimeException(e)) == e); } @Test From eb7dfb98cc9932c77d301b3ae0a3c5b6e8674c78 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 14:04:53 -0400 Subject: [PATCH 10/13] fix(extstore): correct activity storage targets. --- .../ActivityExecutionContextImpl.java | 10 ++- .../activity/HeartbeatContextImpl.java | 2 +- .../ExternalStorageGenericWorkflowClient.java | 10 ++- ...ManualActivityCompletionClientFactory.java | 7 ++ ...alActivityCompletionClientFactoryImpl.java | 28 ++++++ .../ManualActivityCompletionClientImpl.java | 15 ++-- .../internal/worker/ActivityWorker.java | 3 +- .../ActivityExecutionContextImplTest.java | 76 ++++++++++++++++ ...ernalStorageGenericWorkflowClientTest.java | 86 +++++++++++++++++++ .../storage/ExternalStoragePipelineTest.java | 14 +++ 10 files changed, 238 insertions(+), 13 deletions(-) create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/activity/ActivityExecutionContextImplTest.java create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java index 4cb9e66108..9d955547e9 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/ActivityExecutionContextImpl.java @@ -12,6 +12,7 @@ import io.temporal.internal.client.external.ManualActivityCompletionClientFactory; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverActivityInfo; import io.temporal.workflow.Functions; import java.lang.reflect.Type; import java.time.Duration; @@ -159,7 +160,14 @@ public ManualActivityCompletionClient useLocalManualCompletion() { new ActivitySerializationContext(info); return new CompletionAwareManualCompletionClient( manualCompletionClientFactory.getClient( - info.getTaskToken(), metricsScope, activitySerializationContext), + info.getTaskToken(), + metricsScope, + activitySerializationContext, + new StorageDriverActivityInfo( + info.getNamespace(), + info.getActivityId(), + info.getActivityRunId(), + info.getActivityType())), completionHandle); } finally { lock.unlock(); diff --git a/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java index 07ddfafe23..c6018f40d7 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/activity/HeartbeatContextImpl.java @@ -341,7 +341,7 @@ private void checkHeartbeatTimeoutDeadlineLocked() { private StorageDriverTargetInfo activityStorageTarget() { return new StorageDriverActivityInfo( - namespace, info.getActivityId(), info.getWorkflowRunId(), info.getActivityType()); + namespace, info.getActivityId(), info.getActivityRunId(), info.getActivityType()); } private void sendHeartbeatRequest(Object details) { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java index 8ef56b88de..2fcd26f639 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java @@ -5,6 +5,7 @@ import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.api.workflowservice.v1.*; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.payload.storage.StorageDriverActivityInfo; import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.payload.storage.StorageDriverWorkflowInfo; import java.util.concurrent.CompletableFuture; @@ -253,7 +254,14 @@ public ExecuteMultiOperationResponse executeMultiOperation( @Override public StartActivityExecutionResponse startActivity(StartActivityExecutionRequest request) { - return next.startActivity(externalStorage.storeBlocking(request, null)); + return next.startActivity( + externalStorage.storeBlocking( + request, + new StorageDriverActivityInfo( + namespace, + Strings.emptyToNull(request.getActivityId()), + null, + Strings.emptyToNull(request.getActivityType().getName())))); } @Override diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java index 9d5f35e777..c70c07492b 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactory.java @@ -6,6 +6,7 @@ import io.temporal.common.converter.DataConverter; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.WorkflowServiceStubs; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -37,6 +38,12 @@ ManualActivityCompletionClient getClient( @Nonnull Scope metricsScope, @Nullable ActivitySerializationContext activitySerializationContext); + ManualActivityCompletionClient getClient( + @Nonnull byte[] taskToken, + @Nonnull Scope metricsScope, + @Nullable ActivitySerializationContext activitySerializationContext, + @Nullable StorageDriverTargetInfo storageTarget); + ManualActivityCompletionClient getClient( @Nonnull WorkflowExecution execution, @Nonnull String activityId, diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java index 761970b753..bd213c36a0 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientFactoryImpl.java @@ -1,12 +1,15 @@ package io.temporal.internal.client.external; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.uber.m3.tally.Scope; import io.temporal.activity.ManualActivityCompletionClient; import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.common.converter.DataConverter; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.WorkflowServiceStubs; import java.util.Objects; import javax.annotation.Nonnull; @@ -43,6 +46,23 @@ public ManualActivityCompletionClient getClient( @Nonnull byte[] taskToken, @Nonnull Scope metricsScope, @Nullable ActivitySerializationContext activitySerializationContext) { + StorageDriverTargetInfo storageTarget = + activitySerializationContext == null + ? null + : new StorageDriverActivityInfo( + namespace, + null, + null, + Strings.emptyToNull(activitySerializationContext.getActivityType())); + return getClient(taskToken, metricsScope, activitySerializationContext, storageTarget); + } + + @Override + public ManualActivityCompletionClient getClient( + @Nonnull byte[] taskToken, + @Nonnull Scope metricsScope, + @Nullable ActivitySerializationContext activitySerializationContext, + @Nullable StorageDriverTargetInfo storageTarget) { Preconditions.checkNotNull(metricsScope, "metricsScope"); Preconditions.checkNotNull(taskToken, "taskToken"); Preconditions.checkArgument(taskToken.length > 0, "empty taskToken"); @@ -56,6 +76,7 @@ public ManualActivityCompletionClient getClient( null, null, activitySerializationContext, + storageTarget, externalStorage); } @@ -76,6 +97,12 @@ public ManualActivityCompletionClient getClient( Preconditions.checkNotNull(metricsScope, "metricsScope"); Preconditions.checkNotNull(execution, "execution"); Preconditions.checkNotNull(activityId, "activityId"); + String activityRunId = + execution.getWorkflowId().isEmpty() ? Strings.emptyToNull(execution.getRunId()) : null; + String activityType = + activitySerializationContext == null + ? null + : Strings.emptyToNull(activitySerializationContext.getActivityType()); return new ManualActivityCompletionClientImpl( service, namespace, @@ -86,6 +113,7 @@ public ManualActivityCompletionClient getClient( execution, activityId, activitySerializationContext, + new StorageDriverActivityInfo(namespace, activityId, activityRunId, activityType), externalStorage); } } diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java index 7b863b9862..32822bacbd 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java @@ -20,7 +20,6 @@ import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.internal.retryer.GrpcRetryer; import io.temporal.payload.context.ActivitySerializationContext; -import io.temporal.payload.storage.StorageDriverActivityInfo; import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.serviceclient.RpcRetryOptions; import io.temporal.serviceclient.WorkflowServiceStubs; @@ -45,6 +44,7 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie private final byte[] taskToken; private final GrpcRetryer grpcRetryer; private final GrpcRetryer.GrpcRetryerOptions replyGrpcRetryerOptions; + private final @Nullable StorageDriverTargetInfo storageTarget; private final @Nullable ExternalStorageMessageConverter externalStorage; ManualActivityCompletionClientImpl( @@ -57,9 +57,11 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie @Nullable WorkflowExecution execution, @Nullable String activityId, @Nullable ActivitySerializationContext context, + @Nullable StorageDriverTargetInfo storageTarget, @Nullable ExternalStorageMessageConverter externalStorage) { this.service = service; this.externalStorage = externalStorage; + this.storageTarget = storageTarget; this.dataConverterWithActivityExecutionContext = context != null ? dataConverter.withContext(context) : dataConverter; this.namespace = namespace; @@ -85,12 +87,7 @@ class ManualActivityCompletionClientImpl implements ManualActivityCompletionClie private T storeOutbound(T request) { return externalStorage == null ? request - : externalStorage.storeBlocking(request, activityTarget()); - } - - private StorageDriverTargetInfo activityTarget() { - return new StorageDriverActivityInfo( - namespace, activityId, execution != null ? execution.getRunId() : null, null); + : externalStorage.storeBlocking(request, storageTarget); } @Override @@ -211,7 +208,7 @@ public void recordHeartbeat(@Nullable Object details) throws CanceledFailure { dataConverterWithActivityExecutionContext.toPayloads(details), metricsScope, externalStorage, - activityTarget()); + storageTarget); if (status.getCancelRequested()) { throw new ActivityCanceledException(); } else if (status.getActivityReset()) { @@ -230,7 +227,7 @@ public void recordHeartbeat(@Nullable Object details) throws CanceledFailure { dataConverterWithActivityExecutionContext.toPayloads(details), metricsScope, externalStorage, - activityTarget()); + storageTarget); if (status.getCancelRequested()) { throw new ActivityCanceledException(); } else if (status.getActivityReset()) { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java b/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java index 324fe98c8c..eb1d7cdc07 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/worker/ActivityWorker.java @@ -2,6 +2,7 @@ import static io.temporal.serviceclient.MetricsTag.METRICS_TAGS_CALL_OPTIONS_KEY; +import com.google.common.base.Strings; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import com.uber.m3.tally.Scope; @@ -496,7 +497,7 @@ private StorageDriverTargetInfo activityStorageTarget( return new StorageDriverActivityInfo( namespace, pollResponse.getActivityId(), - pollResponse.getWorkflowExecution().getRunId(), + Strings.emptyToNull(pollResponse.getActivityRunId()), pollResponse.getActivityType().getName()); } diff --git a/temporal-sdk/src/test/java/io/temporal/internal/activity/ActivityExecutionContextImplTest.java b/temporal-sdk/src/test/java/io/temporal/internal/activity/ActivityExecutionContextImplTest.java new file mode 100644 index 0000000000..8cff52cad2 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/activity/ActivityExecutionContextImplTest.java @@ -0,0 +1,76 @@ +package io.temporal.internal.activity; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.uber.m3.tally.NoopScope; +import com.uber.m3.tally.Scope; +import io.temporal.activity.ActivityInfo; +import io.temporal.activity.ManualActivityCompletionClient; +import io.temporal.client.WorkflowClient; +import io.temporal.common.converter.GlobalDataConverter; +import io.temporal.internal.client.external.ManualActivityCompletionClientFactory; +import io.temporal.payload.context.ActivitySerializationContext; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverTargetInfo; +import io.temporal.serviceclient.WorkflowServiceStubs; +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import org.junit.Test; + +public class ActivityExecutionContextImplTest { + + @Test + public void localManualCompletionIncludesActivityTarget() { + WorkflowClient client = mock(WorkflowClient.class); + when(client.getWorkflowServiceStubs()).thenReturn(mock(WorkflowServiceStubs.class)); + ActivityInfo info = mock(ActivityInfo.class); + when(info.getNamespace()).thenReturn("test-namespace"); + when(info.getWorkflowId()).thenReturn(null); + when(info.getWorkflowType()).thenReturn(null); + when(info.getActivityId()).thenReturn("activity-id"); + when(info.getActivityRunId()).thenReturn("activity-run-id"); + when(info.getActivityType()).thenReturn("activity-type"); + when(info.getActivityTaskQueue()).thenReturn("task-queue"); + when(info.getTaskToken()).thenReturn(new byte[] {1, 2, 3}); + ManualActivityCompletionClientFactory completionClientFactory = + mock(ManualActivityCompletionClientFactory.class); + when(completionClientFactory.getClient( + any(byte[].class), + any(Scope.class), + any(ActivitySerializationContext.class), + any(StorageDriverTargetInfo.class))) + .thenReturn(mock(ManualActivityCompletionClient.class)); + NoopScope metricsScope = new NoopScope(); + ActivityExecutionContextImpl context = + new ActivityExecutionContextImpl( + client, + "test-namespace", + new Object(), + info, + GlobalDataConverter.get(), + mock(ScheduledExecutorService.class), + completionClientFactory, + () -> {}, + metricsScope, + "test-identity", + Duration.ofSeconds(60), + Duration.ofSeconds(30), + () -> {}, + null); + + context.useLocalManualCompletion(); + + verify(completionClientFactory) + .getClient( + eq(new byte[] {1, 2, 3}), + eq(metricsScope), + any(ActivitySerializationContext.class), + eq( + new StorageDriverActivityInfo( + "test-namespace", "activity-id", "activity-run-id", "activity-type"))); + } +} diff --git a/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java new file mode 100644 index 0000000000..97ab729610 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java @@ -0,0 +1,86 @@ +package io.temporal.internal.client.external; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.temporal.api.common.v1.ActivityType; +import io.temporal.api.common.v1.Payload; +import io.temporal.api.common.v1.Payloads; +import io.temporal.api.workflowservice.v1.StartActivityExecutionRequest; +import io.temporal.api.workflowservice.v1.StartActivityExecutionResponse; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.payload.storage.ExternalStorageOptions; +import io.temporal.payload.storage.StorageDriver; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverClaim; +import io.temporal.payload.storage.StorageDriverRetrieveContext; +import io.temporal.payload.storage.StorageDriverStoreContext; +import io.temporal.payload.storage.StorageDriverTargetInfo; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.junit.Test; + +public class ExternalStorageGenericWorkflowClientTest { + + @Test + public void standaloneActivityStartIncludesKnownTargetInfo() { + GenericWorkflowClient next = mock(GenericWorkflowClient.class); + when(next.startActivity(any())).thenReturn(StartActivityExecutionResponse.getDefaultInstance()); + CapturingDriver driver = new CapturingDriver(); + ExternalStorageGenericWorkflowClient client = + new ExternalStorageGenericWorkflowClient( + next, + ExternalStorageMessageConverter.create( + ExternalStorageOptions.newBuilder() + .setDriver(driver) + .setPayloadSizeThreshold(0) + .build(), + 1), + "test-namespace"); + StartActivityExecutionRequest request = + StartActivityExecutionRequest.newBuilder() + .setActivityId("activity-id") + .setActivityType(ActivityType.newBuilder().setName("activity-type")) + .setInput(Payloads.newBuilder().addPayloads(Payload.getDefaultInstance())) + .build(); + + client.startActivity(request); + + assertEquals( + Collections.singletonList( + new StorageDriverActivityInfo("test-namespace", "activity-id", null, "activity-type")), + driver.targets); + } + + private static final class CapturingDriver implements StorageDriver { + private final List targets = new ArrayList<>(); + + @Override + public String getName() { + return "test"; + } + + @Override + public String getType() { + return "test"; + } + + @Override + public CompletableFuture> store( + StorageDriverStoreContext context, List payloads) { + targets.add(context.getTarget()); + return CompletableFuture.completedFuture( + Collections.singletonList(new StorageDriverClaim(Collections.emptyMap()))); + } + + @Override + public CompletableFuture> retrieve( + StorageDriverRetrieveContext context, List claims) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + } +} diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java index 943dce190f..acdf807377 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePipelineTest.java @@ -16,9 +16,11 @@ import io.temporal.common.WorkflowExecutionHistory; import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriver; +import io.temporal.payload.storage.StorageDriverActivityInfo; import io.temporal.payload.storage.StorageDriverClaim; import io.temporal.payload.storage.StorageDriverRetrieveContext; import io.temporal.payload.storage.StorageDriverStoreContext; +import io.temporal.payload.storage.StorageDriverTargetInfo; import io.temporal.testing.TestEnvironmentOptions; import io.temporal.testing.TestWorkflowEnvironment; import io.temporal.testing.WorkflowReplayer; @@ -72,6 +74,15 @@ public void payloadsRoundTripThroughStorage() { assertEquals("echo: hello", workflow.lastResult()); assertTrue("expected the driver to have stored payloads", driver.stores.get() > 0); assertTrue("expected the driver to have restored payloads", driver.retrieves.get() > 0); + assertTrue( + "expected an activity storage target", + driver.targets.stream().anyMatch(StorageDriverActivityInfo.class::isInstance)); + assertTrue( + "workflow run IDs must not be used as activity run IDs", + driver.targets.stream() + .filter(StorageDriverActivityInfo.class::isInstance) + .map(StorageDriverActivityInfo.class::cast) + .allMatch(target -> target.getRunId() == null)); } @Test @@ -195,6 +206,8 @@ public String echo(String input) { private static final class InMemoryDriver implements StorageDriver { private final String name; private final Map objects = new ConcurrentHashMap<>(); + private final List targets = + Collections.synchronizedList(new ArrayList<>()); final AtomicInteger stores = new AtomicInteger(); final AtomicInteger retrieves = new AtomicInteger(); private final AtomicInteger counter = new AtomicInteger(); @@ -216,6 +229,7 @@ public String getType() { @Override public CompletableFuture> store( StorageDriverStoreContext context, List payloads) { + targets.add(context.getTarget()); List claims = new ArrayList<>(payloads.size()); for (Payload payload : payloads) { stores.incrementAndGet(); From a274ddda4fd432fb750d74ba23f3c3dd24ccc65c Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 14:55:12 -0400 Subject: [PATCH 11/13] fix(extstore): handle manual completion storage failures --- .../ManualActivityCompletionClientImpl.java | 42 +++-- ...anualActivityCompletionClientImplTest.java | 146 ++++++++++++++++++ 2 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 temporal-sdk/src/test/java/io/temporal/internal/client/external/ManualActivityCompletionClientImplTest.java diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java index 32822bacbd..970e14a674 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ManualActivityCompletionClientImpl.java @@ -100,8 +100,8 @@ public void complete(@Nullable Object result) { .setIdentity(identity) .setTaskToken(ByteString.copyFrom(taskToken)); payloads.ifPresent(builder::setResult); - RespondActivityTaskCompletedRequest request = storeOutbound(builder.build()); try { + RespondActivityTaskCompletedRequest request = storeOutbound(builder.build()); grpcRetryer.retry( () -> service @@ -123,8 +123,8 @@ public void complete(@Nullable Object result) { .setWorkflowId(execution.getWorkflowId()) .setRunId(execution.getRunId()); payloads.ifPresent(builder::setResult); - RespondActivityTaskCompletedByIdRequest request = storeOutbound(builder.build()); try { + RespondActivityTaskCompletedByIdRequest request = storeOutbound(builder.build()); grpcRetryer.retry( () -> service @@ -143,15 +143,14 @@ public void fail(@Nonnull Throwable exception) { Preconditions.checkNotNull(exception, "null exception"); // When converting failures reason is class name, details are serialized exception. if (taskToken != null) { - RespondActivityTaskFailedRequest request = - storeOutbound( - RespondActivityTaskFailedRequest.newBuilder() - .setFailure( - dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) - .setNamespace(namespace) - .setTaskToken(ByteString.copyFrom(taskToken)) - .build()); + RespondActivityTaskFailedRequest unstoredRequest = + RespondActivityTaskFailedRequest.newBuilder() + .setFailure(dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) + .setNamespace(namespace) + .setTaskToken(ByteString.copyFrom(taskToken)) + .build(); try { + RespondActivityTaskFailedRequest request = storeOutbound(unstoredRequest); grpcRetryer.retry( () -> service @@ -171,17 +170,16 @@ public void fail(@Nonnull Throwable exception) { if (activityId == null) { throw new IllegalArgumentException("Either activity id or task token are required"); } - RespondActivityTaskFailedByIdRequest request = - storeOutbound( - RespondActivityTaskFailedByIdRequest.newBuilder() - .setFailure( - dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) - .setNamespace(namespace) - .setWorkflowId(execution.getWorkflowId()) - .setRunId(execution.getRunId()) - .setActivityId(activityId) - .build()); + RespondActivityTaskFailedByIdRequest unstoredRequest = + RespondActivityTaskFailedByIdRequest.newBuilder() + .setFailure(dataConverterWithActivityExecutionContext.exceptionToFailure(exception)) + .setNamespace(namespace) + .setWorkflowId(execution.getWorkflowId()) + .setRunId(execution.getRunId()) + .setActivityId(activityId) + .build(); try { + RespondActivityTaskFailedByIdRequest request = storeOutbound(unstoredRequest); grpcRetryer.retry( () -> service @@ -251,8 +249,8 @@ public void reportCancellation(@Nullable Object details) { .setNamespace(namespace) .setTaskToken(ByteString.copyFrom(taskToken)); convertedDetails.ifPresent(builder::setDetails); - RespondActivityTaskCanceledRequest request = storeOutbound(builder.build()); try { + RespondActivityTaskCanceledRequest request = storeOutbound(builder.build()); grpcRetryer.retry( () -> service @@ -276,8 +274,8 @@ public void reportCancellation(@Nullable Object details) { .setRunId(OptionsUtils.safeGet(execution.getRunId())) .setActivityId(activityId); convertedDetails.ifPresent(builder::setDetails); - RespondActivityTaskCanceledByIdRequest request = storeOutbound(builder.build()); try { + RespondActivityTaskCanceledByIdRequest request = storeOutbound(builder.build()); grpcRetryer.retry( () -> service diff --git a/temporal-sdk/src/test/java/io/temporal/internal/client/external/ManualActivityCompletionClientImplTest.java b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ManualActivityCompletionClientImplTest.java new file mode 100644 index 0000000000..a91cdd3938 --- /dev/null +++ b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ManualActivityCompletionClientImplTest.java @@ -0,0 +1,146 @@ +package io.temporal.internal.client.external; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.uber.m3.tally.NoopScope; +import io.temporal.api.common.v1.Payload; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.workflowservice.v1.GetSystemInfoResponse; +import io.temporal.client.ActivityCompletionFailureException; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.failure.ApplicationFailure; +import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; +import io.temporal.payload.storage.ExternalStorageOptions; +import io.temporal.payload.storage.StorageDriver; +import io.temporal.payload.storage.StorageDriverActivityInfo; +import io.temporal.payload.storage.StorageDriverClaim; +import io.temporal.payload.storage.StorageDriverRetrieveContext; +import io.temporal.payload.storage.StorageDriverStoreContext; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; + +public class ManualActivityCompletionClientImplTest { + private final RuntimeException storageFailure = new RuntimeException("storage failed"); + private WorkflowServiceStubs service; + private ExternalStorageMessageConverter externalStorage; + + @Before + public void setUp() { + service = mock(WorkflowServiceStubs.class); + when(service.getServerCapabilities()) + .thenReturn(() -> GetSystemInfoResponse.Capabilities.getDefaultInstance()); + when(service.getOptions()).thenReturn(WorkflowServiceStubsOptions.getDefaultInstance()); + externalStorage = + ExternalStorageMessageConverter.create( + ExternalStorageOptions.newBuilder() + .setDriver(new FailingDriver()) + .setPayloadSizeThreshold(0) + .build(), + 1); + } + + @Test + public void taskTokenCompletionWrapsStorageFailure() { + ManualActivityCompletionClientImpl client = taskTokenClient(); + + ActivityCompletionFailureException failure = + assertThrows(ActivityCompletionFailureException.class, () -> client.complete("result")); + + assertSame(storageFailure, failure.getCause()); + verify(service, never()).blockingStub(); + } + + @Test + public void byIdFailureWrapsStorageFailure() { + ManualActivityCompletionClientImpl client = byIdClient(); + + ActivityCompletionFailureException failure = + assertThrows( + ActivityCompletionFailureException.class, + () -> client.fail(ApplicationFailure.newFailure("activity failed", "test", "details"))); + + assertSame(storageFailure, failure.getCause()); + verify(service, never()).blockingStub(); + } + + @Test + public void taskTokenCancellationIgnoresStorageFailure() { + taskTokenClient().reportCancellation("details"); + + verify(service, never()).blockingStub(); + } + + @Test + public void byIdCancellationIgnoresStorageFailure() { + byIdClient().reportCancellation("details"); + + verify(service, never()).blockingStub(); + } + + private ManualActivityCompletionClientImpl taskTokenClient() { + return new ManualActivityCompletionClientImpl( + service, + "test-namespace", + "test-identity", + DefaultDataConverter.newDefaultInstance(), + new NoopScope(), + new byte[] {1, 2, 3}, + null, + null, + null, + new StorageDriverActivityInfo( + "test-namespace", "activity-id", "activity-run-id", "activity-type"), + externalStorage); + } + + private ManualActivityCompletionClientImpl byIdClient() { + return new ManualActivityCompletionClientImpl( + service, + "test-namespace", + "test-identity", + DefaultDataConverter.newDefaultInstance(), + new NoopScope(), + null, + WorkflowExecution.newBuilder().setRunId("activity-run-id").build(), + "activity-id", + null, + new StorageDriverActivityInfo( + "test-namespace", "activity-id", "activity-run-id", "activity-type"), + externalStorage); + } + + private final class FailingDriver implements StorageDriver { + @Override + public String getName() { + return "test"; + } + + @Override + public String getType() { + return "test"; + } + + @Override + public CompletableFuture> store( + StorageDriverStoreContext context, List payloads) { + CompletableFuture> result = new CompletableFuture<>(); + result.completeExceptionally(storageFailure); + return result; + } + + @Override + public CompletableFuture> retrieve( + StorageDriverRetrieveContext context, List claims) { + throw new UnsupportedOperationException(); + } + } +} From 0bb07a8a856bc6c60db268e3aa26746d3ecd5f84 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 15:10:01 -0400 Subject: [PATCH 12/13] fix(extstore): bound client I/O and honor interrupts. --- .../client/WorkflowClientInternalImpl.java | 2 +- .../ExternalStorageMessageConverter.java | 28 ++- .../payload/storage/StorageDriver.java | 6 +- .../ExternalStorageMessageConverterTest.java | 160 ++++++++++++++++++ 4 files changed, 188 insertions(+), 8 deletions(-) diff --git a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java index fabf75615f..581a70eb0f 100644 --- a/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java +++ b/temporal-sdk/src/main/java/io/temporal/client/WorkflowClientInternalImpl.java @@ -111,7 +111,7 @@ public static WorkflowClient newInstance( .tagged(MetricsTag.defaultTags(options.getNamespace())); ExternalStorageOptions externalStorage = options.getExternalStorage(); ExternalStorageMessageConverter externalStorageConverter = - externalStorage == null ? null : ExternalStorageMessageConverter.create(externalStorage, 1); + externalStorage == null ? null : ExternalStorageMessageConverter.create(externalStorage); GenericWorkflowClient genericClient = new GenericWorkflowClientImpl(workflowServiceStubs, metricsScope); if (externalStorageConverter != null) { diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java index fde415fb66..5521e2e742 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverter.java @@ -9,6 +9,7 @@ import io.temporal.payload.storage.StorageDriverTargetInfo; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; /** @@ -18,9 +19,16 @@ *

Search attributes stay inline because the server indexes and validates their payload values. */ public final class ExternalStorageMessageConverter { + private static final int DEFAULT_PAYLOAD_VISIT_CONCURRENCY = 3; + private final ExternalStoragePayloadConverter payloadConverter; private final int payloadVisitConcurrency; + /** Builds a message converter with the default payload-list visit concurrency. */ + public static ExternalStorageMessageConverter create(ExternalStorageOptions options) { + return create(options, DEFAULT_PAYLOAD_VISIT_CONCURRENCY); + } + /** * Builds a message converter from user-facing options. {@code payloadVisitConcurrency} bounds the * number of concurrent payload-list visits within a single message walk (at least {@code 1}). @@ -59,25 +67,35 @@ public CompletableFuture retrieve(T message) { return PayloadVisitors.visit(message, options); } - /** Blocking variant of {@link #store}, for synchronous worker call sites. */ + /** + * Blocking variant of {@link #store}. Interruption cancels the conversion future, restores the + * thread's interrupt status, and throws a {@link CompletionException}. + */ public T storeBlocking(T message, @Nullable StorageDriverTargetInfo target) { return join(store(message, target)); } - /** Blocking variant of {@link #retrieve}, for synchronous worker call sites. */ + /** + * Blocking variant of {@link #retrieve}. Interruption cancels the conversion future, restores the + * thread's interrupt status, and throws a {@link CompletionException}. + */ public T retrieveBlocking(T message) { return join(retrieve(message)); } private static T join(CompletableFuture future) { try { - return future.join(); - } catch (CompletionException e) { + return future.get(); + } catch (InterruptedException e) { + future.cancel(true); + Thread.currentThread().interrupt(); + throw new CompletionException(e); + } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause != null) { Throwables.throwIfUnchecked(cause); } - throw e; + throw new CompletionException(cause == null ? e : cause); } } } diff --git a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java index 9e286ef97a..42ab9c06c7 100644 --- a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java +++ b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java @@ -31,7 +31,8 @@ public interface StorageDriver { * order. The returned list must be the same length as {@code payloads}. * *

Drivers should use {@link StorageDriverStoreContext#getCancellationToken()} to abort - * in-flight requests. + * in-flight requests and configure an appropriate backend timeout. The SDK does not impose a + * storage-operation timeout. */ @Nonnull CompletableFuture> store( @@ -42,7 +43,8 @@ CompletableFuture> store( * in the same order. The returned list must be the same length as {@code claims}. * *

Drivers should use {@link StorageDriverRetrieveContext#getCancellationToken()} to abort - * in-flight requests. + * in-flight requests and configure an appropriate backend timeout. The SDK does not impose a + * storage-operation timeout. */ @Nonnull CompletableFuture> retrieve( diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverterTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverterTest.java index 5287b68ee1..a11b831593 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverterTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageConverterTest.java @@ -11,6 +11,7 @@ import io.temporal.api.common.v1.Payload; import io.temporal.api.common.v1.Payloads; import io.temporal.api.common.v1.SearchAttributes; +import io.temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest; import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriver; import io.temporal.payload.storage.StorageDriverClaim; @@ -22,6 +23,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.Test; /** Tests external storage message conversion. */ @@ -98,6 +105,74 @@ public void searchAttributesAreNotOffloaded() throws Exception { assertEquals(payload("indexed-value"), indexed); } + @Test + public void defaultConcurrencyBoundsOutstandingVisits() throws Exception { + ControlledDriver driver = new ControlledDriver(); + ExternalStorageMessageConverter converter = + ExternalStorageMessageConverter.create( + ExternalStorageOptions.newBuilder() + .setDriver(driver) + .setPayloadSizeThreshold(0) + .build()); + RespondWorkflowTaskCompletedRequest.Builder request = + RespondWorkflowTaskCompletedRequest.newBuilder(); + for (int i = 0; i < 6; i++) { + request.addCommands( + Command.newBuilder() + .setScheduleActivityTaskCommandAttributes( + ScheduleActivityTaskCommandAttributes.newBuilder() + .setInput(Payloads.newBuilder().addPayloads(payload("input-" + i))))); + } + + CompletableFuture stored = + converter.store(request.build(), null); + + assertEquals(3, driver.started.get()); + assertEquals(3, driver.pendingCount()); + for (int i = 0; i < 6; i++) { + driver.completeNext(); + assertTrue(driver.pendingCount() <= 3); + } + stored.get(); + assertEquals(6, driver.started.get()); + assertEquals(0, driver.pendingCount()); + } + + @Test + public void blockingStorePreservesInterruptStatus() throws Exception { + InterruptibleDriver driver = new InterruptibleDriver(); + ExternalStorageMessageConverter converter = + ExternalStorageMessageConverter.create( + ExternalStorageOptions.newBuilder() + .setDriver(driver) + .setPayloadSizeThreshold(0) + .build()); + AtomicReference failure = new AtomicReference<>(); + AtomicBoolean interrupted = new AtomicBoolean(); + Thread thread = + new Thread( + () -> { + try { + converter.storeBlocking( + Payloads.newBuilder().addPayloads(payload("input")).build(), null); + } catch (Throwable e) { + failure.set(e); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + thread.start(); + assertTrue(driver.started.await(5, TimeUnit.SECONDS)); + + thread.interrupt(); + thread.join(TimeUnit.SECONDS.toMillis(5)); + + assertFalse("blocking store did not respond to interruption", thread.isAlive()); + assertTrue(failure.get() instanceof CompletionException); + assertTrue(failure.get().getCause() instanceof InterruptedException); + assertTrue(interrupted.get()); + driver.complete(); + } + private static ExternalStorageMessageConverter converter(StorageDriver driver, int threshold) { ExternalStoragePayloadConverter payloadConverter = ExternalStoragePayloadConverter.fromOptions( @@ -155,4 +230,89 @@ public synchronized CompletableFuture> retrieve( return CompletableFuture.completedFuture(payloads); } } + + private static final class ControlledDriver implements StorageDriver { + private final List pending = new ArrayList<>(); + private final AtomicInteger started = new AtomicInteger(); + + @Override + public String getName() { + return "controlled"; + } + + @Override + public String getType() { + return "test.controlled"; + } + + @Override + public synchronized CompletableFuture> store( + StorageDriverStoreContext context, List payloads) { + started.incrementAndGet(); + CompletableFuture> future = new CompletableFuture<>(); + pending.add(new PendingStore(future, payloads.size())); + return future; + } + + synchronized int pendingCount() { + return pending.size(); + } + + synchronized void completeNext() { + PendingStore store = pending.remove(0); + List claims = new ArrayList<>(store.payloadCount); + for (int i = 0; i < store.payloadCount; i++) { + claims.add(new StorageDriverClaim(Collections.emptyMap())); + } + store.future.complete(claims); + } + + @Override + public CompletableFuture> retrieve( + StorageDriverRetrieveContext context, List claims) { + throw new UnsupportedOperationException(); + } + } + + private static final class PendingStore { + private final CompletableFuture> future; + private final int payloadCount; + + private PendingStore(CompletableFuture> future, int payloadCount) { + this.future = future; + this.payloadCount = payloadCount; + } + } + + private static final class InterruptibleDriver implements StorageDriver { + private final CountDownLatch started = new CountDownLatch(1); + private final CompletableFuture> future = new CompletableFuture<>(); + + @Override + public String getName() { + return "interruptible"; + } + + @Override + public String getType() { + return "test.interruptible"; + } + + @Override + public CompletableFuture> store( + StorageDriverStoreContext context, List payloads) { + started.countDown(); + return future; + } + + void complete() { + future.complete(Collections.singletonList(new StorageDriverClaim(Collections.emptyMap()))); + } + + @Override + public CompletableFuture> retrieve( + StorageDriverRetrieveContext context, List claims) { + throw new UnsupportedOperationException(); + } + } } From df97c191f6dcf5acfacc892a20a92d9ab1bbf147 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 30 Jul 2026 15:25:48 -0400 Subject: [PATCH 13/13] fix(extstore): add multi-operation workflow target --- .../ExternalStorageGenericWorkflowClient.java | 14 +++++- ...ernalStorageGenericWorkflowClientTest.java | 43 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java index 2fcd26f639..27d1f3503d 100644 --- a/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java +++ b/temporal-sdk/src/main/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClient.java @@ -46,6 +46,17 @@ private StorageDriverTargetInfo workflowTarget(WorkflowExecution execution, Stri return workflowTarget(execution.getWorkflowId(), execution.getRunId(), type); } + @Nullable + private StorageDriverTargetInfo multiOperationTarget(ExecuteMultiOperationRequest request) { + for (ExecuteMultiOperationRequest.Operation operation : request.getOperationsList()) { + if (operation.hasStartWorkflow()) { + StartWorkflowExecutionRequest start = operation.getStartWorkflow(); + return workflowTarget(start.getWorkflowId(), null, start.getWorkflowType().getName()); + } + } + return null; + } + @Override public StartWorkflowExecutionResponse start(StartWorkflowExecutionRequest request) { return next.start( @@ -248,7 +259,8 @@ public UpdateWorkerBuildIdCompatibilityResponse updateWorkerBuildIdCompatability @Override public ExecuteMultiOperationResponse executeMultiOperation( ExecuteMultiOperationRequest request, @Nonnull Deadline deadline) { - ExecuteMultiOperationRequest stored = externalStorage.storeBlocking(request, null); + ExecuteMultiOperationRequest stored = + externalStorage.storeBlocking(request, multiOperationTarget(request)); return externalStorage.retrieveBlocking(next.executeMultiOperation(stored, deadline)); } diff --git a/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java index 97ab729610..04cc02671e 100644 --- a/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java +++ b/temporal-sdk/src/test/java/io/temporal/internal/client/external/ExternalStorageGenericWorkflowClientTest.java @@ -5,11 +5,16 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import io.grpc.Deadline; import io.temporal.api.common.v1.ActivityType; import io.temporal.api.common.v1.Payload; import io.temporal.api.common.v1.Payloads; +import io.temporal.api.common.v1.WorkflowType; +import io.temporal.api.workflowservice.v1.ExecuteMultiOperationRequest; +import io.temporal.api.workflowservice.v1.ExecuteMultiOperationResponse; import io.temporal.api.workflowservice.v1.StartActivityExecutionRequest; import io.temporal.api.workflowservice.v1.StartActivityExecutionResponse; +import io.temporal.api.workflowservice.v1.StartWorkflowExecutionRequest; import io.temporal.internal.payload.storage.ExternalStorageMessageConverter; import io.temporal.payload.storage.ExternalStorageOptions; import io.temporal.payload.storage.StorageDriver; @@ -18,10 +23,12 @@ import io.temporal.payload.storage.StorageDriverRetrieveContext; import io.temporal.payload.storage.StorageDriverStoreContext; import io.temporal.payload.storage.StorageDriverTargetInfo; +import io.temporal.payload.storage.StorageDriverWorkflowInfo; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.junit.Test; public class ExternalStorageGenericWorkflowClientTest { @@ -56,6 +63,42 @@ public void standaloneActivityStartIncludesKnownTargetInfo() { driver.targets); } + @Test + public void multiOperationIncludesWorkflowTargetInfo() { + GenericWorkflowClient next = mock(GenericWorkflowClient.class); + when(next.executeMultiOperation(any(), any())) + .thenReturn(ExecuteMultiOperationResponse.getDefaultInstance()); + CapturingDriver driver = new CapturingDriver(); + ExternalStorageGenericWorkflowClient client = + new ExternalStorageGenericWorkflowClient( + next, + ExternalStorageMessageConverter.create( + ExternalStorageOptions.newBuilder() + .setDriver(driver) + .setPayloadSizeThreshold(0) + .build(), + 1), + "test-namespace"); + ExecuteMultiOperationRequest request = + ExecuteMultiOperationRequest.newBuilder() + .addOperations( + ExecuteMultiOperationRequest.Operation.newBuilder() + .setStartWorkflow( + StartWorkflowExecutionRequest.newBuilder() + .setWorkflowId("workflow-id") + .setWorkflowType(WorkflowType.newBuilder().setName("workflow-type")) + .setInput( + Payloads.newBuilder().addPayloads(Payload.getDefaultInstance())))) + .build(); + + client.executeMultiOperation(request, Deadline.after(1, TimeUnit.SECONDS)); + + assertEquals( + Collections.singletonList( + new StorageDriverWorkflowInfo("test-namespace", "workflow-id", null, "workflow-type")), + driver.targets); + } + private static final class CapturingDriver implements StorageDriver { private final List targets = new ArrayList<>();