diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformer.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformer.java
new file mode 100644
index 000000000..7385f9900
--- /dev/null
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformer.java
@@ -0,0 +1,75 @@
+package io.temporal.internal.payload.storage;
+
+import com.google.protobuf.Message;
+import io.temporal.common.CancellationToken;
+import io.temporal.internal.payload.visitor.PayloadVisitorOptions;
+import io.temporal.internal.payload.visitor.PayloadVisitors;
+import io.temporal.payload.storage.StorageDriverTargetInfo;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import javax.annotation.Nullable;
+
+/**
+ * Transforms payload lists reachable from a proto message by delegating each visited list to {@link
+ * ExternalStoragePayloadTransformer}.
+ *
+ *
Search attributes stay inline because the server indexes and validates their payload values.
+ *
+ *
The {@link Message.Builder} overloads transform in place; the {@link Message} overloads copy
+ * through a builder and complete with the copy.
+ */
+final class ExternalStorageMessageTransformer {
+ private final ExternalStoragePayloadTransformer payloadTransformer;
+ private final int payloadVisitConcurrency;
+
+ ExternalStorageMessageTransformer(
+ ExternalStoragePayloadTransformer payloadTransformer, int payloadVisitConcurrency) {
+ this.payloadTransformer = payloadTransformer;
+ this.payloadVisitConcurrency = payloadVisitConcurrency;
+ }
+
+ CompletableFuture store(
+ T message,
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ return PayloadVisitors.visit(message, storeOptions(target, cancellationToken));
+ }
+
+ CompletableFuture store(
+ Message.Builder builder,
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ return PayloadVisitors.visit(builder, storeOptions(target, cancellationToken));
+ }
+
+ CompletableFuture retrieve(
+ T message, CancellationToken cancellationToken) {
+ return PayloadVisitors.visit(message, retrieveOptions(cancellationToken));
+ }
+
+ CompletableFuture retrieve(
+ Message.Builder builder, CancellationToken cancellationToken) {
+ return PayloadVisitors.visit(builder, retrieveOptions(cancellationToken));
+ }
+
+ private PayloadVisitorOptions storeOptions(
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ return PayloadVisitorOptions.newBuilder(
+ (visitedTarget, payloads) ->
+ payloadTransformer.store(payloads, visitedTarget, cancellationToken))
+ .setInitialContext(target)
+ .setConcurrency(payloadVisitConcurrency)
+ .setSkipSearchAttributes(true)
+ .build();
+ }
+
+ private PayloadVisitorOptions retrieveOptions(
+ CancellationToken cancellationToken) {
+ return PayloadVisitorOptions.newBuilder(
+ (context, payloads) -> payloadTransformer.retrieve(payloads, cancellationToken))
+ .setConcurrency(payloadVisitConcurrency)
+ .setSkipSearchAttributes(true)
+ .build();
+ }
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformer.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformer.java
new file mode 100644
index 000000000..6e0d4d770
--- /dev/null
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformer.java
@@ -0,0 +1,292 @@
+package io.temporal.internal.payload.storage;
+
+import io.temporal.api.common.v1.Payload;
+import io.temporal.common.CancellationToken;
+import io.temporal.internal.common.ListUtils;
+import io.temporal.internal.concurrent.structured.TaskScope;
+import io.temporal.payload.storage.ExternalStorageOptions;
+import io.temporal.payload.storage.StorageDriver;
+import io.temporal.payload.storage.StorageDriverClaim;
+import io.temporal.payload.storage.StorageDriverRetrieveContext;
+import io.temporal.payload.storage.StorageDriverSelector;
+import io.temporal.payload.storage.StorageDriverStoreContext;
+import io.temporal.payload.storage.StorageDriverTargetInfo;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+/**
+ * Transforms one payload list between inline payloads and external-storage references by routing
+ * entries to storage drivers.
+ */
+final class ExternalStoragePayloadTransformer {
+ private final Map driversByName;
+ private final StorageDriverSelector selector;
+ private final int payloadSizeThreshold;
+
+ static ExternalStoragePayloadTransformer fromOptions(ExternalStorageOptions options) {
+ Map driversByName = new LinkedHashMap<>();
+ for (StorageDriver driver : options.getDrivers()) {
+ driversByName.put(driver.getName(), driver);
+ }
+ return new ExternalStoragePayloadTransformer(
+ driversByName, options.getDriverSelector(), options.getPayloadSizeThreshold());
+ }
+
+ private ExternalStoragePayloadTransformer(
+ Map driversByName,
+ StorageDriverSelector selector,
+ int payloadSizeThreshold) {
+ this.driversByName = driversByName;
+ this.selector = selector;
+ this.payloadSizeThreshold = payloadSizeThreshold;
+ }
+
+ CompletableFuture> store(
+ List payloads,
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ StorageDriverStoreContext context =
+ new StorageDriverStoreContextImpl(target, cancellationToken);
+ Map> batches;
+ try {
+ batches = buildStoreBatches(payloads, context);
+ } catch (RuntimeException e) {
+ return failedFuture(e);
+ }
+ if (batches.isEmpty()) {
+ return CompletableFuture.completedFuture(payloads);
+ }
+ return runStoreDrivers(batches, target, cancellationToken)
+ .thenApply(referencePayloads -> applyPayloadReplacements(payloads, referencePayloads));
+ }
+
+ private Map> buildStoreBatches(
+ List payloads, StorageDriverStoreContext context) {
+ Map> batches = new LinkedHashMap<>();
+ for (int i = 0; i < payloads.size(); i++) {
+ Payload payload = payloads.get(i);
+ if (payloadSizeThreshold > 0 && payload.getSerializedSize() < payloadSizeThreshold) {
+ continue;
+ }
+ StorageDriver driver = selector.selectDriver(context, payload);
+ if (driver == null) {
+ continue;
+ }
+ if (driversByName.get(driver.getName()) != driver) {
+ throw new IllegalStateException(
+ "Storage driver selector returned a driver not registered with this external storage: '"
+ + driver.getName()
+ + "'");
+ }
+ batches.computeIfAbsent(driver.getName(), name -> new Batch<>(driver)).add(i, payload);
+ }
+ return batches;
+ }
+
+ private CompletableFuture>> runStoreDrivers(
+ Map> batches,
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ return withDriverScope(
+ cancellationToken,
+ scope -> {
+ StorageDriverStoreContext context =
+ new StorageDriverStoreContextImpl(target, scope.token());
+ for (Batch batch : batches.values()) {
+ scope
+ .attach(batch.driver.store(context, batch.values()))
+ .map(claims -> createReferencePayloads(batch, claims));
+ }
+ return scope.awaitAll(ListUtils::flatten);
+ });
+ }
+
+ /**
+ * Runs {@code body} in a scope that is also cancelled by {@code cancellationToken}, so a caller
+ * abandoning the operation trips the token the drivers observe.
+ */
+ private static CompletableFuture>> withDriverScope(
+ CancellationToken cancellationToken,
+ Function<
+ TaskScope>>,
+ CompletableFuture>>>
+ body) {
+ return TaskScope.withScope(
+ (TaskScope>> scope) -> {
+ CancellationToken.Registration registration =
+ cancellationToken.onCancel(scope::cancelAll);
+ CompletableFuture>> result;
+ try {
+ result = body.apply(scope);
+ } catch (Throwable t) {
+ scope.cancelAll();
+ result = failedFuture(t);
+ }
+ // The registration outlives body(), so it is released only once the work settles.
+ return result.whenComplete((ignored, error) -> registration.close());
+ });
+ }
+
+ private static List> createReferencePayloads(
+ Batch batch, List claims) {
+ if (claims == null || claims.size() != batch.size()) {
+ throw new IllegalStateException(
+ String.format(
+ "Storage driver '%s' returned %d claims for %d payloads",
+ batch.driver.getName(), claims == null ? 0 : claims.size(), batch.size()));
+ }
+ List> replacements = new ArrayList<>(claims.size());
+ for (int batchIndex = 0; batchIndex < claims.size(); batchIndex++) {
+ StorageDriverClaim claim = claims.get(batchIndex);
+ if (claim == null) {
+ throw new IllegalStateException(
+ String.format(
+ "Storage driver '%s' returned a null claim at index %d",
+ batch.driver.getName(), batchIndex));
+ }
+ IndexedValue indexedPayload = batch.get(batchIndex);
+ replacements.add(
+ new IndexedValue<>(
+ indexedPayload.originalIndex,
+ ExternalStorageReferences.toReferencePayload(
+ batch.driver.getName(), claim, indexedPayload.value.getSerializedSize())));
+ }
+ return replacements;
+ }
+
+ CompletableFuture> retrieve(
+ List payloads, CancellationToken cancellationToken) {
+ Map> batches;
+ try {
+ batches = buildRetrieveBatches(payloads);
+ } catch (RuntimeException e) {
+ return failedFuture(e);
+ }
+ if (batches.isEmpty()) {
+ return CompletableFuture.completedFuture(payloads);
+ }
+ return runRetrieveDrivers(batches, cancellationToken)
+ .thenApply(retrievedPayloads -> applyPayloadReplacements(payloads, retrievedPayloads));
+ }
+
+ private Map> buildRetrieveBatches(List payloads) {
+ Map> batches = new LinkedHashMap<>();
+ for (int i = 0; i < payloads.size(); i++) {
+ Payload payload = payloads.get(i);
+ ExternalStorageReferences.ParsedReference reference =
+ ExternalStorageReferences.tryParseReference(payload);
+ if (reference == null) {
+ continue;
+ }
+ StorageDriver driver = driversByName.get(reference.driverName);
+ if (driver == null) {
+ throw new IllegalStateException(
+ "No storage driver registered with name '" + reference.driverName + "'");
+ }
+ batches
+ .computeIfAbsent(reference.driverName, name -> new Batch<>(driver))
+ .add(i, reference.claim);
+ }
+ return batches;
+ }
+
+ private CompletableFuture>> runRetrieveDrivers(
+ Map> batches,
+ CancellationToken cancellationToken) {
+ return withDriverScope(
+ cancellationToken,
+ scope -> {
+ StorageDriverRetrieveContext context =
+ new StorageDriverRetrieveContextImpl(scope.token());
+ for (Batch batch : batches.values()) {
+ scope
+ .attach(batch.driver.retrieve(context, batch.values()))
+ .map(payloads -> mapPayloadsToOriginalPositions(batch, payloads));
+ }
+ return scope.awaitAll(ListUtils::flatten);
+ });
+ }
+
+ private static List> mapPayloadsToOriginalPositions(
+ Batch batch, List payloads) {
+ if (payloads == null || payloads.size() != batch.size()) {
+ throw new IllegalStateException(
+ String.format(
+ "Storage driver '%s' returned %d payloads for %d claims",
+ batch.driver.getName(), payloads == null ? 0 : payloads.size(), batch.size()));
+ }
+ List> replacements = new ArrayList<>(payloads.size());
+ for (int batchIndex = 0; batchIndex < payloads.size(); batchIndex++) {
+ Payload payload = payloads.get(batchIndex);
+ if (payload == null) {
+ throw new IllegalStateException(
+ String.format(
+ "Storage driver '%s' returned a null payload at index %d",
+ batch.driver.getName(), batchIndex));
+ }
+ replacements.add(new IndexedValue<>(batch.get(batchIndex).originalIndex, payload));
+ }
+ return replacements;
+ }
+
+ private static CompletableFuture failedFuture(Throwable t) {
+ CompletableFuture future = new CompletableFuture<>();
+ future.completeExceptionally(t);
+ return future;
+ }
+
+ private static List applyPayloadReplacements(
+ List payloads, List> replacements) {
+ Payload[] updatedPayloads = payloads.toArray(new Payload[0]);
+ for (IndexedValue replacement : replacements) {
+ updatedPayloads[replacement.originalIndex] = replacement.value;
+ }
+ return Arrays.asList(updatedPayloads);
+ }
+
+ private static final class IndexedValue {
+ final int originalIndex;
+ final T value;
+
+ IndexedValue(int originalIndex, T value) {
+ this.originalIndex = originalIndex;
+ this.value = value;
+ }
+ }
+
+ private static final class Batch {
+ final StorageDriver driver;
+ private final List> indexedValues = new ArrayList<>();
+
+ Batch(StorageDriver driver) {
+ this.driver = driver;
+ }
+
+ void add(int originalIndex, T value) {
+ indexedValues.add(new IndexedValue<>(originalIndex, value));
+ }
+
+ int size() {
+ return indexedValues.size();
+ }
+
+ IndexedValue get(int batchIndex) {
+ return indexedValues.get(batchIndex);
+ }
+
+ List values() {
+ List values = new ArrayList<>(indexedValues.size());
+ for (IndexedValue indexedValue : indexedValues) {
+ values.add(indexedValue.value);
+ }
+ return values;
+ }
+ }
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageReferences.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageReferences.java
new file mode 100644
index 000000000..3a68c6bb6
--- /dev/null
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/ExternalStorageReferences.java
@@ -0,0 +1,88 @@
+package io.temporal.internal.payload.storage;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.util.JsonFormat;
+import io.temporal.api.common.v1.Payload;
+import io.temporal.api.sdk.v1.ExternalStorageReference;
+import io.temporal.common.converter.EncodingKeys;
+import io.temporal.payload.storage.StorageDriverClaim;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+final class ExternalStorageReferences {
+ private static final String ENCODING_PROTOBUF_JSON = "json/protobuf";
+ private static final String REFERENCE_MESSAGE_TYPE =
+ ExternalStorageReference.getDescriptor().getFullName();
+
+ private static final JsonFormat.Printer PRINTER = JsonFormat.printer();
+ private static final JsonFormat.Parser PARSER = JsonFormat.parser().ignoringUnknownFields();
+
+ static final class ParsedReference {
+ final String driverName;
+ final StorageDriverClaim claim;
+
+ ParsedReference(String driverName, StorageDriverClaim claim) {
+ this.driverName = driverName;
+ this.claim = claim;
+ }
+ }
+
+ static Payload toReferencePayload(
+ @Nonnull String driverName,
+ @Nonnull StorageDriverClaim claim,
+ long originalPayloadSizeBytes) {
+ ExternalStorageReference reference =
+ ExternalStorageReference.newBuilder()
+ .setDriverName(driverName)
+ .putAllClaimData(claim.getClaimData())
+ .build();
+ String json;
+ try {
+ json = PRINTER.print(reference);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to serialize external storage reference", e);
+ }
+ return Payload.newBuilder()
+ .putMetadata(
+ EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8(ENCODING_PROTOBUF_JSON))
+ .putMetadata(
+ EncodingKeys.METADATA_MESSAGE_TYPE_KEY, ByteString.copyFromUtf8(REFERENCE_MESSAGE_TYPE))
+ .setData(ByteString.copyFromUtf8(json))
+ .addExternalPayloads(
+ Payload.ExternalPayloadDetails.newBuilder()
+ .setSizeBytes(originalPayloadSizeBytes)
+ .build())
+ .build();
+ }
+
+ /**
+ * Returns the reference encoded in {@code payload}, or null if the payload is not an external
+ * storage reference this SDK understands.
+ *
+ * Only the encoding and message type identify a reference. {@code external_payloads} records
+ * the original size for the server's benefit and is not part of the exchange contract, so a
+ * producer that omits it still yields a readable reference.
+ */
+ static @Nullable ParsedReference tryParseReference(@Nonnull Payload payload) {
+ if (!hasMetadata(payload, EncodingKeys.METADATA_ENCODING_KEY, ENCODING_PROTOBUF_JSON)
+ || !hasMetadata(payload, EncodingKeys.METADATA_MESSAGE_TYPE_KEY, REFERENCE_MESSAGE_TYPE)) {
+ return null;
+ }
+ ExternalStorageReference.Builder builder = ExternalStorageReference.newBuilder();
+ try {
+ PARSER.merge(payload.getData().toStringUtf8(), builder);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to parse external storage reference", e);
+ }
+ ExternalStorageReference reference = builder.build();
+ return new ParsedReference(
+ reference.getDriverName(), new StorageDriverClaim(reference.getClaimDataMap()));
+ }
+
+ private static boolean hasMetadata(Payload payload, String key, String expected) {
+ ByteString value = payload.getMetadataMap().get(key);
+ return value != null && expected.equals(value.toStringUtf8());
+ }
+
+ private ExternalStorageReferences() {}
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverRetrieveContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverRetrieveContextImpl.java
new file mode 100644
index 000000000..84374b2cd
--- /dev/null
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverRetrieveContextImpl.java
@@ -0,0 +1,21 @@
+package io.temporal.internal.payload.storage;
+
+import io.temporal.common.CancellationToken;
+import io.temporal.payload.storage.StorageDriverRetrieveContext;
+import java.util.Objects;
+import java.util.concurrent.CancellationException;
+import javax.annotation.Nonnull;
+
+final class StorageDriverRetrieveContextImpl implements StorageDriverRetrieveContext {
+ private final CancellationToken cancellationToken;
+
+ StorageDriverRetrieveContextImpl(CancellationToken cancellationToken) {
+ this.cancellationToken = Objects.requireNonNull(cancellationToken, "cancellationToken");
+ }
+
+ @Nonnull
+ @Override
+ public CancellationToken getCancellationToken() {
+ return cancellationToken;
+ }
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverStoreContextImpl.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverStoreContextImpl.java
new file mode 100644
index 000000000..c28ea0f63
--- /dev/null
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/storage/StorageDriverStoreContextImpl.java
@@ -0,0 +1,33 @@
+package io.temporal.internal.payload.storage;
+
+import io.temporal.common.CancellationToken;
+import io.temporal.payload.storage.StorageDriverStoreContext;
+import io.temporal.payload.storage.StorageDriverTargetInfo;
+import java.util.Objects;
+import java.util.concurrent.CancellationException;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+final class StorageDriverStoreContextImpl implements StorageDriverStoreContext {
+ private final @Nullable StorageDriverTargetInfo target;
+ private final CancellationToken cancellationToken;
+
+ StorageDriverStoreContextImpl(
+ @Nullable StorageDriverTargetInfo target,
+ CancellationToken cancellationToken) {
+ this.target = target;
+ this.cancellationToken = Objects.requireNonNull(cancellationToken, "cancellationToken");
+ }
+
+ @Nullable
+ @Override
+ public StorageDriverTargetInfo getTarget() {
+ return target;
+ }
+
+ @Nonnull
+ @Override
+ public CancellationToken getCancellationToken() {
+ return cancellationToken;
+ }
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitor.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitor.java
index 8872a44cd..70a2c1cc4 100644
--- a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitor.java
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitor.java
@@ -17,6 +17,6 @@
* @param type of the contextual value supplied to each visit
*/
@FunctionalInterface
-interface PayloadVisitor {
+public interface PayloadVisitor {
CompletableFuture> visit(C context, List payloads);
}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitorOptions.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitorOptions.java
index c834c2ec8..4eac39be4 100644
--- a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitorOptions.java
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitorOptions.java
@@ -9,7 +9,7 @@
*
* @param type of the contextual value supplied to the visitor
*/
-final class PayloadVisitorOptions {
+public final class PayloadVisitorOptions {
private final @Nonnull PayloadVisitor payloadVisitor;
private final @Nullable MessageVisitor messageVisitor;
private final @Nullable C initialContext;
@@ -36,7 +36,7 @@ public PayloadVisitor getPayloadVisitor() {
}
@Nullable
- public MessageVisitor getMessageVisitor() {
+ MessageVisitor getMessageVisitor() {
return messageVisitor;
}
@@ -69,7 +69,7 @@ private Builder(@Nonnull PayloadVisitor payloadVisitor) {
this.payloadVisitor = Objects.requireNonNull(payloadVisitor, "payloadVisitor");
}
- public Builder setMessageVisitor(@Nullable MessageVisitor messageVisitor) {
+ Builder setMessageVisitor(@Nullable MessageVisitor messageVisitor) {
this.messageVisitor = messageVisitor;
return this;
}
diff --git a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitors.java b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitors.java
index 69e992412..c6f6179fa 100644
--- a/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitors.java
+++ b/temporal-sdk/src/main/java/io/temporal/internal/payload/visitor/PayloadVisitors.java
@@ -5,7 +5,7 @@
import javax.annotation.Nonnull;
/** Visits every payload within a proto message. */
-final class PayloadVisitors {
+public final class PayloadVisitors {
private PayloadVisitors() {}
/** Visits the payloads in {@code builder} in place. */
diff --git a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java
index 323925075..01d851fbe 100644
--- a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java
+++ b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriver.java
@@ -29,6 +29,9 @@ public interface StorageDriver {
/**
* Stores {@code payloads} and returns one {@link StorageDriverClaim} per payload, in the same
* order. The returned list must be the same length as {@code payloads}.
+ *
+ * Observe {@link StorageDriverStoreContext#getCancellationToken()} to learn when the SDK has
+ * abandoned this operation, and abandon in-flight requests accordingly.
*/
@Nonnull
CompletableFuture> store(
@@ -37,6 +40,9 @@ CompletableFuture> store(
/**
* Retrieves the payloads identified by {@code claims} and returns one {@link Payload} per claim,
* in the same order. The returned list must be the same length as {@code claims}.
+ *
+ * Observe {@link StorageDriverRetrieveContext#getCancellationToken()} to learn when the SDK
+ * has abandoned this operation, and abandon in-flight requests accordingly.
*/
@Nonnull
CompletableFuture> retrieve(
diff --git a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverRetrieveContext.java b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverRetrieveContext.java
index 77f11c750..ae2b1ffbe 100644
--- a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverRetrieveContext.java
+++ b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverRetrieveContext.java
@@ -1,12 +1,25 @@
package io.temporal.payload.storage;
+import io.temporal.common.CancellationToken;
import io.temporal.common.Experimental;
+import java.util.concurrent.CancellationException;
+import javax.annotation.Nonnull;
/**
* Context passed to {@link StorageDriver#retrieve}.
*
- * Implemented by the SDK and passed to the driver. Driver authors do not implement this in
- * production code, only when constructing instances for their own tests.
+ *
The SDK supplies the instance a driver receives. Members added here in later releases will
+ * carry a default, so an existing driver-side implementation keeps compiling and behaves as though
+ * the new member were absent.
*/
@Experimental
-public interface StorageDriverRetrieveContext {}
+public interface StorageDriverRetrieveContext {
+ /**
+ * Token cancelled when the SDK abandons this retrieve operation. Defaults to a token that is
+ * never cancelled.
+ */
+ @Nonnull
+ default CancellationToken getCancellationToken() {
+ return CancellationToken.none();
+ }
+}
diff --git a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverStoreContext.java b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverStoreContext.java
index 723655bf1..f001adc57 100644
--- a/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverStoreContext.java
+++ b/temporal-sdk/src/main/java/io/temporal/payload/storage/StorageDriverStoreContext.java
@@ -1,13 +1,17 @@
package io.temporal.payload.storage;
+import io.temporal.common.CancellationToken;
import io.temporal.common.Experimental;
+import java.util.concurrent.CancellationException;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/**
* Context passed to {@link StorageDriver#store} and {@link StorageDriverSelector}.
*
- * Implemented by the SDK and passed to the driver. Driver authors do not implement this in
- * production code, only when constructing instances for their own tests.
+ *
The SDK supplies the instance a driver receives. Members added here in later releases will
+ * carry a default, so an existing driver-side implementation keeps compiling and behaves as though
+ * the new member were absent.
*/
@Experimental
public interface StorageDriverStoreContext {
@@ -17,4 +21,13 @@ public interface StorageDriverStoreContext {
*/
@Nullable
StorageDriverTargetInfo getTarget();
+
+ /**
+ * Token cancelled when the SDK abandons this store operation. Defaults to a token that is never
+ * cancelled.
+ */
+ @Nonnull
+ default CancellationToken getCancellationToken() {
+ return CancellationToken.none();
+ }
}
diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformerTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformerTest.java
new file mode 100644
index 000000000..f17bcff47
--- /dev/null
+++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageMessageTransformerTest.java
@@ -0,0 +1,161 @@
+package io.temporal.internal.payload.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.protobuf.ByteString;
+import io.temporal.api.command.v1.Command;
+import io.temporal.api.command.v1.ScheduleActivityTaskCommandAttributes;
+import io.temporal.api.command.v1.StartChildWorkflowExecutionCommandAttributes;
+import io.temporal.api.common.v1.Payload;
+import io.temporal.api.common.v1.Payloads;
+import io.temporal.api.common.v1.SearchAttributes;
+import io.temporal.common.CancellationToken;
+import io.temporal.payload.storage.ExternalStorageOptions;
+import io.temporal.payload.storage.StorageDriver;
+import io.temporal.payload.storage.StorageDriverClaim;
+import io.temporal.payload.storage.StorageDriverRetrieveContext;
+import io.temporal.payload.storage.StorageDriverStoreContext;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.junit.Test;
+
+/** Tests external storage message conversion. */
+public class ExternalStorageMessageTransformerTest {
+
+ @Test
+ public void storeAndRetrieveRoundTripsOverAMessage() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStorageMessageTransformer transformer = transformer(driver, 0);
+ Payloads message =
+ Payloads.newBuilder().addPayloads(payload("a")).addPayloads(payload("b")).build();
+
+ Payloads stored = transformer.store(message, null, CancellationToken.none()).get();
+
+ assertNotNull(ExternalStorageReferences.tryParseReference(stored.getPayloads(0)));
+ assertNotNull(ExternalStorageReferences.tryParseReference(stored.getPayloads(1)));
+
+ Payloads retrieved = transformer.retrieve(stored, CancellationToken.none()).get();
+ assertEquals(message, retrieved);
+ }
+
+ @Test
+ public void walksNestedPayloads() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStorageMessageTransformer transformer = transformer(driver, 0);
+ Command command =
+ Command.newBuilder()
+ .setScheduleActivityTaskCommandAttributes(
+ ScheduleActivityTaskCommandAttributes.newBuilder()
+ .setInput(Payloads.newBuilder().addPayloads(payload("deep"))))
+ .build();
+
+ Command stored = transformer.store(command, null, CancellationToken.none()).get();
+
+ Payload nested = stored.getScheduleActivityTaskCommandAttributes().getInput().getPayloads(0);
+ assertNotNull(ExternalStorageReferences.tryParseReference(nested));
+ assertEquals(command, transformer.retrieve(stored, CancellationToken.none()).get());
+ }
+
+ @Test
+ public void payloadBelowThresholdLeavesMessageUnchanged() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStorageMessageTransformer transformer = transformer(driver, 1024);
+ Payloads message = Payloads.newBuilder().addPayloads(payload("small")).build();
+
+ Payloads stored = transformer.store(message, null, CancellationToken.none()).get();
+
+ assertNull(ExternalStorageReferences.tryParseReference(stored.getPayloads(0)));
+ assertEquals(message, stored);
+ assertTrue(driver.storeBatchSizes.isEmpty());
+ }
+
+ @Test
+ public void searchAttributesAreNotOffloaded() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStorageMessageTransformer transformer = transformer(driver, 0);
+ Command command =
+ Command.newBuilder()
+ .setStartChildWorkflowExecutionCommandAttributes(
+ StartChildWorkflowExecutionCommandAttributes.newBuilder()
+ .setInput(Payloads.newBuilder().addPayloads(payload("input")))
+ .setSearchAttributes(
+ SearchAttributes.newBuilder()
+ .putIndexedFields("k", payload("indexed-value"))))
+ .build();
+
+ Command stored = transformer.store(command, null, CancellationToken.none()).get();
+
+ StartChildWorkflowExecutionCommandAttributes attrs =
+ stored.getStartChildWorkflowExecutionCommandAttributes();
+ assertNotNull(ExternalStorageReferences.tryParseReference(attrs.getInput().getPayloads(0)));
+ Payload indexed = attrs.getSearchAttributes().getIndexedFieldsOrThrow("k");
+ assertNull(ExternalStorageReferences.tryParseReference(indexed));
+ assertEquals(payload("indexed-value"), indexed);
+ }
+
+ private static ExternalStorageMessageTransformer transformer(
+ StorageDriver driver, int threshold) {
+ ExternalStoragePayloadTransformer payloadTransformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDriver(driver)
+ .setPayloadSizeThreshold(threshold)
+ .build());
+ return new ExternalStorageMessageTransformer(payloadTransformer, 4);
+ }
+
+ private static Payload payload(String data) {
+ return Payload.newBuilder().setData(ByteString.copyFromUtf8(data)).build();
+ }
+
+ private static final class InMemoryDriver implements StorageDriver {
+ private final String name;
+ private final Map objects = new HashMap<>();
+ final List storeBatchSizes = new ArrayList<>();
+ private int counter = 0;
+
+ InMemoryDriver(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getType() {
+ return "test.inmemory";
+ }
+
+ @Override
+ public synchronized CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ storeBatchSizes.add(payloads.size());
+ List claims = new ArrayList<>();
+ for (Payload payload : payloads) {
+ String key = name + "-" + (counter++);
+ objects.put(key, payload);
+ claims.add(new StorageDriverClaim(Collections.singletonMap("key", key)));
+ }
+ return CompletableFuture.completedFuture(claims);
+ }
+
+ @Override
+ public synchronized CompletableFuture> retrieve(
+ StorageDriverRetrieveContext context, List claims) {
+ List payloads = new ArrayList<>();
+ for (StorageDriverClaim claim : claims) {
+ payloads.add(objects.get(claim.getClaimData().get("key")));
+ }
+ return CompletableFuture.completedFuture(payloads);
+ }
+ }
+}
diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformerTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformerTest.java
new file mode 100644
index 000000000..f1632ca81
--- /dev/null
+++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStoragePayloadTransformerTest.java
@@ -0,0 +1,441 @@
+package io.temporal.internal.payload.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.protobuf.ByteString;
+import io.temporal.api.common.v1.Payload;
+import io.temporal.common.CancellationToken;
+import io.temporal.internal.concurrent.structured.CancelSource;
+import io.temporal.payload.storage.ExternalStorageOptions;
+import io.temporal.payload.storage.StorageDriver;
+import io.temporal.payload.storage.StorageDriverClaim;
+import io.temporal.payload.storage.StorageDriverRetrieveContext;
+import io.temporal.payload.storage.StorageDriverSelector;
+import io.temporal.payload.storage.StorageDriverStoreContext;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.Test;
+
+/** Tests external storage payload-list conversion. */
+public class ExternalStoragePayloadTransformerTest {
+
+ @Test
+ public void storesAndRetrievesRoundTrip() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 0);
+ List input = Arrays.asList(payload("a"), payload("b"));
+
+ List stored = transformer.store(input, null, CancellationToken.none()).get();
+
+ assertEquals(2, stored.size());
+ assertNotNull(ExternalStorageReferences.tryParseReference(stored.get(0)));
+ assertNotNull(ExternalStorageReferences.tryParseReference(stored.get(1)));
+ assertEquals(Collections.singletonList(2), driver.storeBatchSizes);
+ assertEquals(
+ input.get(0).getSerializedSize(), stored.get(0).getExternalPayloads(0).getSizeBytes());
+
+ List retrieved = transformer.retrieve(stored, CancellationToken.none()).get();
+ assertEquals(input, retrieved);
+ assertEquals(Collections.singletonList(2), driver.retrieveBatchSizes);
+ }
+
+ @Test
+ public void payloadBelowThresholdStaysInline() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 100);
+ Payload small = payload("x");
+ Payload large = payload(repeat("y", 200));
+
+ List stored =
+ transformer.store(Arrays.asList(small, large), null, CancellationToken.none()).get();
+
+ assertNull(ExternalStorageReferences.tryParseReference(stored.get(0)));
+ assertEquals(small, stored.get(0));
+ assertNotNull(ExternalStorageReferences.tryParseReference(stored.get(1)));
+ assertEquals(Collections.singletonList(1), driver.storeBatchSizes);
+ }
+
+ @Test
+ public void selectorReturningNullKeepsInline() throws Exception {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStoragePayloadTransformer transformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDriver(driver)
+ .setDriverSelector((context, payload) -> null)
+ .setPayloadSizeThreshold(0)
+ .build());
+
+ List stored =
+ transformer
+ .store(Collections.singletonList(payload("a")), null, CancellationToken.none())
+ .get();
+
+ assertEquals(payload("a"), stored.get(0));
+ assertTrue(driver.storeBatchSizes.isEmpty());
+ }
+
+ @Test
+ public void multipleDriversBatchPerDriverAndPreserveOrder() throws Exception {
+ InMemoryDriver d1 = new InMemoryDriver("d1");
+ InMemoryDriver d2 = new InMemoryDriver("d2");
+ Map byPrefix = new HashMap<>();
+ byPrefix.put("1", d1);
+ byPrefix.put("2", d2);
+ StorageDriverSelector selector =
+ (context, payload) -> byPrefix.get(payload.getData().toStringUtf8().substring(0, 1));
+ ExternalStoragePayloadTransformer transformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDrivers(Arrays.asList(d1, d2))
+ .setDriverSelector(selector)
+ .setPayloadSizeThreshold(0)
+ .build());
+ List input = Arrays.asList(payload("1-a"), payload("2-b"), payload("1-c"));
+
+ List stored = transformer.store(input, null, CancellationToken.none()).get();
+
+ assertEquals(Collections.singletonList(2), d1.storeBatchSizes);
+ assertEquals(Collections.singletonList(1), d2.storeBatchSizes);
+ assertEquals(input, transformer.retrieve(stored, CancellationToken.none()).get());
+ }
+
+ @Test
+ public void arityMismatchFails() {
+ StorageDriver driver =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ return CompletableFuture.completedFuture(Collections.emptyList());
+ }
+ };
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 0);
+
+ Throwable cause =
+ causeOf(
+ transformer.store(
+ Collections.singletonList(payload("a")), null, CancellationToken.none()));
+ assertTrue(cause instanceof IllegalStateException);
+ assertTrue(cause.getMessage().contains("returned 0 claims for 1 payloads"));
+ }
+
+ @Test
+ public void nullClaimFromDriverFails() {
+ StorageDriver driver =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ return CompletableFuture.completedFuture(Collections.singletonList(null));
+ }
+ };
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 0);
+
+ Throwable cause =
+ causeOf(
+ transformer.store(
+ Collections.singletonList(payload("a")), null, CancellationToken.none()));
+ assertTrue(cause instanceof IllegalStateException);
+ assertTrue(cause.getMessage().contains("returned a null claim at index 0"));
+ }
+
+ @Test
+ public void nullPayloadFromDriverFails() {
+ StorageDriver driver =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> retrieve(
+ StorageDriverRetrieveContext context, List claims) {
+ return CompletableFuture.completedFuture(Collections.singletonList(null));
+ }
+ };
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 0);
+ Payload reference =
+ ExternalStorageReferences.toReferencePayload(
+ "d1", new StorageDriverClaim(Collections.singletonMap("key", "k")), 1L);
+
+ Throwable cause =
+ causeOf(
+ transformer.retrieve(Collections.singletonList(reference), CancellationToken.none()));
+ assertTrue(cause instanceof IllegalStateException);
+ assertTrue(cause.getMessage().contains("returned a null payload at index 0"));
+ }
+
+ @Test
+ public void unknownDriverOnRetrieveFails() {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ ExternalStoragePayloadTransformer transformer = transformer(driver, 0);
+ Payload reference =
+ ExternalStorageReferences.toReferencePayload(
+ "ghost", new StorageDriverClaim(Collections.singletonMap("key", "k")), 1L);
+
+ Throwable cause =
+ causeOf(
+ transformer.retrieve(Collections.singletonList(reference), CancellationToken.none()));
+ assertTrue(cause instanceof IllegalStateException);
+ assertTrue(cause.getMessage().contains("No storage driver registered with name 'ghost'"));
+ }
+
+ @Test
+ public void selectorReturningUnregisteredDriverFails() {
+ InMemoryDriver registered = new InMemoryDriver("d1");
+ InMemoryDriver stranger = new InMemoryDriver("d2");
+ ExternalStoragePayloadTransformer transformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDriver(registered)
+ .setDriverSelector((context, payload) -> stranger)
+ .setPayloadSizeThreshold(0)
+ .build());
+
+ Throwable cause =
+ causeOf(
+ transformer.store(
+ Collections.singletonList(payload("a")), null, CancellationToken.none()));
+ assertTrue(cause instanceof IllegalStateException);
+ assertTrue(cause.getMessage().contains("not registered"));
+ }
+
+ @Test
+ public void firstErrorRequestsCancellationOfOutstandingDriverCalls() {
+ CompletableFuture> inFlight = new CompletableFuture<>();
+ CompletableFuture> failing = new CompletableFuture<>();
+ AtomicBoolean cancellationRequested = new AtomicBoolean(false);
+ StorageDriver slow =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ context.getCancellationToken().onCancel(() -> cancellationRequested.set(true));
+ return inFlight;
+ }
+ };
+ StorageDriver doomed = controlledStore("d2", failing);
+ Map byPrefix = new HashMap<>();
+ byPrefix.put("1", slow);
+ byPrefix.put("2", doomed);
+ ExternalStoragePayloadTransformer transformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDrivers(Arrays.asList(slow, doomed))
+ .setDriverSelector(
+ (context, payload) ->
+ byPrefix.get(payload.getData().toStringUtf8().substring(0, 1)))
+ .setPayloadSizeThreshold(0)
+ .build());
+
+ CompletableFuture> result =
+ transformer.store(
+ Arrays.asList(payload("1-a"), payload("2-b")), null, CancellationToken.none());
+ assertFalse(result.isDone());
+
+ failing.completeExceptionally(new RuntimeException("boom"));
+
+ assertTrue(result.isCompletedExceptionally());
+ assertTrue(cancellationRequested.get());
+ assertTrue(inFlight.isCancelled());
+ }
+
+ @Test
+ public void callerCancellationRequestsCancellationOfInFlightStore() {
+ CompletableFuture> inFlight = new CompletableFuture<>();
+ AtomicBoolean cancellationRequested = new AtomicBoolean(false);
+ StorageDriver slow =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ context.getCancellationToken().onCancel(() -> cancellationRequested.set(true));
+ return inFlight;
+ }
+ };
+ CancelSource caller = new CancelSource<>(CancellationException::new);
+
+ CompletableFuture> result =
+ transformer(slow, 0).store(Collections.singletonList(payload("a")), null, caller.token());
+ assertFalse(result.isDone());
+
+ caller.cancel();
+
+ assertTrue(cancellationRequested.get());
+ assertTrue(inFlight.isCancelled());
+ assertTrue(result.isCompletedExceptionally());
+ }
+
+ @Test
+ public void callerCancellationRequestsCancellationOfInFlightRetrieve() {
+ CompletableFuture> inFlight = new CompletableFuture<>();
+ AtomicBoolean cancellationRequested = new AtomicBoolean(false);
+ StorageDriver slow =
+ new FakeDriver("d1") {
+ @Override
+ public CompletableFuture> retrieve(
+ StorageDriverRetrieveContext context, List claims) {
+ context.getCancellationToken().onCancel(() -> cancellationRequested.set(true));
+ return inFlight;
+ }
+ };
+ CancelSource caller = new CancelSource<>(CancellationException::new);
+ Payload reference =
+ ExternalStorageReferences.toReferencePayload(
+ "d1", new StorageDriverClaim(Collections.singletonMap("key", "k")), 1L);
+
+ CompletableFuture> result =
+ transformer(slow, 0).retrieve(Collections.singletonList(reference), caller.token());
+ assertFalse(result.isDone());
+
+ caller.cancel();
+
+ assertTrue(cancellationRequested.get());
+ assertTrue(inFlight.isCancelled());
+ assertTrue(result.isCompletedExceptionally());
+ }
+
+ @Test
+ public void selectorObservesCallerCancellationToken() {
+ InMemoryDriver driver = new InMemoryDriver("d1");
+ CancelSource caller = new CancelSource<>(CancellationException::new);
+ AtomicReference> observed = new AtomicReference<>();
+ ExternalStoragePayloadTransformer transformer =
+ ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDriver(driver)
+ .setDriverSelector(
+ (context, payload) -> {
+ observed.set(context.getCancellationToken());
+ return driver;
+ })
+ .setPayloadSizeThreshold(0)
+ .build());
+
+ transformer.store(Collections.singletonList(payload("a")), null, caller.token());
+
+ assertSame(caller.token(), observed.get());
+ }
+
+ private static ExternalStoragePayloadTransformer transformer(
+ StorageDriver driver, int threshold) {
+ return ExternalStoragePayloadTransformer.fromOptions(
+ ExternalStorageOptions.newBuilder()
+ .setDriver(driver)
+ .setPayloadSizeThreshold(threshold)
+ .build());
+ }
+
+ private static Payload payload(String data) {
+ return Payload.newBuilder().setData(ByteString.copyFromUtf8(data)).build();
+ }
+
+ private static String repeat(String s, int n) {
+ StringBuilder sb = new StringBuilder(s.length() * n);
+ for (int i = 0; i < n; i++) {
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
+ private static Throwable causeOf(CompletableFuture> future) {
+ try {
+ future.get();
+ fail("expected failure");
+ return null;
+ } catch (ExecutionException e) {
+ return e.getCause();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static StorageDriver controlledStore(
+ String name, CompletableFuture> future) {
+ return new FakeDriver(name) {
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ return future;
+ }
+ };
+ }
+
+ private static class FakeDriver implements StorageDriver {
+ private final String name;
+
+ FakeDriver(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getType() {
+ return "test.fake";
+ }
+
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public CompletableFuture> retrieve(
+ StorageDriverRetrieveContext context, List claims) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class InMemoryDriver extends FakeDriver {
+ final Map objects = new HashMap<>();
+ final List storeBatchSizes = new ArrayList<>();
+ final List retrieveBatchSizes = new ArrayList<>();
+ private int counter = 0;
+
+ InMemoryDriver(String name) {
+ super(name);
+ }
+
+ @Override
+ public CompletableFuture> store(
+ StorageDriverStoreContext context, List payloads) {
+ storeBatchSizes.add(payloads.size());
+ List claims = new ArrayList<>();
+ for (Payload payload : payloads) {
+ String key = getName() + "-" + (counter++);
+ objects.put(key, payload);
+ claims.add(new StorageDriverClaim(Collections.singletonMap("key", key)));
+ }
+ return CompletableFuture.completedFuture(claims);
+ }
+
+ @Override
+ public CompletableFuture> retrieve(
+ StorageDriverRetrieveContext context, List claims) {
+ retrieveBatchSizes.add(claims.size());
+ List payloads = new ArrayList<>();
+ for (StorageDriverClaim claim : claims) {
+ payloads.add(objects.get(claim.getClaimData().get("key")));
+ }
+ return CompletableFuture.completedFuture(payloads);
+ }
+ }
+}
diff --git a/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageReferencesTest.java b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageReferencesTest.java
new file mode 100644
index 000000000..3f7b6b948
--- /dev/null
+++ b/temporal-sdk/src/test/java/io/temporal/internal/payload/storage/ExternalStorageReferencesTest.java
@@ -0,0 +1,134 @@
+package io.temporal.internal.payload.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import com.google.protobuf.ByteString;
+import io.temporal.api.common.v1.Payload;
+import io.temporal.api.sdk.v1.ExternalStorageReference;
+import io.temporal.common.converter.EncodingKeys;
+import io.temporal.payload.storage.StorageDriverClaim;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Test;
+
+/** Tests external storage reference encoding and decoding. */
+public class ExternalStorageReferencesTest {
+
+ @Test
+ public void currentFormatRoundTrips() {
+ Map claimData = new HashMap<>();
+ claimData.put("bucket", "my-bucket");
+ claimData.put("key", "abc123");
+ StorageDriverClaim claim = new StorageDriverClaim(claimData);
+
+ Payload reference = ExternalStorageReferences.toReferencePayload("driver-1", claim, 4096L);
+
+ assertEquals(1, reference.getExternalPayloadsCount());
+ assertEquals(4096L, reference.getExternalPayloads(0).getSizeBytes());
+ assertEquals(
+ "json/protobuf",
+ reference.getMetadataMap().get(EncodingKeys.METADATA_ENCODING_KEY).toStringUtf8());
+
+ ExternalStorageReferences.ParsedReference parsed =
+ ExternalStorageReferences.tryParseReference(reference);
+ assertNotNull(parsed);
+ assertEquals("driver-1", parsed.driverName);
+ assertEquals(claim, parsed.claim);
+ }
+
+ @Test
+ public void inlinePayloadIsNotAReference() {
+ Payload inline =
+ Payload.newBuilder()
+ .putMetadata(EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/plain"))
+ .setData(ByteString.copyFromUtf8("\"hello\""))
+ .build();
+ assertNull(ExternalStorageReferences.tryParseReference(inline));
+ }
+
+ /**
+ * {@code external_payloads} records the original size for the server and is not part of the
+ * exchange contract, so a producer that omits it still writes a readable reference.
+ */
+ @Test
+ public void referenceWithoutExternalPayloadsIsStillAReference() {
+ Payload reference =
+ Payload.newBuilder()
+ .putMetadata(
+ EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/protobuf"))
+ .putMetadata(
+ EncodingKeys.METADATA_MESSAGE_TYPE_KEY,
+ ByteString.copyFromUtf8(ExternalStorageReference.getDescriptor().getFullName()))
+ .setData(ByteString.copyFromUtf8("{\"driverName\":\"driver-1\"}"))
+ .build();
+
+ ExternalStorageReferences.ParsedReference parsed =
+ ExternalStorageReferences.tryParseReference(reference);
+ assertNotNull(parsed);
+ assertEquals("driver-1", parsed.driverName);
+ }
+
+ @Test
+ public void payloadWithReferenceMessageTypeButForeignEncodingIsNotAReference() {
+ Payload foreign =
+ Payload.newBuilder()
+ .putMetadata(EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/plain"))
+ .putMetadata(
+ EncodingKeys.METADATA_MESSAGE_TYPE_KEY,
+ ByteString.copyFromUtf8(ExternalStorageReference.getDescriptor().getFullName()))
+ .setData(ByteString.copyFromUtf8("{\"driverName\":\"driver-1\"}"))
+ .addExternalPayloads(
+ Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(4096L).build())
+ .build();
+
+ assertNull(ExternalStorageReferences.tryParseReference(foreign));
+ }
+
+ @Test
+ public void payloadWithExternalPayloadsButForeignMessageTypeIsNotAReference() {
+ Payload foreign =
+ Payload.newBuilder()
+ .putMetadata(
+ EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/protobuf"))
+ .putMetadata(
+ EncodingKeys.METADATA_MESSAGE_TYPE_KEY,
+ ByteString.copyFromUtf8("some.other.sdk.v1.ExternalStorageReference"))
+ .setData(ByteString.copyFromUtf8("{\"foo\":1}"))
+ .addExternalPayloads(
+ Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(4096L).build())
+ .build();
+
+ assertNull(ExternalStorageReferences.tryParseReference(foreign));
+ }
+
+ /**
+ * References written by other SDKs must stay readable, so parsing tolerates snake_case field
+ * names and fields added to the proto after this release.
+ */
+ @Test
+ public void parsesReferenceWrittenByAnotherSdk() {
+ Payload reference =
+ Payload.newBuilder()
+ .putMetadata(
+ EncodingKeys.METADATA_ENCODING_KEY, ByteString.copyFromUtf8("json/protobuf"))
+ .putMetadata(
+ EncodingKeys.METADATA_MESSAGE_TYPE_KEY,
+ ByteString.copyFromUtf8(ExternalStorageReference.getDescriptor().getFullName()))
+ .setData(
+ ByteString.copyFromUtf8(
+ "{\"driver_name\":\"driver-1\",\"claim_data\":{\"key\":\"abc123\"},"
+ + "\"field_added_later\":\"ignored\"}"))
+ .addExternalPayloads(
+ Payload.ExternalPayloadDetails.newBuilder().setSizeBytes(4096L).build())
+ .build();
+
+ ExternalStorageReferences.ParsedReference parsed =
+ ExternalStorageReferences.tryParseReference(reference);
+ assertNotNull(parsed);
+ assertEquals("driver-1", parsed.driverName);
+ assertEquals(new StorageDriverClaim(Collections.singletonMap("key", "abc123")), parsed.claim);
+ }
+}
diff --git a/temporal-sdk/src/test/java/io/temporal/payload/storage/ExternalStorageOptionsTest.java b/temporal-sdk/src/test/java/io/temporal/payload/storage/ExternalStorageOptionsTest.java
index bc2ed1bc5..2c7ffc782 100644
--- a/temporal-sdk/src/test/java/io/temporal/payload/storage/ExternalStorageOptionsTest.java
+++ b/temporal-sdk/src/test/java/io/temporal/payload/storage/ExternalStorageOptionsTest.java
@@ -11,6 +11,7 @@
import java.util.concurrent.CompletableFuture;
import org.junit.Test;
+/** Tests external storage option validation and defaults. */
public class ExternalStorageOptionsTest {
private static StorageDriverStoreContext storeContext(StorageDriverTargetInfo target) {