feat: add application-level keepalive to prevent ALB idle connection timeouts

AWS ALBs do not forward HTTP/2 PING frames, causing idle gRPC connections
to be closed. This adds a background loop that periodically calls the
existing Hello RPC as application-level traffic to keep the connection
alive through L7 load balancers.

Signed-off-by: joshvanl <me@joshvanl.dev>

Author: JoshVanL

durabletask-client/src/main/java/io/dapr/durabletask/DurableTaskGrpcWorker.java

@@ -13,6 +13,7 @@
 
 package io.dapr.durabletask;
 
+import com.google.protobuf.Empty;
 import io.dapr.durabletask.implementation.protobuf.OrchestratorService;
 import io.dapr.durabletask.implementation.protobuf.TaskHubSidecarServiceGrpc;
 import io.dapr.durabletask.orchestration.TaskOrchestrationFactories;
@@ -35,6 +36,7 @@
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -48,6 +50,7 @@ public final class DurableTaskGrpcWorker implements AutoCloseable {
   private static final int DEFAULT_PORT = 4001;
   private static final Logger logger = Logger.getLogger(DurableTaskGrpcWorker.class.getPackage().getName());
   private static final Duration DEFAULT_MAXIMUM_TIMER_INTERVAL = Duration.ofDays(3);
+  private static final long KEEPALIVE_INTERVAL_SECONDS = 30;
 
   private final TaskOrchestrationFactories orchestrationFactories;
 
@@ -63,6 +66,7 @@ public final class DurableTaskGrpcWorker implements AutoCloseable {
   private final TaskHubSidecarServiceGrpc.TaskHubSidecarServiceBlockingStub sidecarClient;
   private final boolean isExecutorServiceManaged;
   private volatile boolean isNormalShutdown = false;
+  private ScheduledExecutorService keepaliveScheduler;
   private Thread workerThread;
 
   DurableTaskGrpcWorker(DurableTaskGrpcWorkerBuilder builder) {
@@ -136,6 +140,7 @@ public void close() {
       this.workerThread.interrupt();
     }
     this.isNormalShutdown = true;
+    this.stopKeepaliveLoop();
     this.shutDownWorkerPool();
     this.closeSideCarChannel();
   }
@@ -175,6 +180,7 @@ public void startAndBlock() {
         OrchestratorService.GetWorkItemsRequest getWorkItemsRequest = OrchestratorService.GetWorkItemsRequest
             .newBuilder().build();
         Iterator<OrchestratorService.WorkItem> workItemStream = this.sidecarClient.getWorkItems(getWorkItemsRequest);
+        startKeepaliveLoop();
         while (workItemStream.hasNext()) {
           OrchestratorService.WorkItem workItem = workItemStream.next();
           OrchestratorService.WorkItem.RequestCase requestType = workItem.getRequestCase();
@@ -214,6 +220,8 @@ public void startAndBlock() {
               String.format("Unexpected failure connecting to %s", this.getSidecarAddress()), e);
         }
 
+        stopKeepaliveLoop();
+
         // Retry after 5 seconds
         try {
           Thread.sleep(5000);
@@ -262,6 +270,37 @@ private void shutDownWorkerPool() {
     }
   }
 
+  /**
+   * Starts a background keepalive loop to keep the gRPC connection alive.
+   * This is an application-level keepalive to prevent AWS ALBs from
+   * killing idle HTTP/2 connections.
+   */
+  private void startKeepaliveLoop() {
+    stopKeepaliveLoop();
+    this.keepaliveScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+      Thread t = new Thread(r, "durabletask-keepalive");
+      t.setDaemon(true);
+      return t;
+    });
+    this.keepaliveScheduler.scheduleAtFixedRate(() -> {
+      try {
+        this.sidecarClient.hello(Empty.getDefaultInstance());
+      } catch (StatusRuntimeException e) {
+        logger.log(Level.FINE, "keepalive failed", e);
+      }
+    }, KEEPALIVE_INTERVAL_SECONDS, KEEPALIVE_INTERVAL_SECONDS, TimeUnit.SECONDS);
+  }
+
+  /**
+   * Stops the background keepalive loop if one is running.
+   */
+  private void stopKeepaliveLoop() {
+    if (this.keepaliveScheduler != null) {
+      this.keepaliveScheduler.shutdownNow();
+      this.keepaliveScheduler = null;
+    }
+  }
+
   private String getSidecarAddress() {
     return this.sidecarClient.getChannel().authority();
   }