add backoff for recycling a channel

Author: nimf

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/channels/ChannelPoolDpImpl.java

@@ -65,7 +65,8 @@ public class ChannelPoolDpImpl implements ChannelPool {
   private static final String DEFAULT_LOG_NAME = "pool";
   private static final AtomicInteger INDEX = new AtomicInteger();
 
-  private static final int CONSECUTIVE_OPEN_FAILURE_THRESHOLD = 5;
+  // TODO: Move to client configuration.
+  private static final int CONSECUTIVE_OPEN_SESSION_FAILURE_THRESHOLD = 5;
 
   private final String poolLogId;
 
@@ -97,6 +98,12 @@ public class ChannelPoolDpImpl implements ChannelPool {
   @GuardedBy("this")
   private boolean closed = false;
 
+  @GuardedBy("this")
+  private long lastRecycleNano = 0;
+
+  @GuardedBy("this")
+  private Duration recycleBackoff = Duration.ofMillis(1);
+
   public ChannelPoolDpImpl(
       Supplier<ManagedChannel> channelSupplier,
       ChannelPoolConfiguration config,
@@ -224,6 +231,7 @@ public void onBeforeSessionStart(PeerInfo peerInfo) {
                 afeId = AfeId.extract(peerInfo);
                 synchronized (ChannelPoolDpImpl.this) {
                   channelWrapper.consecutiveFailures = 0;
+                  recycleBackoff = Duration.ofMillis(1);
                   rehomeChannel(channelWrapper, afeId);
                   sessionsPerAfeId.add(afeId);
                 }
@@ -327,7 +335,7 @@ private static boolean shouldRecycleChannel(ChannelWrapper channelWrapper, Statu
       return true;
     }
 
-    if (channelWrapper.consecutiveFailures >= CONSECUTIVE_OPEN_FAILURE_THRESHOLD) {
+    if (channelWrapper.consecutiveFailures >= CONSECUTIVE_OPEN_SESSION_FAILURE_THRESHOLD) {
       return true;
     }
 
@@ -341,6 +349,13 @@ private void recycleChannel(ChannelWrapper channelWrapper) {
       return;
     }
 
+    if (lastRecycleNano > System.nanoTime() - recycleBackoff.toNanos()) {
+      return;
+    }
+
+    lastRecycleNano = System.nanoTime();
+    recycleBackoff = recycleBackoff.multipliedBy(2);
+
     channelWrapper.group.channels.remove(channelWrapper);
     channelWrapper.channel.shutdown();
     // Checking for starting group because we don't want to delete the stating group.