From 55d48b16fce6055b7abe7fa68a95356c6324641b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= <janhoy@apache.org>
Date: Fri, 7 Aug 2026 01:30:00 +0200
Subject: [PATCH] Make StoppableIndexingThread retry idempotent updates on
 transient transport errors

The nightly ChaosMonkeySafeLeaderWithPullReplicasTest (and other chaos
tests) can fail with getFailCount()==1 when the chaos monkey stops a node
while an update is in flight: the Jetty HTTP/2 client surfaces this as an
ambiguous IOException (e.g. cancel_stream_error/input_shutdown,
ClosedChannelException) that CloudSolrClient does not retry, so the
indexing thread records a failure and the test's assertEquals(0,
getFailCount()) fails. Seen on Jenkins for both main and branch_10x.

Rather than broaden CloudSolrClient's retry classification -- which for
non-idempotent updates risks double-processing, since these HTTP/2 errors
do not guarantee the server never handled the request -- retry in the
test thread, which only sends idempotent operations (adds of docs with
unique ids and delete-by-id). Bounded retries (5, with backoff) are
applied when the root cause is an IOException; Solr-level errors are not
retried.
---
 .../solr/cloud/StoppableIndexingThread.java   | 47 +++++++++++++++++--
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java
index 5ed3b89337b0..3e1fad29e7f7 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/StoppableIndexingThread.java
@@ -25,6 +25,7 @@
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,6 +33,8 @@
 public class StoppableIndexingThread extends AbstractFullDistribZkTestBase.StoppableThread {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  private static final int MAX_TRANSIENT_RETRIES = 5;
+
   static String t1 = "a_t";
   static String i1 = "a_i";
   private volatile boolean stop = false;
@@ -99,10 +102,12 @@ public void run() {
             UpdateRequest req = new UpdateRequest();
             req.deleteById(deleteId);
             req.setParam("CONTROL", "TRUE");
-            req.process(controlClient);
+            processWithRetry(req, controlClient);
           }
 
-          cloudClient.deleteById(deleteId);
+          UpdateRequest cloudReq = new UpdateRequest();
+          cloudReq.deleteById(deleteId);
+          processWithRetry(cloudReq, cloudClient);
         } catch (Exception e) {
           log.error("REQUEST FAILED for id={}", deleteId, e);
           if (e instanceof SolrServerException) {
@@ -185,11 +190,45 @@ protected void indexDocs(List<SolrInputDocument> docs) throws IOException, SolrS
       UpdateRequest req = new UpdateRequest();
       req.add(docs);
       req.setParam("CONTROL", "TRUE");
-      req.process(controlClient);
+      processWithRetry(req, controlClient);
     }
 
     UpdateRequest ureq = new UpdateRequest();
     ureq.add(docs);
-    ureq.process(cloudClient);
+    processWithRetry(ureq, cloudClient);
+  }
+
+  /**
+   * Process the request, retrying a bounded number of times when the failure is an ambiguous
+   * transport-level error (root cause is an {@link IOException}), such as a node being stopped by
+   * the chaos monkey while the request was in flight. Everything this thread sends is idempotent
+   * (adds of docs with unique ids and delete-by-id), so at-least-once delivery is safe, and
+   * retrying is what a well-behaved indexing application would do. Solr-level errors (e.g. a 4xx
+   * response) are not retried.
+   */
+  private void processWithRetry(UpdateRequest req, SolrClient client)
+      throws IOException, SolrServerException {
+    for (int attempt = 1; ; attempt++) {
+      try {
+        req.process(client);
+        return;
+      } catch (SolrServerException | IOException e) {
+        Throwable rootCause = SolrException.getRootCause(e);
+        if (attempt >= MAX_TRANSIENT_RETRIES || !(rootCause instanceof IOException)) {
+          throw e;
+        }
+        log.info(
+            "Retrying update after transient error (attempt {}/{})",
+            attempt,
+            MAX_TRANSIENT_RETRIES,
+            rootCause);
+        try {
+          Thread.sleep(250L * attempt);
+        } catch (InterruptedException ie) {
+          Thread.currentThread().interrupt();
+          throw e;
+        }
+      }
+    }
   }
 }