From d0b5dbcc62bf88aa08a41c846a7645c77e09faa1 Mon Sep 17 00:00:00 2001 From: gilseara <> Date: Tue, 3 Mar 2026 11:46:32 +0100 Subject: [PATCH 1/2] Add exponential backoff retry for SSC 502/503 errors Add SSCRetryStrategy implementing Apache HttpClient's ServiceUnavailableRetryStrategy to retry on HTTP 502 (Bad Gateway) and 503 (Service Unavailable) with exponential backoff (1s, 2s, 4s) and random jitter (0-500ms). Wire it into the SSC unirest instance via a custom ApacheClient, following the existing FoD retry pattern. --- .../SSCAndScanCentralUnirestHelper.java | 3 ++ .../_common/rest/helper/SSCRetryStrategy.java | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java index 38b099aeb7..034f4ecde6 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java @@ -22,9 +22,12 @@ import com.fortify.cli.ssc._common.session.helper.SSCAndScanCentralSessionDescriptor; import kong.unirest.UnirestInstance; +import kong.unirest.apache.ApacheClient; public class SSCAndScanCentralUnirestHelper { public static final void configureSscUnirestInstance(UnirestInstance unirest, SSCAndScanCentralSessionDescriptor sessionDescriptor) { + unirest.config().httpClient(config -> new ApacheClient(config, cb -> + cb.setServiceUnavailableRetryStrategy(new SSCRetryStrategy()))); UnirestUnexpectedHttpResponseConfigurer.configure(unirest); UnirestJsonHeaderConfigurer.configure(unirest); UnirestUrlConfigConfigurer.configure(unirest, sessionDescriptor.getSscUrlConfig()); diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java new file mode 100644 index 0000000000..57f21e6099 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.ssc._common.rest.helper; + +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.http.HttpResponse; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.protocol.HttpContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class implements an Apache HttpClient 4.x {@link ServiceUnavailableRetryStrategy} + * that will retry a request if the server responds with an HTTP 502 (Bad Gateway) + * or 503 (Service Unavailable) response, using exponential backoff with jitter. + */ +public final class SSCRetryStrategy implements ServiceUnavailableRetryStrategy { + private static final Logger LOG = LoggerFactory.getLogger(SSCRetryStrategy.class); + private static final int MAX_RETRIES = 3; + private static final long BASE_DELAY_MS = 1000; + private static final long MAX_JITTER_MS = 500; + private final ThreadLocal interval = new ThreadLocal(); + + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { + int statusCode = response.getStatusLine().getStatusCode(); + if ( executionCount <= MAX_RETRIES && (statusCode == 502 || statusCode == 503) ) { + long delay = BASE_DELAY_MS * (1L << (executionCount - 1)); + long jitter = ThreadLocalRandom.current().nextLong(MAX_JITTER_MS + 1); + long totalDelay = delay + jitter; + LOG.debug("SSC returned {}; retrying (attempt {}/{}) after {} ms", statusCode, executionCount, MAX_RETRIES, totalDelay); + interval.set(totalDelay); + return true; + } + return false; + } + + public long getRetryInterval() { + Long result = interval.get(); + return result == null ? -1 : result; + } +} From a4374dd415a81049bbf99ae04df86165c1675d33 Mon Sep 17 00:00:00 2001 From: gilseara <> Date: Tue, 24 Mar 2026 13:37:27 +0100 Subject: [PATCH 2/2] Limit 502/503 retry to GET requests; apply to SC-SAST, SC-DAST, and FoD Only retry GET requests on 502/503 to avoid duplicate side effects from retrying non-idempotent operations. Apply the same retry strategy to SC-SAST and SC-DAST unirest instances, and add 502/503 GET-only retry to FoDRetryStrategy alongside the existing 404/429 handling. --- .../_common/rest/helper/FoDRetryStrategy.java | 31 ++++++++++++++++--- .../SSCAndScanCentralUnirestHelper.java | 6 +++- .../_common/rest/helper/SSCRetryStrategy.java | 14 +++++++-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRetryStrategy.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRetryStrategy.java index 88a72c0e9c..015b0fadd7 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRetryStrategy.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/helper/FoDRetryStrategy.java @@ -12,23 +12,30 @@ */ package com.fortify.cli.fod._common.rest.helper; +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.client.ServiceUnavailableRetryStrategy; import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class implements an Apache HttpClient 4.x {@link ServiceUnavailableRetryStrategy} * that will retry a request if the server responds with an HTTP 429 (TOO_MANY_REQUESTS) - * response. + * response, and will retry GET requests on HTTP 502 (Bad Gateway) or 503 + * (Service Unavailable) with exponential backoff and jitter. */ public final class FoDRetryStrategy implements ServiceUnavailableRetryStrategy { private static final Logger LOG = LoggerFactory.getLogger(FoDRetryStrategy.class); - private final String HEADER_NAME = "X-Rate-Limit-Reset"; + private static final String HEADER_NAME = "X-Rate-Limit-Reset"; + private static final long BASE_DELAY_MS = 1000; + private static final long MAX_JITTER_MS = 500; private int maxRetries = 2; private final ThreadLocal interval = new ThreadLocal(); - + public FoDRetryStrategy maxRetries(int maxRetries) { this.maxRetries = maxRetries; return this; @@ -36,16 +43,30 @@ public FoDRetryStrategy maxRetries(int maxRetries) { public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { if ( executionCount < maxRetries+1 ) { - if ( response.getStatusLine().getStatusCode()==404 ) { + int statusCode = response.getStatusLine().getStatusCode(); + if ( statusCode==404 ) { // Sometimes it can take a bit of time for FoD to properly register a scan request and // possibly other newly created resources, hence we also retry on 404 errors. interval.set((long)5000); return true; - } else if ( response.getStatusLine().getStatusCode()==429 ) { + } else if ( statusCode==429 ) { int retrySeconds = Integer.parseInt(response.getFirstHeader(HEADER_NAME).getValue()); LOG.debug("Rate-limited request will be retried after "+retrySeconds+" seconds"); interval.set((long)retrySeconds*1000); return true; + } else if ( statusCode==502 || statusCode==503 ) { + HttpRequest request = (HttpRequest) context.getAttribute(HttpCoreContext.HTTP_REQUEST); + String method = request.getRequestLine().getMethod(); + if ( !"GET".equalsIgnoreCase(method) ) { + LOG.debug("FoD returned {}; not retrying non-GET request ({} {})", statusCode, method, request.getRequestLine().getUri()); + return false; + } + long delay = BASE_DELAY_MS * (1L << (executionCount - 1)); + long jitter = ThreadLocalRandom.current().nextLong(MAX_JITTER_MS + 1); + long totalDelay = delay + jitter; + LOG.debug("FoD returned {}; retrying GET request (attempt {}/{}) after {} ms", statusCode, executionCount, maxRetries, totalDelay); + interval.set(totalDelay); + return true; } } return false; diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java index 034f4ecde6..0bb0c8ff44 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCAndScanCentralUnirestHelper.java @@ -38,15 +38,19 @@ public static final void configureSscUnirestInstance(UnirestInstance unirest, SS public static final void configureScSastControllerUnirestInstance(UnirestInstance unirest, SSCAndScanCentralSessionDescriptor sessionDescriptor) { checkEnabled("SC-SAST", sessionDescriptor.getScSastDisabledReason()); + unirest.config().httpClient(config -> new ApacheClient(config, cb -> + cb.setServiceUnavailableRetryStrategy(new SSCRetryStrategy()))); UnirestUnexpectedHttpResponseConfigurer.configure(unirest); UnirestJsonHeaderConfigurer.configure(unirest); UnirestUrlConfigConfigurer.configure(unirest, sessionDescriptor.getScSastUrlConfig()); ProxyHelper.configureProxy(unirest, "sc-sast", sessionDescriptor.getScSastUrlConfig().getUrl()); unirest.config().setDefaultHeader("fortify-client", String.valueOf(sessionDescriptor.getScSastClientAuthToken())); } - + public static final void configureScDastControllerUnirestInstance(UnirestInstance unirest, SSCAndScanCentralSessionDescriptor sessionDescriptor) { checkEnabled("SC-DAST", sessionDescriptor.getScDastDisabledReason()); + unirest.config().httpClient(config -> new ApacheClient(config, cb -> + cb.setServiceUnavailableRetryStrategy(new SSCRetryStrategy()))); UnirestUnexpectedHttpResponseConfigurer.configure(unirest); UnirestJsonHeaderConfigurer.configure(unirest); UnirestUrlConfigConfigurer.configure(unirest, sessionDescriptor.getScDastUrlConfig()); diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java index 57f21e6099..cbdc5dc0dd 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/helper/SSCRetryStrategy.java @@ -14,16 +14,20 @@ import java.util.concurrent.ThreadLocalRandom; +import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.client.ServiceUnavailableRetryStrategy; import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class implements an Apache HttpClient 4.x {@link ServiceUnavailableRetryStrategy} - * that will retry a request if the server responds with an HTTP 502 (Bad Gateway) + * that will retry GET requests if the server responds with an HTTP 502 (Bad Gateway) * or 503 (Service Unavailable) response, using exponential backoff with jitter. + * Non-GET requests are not retried to avoid the risk of duplicate side effects + * (e.g., creating duplicate entities). */ public final class SSCRetryStrategy implements ServiceUnavailableRetryStrategy { private static final Logger LOG = LoggerFactory.getLogger(SSCRetryStrategy.class); @@ -35,10 +39,16 @@ public final class SSCRetryStrategy implements ServiceUnavailableRetryStrategy { public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { int statusCode = response.getStatusLine().getStatusCode(); if ( executionCount <= MAX_RETRIES && (statusCode == 502 || statusCode == 503) ) { + HttpRequest request = (HttpRequest) context.getAttribute(HttpCoreContext.HTTP_REQUEST); + String method = request.getRequestLine().getMethod(); + if ( !"GET".equalsIgnoreCase(method) ) { + LOG.debug("SSC returned {}; not retrying non-GET request ({} {})", statusCode, method, request.getRequestLine().getUri()); + return false; + } long delay = BASE_DELAY_MS * (1L << (executionCount - 1)); long jitter = ThreadLocalRandom.current().nextLong(MAX_JITTER_MS + 1); long totalDelay = delay + jitter; - LOG.debug("SSC returned {}; retrying (attempt {}/{}) after {} ms", statusCode, executionCount, MAX_RETRIES, totalDelay); + LOG.debug("SSC returned {}; retrying GET request (attempt {}/{}) after {} ms", statusCode, executionCount, MAX_RETRIES, totalDelay); interval.set(totalDelay); return true; }