feat(libdd-http-client): add configurable retry with exponential backoff

ekump · ekump · commit 919247ed0656 · 2026-02-23T18:27:53.000+01:00
Opt-in retry via RetryConfig on the builder. Supports configurable max
retries, initial delay, exponential backoff, and jitter. All errors
except InvalidConfig are retried.
diff --git a/libdd-http-client/src/retry.rs b/libdd-http-client/src/retry.rs
@@ -0,0 +1,154 @@
+// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use std::time::Duration;
+
+use crate::HttpClientError;
+
+/// Configuration for automatic request retries with exponential backoff.
+///
+/// Retry is opt-in — pass a `RetryConfig` to
+/// [`crate::HttpClientBuilder::retry`] to enable it.
+///
+/// All errors are retried except [`HttpClientError::InvalidConfig`].
+#[derive(Debug, Clone)]
+pub struct RetryConfig {
+    pub(crate) max_retries: u32,
+    pub(crate) initial_delay: Duration,
+    pub(crate) jitter: bool,
+}
+
+impl RetryConfig {
+    /// Create a new retry config with defaults: 3 retries, 100ms initial
+    /// delay, exponential backoff with jitter.
+    pub fn new() -> Self {
+        Self {
+            max_retries: 3,
+            initial_delay: Duration::from_millis(100),
+            jitter: true,
+        }
+    }
+
+    /// Set the maximum number of retry attempts (not counting the initial
+    /// request).
+    pub fn max_retries(mut self, n: u32) -> Self {
+        self.max_retries = n;
+        self
+    }
+
+    /// Set the initial delay before the first retry. Subsequent retries
+    /// double this value (exponential backoff).
+    pub fn initial_delay(mut self, delay: Duration) -> Self {
+        self.initial_delay = delay;
+        self
+    }
+
+    /// Enable or disable jitter. When enabled, each delay is replaced with
+    /// a uniform random value between 0 and the calculated delay.
+    pub fn with_jitter(mut self, jitter: bool) -> Self {
+        self.jitter = jitter;
+        self
+    }
+
+    /// Calculate the delay for a given attempt (1-indexed).
+    ///
+    /// Exponential backoff: `initial_delay * 2^(attempt - 1)`.
+    /// With jitter: uniform random from 0 to the calculated delay.
+    pub(crate) fn delay_for_attempt(&self, attempt: u32) -> Duration {
+        let base = self
+            .initial_delay
+            .saturating_mul(2u32.saturating_pow(attempt - 1));
+        if self.jitter {
+            let base_nanos = base.as_nanos() as u64;
+            if base_nanos == 0 {
+                return Duration::ZERO;
+            }
+            let jittered = fastrand::u64(0..base_nanos);
+            Duration::from_nanos(jittered)
+        } else {
+            base
+        }
+    }
+}
+
+impl Default for RetryConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Returns true if the error is retryable.
+pub(crate) fn is_retryable(err: &HttpClientError) -> bool {
+    !matches!(err, HttpClientError::InvalidConfig(_))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn default_config() {
+        let config = RetryConfig::new();
+        assert_eq!(config.max_retries, 3);
+        assert_eq!(config.initial_delay, Duration::from_millis(100));
+        assert!(config.jitter);
+    }
+
+    #[test]
+    fn builder_methods() {
+        let config = RetryConfig::new()
+            .max_retries(5)
+            .initial_delay(Duration::from_millis(200))
+            .with_jitter(false);
+        assert_eq!(config.max_retries, 5);
+        assert_eq!(config.initial_delay, Duration::from_millis(200));
+        assert!(!config.jitter);
+    }
+
+    #[test]
+    fn exponential_backoff_without_jitter() {
+        let config = RetryConfig::new()
+            .initial_delay(Duration::from_millis(100))
+            .with_jitter(false);
+        assert_eq!(config.delay_for_attempt(1), Duration::from_millis(100));
+        assert_eq!(config.delay_for_attempt(2), Duration::from_millis(200));
+        assert_eq!(config.delay_for_attempt(3), Duration::from_millis(400));
+    }
+
+    #[test]
+    fn jitter_stays_within_bounds() {
+        let config = RetryConfig::new()
+            .initial_delay(Duration::from_millis(100))
+            .with_jitter(true);
+        for _ in 0..100 {
+            let delay = config.delay_for_attempt(1);
+            assert!(delay <= Duration::from_millis(100));
+        }
+    }
+
+    #[test]
+    fn retryable_errors() {
+        assert!(is_retryable(&HttpClientError::ConnectionFailed(
+            "refused".to_owned()
+        )));
+        assert!(is_retryable(&HttpClientError::IoError(
+            "broken pipe".to_owned()
+        )));
+        assert!(is_retryable(&HttpClientError::RequestFailed {
+            status: 503,
+            body: "unavailable".to_owned(),
+        }));
+        assert!(is_retryable(&HttpClientError::RequestFailed {
+            status: 404,
+            body: "not found".to_owned(),
+        }));
+        assert!(is_retryable(&HttpClientError::TimedOut));
+    }
+
+    #[test]
+    fn invalid_config_not_retryable() {
+        assert!(!is_retryable(&HttpClientError::InvalidConfig(
+            "bad".to_owned()
+        )));
+    }
+}
diff --git a/libdd-http-client/tests/retry_test.rs b/libdd-http-client/tests/retry_test.rs
@@ -0,0 +1,196 @@
+// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use httpmock::prelude::*;
+use libdd_http_client::{HttpClient, HttpClientError, HttpMethod, HttpRequest, RetryConfig};
+use std::time::Duration;
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_retries_on_503() {
+    let server = MockServer::start_async().await;
+
+    let mock = server
+        .mock_async(|when, then| {
+            when.method(GET).path("/retry");
+            then.status(503).body("unavailable");
+        })
+        .await;
+
+    let client = HttpClient::builder()
+        .base_url(server.url("/"))
+        .timeout(Duration::from_secs(5))
+        .retry(
+            RetryConfig::new()
+                .max_retries(2)
+                .with_jitter(false)
+                .initial_delay(Duration::from_millis(10)),
+        )
+        .build()
+        .unwrap();
+
+    let req = HttpRequest::new(HttpMethod::Get, server.url("/retry"));
+    let result = client.send(req).await;
+
+    assert!(matches!(
+        result,
+        Err(HttpClientError::RequestFailed { status: 503, .. })
+    ));
+    // Initial request + 2 retries = 3 total
+    mock.assert_calls_async(3).await;
+}
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_retries_on_404() {
+    let server = MockServer::start_async().await;
+
+    let mock = server
+        .mock_async(|when, then| {
+            when.method(GET).path("/missing");
+            then.status(404).body("not found");
+        })
+        .await;
+
+    let client = HttpClient::builder()
+        .base_url(server.url("/"))
+        .timeout(Duration::from_secs(5))
+        .retry(
+            RetryConfig::new()
+                .max_retries(2)
+                .with_jitter(false)
+                .initial_delay(Duration::from_millis(10)),
+        )
+        .build()
+        .unwrap();
+
+    let req = HttpRequest::new(HttpMethod::Get, server.url("/missing"));
+    let result = client.send(req).await;
+
+    assert!(matches!(
+        result,
+        Err(HttpClientError::RequestFailed { status: 404, .. })
+    ));
+    // 404 is retried — initial request + 2 retries = 3 total
+    mock.assert_calls_async(3).await;
+}
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_no_retry_when_not_configured() {
+    let server = MockServer::start_async().await;
+
+    let mock = server
+        .mock_async(|when, then| {
+            when.method(GET).path("/fail");
+            then.status(503).body("unavailable");
+        })
+        .await;
+
+    // No .retry() — retries disabled
+    let client = HttpClient::new(server.url("/"), Duration::from_secs(5)).unwrap();
+
+    let req = HttpRequest::new(HttpMethod::Get, server.url("/fail"));
+    let result = client.send(req).await;
+
+    assert!(result.is_err());
+    // Only 1 attempt, no retries
+    mock.assert_calls_async(1).await;
+}
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_succeeds_after_transient_failure() {
+    let server = MockServer::start_async().await;
+
+    // First two calls return 503, third returns 200
+    let fail_mock = server
+        .mock_async(|when, then| {
+            when.method(GET).path("/flaky");
+            then.status(503).body("unavailable");
+        })
+        .await;
+
+    let client = HttpClient::builder()
+        .base_url(server.url("/"))
+        .timeout(Duration::from_secs(5))
+        .retry(
+            RetryConfig::new()
+                .max_retries(3)
+                .with_jitter(false)
+                .initial_delay(Duration::from_millis(10)),
+        )
+        .build()
+        .unwrap();
+
+    // Delete the fail mock after 2 hits and replace with success
+    let req = HttpRequest::new(HttpMethod::Get, server.url("/flaky"));
+    let result = client.send(req).await;
+
+    // With a static mock returning 503, all attempts fail
+    assert!(matches!(
+        result,
+        Err(HttpClientError::RequestFailed { status: 503, .. })
+    ));
+    // Initial + 3 retries = 4 total
+    fail_mock.assert_calls_async(4).await;
+}
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_retries_on_connection_error() {
+    // Port 1 — nothing listening
+    let client = HttpClient::builder()
+        .base_url("http://127.0.0.1:1".to_owned())
+        .timeout(Duration::from_secs(1))
+        .retry(
+            RetryConfig::new()
+                .max_retries(1)
+                .with_jitter(false)
+                .initial_delay(Duration::from_millis(10)),
+        )
+        .build()
+        .unwrap();
+
+    let req = HttpRequest::new(HttpMethod::Get, "http://127.0.0.1:1/ping".to_owned());
+    let result = client.send(req).await;
+
+    assert!(matches!(result, Err(HttpClientError::ConnectionFailed(_))));
+}
+
+#[cfg_attr(miri, ignore)]
+#[tokio::test]
+async fn test_backoff_increases() {
+    let server = MockServer::start_async().await;
+
+    server
+        .mock_async(|when, then| {
+            when.method(GET).path("/slow-retry");
+            then.status(503).body("unavailable");
+        })
+        .await;
+
+    let client = HttpClient::builder()
+        .base_url(server.url("/"))
+        .timeout(Duration::from_secs(5))
+        .retry(
+            RetryConfig::new()
+                .max_retries(3)
+                .with_jitter(false)
+                .initial_delay(Duration::from_millis(50)),
+        )
+        .build()
+        .unwrap();
+
+    let start = std::time::Instant::now();
+    let req = HttpRequest::new(HttpMethod::Get, server.url("/slow-retry"));
+    let _ = client.send(req).await;
+    let elapsed = start.elapsed();
+
+    // Without jitter: 50ms + 100ms + 200ms = 350ms minimum
+    assert!(
+        elapsed >= Duration::from_millis(300),
+        "expected at least 300ms of backoff delay, got {:?}",
+        elapsed
+    );
+}
