Skip to content

fix(traces): add retry strategy #1116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
duncanista merged 6 commits into from
Mar 19, 2026
Merged

fix(traces): add retry strategy #1116

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e4d6c42
add retry strategy
duncanista Mar 19, 2026
4d03c5e
add retries for stats
duncanista Mar 19, 2026
406a09c
consistent retries to 3
duncanista Mar 19, 2026
95933b4
consistent env var across crates
duncanista Mar 19, 2026
812520b
remove expect
duncanista Mar 19, 2026
5910350
reduce logging on one failure in stats
duncanista Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions bottlecap/src/traces/trace_flusher.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use libdd_common::Endpoint;
use libdd_trace_utils::{
config_utils::trace_intake_url_prefixed,
send_data::SendData,
send_with_retry::{RetryBackoffType, RetryStrategy},
trace_utils::{self},
tracer_payload::TracerPayloadCollection,
};
Expand All @@ -19,6 +20,11 @@ use crate::lifecycle::invocation::processor::S_TO_MS;
use crate::traces::http_client::HttpClient;
use crate::traces::trace_aggregator_service::AggregatorHandle;

/// Retry strategy for trace flushing: 5 retries with no delay between attempts.
fn trace_retry_strategy() -> RetryStrategy {
RetryStrategy::new(5, 0, RetryBackoffType::Constant, None)
Comment thread
duncanista marked this conversation as resolved.
Outdated
}

pub struct TraceFlusher {
pub aggregator_handle: AggregatorHandle,
pub config: Arc<Config>,
Expand Down Expand Up @@ -113,7 +119,11 @@ impl TraceFlusher {
let traces_with_tags: Vec<_> = trace_builders
.into_iter()
.map(|info| {
let trace = info.builder.with_api_key(api_key.as_str()).build();
let trace = info
.builder
.with_api_key(api_key.as_str())
.with_retry_strategy(trace_retry_strategy())
.build();
(trace, info.header_tags)
})
.collect();
Expand All @@ -125,12 +135,16 @@ impl TraceFlusher {
let additional_traces: Vec<_> = traces_with_tags
.iter()
.filter_map(|(trace, tags)| match trace.get_payloads() {
TracerPayloadCollection::V07(payloads) => Some(SendData::new(
trace.len(),
TracerPayloadCollection::V07(payloads.clone()),
tags.to_tracer_header_tags(),
&endpoint,
)),
TracerPayloadCollection::V07(payloads) => {
let mut send_data = SendData::new(
trace.len(),
TracerPayloadCollection::V07(payloads.clone()),
tags.to_tracer_header_tags(),
&endpoint,
);
send_data.set_retry_strategy(trace_retry_strategy());
Some(send_data)
}
// All payloads in the extension are V07 (produced by
// collect_pb_trace_chunks), so this branch is unreachable.
_ => None,
Expand Down
Loading