feat(query retries): handle transient or retryable DB errors

benches/store_bench.rs

@@ -32,6 +32,10 @@ async fn get_pending_activations(num_activations: u32, num_workers: u32) {
                 processing_deadline_grace_sec: 3,
                 claim_lease_ms: 5000,
                 enable_sqlite_status_metrics: false,
+                retry_config: taskbroker::store::retry::RetryConfig {
+                    max_retries: 0,
+                    retry_delay: std::time::Duration::from_millis(0),
+                },
             },
         )
         .await
@@ -97,6 +101,10 @@ async fn set_status(num_activations: u32, num_workers: u32) {
                 processing_deadline_grace_sec: 3,
                 claim_lease_ms: 5000,
                 enable_sqlite_status_metrics: false,
+                retry_config: taskbroker::store::retry::RetryConfig {
+                    max_retries: 0,
+                    retry_delay: std::time::Duration::from_millis(0),
+                },
             },
         )
         .await

src/config.rs

@@ -180,6 +180,13 @@ pub struct Config {
     /// The amount of time to wait before retrying writes to db when write fails.
     pub db_write_failure_backoff_ms: u64,
 
+    /// The maximum number of times to retry a transient database query error
+    /// before surfacing the error. When None, queries are not retried.
+    pub db_query_max_retries: Option<u32>,
+
+    /// The delay in milliseconds between query retry attempts.
+    pub db_query_retry_delay_ms: u64,
+
     /// The maximum number of tasks that are buffered
     /// before being written to InflightTaskStore (sqlite).
     pub db_insert_batch_max_len: usize,
@@ -378,6 +385,8 @@ impl Default for Config {
             pg_default_database_name: "postgres".to_owned(),
             pg_extra_query_params: None,
             db_write_failure_backoff_ms: 4000,
+            db_query_max_retries: None,
+            db_query_retry_delay_ms: 100,
             db_insert_batch_max_len: 256,
             db_insert_batch_max_size: 16_000_000,
             db_insert_batch_max_time_ms: 1000,

src/store.rs

@@ -1,5 +1,6 @@
 pub mod activation;
 pub mod adapters;
+pub mod retry;
 pub mod traits;
 pub mod types;