fix(redis-worker): jittered reconnect backoff for mollifier buffer client

The buffer client's ioredis retryStrategy used a fixed Math.min(times*50, 1000) schedule with no jitter, so a fleet of webapp instances reconnecting after the same Redis blip retried in lockstep and stampeded Redis on recovery (thundering herd) — the same lockstep-load pattern the mutate-fallback wait loop was changed to avoid.

Extract mollifierReconnectDelayMs(times, random) and apply equal jitter: a uniform pick in [base/2, base]. Bounded by the original 1s cap, so it is never slower than before, just decorrelated. Pure unit tests pin the band, the cap, and that draws spread.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: d-cs

.changeset/mollifier-buffer-extensions.md

@@ -2,4 +2,4 @@
 "@trigger.dev/redis-worker": minor
 ---
 
-Mollifier buffer extensions: idempotency dedup, an atomic `mutateSnapshot` API, metadata CAS, claim primitives, and a `MollifierSnapshot` type.
+Mollifier buffer extensions: idempotency dedup, an atomic `mutateSnapshot` API, metadata CAS, claim primitives, and a `MollifierSnapshot` type. The buffer's Redis client now reconnects with jittered backoff so a fleet of clients doesn't stampede Redis in lockstep after a blip.

packages/redis-worker/src/mollifier/buffer.test.ts

@@ -2,7 +2,52 @@ import { describe, expect, it } from "vitest";
 import { BufferEntrySchema, serialiseSnapshot, deserialiseSnapshot } from "./schemas.js";
 import { redisTest } from "@internal/testcontainers";
 import { Logger } from "@trigger.dev/core/logger";
-import { MollifierBuffer, idempotencyLookupKeyFor, makeIdempotencyClaimKey } from "./buffer.js";
+import {
+  MollifierBuffer,
+  idempotencyLookupKeyFor,
+  makeIdempotencyClaimKey,
+  mollifierReconnectDelayMs,
+} from "./buffer.js";
+
+describe("mollifierReconnectDelayMs", () => {
+  it("grows linearly with the attempt count and caps the base at 1s", () => {
+    // random=()=>1 yields the top of the equal-jitter band (== base).
+    const top = (times: number) => mollifierReconnectDelayMs(times, () => 1);
+    expect(top(1)).toBe(50);
+    expect(top(4)).toBe(200);
+    expect(top(20)).toBe(1000);
+    // Past the cap the base stays at 1000.
+    expect(top(100)).toBe(1000);
+  });
+
+  it("applies equal jitter: result is uniform in [base/2, base]", () => {
+    // base for times=10 is 500, so the band is [250, 500].
+    expect(mollifierReconnectDelayMs(10, () => 0)).toBe(250); // floor of band
+    expect(mollifierReconnectDelayMs(10, () => 0.999999)).toBe(500); // top of band
+    const mid = mollifierReconnectDelayMs(10, () => 0.5);
+    expect(mid).toBeGreaterThanOrEqual(250);
+    expect(mid).toBeLessThanOrEqual(500);
+  });
+
+  it("never exceeds the original fixed-schedule envelope (strictly an improvement)", () => {
+    for (const times of [1, 2, 5, 10, 20, 50]) {
+      const cap = Math.min(times * 50, 1000);
+      for (const r of [0, 0.25, 0.5, 0.75, 0.999999]) {
+        const delay = mollifierReconnectDelayMs(times, () => r);
+        expect(delay).toBeLessThanOrEqual(cap);
+        expect(delay).toBeGreaterThanOrEqual(Math.floor(cap / 2));
+      }
+    }
+  });
+
+  it("decorrelates concurrent reconnects (distinct values across random draws)", () => {
+    const draws = [0.05, 0.3, 0.55, 0.8, 0.95].map((r) =>
+      mollifierReconnectDelayMs(20, () => r),
+    );
+    // Lockstep would collapse to a single value; jitter spreads them.
+    expect(new Set(draws).size).toBeGreaterThan(1);
+  });
+});
 
 describe("schemas", () => {
   it("serialiseSnapshot then deserialiseSnapshot is identity for plain objects", () => {

packages/redis-worker/src/mollifier/buffer.ts

@@ -18,6 +18,24 @@ export type MollifierBufferOptions = {
 // have a safety net while PG replica lag settles. Q1 D2.
 const ACK_GRACE_TTL_SECONDS = 30;
 
+// ioredis reconnect backoff for the mollifier buffer client. The base
+// grows linearly with the attempt count and is capped at 1s (the same
+// envelope as the previous fixed `Math.min(times * 50, 1000)` schedule).
+// We then apply equal jitter — a uniform pick in `[base/2, base]` — so a
+// fleet of webapp instances reconnecting after the same Redis blip don't
+// retry in lockstep and stampede Redis on recovery (thundering herd).
+// Because the jittered value never exceeds the original cap, this is never
+// slower than before — just decorrelated. Mirrors the jittered-backoff
+// approach the mutate-fallback wait loop adopted for the same reason.
+export function mollifierReconnectDelayMs(
+  times: number,
+  random: () => number = Math.random,
+): number {
+  const base = Math.min(times * 50, 1000);
+  const half = Math.floor(base / 2);
+  return half + Math.round(random() * (base - half));
+}
+
 export type SnapshotPatch =
   | { type: "append_tags"; tags: string[] }
   | { type: "set_metadata"; metadata: string; metadataType: string }
@@ -95,8 +113,7 @@ export class MollifierBuffer {
       {
         ...options.redisOptions,
         retryStrategy(times) {
-          const delay = Math.min(times * 50, 1000);
-          return delay;
+          return mollifierReconnectDelayMs(times);
         },
         maxRetriesPerRequest: 20,
       },