Retry mechanism with retry braking system

API-INTERNAL.md

@@ -79,12 +79,18 @@ If the requested key is a collection, it will return an object with all the coll
 <dd><p>Remove a key from Onyx and update the subscribers</p>
 </dd>
 <dt><a href="#retryOperation">retryOperation()</a></dt>
-<dd><p>Handles storage operation failures based on the error type:</p>
+<dd><p>Handles storage operation failures based on the error class (see lib/storage/errors.ts).
+The connection layer (createStore) owns connection/transport recovery; this operation layer owns
+capacity recovery (eviction) so that a given failure is retried by exactly one layer:</p>
 <ul>
-<li>Storage capacity errors: evicts data and retries the operation</li>
-<li>Invalid data errors: logs an alert and throws an error</li>
-<li>Non-retriable errors: logs an alert and resolves without retrying</li>
-<li>Other errors: retries the operation</li>
+<li>INVALID_DATA: logs an alert and throws (the same data will always fail).</li>
+<li>TRANSIENT / FATAL: the connection layer already retried (transient) or exhausted its heal budget
+and alerted (fatal). Retrying here would only re-amplify, so we skip the write quietly.</li>
+<li>CAPACITY: evicts the least recently accessed evictable key and retries, under a session-level
+circuit breaker (see lib/StorageCircuitBreaker.ts) that halts the loop once eviction stops making
+progress or failures storm — the per-operation budget alone cannot stop a session-wide storm.</li>
+<li>UNKNOWN: the provider couldn&#39;t classify it — log the full error shape (name + message +
+provider) once so it&#39;s visible, then bounded retry without eviction.</li>
 </ul>
 </dd>
 <dt><a href="#broadcastUpdate">broadcastUpdate()</a></dt>
@@ -318,11 +324,17 @@ Remove a key from Onyx and update the subscribers
 <a name="retryOperation"></a>
 
 ## retryOperation()
-Handles storage operation failures based on the error type:
-- Storage capacity errors: evicts data and retries the operation
-- Invalid data errors: logs an alert and throws an error
-- Non-retriable errors: logs an alert and resolves without retrying
-- Other errors: retries the operation
+Handles storage operation failures based on the error class (see lib/storage/errors.ts).
+The connection layer (createStore) owns connection/transport recovery; this operation layer owns
+capacity recovery (eviction) so that a given failure is retried by exactly one layer:
+- INVALID_DATA: logs an alert and throws (the same data will always fail).
+- TRANSIENT / FATAL: the connection layer already retried (transient) or exhausted its heal budget
+  and alerted (fatal). Retrying here would only re-amplify, so we skip the write quietly.
+- CAPACITY: evicts the least recently accessed evictable key and retries, under a session-level
+  circuit breaker (see lib/StorageCircuitBreaker.ts) that halts the loop once eviction stops making
+  progress or failures storm — the per-operation budget alone cannot stop a session-wide storm.
+- UNKNOWN: the provider couldn't classify it — log the full error shape (name + message +
+  provider) once so it's visible, then bounded retry without eviction.
 
 **Kind**: global function  
 <a name="broadcastUpdate"></a>

lib/CircuitBreaker/AbstractCircuitBreaker.ts

@@ -0,0 +1,199 @@
+import StateMachine from '../StateMachine';
+import {CIRCUIT_BREAKER_TRANSITIONS} from './types';
+import type {CircuitBreakerOptions, CircuitBreakerState} from './types';
+
+/**
+ * Generic circuit breaker built on {@link StateMachine}.
+ *
+ * - **closed**: requests are allowed; failures are counted.
+ * - **open**: requests are rejected until {@link resetTimeoutMs} elapses.
+ * - **half-open**: the recovery-probe state. After the open timeout, the breaker admits exactly ONE
+ *   probe request: success means the dependency recovered, so the circuit closes. Failure means it's
+ *   still down, so the circuit reopens. This single-request probe prevents a "thundering herd" where
+ *   every caller fails loudly when the service hasn't recovered yet.
+ *
+ * Subclasses implement the failure-counting policy by overriding {@link recordFailureInClosed} (and
+ * friends) — e.g. counting consecutive failures, or failures within a rolling time window, or any
+ * combination of those.
+ *
+ * @example
+ * class MyBreaker extends AbstractCircuitBreaker {
+ *     private failures = 0;
+ *     protected recordFailureInClosed() {
+ *         this.failures += 1;
+ *         return this.failures >= 3 ? `${this.failures} failures` : null;
+ *     }
+ *     protected recordSuccessInClosed() { this.failures = 0; }
+ *     protected resetFailureState() { this.failures = 0; }
+ * }
+ *
+ * const breaker = new MyBreaker({resetTimeoutMs: 30_000});
+ * if (breaker.isAllowed()) {
+ *     try {
+ *         doWork();
+ *         breaker.recordSuccess();
+ *     } catch {
+ *         breaker.recordFailure();
+ *     }
+ * }
+ */
+abstract class AbstractCircuitBreaker {
+    private machine: StateMachine<typeof CIRCUIT_BREAKER_TRANSITIONS, CircuitBreakerState>;
+
+    private openedAt = 0;
+
+    private isProbeInFlight = false;
+
+    private readonly resetTimeoutMs: number;
+
+    private readonly onTrip?: (reason: string) => void;
+
+    private readonly onClose?: () => void;
+
+    constructor(options: CircuitBreakerOptions = {}) {
+        this.resetTimeoutMs = options.resetTimeoutMs ?? 60_000;
+        this.onTrip = options.onTrip;
+        this.onClose = options.onClose;
+        this.machine = new StateMachine('closed', CIRCUIT_BREAKER_TRANSITIONS);
+    }
+
+    /** Record a failure while the circuit is closed. Returns a trip reason when the threshold is exceeded. */
+    protected abstract recordFailureInClosed(): string | null;
+
+    /** Update failure state after a successful request while the circuit is closed. */
+    protected abstract recordSuccessInClosed(): void;
+
+    /** Clear accumulated failure state without changing circuit state. */
+    protected abstract resetFailureState(): void;
+
+    /**
+     * Whether a request may proceed.
+     *
+     * Returns `false` while open. In half-open, the FIRST caller is admitted as the recovery probe and
+     * `isProbeInFlight` is latched so every subsequent caller is rejected until that probe resolves
+     * (via {@link recordSuccess} → close, or {@link recordFailure} → reopen). That single-probe gate is
+     * the whole point of half-open: it tests recovery with one request instead of letting a herd of
+     * waiting callers stampede a dependency that may still be down.
+     */
+    isAllowed(): boolean {
+        const currentState = this.getCurrentState();
+
+        if (currentState === 'open') {
+            return false;
+        }
+
+        if (currentState === 'half-open') {
+            if (this.isProbeInFlight) {
+                return false;
+            }
+            this.isProbeInFlight = true;
+        }
+
+        return true;
+    }
+
+    /**
+     * Record a failed request. May open the circuit from closed or half-open.
+     * @returns `true` when the circuit is open after recording (the request must not proceed).
+     */
+    recordFailure(): boolean {
+        if (this.machine.state === 'open') {
+            return true;
+        }
+
+        if (this.machine.state === 'half-open') {
+            this.trip();
+            return true;
+        }
+
+        const reason = this.recordFailureInClosed();
+        if (reason) {
+            this.trip(reason);
+            return true;
+        }
+
+        return false;
+    }
+
+    /** Record a successful request. Closes the circuit from half-open and clears failure counts. */
+    recordSuccess(): void {
+        if (this.machine.state === 'half-open') {
+            this.close();
+            return;
+        }
+
+        if (this.machine.state === 'closed') {
+            this.recordSuccessInClosed();
+        }
+    }
+
+    /**
+     * The current state WITHOUT advancing recovery — a pure query, safe to call without side effects.
+     * The open→half-open transition is applied only at the admission point ({@link isAllowed}); by the
+     * time a caller queries state after being admitted, that transition has already happened.
+     */
+    peekState(): CircuitBreakerState {
+        return this.machine.state;
+    }
+
+    /**
+     * Force the circuit back to closed from ANY state. This is a reset, not a transition, so it
+     * deliberately bypasses the transition graph (and does not fire {@link onClose}). Use only to wipe
+     * all state — e.g. between tests or sessions.
+     */
+    protected hardReset(): void {
+        this.machine = new StateMachine('closed', CIRCUIT_BREAKER_TRANSITIONS);
+        this.openedAt = 0;
+        this.isProbeInFlight = false;
+        this.resetFailureState();
+    }
+
+    private getCurrentState(): CircuitBreakerState {
+        this.maybeRecover();
+        return this.machine.state;
+    }
+
+    private trip(reason = ''): void {
+        if (this.machine.state === 'open') {
+            return;
+        }
+
+        this.machine = this.machine.transition('open');
+        this.openedAt = Date.now();
+        this.isProbeInFlight = false;
+        this.resetFailureState();
+        this.onTrip?.(reason);
+    }
+
+    private close(): void {
+        // close() only ever runs from half-open (see recordSuccess), and half-open → closed is the one
+        // legal closing transition — so go through transition() to keep the illegal open → closed jump
+        // an error rather than silently constructing a fresh closed machine.
+        this.machine = this.machine.transition('closed');
+        this.openedAt = 0;
+        this.isProbeInFlight = false;
+        this.resetFailureState();
+        this.onClose?.();
+    }
+
+    /**
+     * Lazily advance open → half-open once the reset timeout has elapsed. This is checked on read
+     * (via {@link getCurrentState}) rather than on a timer, so there's nothing to schedule or clean up:
+     * the transition simply becomes visible to the next caller after the window. Entering half-open
+     * clears `isProbeInFlight` so the next admitted request becomes the recovery probe.
+     */
+    private maybeRecover(): void {
+        if (this.machine.state !== 'open') {
+            return;
+        }
+
+        if (Date.now() - this.openedAt < this.resetTimeoutMs) {
+            return;
+        }
+
+        this.machine = this.machine.transition('half-open');
+        this.isProbeInFlight = false;
+    }
+}
+
+export default AbstractCircuitBreaker;

lib/CircuitBreaker/types.ts

@@ -0,0 +1,43 @@
+/**
+ * States of the circuit breaker.
+ *
+ * - **closed**: normal operation; requests flow and failures are counted.
+ * - **open**: tripped; requests are rejected outright so a known-bad dependency isn't hammered.
+ * - **half-open**: a trial state entered after the open timeout — see {@link CIRCUIT_BREAKER_TRANSITIONS}.
+ */
+type CircuitBreakerState = 'closed' | 'open' | 'half-open';
+
+/**
+ * Legal state transitions. The flow is closed → open → half-open → (closed | open).
+ *
+ * The **half-open** state exists so the breaker can test whether the dependency has recovered WITHOUT
+ * flipping straight back to fully closed. Going open → closed blindly would, on a dependency that is
+ * still down, immediately re-admit the full load and re-trip — flapping between open and closed every
+ * window. Instead, after the open timeout the breaker moves to half-open and admits a single trial
+ * ("probe") request:
+ *  - probe succeeds → the dependency is healthy again → transition to **closed** (resume normal flow).
+ *  - probe fails    → still broken → transition back to **open** for another timeout window.
+ *
+ * Admitting exactly one probe (rather than reopening the floodgates) is also what prevents the
+ * "thundering herd": many callers retrying at once the instant the timeout elapses, re-overwhelming a
+ * dependency that was just starting to recover.
+ */
+const CIRCUIT_BREAKER_TRANSITIONS = {
+    closed: ['open'],
+    open: ['half-open'],
+    'half-open': ['closed', 'open'],
+} as const satisfies Record<CircuitBreakerState, readonly CircuitBreakerState[]>;
+
+type CircuitBreakerOptions = {
+    /** Time in milliseconds the circuit stays open before moving to half-open. */
+    resetTimeoutMs?: number;
+
+    /** Called once each time the circuit opens. */
+    onTrip?: (reason: string) => void;
+
+    /** Called when the circuit closes. */
+    onClose?: () => void;
+};
+
+export type {CircuitBreakerOptions, CircuitBreakerState};
+export {CIRCUIT_BREAKER_TRANSITIONS};