From 30ea91f3f678a8c046477bd7623996790e6d776f Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Wed, 8 Jul 2026 23:09:17 +0100 Subject: [PATCH 01/13] perf(durable-streams-rust): cardinality-cliff sidecar fix + wal/server telemetry Skip the per-append .meta sidecar flush for plain appends (memory + wal), eliminating the memory-mode cardinality cliff (+104% @ 50k local, +14-16% NVMe) and reducing wal checkpoint work. Adds --server-stats (SRV_STATS: cpu/inflight/ svc/durwait) and --wal-fsync-parallel telemetry/knobs used to characterize the wal bottleneck as device-fdatasync-bound (not shards/CPU). Findings in CARDINALITY_CLIFF_FIX.md. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- .../CARDINALITY_CLIFF_CAUSES.md | 190 +++++++++++++ .../CARDINALITY_CLIFF_FIX.md | 250 ++++++++++++++++++ .../WRITE_BOTTLENECKS_1M.md | 192 ++++++++++++++ packages/durable-streams-rust/src/handlers.rs | 96 ++++++- packages/durable-streams-rust/src/main.rs | 54 ++++ packages/durable-streams-rust/src/srvstats.rs | 140 ++++++++++ packages/durable-streams-rust/src/store.rs | 58 +++- .../durable-streams-rust/src/wal/shard.rs | 58 +++- 8 files changed, 1032 insertions(+), 6 deletions(-) create mode 100644 packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md create mode 100644 packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md create mode 100644 packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md create mode 100644 packages/durable-streams-rust/src/srvstats.rs diff --git a/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md b/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md new file mode 100644 index 0000000000..9e5f3d8f91 --- /dev/null +++ b/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md @@ -0,0 +1,190 @@ +# The cardinality cliff — code-level cause research + +**Audience:** an engineer/agent picking up the write-cardinality-cliff work with no +prior context. **Date:** 2026-07-08. **Status:** code research only — ranked +hypotheses with file:line evidence and cheap confirmation experiments; no fixes +applied or measured yet. **Companion docs:** +[`CARDINALITY_CLIFF_REPRO.md`](CARDINALITY_CLIFF_REPRO.md) (how to reproduce +locally in ~15-min cycles), [`WRITE_BOTTLENECKS_1M.md`](WRITE_BOTTLENECKS_1M.md) +(background; bottleneck #2 is this cliff). + +## The symptom being explained + +At fixed offered load, append throughput falls steeply and *per-request* latency +(measured below the knee — not queueing) rises as the number of distinct streams +grows. Local kind repro (2-vCPU server): + +| streams | wal thr (rel) | memory thr (rel) | +|---|---|---| +| 1k | 55.4k (100 %) | 112.5k (100 %) | +| 10k | 48.2k (87 %) | 63.7k (57 %) | +| 50k | 27.8k (50 %) | 22.7k (20 %) | + +Two discriminating facts any explanation must fit: + +1. **The cliff exists in BOTH wal and memory mode** → the dominant mechanism is + in the shared append path or shared background machinery, not fsync/WAL + coordination. +2. **Memory mode degrades HARDER than wal locally, ending below it in absolute + terms at 50k** (22.7k vs 27.8k) → whatever memory mode does *instead of* the + WAL checkpoint must be more expensive per distinct stream, not less. + +The WAL contention counters (`--wal-stats`) stay flat across cardinality, ruling +out lock/wakeup coordination — consistent with everything below. + +--- + +## Ranked causes + +### #1 — Per-stream sidecar flush work stops amortizing (both modes; strongest cliff-shaped suspect) + +Every append marks its stream's `.meta` JSON sidecar dirty; a periodic task then +rewrites one sidecar per dirty stream. A rewrite is `Meta::capture` (clones path, +content-type, producers map, manifest) + `serde_json::to_vec` + `File::create(tmp)` ++ `write_all` + `rename` — all into the **single shared `streams/` directory** +(`write_meta_sync`, `store.rs:1122`). The `meta_dirty` CAS dedup only helps while +a stream is appended **more than once per flush interval**. So: + +``` +sidecar writes/sec ≈ min(append rate, N_streams / interval) +``` + +- At 1k streams / 100k ops/s: ≤1k writes/s — <1 % overhead, invisible. +- Once `N_streams > ops/s × interval`: **every append pays a full sidecar + rewrite** — the server's filesystem-metadata work roughly doubles per append, + keyed purely to stream cardinality at fixed load. That is a cliff with exactly + the observed onset shape. + +Mode-specific plumbing (this asymmetry explains why memory ends up *worse*): + +- **Memory mode:** `handle_append_inner` → `store.mark_meta_dirty` + (`handlers.rs:1174` → `store.rs:1170`). The **1 s** sweeper + (`META_SWEEP_INTERVAL`, `main.rs:455`; loop `main.rs:461-476`) drains the queue + and writes *all* dirty sidecars **serially inside one `spawn_blocking` task** + (`store.rs:1184` `sweep_meta_once`). At 50k streams that is up to ~50k + create+rename pairs per second on one blocking thread of a 2-vCPU server, all + contending the `streams/` directory inode rwsem (the pre-batching design's + per-append version of this was measured at ~40 % of server CPU — see the + comment at `handlers.rs:1160-1168`). On top of that, `mark_meta_dirty` takes a + **global** `StdMutex>>` (`Store.meta_sweep`, + `store.rs:389`) on every first-touch-per-cycle — and at high cardinality the + first-touch fraction approaches 100 % of appends, so a global hot-path lock + reappears that wal mode avoids with a plain atomic store (`handlers.rs:1168`). +- **wal mode:** the **3 s** shard checkpoint owns the flush — O(touched streams) + `write_meta_sync` calls (`wal/shard.rs:877-883`) **plus** one `barrier_fsync` + per touched per-stream data file (`wal/shard.rs:837-839`): ~16k fdatasyncs/s + at 50k streams touched per interval. Sharded + concurrent (JoinSet, + `main.rs:437-446`) + 3 s cadence, vs memory's serial global 1 s sweep — hence + the milder wal curve. + +### #2 — wal-only: `persist_durable_tails` is O(total streams ever touched), every checkpoint + +Each checkpoint also rewrites the shard's `tails` file from a **cumulative** +resident map of every stream ever touched on that shard (`tails_cache`, +`wal/shard.rs:354`): collect the whole map → `sort_unstable` → serialize every +entry → write → `sync_all` → rename (`wal/shard.rs:922-957`). Cost is +**O(total distinct streams per shard) regardless of how many were touched this +tick**, grows monotonically (idle streams never leave), and runs on a blocking +thread every 3 s. The field doc estimates ~20 ms/tick at 400k streams. A strong +contributor to the wal curve's steepening and a direct competitor for the +blocking pool / disk bandwidth the committer needs. + +### #3 — Shared per-append working-set physics (both modes; explains the below-knee latency rise) + +- **Registry lookup:** `Store.streams` is `DashMap>` + with the **default SipHash hasher and no capacity hint** (`store.rs:369`, + `:423`); one lookup per append (`store.rs:667`, called at `handlers.rs:899`). + Fixed instruction cost, but at 50k–1M entries the map plus its `Arc` targets + outgrow L2/L3, so each lookup pays several main-memory misses. (The code + already knows: see the comment at `handlers.rs:889-891` about avoiding a + second lookup.) +- **Per-stream cacheline spray:** each append touches the stream's `appender` + AsyncMutex, the `shared` RwLock **six-to-eight separate times** (lookup check; + soft-delete check `handlers.rs:909`; closed check `handlers.rs:958`; + `write_wire`'s write `handlers.rs:741`; the producer/seq write at + `handlers.rs:1087` — taken even when there are no producer headers; + `wal_stream_offset`'s read `handlers.rs:679`; `publish_durable_tail`'s write + `handlers.rs:762`; the final `tail()` read `handlers.rs:1180`), plus the + `last_chunk` RwLock and the `tail_tx` watch. On a hot stream these lines are + cache-resident; at high cardinality every one is a miss. Per-request latency + therefore rises with N with no queueing involved — matching the knee-p50 trend + in both modes, local and remote. +- **Kernel-side one-file-per-stream:** at fixed byte volume, N distinct streams + dirty N distinct inodes per writeback interval — inode writeback, ext4 journal + handles, dentry/page-cache pressure all scale with distinct files touched, + plus ~1 persistent fd per stream (`Shared.file`, `store.rs:72`). + +### #4 — wal-only, secondary: `register_dirty` first-touch transitions + +The hot path is lock-free (`dirty_epoch` compare, `wal/shard.rs:722-726`), but +the first touch per stream per checkpoint interval CASes the epoch, takes the +shard `dirty` mutex and pushes an `Arc` clone (`wal/shard.rs:737-749`). At fixed +load, growing cardinality raises the first-touch fraction toward 100 % of +appends — the same "dedup stops working" arithmetic as #1, but sharded and much +cheaper per event. Real, but secondary. + +--- + +## Ruled out (checked, with evidence) + +- **SSE reactor:** `wake_stream` early-returns when a stream has no subscribers + (`sse_reactor.rs:144-146`); no reactor thread is even spawned without a + `register`. O(1) per append in a write-only benchmark. +- **Telemetry:** default build compiles the recorders to no-ops + (`telemetry.rs:353-398`); with the feature on, label sets are bounded/static — + never keyed per stream. `wal/telemetry.rs` is per-shard, not per-stream. +- **Tiering/blobstore:** fully inert without `--tier` (`maybe_seal_bg` gate, + `handlers.rs:663`; `enabled()` gates inside tier.rs). +- **WAL group commit & durability wakeups:** per-commit work is O(batch) + + O(satisfied waiters) (`wal/shard.rs:1263-1305`, `:1322-1336`); the waiter + registry is one oneshot per in-flight request keyed by LSN — scales with + offered load, not stream count. Matches the flat `*_wait_load` counters. +- **WAL record encoding:** records carry an interned `stream_id: u64` + (`wal/codec.rs:78-84`), never the path string; frame is a fixed 38-byte header + + payload. +- **Full-registry scans:** none exist. `Store.streams` is only ever + point-accessed (get/insert/remove); no background loop iterates it. +- **Segment recycling:** `read_dir` costs scale with byte throughput / segment + count, not stream count. +- **HTTP layer (`engine_raw.rs` / `http1.rs` / `api.rs`):** no per-stream state, + no stream-keyed maps; header access is O(request headers). + +--- + +## Cheap confirmation experiments (in suspect order) + +1. **Memory mode / #1:** temporarily neuter the sweeper's sidecar write (or just + log `sweep_meta_once`'s returned count + wall time per tick, call site + `main.rs:473`). If the 1k→50k memory curve flattens dramatically with writes + disabled, #1 is confirmed for memory mode. (The producer-dedup lag this + introduces is already a documented non-durable window — fine for a bench.) +2. **wal mode / #1+#2:** run with `--wal-stats 1` and read the existing + `WAL_CKPT` line (`wal/shard.rs:887-899`) — `touched=`, `meta_us=`, and the + tails entry count should climb with cardinality while commit-path counters + stay flat. +3. **Shared baseline / #3:** swap the registry hasher (ahash) or key by interned + id, add a capacity hint, and re-measure — isolates the lookup-miss share of + the residual cliff that would remain after #1/#2 are fixed. + +## Fix directions + +> **#1 is DONE and measured — see [`CARDINALITY_CLIFF_FIX.md`](CARDINALITY_CLIFF_FIX.md).** +> A plain non-TTL append no longer writes/queues a sidecar rewrite in either mode +> (the durable tail is carried by the data-file length in memory mode / the +> per-shard `tails` map in wal mode; `last_access` only gates TTL). Result: memory +> cliff essentially eliminated (50k streams **+104%**, 44%→92% of the 1k rate); wal +> a modest, correct win (50k **+11%** throughput, knee p50 4.0→2.5 ms) — the wal +> cliff is fsync/`tails`-bound (#2 below), not the meta write. The items below +> remain. + +- Spread the meta sweep over the interval and shard its queue; make memory-mode + dirty-tracking lock-free like wal's (atomic + per-shard queues). *(Largely moot + now — the common case queues nothing; only producer/seq/TTL appends do.)* +- ~~Skip sidecar rewrites whose captured contents are unchanged (append-only + streams with no producers churn only `durable_tail`/`last_access`).~~ **Done (#1).** +- Incremental/delta `tails` persistence instead of the full-map rewrite per + checkpoint (#2). +- Batch/fan out the checkpoint fdatasyncs; consider syncing only files whose + WAL records are about to be recycled. +- Interned stream ids + faster hasher + sharded registry; pack the per-append + hot fields of `StreamState` into one cacheline (#3). diff --git a/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md b/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md new file mode 100644 index 0000000000..1e8151d059 --- /dev/null +++ b/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md @@ -0,0 +1,250 @@ +# The cardinality cliff — fix & measurements + +**Audience:** whoever picks up the write-cardinality-cliff work. **Date:** +2026-07-08. **Status:** memory-mode fix implemented, validated, and low-risk; +wal-mode gate implemented, crash-sim-correct, but a modest local win. **Companion +docs:** [`CARDINALITY_CLIFF_CAUSES.md`](CARDINALITY_CLIFF_CAUSES.md) (the ranked +hypotheses this acts on — the fix targets **#1**), +[`WRITE_BOTTLENECKS_1M.md`](WRITE_BOTTLENECKS_1M.md). + +## What was done + +Hypothesis **#1** (per-stream `.meta` sidecar flush work stops amortizing) was +confirmed and fixed. The core change is one predicate in the append path +(`handlers.rs`): + +```rust +// A plain append to a non-TTL stream changes only durable_tail/last_access. +// The durable tail is carried elsewhere (memory: re-derived from the data-file +// length on restart; wal: the checkpoint's per-shard `tails` map), and +// last_access only gates TTL — so a plain non-TTL append needs NO sidecar flush. +let meta_persist_needed = + producer.is_some() || seq_header.is_some() || st.config.ttl_seconds.is_some(); +``` + +- **Memory mode:** only `store.mark_meta_dirty(&st)` when `meta_persist_needed`. + Previously *every* memory-mode append queued a sidecar rewrite for the 1 s + sweeper; at high cardinality the dedup stopped working and the sweep wrote up to + ~N sidecars/s on one blocking thread, all contending the shared `streams/` + directory inode. +- **Wal mode:** only `st.meta_dirty.store(true)` when `meta_persist_needed` (or the + `--wal-meta-gate off` A/B toggle is set). The `~3 s` checkpoint then skips + `write_meta_sync` for plain-append streams. Their `durable_tail` is still fsync'd + and recorded in the per-shard `tails` map (`persist_durable_tails`, independent of + this flag) — the authoritative durable-tail proof recovery reconciles against. + +### Why it is safe + +- **Memory recovery reads neither field.** `boot_meta_durable_tail` / + `meta.durable_tail` is consumed only by `wal/recovery.rs`; memory-mode + `Store::new_with_tier` derives the tail from `file.metadata().len()`. The + pre-existing e2e test `memory_mode_data_survives_restart_via_sidecar` appends 10 + bytes, never flushes the sidecar, reopens, and recovers `tail == 10` — it already + proved this. +- **Wal recovery uses the `tails` map, not the sidecar `durable_tail`.** The map is + persisted every checkpoint regardless of the meta-dirty flag; the sidecar's + `durable_tail` is redundant with it. The randomized crash-recovery simulation + (`crash_recovery_randomized_simulation`, which uses **plain** acked appends) and + `e2e_recycled_first_segment_acked_records_survive_crash` pass unchanged. +- **Compaction / seal / tier / close / delete are unaffected** — they call + `write_meta_sync(.., durable=true)` directly, never via the deferred `meta_dirty` + flag. `last_access` for TTL streams still flushes (the predicate keeps them). + +New test `memory_plain_append_skips_sidecar_flush` locks in the behavior. Full +suite: **104 passed** (0 failed). + +## Measurements (local kind, 2 vCPU, plain 256 B appends, `ds-bench`) + +Normalized to each config's own 1k-stream throughput (the cliff is the *decline*). + +### Memory mode — cliff essentially eliminated + +| streams | baseline | **fixed** | throughput Δ | knee p50 | +|---|---|---|---|---| +| 1k | 123.5k (100%) | 120.8k (100%) | — | 0.5 → 0.4 ms | +| 10k | 86.9k (70%) | 127.6k (106%) | **+47%** | 0.5 → 0.5 ms | +| 50k | 54.6k (44%) | 111.6k (**92%**) | **+104%** | 2.1 → 2.2 ms | + +At 50k streams throughput more than **doubles** (54.6k → 111.6k) and the cliff +flattens from 44% to 92% of the 1k rate. The fixed default build lands on the +`--meta-sweep-disable` neuter ceiling (102k), confirming the fix captures the full +available win. The residual 8% decline is #3 working-set physics (registry / cache +/ page-cache), which this fix does not address. + +### Wal mode — correct, reduces checkpoint work, modest throughput win + +Same-binary A/B (`--wal-meta-gate on|off`, `repeats: 2`, so no cross-image drift): + +| streams | baseline | **gated** | throughput Δ | knee p50 | +|---|---|---|---|---| +| 1k | 54.6k | 66.9k | +23% | 1.7 → 2.4 ms | +| 10k | 57.1k | 54.7k | −4% (noise) | 2.5 → 2.8 ms | +| 50k | 34.1k | 38.0k | **+11%** | 4.0 → **2.5 ms** | + +The wal cliff is fsync-lane bound: the checkpoint's meta write is one of *three* +costs, and the per-stream `fdatasync`s + the O(cumulative) `tails`-map rewrite (#2) +dominate — this gate doesn't touch them. The clean signal is **knee p50 at 50k +dropping 4.0 → 2.5 ms** (the checkpoint does less work); throughput barely moves +because it is disk-bound. The gate is worth keeping — it's free, correct, cuts +write amplification, and may matter more on real NVMe — but **it is not the wal +cliff fix**. The real wal lever is **#2: incremental/delta `tails` persistence** +instead of the full-map rewrite per checkpoint. + +## Dev/bench toggles added (all default-off / gate default-on) + +- `--meta-sweep-stats` — log a `META_SWEEP queued=/wrote=/elapsed_us=` line per tick. +- `--meta-sweep-disable` — drain the sweep queue but skip the sidecar write (the #1 + confirmation neuter; makes the sidecar permanently stale — bench only). +- `--wal-meta-gate on|off` — toggle the wal-mode gate for a same-binary A/B. +- `--mem-meta-gate on|off` — memory counterpart, to A/B the #1 fix on one binary. +- `--wal-fsync-parallel N` — checkpoint fdatasync fan-out (H4; default 1 = serial). +- `--server-stats N` — periodic `SRV_STATS` line (both modes) for bottleneck + analysis: `cpu_cores` (process CPU utilization; ≈ cgroup quota ⇒ CPU-bound), + `inflight` (append queue depth), `svc_us` (handler service time), `applock_us` + (appender-lock wait), `durwait_us` (WAL fsync wait). Linux only for `cpu_cores`. + +Reproduce: `ds-bench` suites `write-cliff-confirm` (memory neuter A/B), +`write-cliff-validate` (fixed default vs wal baseline), `write-cliff-wal-ab` +(same-binary wal gate A/B). + +## Bottleneck analysis — is it CPU-bound or fsync-bound? (measured) + +Added `--server-stats` and profiled both modes in a Linux Docker container (4 CPU, +50k streams, 512 connections, container-to-container networking to avoid the +Docker Desktop port-forward proxy) — plus a real-NVMe cross-check on +`c4d-standard-16-lssd`. + +**Memory mode = CPU-bound.** `SRV_STATS cpu_cores≈3.0–3.5` (of 4), `inflight≈0`, +`svc_us≈8–15`, `durwait=0`, `applock=0`. The server saturates cores with no +queueing; throughput scales with cores (documented 315k@4→526k@8 vCPU). The #1 +sidecar fix helps here precisely because it removes per-append CPU work — measured +**+16% @ 100k, +14% @ 500k on real NVMe** (memory-nogate vs memory-gated: +318k→370k, 237k→270k). + +**WAL mode = fsync/durability-bound, NOT CPU-bound.** `SRV_STATS cpu_cores≈0.9–1.5` +(of 4 — 3 cores idle!), `inflight≈260–500` (deep queue), `svc_us≈8–13ms` of which +`durwait_us` is **~99%**. Hundreds of appends pile up in `wait_durable_lsn` while +the CPU sits idle — the WAL group-commit fsync is the wall. Real-NVMe baseline: +66k @ 100k, 48k @ 500k (4 vCPU / 4 shards). + +**Implications for "should we allocate more CPU?"** +- **Memory: yes** — it is CPU-bound and scales with cores. +- **WAL: no** — it is fsync-bound with 3 of 4 cores idle. More vCPU is wasted spend. + The wal lever is fsync throughput = **parallel fsync lanes = more `--wal-shards`** + (each shard commits/fsyncs independently), NOT CPU and NOT `--wal-fsync-parallel` + (which fans the CHECKPOINT fsync — regressed −11% on NVMe: fsync16 59k vs fsync1 + 66k @ 100k, same CPU-oversubscription failure as local). + +Caveat on local harness: Docker Desktop virtiofs serializes fsync (so wal +shard-scaling is invisible locally) and a single client container caps memory at +~150k (so memory CPU-scaling is invisible locally). The STRUCTURE (which resource +saturates) is valid locally; the SCALING numbers require real NVMe + a client fleet. + +## H2 / H4 investigation — wal checkpoint breakdown (measured, mostly negative) + +Followed up on the two wal-checkpoint hypotheses by capturing the `WAL_CKPT` +phase breakdown under 50k-stream load (`--wal-stats`). **Iterate on a Linux +container, not bare macOS:** macOS `fsync` is `F_FULLFSYNC` (a true drive barrier, +~7 ms, globally serialized), so bare-metal wal throughput is a ~30× artifact. +`DS_BENCH_FAST_FSYNC=1` swaps in plain `fsync` for a faster loop, but the +representative signal comes from running the server in Docker (Linux `fdatasync`). +Local loop: `target/release/durable-streams-server … --wal-stats 1` driven by +`ds-bench multi-stream --api-style durable --streams 50000`. + +Checkpoint phase split at 50k streams (per checkpoint, Linux container): + +``` +touched=21274 fsync_us≈2,000,000 (≈99%) tails_us≈10,000 (0.5%) meta_us≈300 (~0%) +``` + +- **H2 (incremental `tails` persistence): NOT WORTH DOING.** The cumulative + `tails`-map rewrite is **~0.5%** of checkpoint time even as it grows — optimizing + it is optimizing noise. (`meta_us≈0` also confirms the #1 wal gate zeroed the + sidecar phase: `meta=0` on every line.) +- **H4 (checkpoint `fdatasync`): dominant *within* the checkpoint, but NOT the wal + throughput ceiling on Linux.** The commit path batches healthily + (`WAL_CONT batch_avg≈12`, staged/s 8–18k) and the checkpoint runs on separate + threads, so `fdatasync` overlaps commits instead of blocking them. The real + ceiling is the **commit-path group-commit fsync rate** (~1000 fsync/s × batch + ≈ throughput), not the checkpoint. (Bare macOS mis-attributed this to the + checkpoint because `F_FULLFSYNC` serializes *all* fsyncs globally.) +- **Parallelizing the checkpoint `fdatasync` REGRESSES on constrained hardware.** + Added `--wal-fsync-parallel N` (fan the per-stream syncs across N OS threads; all + complete before recycle, crash-ordering preserved, crash-sim green). Measured on + a 2-vCPU Linux container, 50k streams: + + | fanout | ops/s | vs serial | + |---|---|---| + | 1 (serial) | 16.4k | — | + | 2 | 15.8k | −4% | + | 4 | 13.8k | −16% | + | 16 | 10.9k | −19% | + + More fan-out steals CPU from the runtime (slowing the committer until checkpoints + fall behind) and virtiofs doesn't do concurrent fsync. **Default is therefore + serial (`fanout=1`, a no-op);** the parallel path is opt-in, pending validation on + real NVMe (deep device queue + spare cores) where it may pay off. On current + evidence, do not enable it. + +**Net:** neither H2 nor H4 is a wal throughput win on Linux. The next wal lever is +the **commit-path group commit** (raise batch / fsync efficiency), not the +checkpoint — a separate investigation. + +## NVMe shard + fanout sweep — optimal wal config (measured, definitive) + +Controlled sweeps on real Titanium NVMe (`c4d-standard-16-lssd`, **8 vCPU pin**, +200k streams, 256 conns/pod, 256 B; ds-bench `wal-shard-sweep` + `wal-fanout-sweep`). +Both suites are barrier-aligned, per-cell resumable, and ran to completion. + +**`--wal-shards` sweep — flat within 5 %:** + +| shards | 1 | 4 | 8 | 16 | 24 | +|---|---|---|---|---|---| +| peak ops/s | 72k | **75k** | 73k | 73k | 71k | + +**`--wal-fsync-parallel` sweep (at s4) — flat then regresses:** + +| fanout | 1 | 2 | 4 | 8 | +|---|---|---|---|---| +| peak ops/s | 75k | 73k | 75k | **67k** | + +Live telemetry at *every* shard count: SRV_STATS `cpu_cores≈1.1–1.8/8` (idle), +`durwait_us ≈ 97–99 % of svc_us`; WAL_CONT `fsync/s≈900–1000`, `batch_avg` 53→88; +WAL_CKPT `touched≈580`, `fsync_us≈1.4 s` (the per-stream fdatasync storm). + +**Conclusions:** +- **Neither shards nor fsync-parallel is a throughput lever.** wal is bounded by the + disk's `fdatasync`/s rate (~1000), a single shared resource — not per-shard lanes. + This **overturns the old "shards is the knob / s16 ≈ 380k"** number (does not + reproduce). More shards *hurt* at low load (s24 @1 pod = 34k vs s4 = 49k: thinner + per-shard group-commit batches). `--wal-fsync-parallel ≥ 8` regresses (it just adds + concurrent checkpoint fsyncs that steal device budget from commits). +- **Answers the earlier open NVMe question** (follow-up #3 below): no, `--wal-fsync-parallel > 1` + does **not** help on real NVMe either — the deep device queue doesn't rescue it. +- **Optimal wal config:** `--wal-shards 4` (or `min(cores,4)`), `--wal-fsync-parallel 1` + (default), `server_cpus` 4–8 (CPU is idle — don't overspend). **The real throughput + knob is `connections`** (offered load → bigger `batch_avg`), traded against latency + (36 ms p50 at 3072 in-flight). Server default `shards = core count` is too high on + many-core boxes; it should be a small fixed value. + +## Recommended follow-ups (not done) + +Ranked by evidence after the NVMe sweeps above: + +1. **Collapse the checkpoint per-stream durability barrier from O(N_touched) syscalls + to O(1)** — the real cardinality lever. `checkpoint()` (`wal/shard.rs:797`, step 2) + does one `fdatasync` per touched stream (~580/shard/interval → `fsync_us≈1.4 s`), + which steals device flush budget from the commit fsyncs and *is* the cliff. + A single `syncfs()` (or `sync_file_range` batch + one barrier) at checkpoint would + flush all touched files in one shot. No `syncfs` in the tree today; the per-stream + loop is hardcoded (no checkpoint-interval flag). This is the highest-value change. +2. **Wal commit-path group commit** (the throughput ceiling): throughput ≈ + `fsync/s × batch_avg`. `fsync/s` is hardware; lift `batch_avg` without needing + more in-flight (e.g. a small commit-coalescing delay) to raise throughput at lower + latency. NOT the checkpoint. Supersedes the old "#2 tails" idea (disproved: ~0.5%). +3. **#3 — per-append working-set physics** (residual in both modes): interned stream + ids + ahash + capacity hint on `Store.streams`; pack hot `StreamState` fields into + one cacheline. Likely the memory-mode ~8% residual after #1. +4. ~~Validate on remote NVMe / does `--wal-fsync-parallel > 1` help there?~~ — + **done, answered: no** (see the NVMe sweep above; f8 regresses to 67k). +5. ~~#2 incremental `tails` persistence~~ — **dropped.** ~0.5% of checkpoint time. diff --git a/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md b/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md new file mode 100644 index 0000000000..468ff64493 --- /dev/null +++ b/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md @@ -0,0 +1,192 @@ +# Write bottlenecks at 1M streams — findings brief + +**Audience:** an engineer/agent picking up WAL write-path performance work with no prior +context. **Scope:** single-node write throughput + tail latency of the Rust +`durable-streams` server (`packages/durable-streams-rust`) at high stream cardinality +(200k–1M streams). **Date:** 2026-07-02. **Server under test:** `perf/combined-t1a-t1c-t2a` +(PR: electric-sql/electric#4675), vs the 2026-06-30 baseline `durable-streams:dev`. + +--- + +## TL;DR + +1. **The write ceiling is coordination, not compute or fsync.** Under write saturation the + server plateaus at **~80% CPU** and then *declines* under more load. The cost is the + **group-commit committer scheduling + a durability-wakeup thundering herd**, not lock + contention (at shards ≈ cores) and not fsync. +2. **Fixed the coordination** (`combined` stack): dedicated committer thread (T2a), + coalesced wakeups (T1c), lock-free dirty registration (T1a). Measured **−21× wakeups + per commit**, **~10× more committer activity**, **+60% throughput @ 32 vCPU**, and at + 16 vCPU / 1M streams **p99 80 ms → 32 ms**. +3. **Stream cardinality (200k→1M) adds a second, independent cost** that is *not* + coordination: per-op memory/working-set physics (registry + per-stream state + page + cache + checkpoint fsync fan-out + ~1 fd/stream). This is present in every build and is + the next frontier. +4. **All results are write-only.** The read path (tail-cache, sendfile/splice, live + tailing) is unmeasured. A read+write mix is the recommended next experiment. + +--- + +## Test setup (how to reproduce) + +- **Bench harness:** `ds-bench` (sibling repo). Remote target = GKE; `scripts/bench + suites/.json run`. Golden rule: **always tear clusters down** (`bench` self-tears + on clean completion; arm `scripts/teardown-watchdog.sh`). +- **Server node:** `c4d-standard-16-lssd`, pinned to N vCPU via `SERVER_CPUS=N` (sets + cgroup `cpu.max`; the server reads it through `available_parallelism()`). +- **Server flags:** `--wal-shards N --worker-threads N` (shards = workers = cores). +- **Client:** pool model, `n2d-standard-32` fleet, `CONNS_PER_POD=256`, batch 1, 256 B + payload. **Load knob = total connections = pods × 256.** This is a *fixed-concurrency* + workload, so `throughput ≈ connections ÷ latency` (Little's law) — remember this when + reading the numbers. +- **Telemetry:** `--wal-stats ` makes the server emit a `WAL_CONT` line per interval: + `staged/s`, `fsync/s`, `batch_avg`, `inner_wait_load`, `dirty_wait_load`, + `waiters_woken_avg`. `*_wait_load` = fraction of a core-second spent *parking* on that + lock. Off by default (one relaxed atomic load); do **not** enable it during a latency run + (the hot-path clock reads perturb the tail). + +--- + +## Bottleneck #1 — the coordination ceiling (FIXED by `combined`) + +### Mechanism (per append, in `src/wal/`) + +Every append touches per-shard shared state on the shard its `stream_id` hashes to: + +- `shard.inner` `Mutex` — twice per append (reserve LSN/write-pos, then mark-written). +- `shard.dirty` `Mutex` — a HashMap **insert on every append** (`register_dirty`). +- `durable_tx` `watch` — on each commit, `publish_durable` wakes **every** waiter parked on + the shard (durability barrier), not just the ones whose LSN is now durable. + +Plus the committer itself hopped through the shared Tokio runtime via `spawn_blocking` +per commit — a scheduling round-trip that capped commit *cadence*. + +### Direct evidence (4 vCPU / 4 shards, `--wal-stats`, instrumented baseline vs combined) + +| counter | baseline | combined | note | +|---|---|---|---| +| `waiters_woken` / commit | 340–378 | **16–18** | **~21× fewer** — the herd (T1c) | +| `fsync/s` (commit cadence) | 83–192 | **1021–1178** | **~10× more** — committer freed (T2a) | +| `batch_avg` | 168–189 | 16–18 | baseline commits rarely in huge batches | +| `inner_wait_load` | ~0.000 | ~0.002 | **locks are NOT the gate here** | +| `dirty_wait_load` | ≤0.018 | ~0.000 | ditto | + +**Key insight:** at shards ≈ cores there is ≈1 worker per shard, so the per-shard *locks +barely contend* (`*_wait_load ≈ 0`). The ceiling is the **committer round-trip + wakeup +broadcast**. That is what `combined` removes: + +- **T2a** — dedicated committer OS thread, drop per-commit `spawn_blocking`. *Biggest single + win.* Direct proof: `fsync/s` ~doubles/10×'s — the committer was blocked on runtime + round-trips, not fsync. +- **T1c** — coalesced wakeups (wake only satisfied waiters). Kills the O(waiters) run-queue + storm. At 32 cores the un-coalesced herd is ~415 waiters/commit (≈13.3k in-flight ÷ 32 + shards); T2a's faster committer makes the un-coalesced herd *worse*, so T1c is needed + precisely because T2a helps. +- **T1a** — epoch-gated lock-free `register_dirty` (no per-append `Mutex` insert). + +### Throughput/latency impact + +Pool saturation ladder vs 2026-06-30 baseline: + +| scale | baseline | combined | +|---|---|---| +| 32 vCPU, 200k | 1.48M @ 52 pods, p99 41 ms | **2.37M @ 40 pods (+60%), p99 12 ms** | +| 32 vCPU, 500k | 1.15M @ 52 pods | **≥1.78M (+55%, unsaturated)** | + +Lock-free/committer changes scale with core count: **+11% @ 6 cores → +60% @ 32 cores**, +because the herd and lock pressure only dominate at high core counts. + +--- + +## Bottleneck #2 — stream cardinality (200k → 1M) — NOT yet solved + +This is a **separate axis** from coordination and is present in *every* build. At the same +offered load, more distinct streams cost more per op: + +- **Registry lookups.** `Store.streams: DashMap>` (`store.rs:365`), + hot-path `get(path)` per append (`store.rs:~654`). At 1M entries the map + values exceed + L2/L3 → more cache/TLB misses per append. +- **Per-stream working set.** Each stream is a `StreamState` (`store.rs:164`): an + `AsyncMutex`, a `RwLock`, a `watch::Sender`, a `last_chunk` + cache, and an `Arc`. 1M of these is a large resident set (measured server RSS ~1.16 + GB at 1M — modest, but the *cache-miss* cost is the real tax, not RSS). +- **Page-cache locality.** One data file per stream ⇒ 1M distinct hot tails ⇒ worse + locality, more page-cache churn. +- **Checkpoint fsync fan-out.** The per-shard checkpoint `fdatasync`s *every touched stream + file* before recycling the WAL ⇒ fsync count scales with distinct streams touched per + interval. +- **File descriptors.** ~1 open fd per stream. The server raises `RLIMIT_NOFILE` to the + hard limit on startup (`main.rs:89` `raise_nofile_limit`). On GKE that hard limit is + typically **1,048,576**, so **1M streams sits ~38k below the fd ceiling** once you add + ~8k client sockets + WAL segment/epoll fds. **This is a hard wall just above 1M** — going + materially past 1M streams per node needs a raised limit or fewer fds/stream (e.g. + open-on-demand instead of a persistent `Arc` per stream). + +**Caveat on how cardinality shows up in the metrics:** the WAL contention counters do *not* +attribute the 200k→1M degradation to a lock/wakeup source (they stay flat/low). So this +cost is the memory/IO physics above, not coordination — a different class of fix +(sharded registry, open-on-demand fds, working-set reduction, checkpoint batching). + +--- + +## 1M-stream write scaling (combined, fixed load = 32 pods = 8,192 connections) + +Single fixed rung, 60 s measure, no ladder: + +| server | throughput | p50 | p90 | p99 | p99.9 | max | server CPU | +|---|---|---|---|---|---|---|---| +| **8 vCPU** | 454k ops/s | 2.9 ms | 68 ms | **80 ms** | 94 ms | 3348 ms | **86% of 8** | +| **16 vCPU** | **862k ops/s** | 3.3 ms | 25 ms | **32 ms** | 88 ms | **405 ms** | **77% of 16** | + +Reading it: +- **8 vCPU was capacity-bound at this load.** Doubling cores nearly **doubled throughput + (454k→862k)** at the *same* 8,192 connections — the signature of fixed-concurrency: the + server clears each write faster, so every connection completes more ops/s. (Little's law + checks both ways: ≈8,190 in-flight.) +- **The tail was the story, and it collapsed.** p99 80→32 ms, and the multi-second + outliers vanished (max 3348→405 ms). At 8 vCPU an 86%-CPU redline meant a real slice of + writes queued behind group-commit; 16 vCPU (77%) has slack, so far fewer wait. +- **Median was never the problem** (2.9→3.3 ms). The win is entirely tail + headroom. + +**Implication:** at 1M streams, ~8,000 concurrent writers want **~12–16 vCPU** to keep p99 +in the tens-of-ms range with headroom. 8 vCPU is under-provisioned for that concurrency. + +--- + +## What's fixed vs open + +**Fixed (in `combined`, PR #4675):** the coordination ceiling — dedicated committer (T2a), +coalesced wakeups (T1c), lock-free dirty (T1a), plus `--wal-stats` telemetry and +`--worker-threads`. + +**Deferred, with reasons:** +- **T1b (atomic WAL reserve):** `inner_wait_load ≈ 0` at 4–16 vCPU (shards ≈ cores) ⇒ + lock-restructuring risk for ~no gain at our operating points. Revisit only at + high-core/low-shard configs where the inner lock genuinely contends. +- **T3 (shared-nothing / thread-per-core):** design-only; confirmed to pay off only at + ≥16–32 cores and not needed to capture the current win. The pool client touching all + shards per connection forces a cross-core SPSC handoff that is otherwise net-negative. + +**Open (recommended next work, in priority order):** +1. **Read + write mixed workload.** Everything above is write-only. Reads exercise + tail-cache, sendfile/splice, the `RwLock` read path, and live tailing — none + stressed here. Combined only touches the WAL commit/wakeup path, so read-regression risk + is low and T1c may *help* live tailers (durability wakeups also wake subscribers) — but + verify before calling the write win "done." +2. **Cardinality cost reduction (bottleneck #2).** Sharded/segmented registry, checkpoint + `fdatasync` batching, and reducing per-stream fixed cost (esp. fds — open-on-demand vs + persistent `Arc`) to get past the ~1M fd wall. +3. **Find 16 vCPU's own redline.** These 1M numbers are one fixed rung (77% CPU). A heavier + rung (e.g. 64 pods) pins where 16 vCPU saturates and its p99 there. + +--- + +## Provenance + +- Full investigation + telemetry design: `CONTENTION_INVESTIGATION.md` (same directory). +- Baseline that motivated this: `ds-bench/results/run-durable-pool2/FINDINGS.md`. +- 4-vCPU counter A/B: `ds-bench/suites/run-durable-cpu4-stats-{base,combined}.json`. +- 1M p99 runs: `ds-bench/suites/run-durable-cpu{8,16}-1m-p99-combined.json`. +- Code anchors: `src/wal/shard.rs` (register_dirty / reserve_and_stage / publish_durable), + `src/wal/telemetry.rs` (WAL_CONT), `src/store.rs:164,365` (StreamState / registry), + `src/main.rs:89` (raise_nofile_limit). diff --git a/packages/durable-streams-rust/src/handlers.rs b/packages/durable-streams-rust/src/handlers.rs index a4d2c74e04..eeed3b9888 100644 --- a/packages/durable-streams-rust/src/handlers.rs +++ b/packages/durable-streams-rust/src/handlers.rs @@ -896,6 +896,9 @@ async fn handle_append(store: Arc, req: Req, path: String) -> Resp { async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp, AppendOutcome, bool) { use AppendOutcome::*; + // Load-telemetry probe: bumps the in-flight gauge and records service time on + // drop (covers every return path). No-op unless `--server-stats` is on. + let _probe = crate::srvstats::AppendProbe::start(); let st = match store.get(&path) { Some(s) => s, None => return (text_response(404, "stream not found"), Conflict, false), @@ -950,8 +953,10 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp // Serialize per stream: producer validation + write + state update under one // lock. Time the wait separately — lock contention is a key bottleneck. let lock_t0 = crate::telemetry::Timer::start(); + let srv_lock_t0 = std::time::Instant::now(); let mut ap = st.appender.lock().await; crate::telemetry::record_append_lock_wait(lock_t0.elapsed_secs()); + crate::srvstats::record_applock_wait(srv_lock_t0.elapsed()); // Closed checks (precedence: closed → seq regression → gap). { @@ -1083,6 +1088,16 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp Err(_) => ret!(text_response(500, "write failed"), Conflict), } } + // Does this append change state the memory-mode sidecar must persist? Captured + // BEFORE `seq_header` is consumed below. Producer/seq updates are idempotency + // state; a TTL stream's sliding `last_access` must survive restart (mirrors the + // read path, which marks dirty only for TTL streams). A plain append to a non-TTL stream changes + // only `durable_tail`/`last_access`. In BOTH modes the durable tail is carried + // elsewhere (memory: re-derived from the data-file length on restart; wal: the + // checkpoint's per-shard `tails` map), and `last_access` only gates TTL — so a + // plain non-TTL append needs no sidecar flush at all (cardinality-cliff #1). + let meta_persist_needed = + producer.is_some() || seq_header.is_some() || st.config.ttl_seconds.is_some(); { let mut s = st.shared.write().unwrap(); if let Some(p) = &producer { @@ -1130,7 +1145,9 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp // Wait for durability off the lock before exposing the bytes. if let Some(lsn) = staged_lsn { + let dur_t0 = std::time::Instant::now(); wait_durable_lsn(&store, &st, lsn).await; + crate::srvstats::record_durwait(dur_t0.elapsed()); } // Durable now (wal) / page-cache written (memory): expose the new bytes to @@ -1165,12 +1182,43 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp // saturation) plus a timer task OFF the per-append path. Producer/access // updates are already documented as a non-durable, lagging flush; the lag // bound moves from the 100 ms debounce to the checkpoint cadence. - st.meta_dirty.store(true, std::sync::atomic::Ordering::Release); - } else { + // + // GATED (cardinality-cliff #1): only mark when the append changed state + // the sidecar must persist — producer/seq idempotency or a sliding TTL. + // A plain append still gets its fdatasync AND its `durable_tail` recorded + // in the checkpoint's per-shard `tails` map (register_dirty + the + // unconditional `persist_durable_tails`, independent of this flag) — and + // that map, not the sidecar, is the authoritative durable-tail proof + // recovery reconciles against (see wal/shard.rs step 3a, wal/recovery.rs). + // `last_access` only gates TTL. So a plain non-TTL append needs no sidecar + // rewrite here — dropping it removes the O(touched) `write_meta_sync` calls + // that dominate the checkpoint's meta phase at high stream cardinality. + // `wal_meta_gate()` (default on) can be turned off to restore always-mark + // for a same-binary A/B of the gate. + if meta_persist_needed || !crate::store::wal_meta_gate() { + st.meta_dirty.store(true, std::sync::atomic::Ordering::Release); + } + } else if meta_persist_needed { // No WAL record staged (memory durability): no checkpoint will flush // the sidecar — queue it for the store-level periodic sweeper. Same // batched treatment the wal branch above gets from the checkpoint: no // per-stream timer task, no per-append sidecar rewrite (#4691). + // + // GATED (cardinality-cliff fix): only queue when the append actually + // changed state the sidecar must persist — producer/seq idempotency or a + // sliding TTL (see `meta_persist_needed`). A plain append to a non-TTL + // stream changes only `durable_tail`/`last_access`, and memory-mode + // recovery reads NEITHER (the tail is re-derived from the data-file + // length in `Store::new_with_tier`; `last_access` only gates TTL expiry, + // which these streams don't have). Skipping the queue for that common + // case removes the per-append sidecar rewrite whose cost stops amortizing + // at high stream cardinality (CARDINALITY_CLIFF_CAUSES.md #1). + // + // `mem_meta_gate()` (default on) can be turned off to restore always-queue + // for a same-binary A/B of the fix. + store.mark_meta_dirty(&st); + } else if !crate::store::mem_meta_gate() { + // A/B baseline (gate off): old behavior — queue every memory-mode append. store.mark_meta_dirty(&st); } if !wire.is_empty() { @@ -2298,5 +2346,49 @@ mod memory_mode_tests { let _ = std::fs::remove_dir_all(&dir); } + + /// Cardinality-cliff fix (#1): a PLAIN append (no producer/seq, non-TTL + /// stream) in memory mode must NOT queue a sidecar flush. The tail is + /// recovered from the data-file length and `last_access` only gates TTL, so + /// the per-append sidecar rewrite is pure overhead whose cost stops + /// amortizing at high stream cardinality. Contrast + /// `memory_append_defers_sidecar_to_store_sweep`, which uses a producer + /// append and therefore still marks dirty. + #[tokio::test] + async fn memory_plain_append_skips_sidecar_flush() { + let _guard = crate::handlers::test_support::DurabilityGuard::memory(); + let dir = tmp("mem-plain-noflush"); + let store = + Arc::new(Store::new_with_tier(dir.clone(), TierConfig::default()).unwrap()); + + let resp = handle(Arc::clone(&store), put_req("m/p", "application/octet-stream")).await; + assert!((200..300).contains(&resp.status), "create: {}", resp.status); + // Drain anything the create queued so we measure only the append's effect. + let store2 = Arc::clone(&store); + let _ = tokio::task::spawn_blocking(move || store2.sweep_meta_once()) + .await + .unwrap(); + + // Plain append: no producer headers, non-TTL stream. + let resp = handle( + Arc::clone(&store), + post_req("m/p", "application/octet-stream", b"payload"), + ) + .await; + assert!((200..300).contains(&resp.status), "append: {}", resp.status); + + let flushed = tokio::task::spawn_blocking({ + let store = Arc::clone(&store); + move || store.sweep_meta_once() + }) + .await + .unwrap(); + assert_eq!( + flushed, 0, + "a plain non-TTL memory-mode append must not queue a sidecar flush" + ); + + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index 721240c4ec..609653029d 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -3,6 +3,7 @@ mod blobstore; mod engine_raw; mod handlers; mod http1; +mod srvstats; #[cfg(target_os = "linux")] mod sse_reactor; mod store; @@ -137,6 +138,7 @@ fn main() { // append path). Dependency-free — the measurement vehicle for the contention // investigation, independent of the heavy `telemetry` OTLP feature. let mut wal_stats_secs: Option = None; + let mut server_stats_secs: Option = None; let mut args = std::env::args().skip(1); while let Some(a) = args.next() { match a.as_str() { @@ -238,6 +240,53 @@ fn main() { } } } + // Dev/benchmark toggles for the cardinality-cliff investigation. + // `--meta-sweep-disable` drops the memory-mode sidecar write (makes + // the sidecar permanently stale — bench-only); `--meta-sweep-stats` + // logs a META_SWEEP line per tick. + "--meta-sweep-disable" => store::set_meta_sweep_disable(true), + "--meta-sweep-stats" => store::set_meta_sweep_stats(true), + // Periodic SRV_STATS line (both modes): cpu_cores / inflight / service + // + appender-lock + durability wait — bottleneck analysis. + "--server-stats" => { + let n: u64 = parse_val(args.next(), "--server-stats"); + if n == 0 { + eprintln!("--server-stats must be ≥ 1 (seconds)"); + std::process::exit(2); + } + server_stats_secs = Some(n); + } + "--wal-meta-gate" => { + let v = val(args.next(), "--wal-meta-gate"); + match v.as_str() { + "on" => store::set_wal_meta_gate(true), + "off" => store::set_wal_meta_gate(false), + _ => { + eprintln!("--wal-meta-gate must be on|off"); + std::process::exit(2); + } + } + } + "--mem-meta-gate" => { + let v = val(args.next(), "--mem-meta-gate"); + match v.as_str() { + "on" => store::set_mem_meta_gate(true), + "off" => store::set_mem_meta_gate(false), + _ => { + eprintln!("--mem-meta-gate must be on|off"); + std::process::exit(2); + } + } + } + // Checkpoint fdatasync fan-out (H4). `1` = serial baseline. + "--wal-fsync-parallel" => { + let n: u64 = parse_val(args.next(), "--wal-fsync-parallel"); + if n == 0 { + eprintln!("--wal-fsync-parallel must be ≥ 1"); + std::process::exit(2); + } + wal::shard::set_fsync_fanout(n); + } other => { eprintln!("unknown argument: {other}"); std::process::exit(2); @@ -303,6 +352,11 @@ fn main() { // touches here (its append path flushes via the checkpoint instead). spawn_meta_sweeper(Arc::clone(&store)); + // Server load telemetry (both modes) for bottleneck analysis. + if let Some(secs) = server_stats_secs { + srvstats::spawn(secs); + } + // ---- WAL wiring (Wal mode only) ---- // // Skipped entirely in `--durability memory` mode — no WAL is opened, diff --git a/packages/durable-streams-rust/src/srvstats.rs b/packages/durable-streams-rust/src/srvstats.rs new file mode 100644 index 0000000000..aee1a396e2 --- /dev/null +++ b/packages/durable-streams-rust/src/srvstats.rs @@ -0,0 +1,140 @@ +//! Lightweight, always-cheap server-side load telemetry for bottleneck analysis +//! (cardinality-cliff performance work). Answers the core question — is the server +//! CPU-bound, fsync/durability-bound, or lock-bound? — for BOTH wal and memory +//! modes, which the WAL-only `--wal-stats` counters cannot. +//! +//! Enabled by `--server-stats N` (seconds). Off by default and gated by +//! `STATS_ON`, so the hot-path instrumentation is a single relaxed load + branch +//! when disabled. Each tick prints a `SRV_STATS` line and resets the interval +//! accumulators. +//! +//! Fields: +//! - `cpu_cores` process CPU utilization in cores (utime+stime delta / wall); +//! ≈ the cgroup cpu quota ⇒ CPU-bound. Linux only (`-1` elsewhere). +//! - `appends_s` acked appends/sec over the interval. +//! - `inflight` in-flight append handlers sampled at tick time (queue depth). +//! - `svc_us` mean append handler wall time (service time). +//! - `applock_us` mean time waiting to acquire the per-stream appender lock. +//! - `durwait_us` mean time in `wait_durable_lsn` (WAL fsync wait; ~0 in memory). + +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::time::Instant; + +static STATS_ON: AtomicBool = AtomicBool::new(false); +static APPENDS: AtomicU64 = AtomicU64::new(0); +static INFLIGHT: AtomicI64 = AtomicI64::new(0); +static SVC_US: AtomicU64 = AtomicU64::new(0); +static APPLOCK_US: AtomicU64 = AtomicU64::new(0); +static DURWAIT_US: AtomicU64 = AtomicU64::new(0); + +pub fn set_enabled(v: bool) { + STATS_ON.store(v, Ordering::Relaxed); +} +#[inline] +pub fn enabled() -> bool { + STATS_ON.load(Ordering::Relaxed) +} + +/// RAII probe for one append handler: bumps the in-flight gauge on creation and, +/// on drop (covering every early return), records the service time and counts the +/// append. Create it once `enabled()` is true. +pub struct AppendProbe { + start: Instant, +} +impl AppendProbe { + #[inline] + pub fn start() -> Option { + if !enabled() { + return None; + } + INFLIGHT.fetch_add(1, Ordering::Relaxed); + Some(Self { start: Instant::now() }) + } +} +impl Drop for AppendProbe { + fn drop(&mut self) { + INFLIGHT.fetch_sub(1, Ordering::Relaxed); + SVC_US.fetch_add(self.start.elapsed().as_micros() as u64, Ordering::Relaxed); + APPENDS.fetch_add(1, Ordering::Relaxed); + } +} + +#[inline] +pub fn record_applock_wait(d: std::time::Duration) { + if enabled() { + APPLOCK_US.fetch_add(d.as_micros() as u64, Ordering::Relaxed); + } +} +#[inline] +pub fn record_durwait(d: std::time::Duration) { + if enabled() { + DURWAIT_US.fetch_add(d.as_micros() as u64, Ordering::Relaxed); + } +} + +/// Read process CPU time (utime+stime) in seconds from `/proc/self/stat`. Linux +/// only; `None` elsewhere (macOS dev boxes — use a Linux container to profile). +#[cfg(target_os = "linux")] +fn cpu_secs() -> Option { + let s = std::fs::read_to_string("/proc/self/stat").ok()?; + // comm (field 2) may contain spaces/parens — parse after the final ')'. + let rest = &s[s.rfind(')')? + 1..]; + let f: Vec<&str> = rest.split_whitespace().collect(); + // After ')': [state, ppid, ...]; utime is field 14 ⇒ index 11, stime ⇒ 12. + let utime: f64 = f.get(11)?.parse().ok()?; + let stime: f64 = f.get(12)?.parse().ok()?; + let hz = 100.0; // _SC_CLK_TCK is 100 on all our targets. + Some((utime + stime) / hz) +} +#[cfg(not(target_os = "linux"))] +fn cpu_secs() -> Option { + None +} + +/// Spawn the periodic printer. Each tick emits one `SRV_STATS` line and resets the +/// interval accumulators. +pub fn spawn(secs: u64) { + set_enabled(true); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(std::time::Duration::from_secs(secs)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + ticker.tick().await; + let mut last_cpu = cpu_secs(); + let mut last = Instant::now(); + loop { + ticker.tick().await; + let now = Instant::now(); + let wall = now.duration_since(last).as_secs_f64(); + last = now; + + let appends = APPENDS.swap(0, Ordering::Relaxed); + let svc = SVC_US.swap(0, Ordering::Relaxed); + let applock = APPLOCK_US.swap(0, Ordering::Relaxed); + let durwait = DURWAIT_US.swap(0, Ordering::Relaxed); + let inflight = INFLIGHT.load(Ordering::Relaxed); + + let cpu_cores = match (cpu_secs(), last_cpu) { + (Some(c), Some(p)) if wall > 0.0 => { + last_cpu = Some(c); + (c - p) / wall + } + (Some(c), _) => { + last_cpu = Some(c); + -1.0 + } + _ => -1.0, + }; + + let n = appends.max(1) as f64; + eprintln!( + "SRV_STATS cpu_cores={:.2} appends_s={:.0} inflight={} svc_us={:.0} applock_us={:.1} durwait_us={:.1}", + cpu_cores, + appends as f64 / wall.max(0.001), + inflight, + svc as f64 / n, + applock as f64 / n, + durwait as f64 / n, + ); + } + }); +} diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index 1b69a5ab30..6ec863a6e2 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -290,6 +290,48 @@ pub fn tail_cache_bytes() -> usize { TAIL_CACHE_BYTES.load(Ordering::Relaxed) } +/// Dev/benchmark toggle: when set, `sweep_meta_once` drains its queue and clears +/// `meta_dirty` but SKIPS the actual sidecar `write_meta_sync`. Used to confirm +/// the cardinality-cliff hypothesis #1 (per-stream sidecar flush cost) in memory +/// mode — if the 1k→50k throughput curve flattens with this on, the sidecar +/// write dominates. NOT for production: it makes the producer-dedup/tail sidecar +/// permanently stale. +static META_SWEEP_DISABLE: AtomicBool = AtomicBool::new(false); +/// Dev/benchmark toggle: when set, emit a `META_SWEEP` line per tick with the +/// number of sidecars written and the wall time spent. +static META_SWEEP_STATS: AtomicBool = AtomicBool::new(false); + +pub fn set_meta_sweep_disable(v: bool) { + META_SWEEP_DISABLE.store(v, Ordering::Relaxed); +} +pub fn set_meta_sweep_stats(v: bool) { + META_SWEEP_STATS.store(v, Ordering::Relaxed); +} + +/// When true (default), a plain non-TTL wal-mode append does NOT mark its stream +/// meta-dirty, so the checkpoint skips the redundant sidecar rewrite (the durable +/// tail is carried by the per-shard `tails` map). Set false to restore the old +/// always-mark behavior — used ONLY to A/B the gate on one binary, since laptop +/// wal throughput is too noisy for a trustworthy cross-image comparison. +static WAL_META_GATE: AtomicBool = AtomicBool::new(true); +pub fn set_wal_meta_gate(v: bool) { + WAL_META_GATE.store(v, Ordering::Relaxed); +} +pub fn wal_meta_gate() -> bool { + WAL_META_GATE.load(Ordering::Relaxed) +} + +/// Memory-mode counterpart of [`wal_meta_gate`]: when true (default), a plain +/// non-TTL memory-mode append does not queue a sidecar sweep. Set false to restore +/// the old always-queue behavior — used only to A/B the #1 fix on one binary. +static MEM_META_GATE: AtomicBool = AtomicBool::new(true); +pub fn set_mem_meta_gate(v: bool) { + MEM_META_GATE.store(v, Ordering::Relaxed); +} +pub fn mem_meta_gate() -> bool { + MEM_META_GATE.load(Ordering::Relaxed) +} + impl StreamState { /// Record the just-appended wire chunk as the resident tail. `start` is the /// logical offset where `bytes` begins. Chunks larger than the tail-cache cap @@ -1184,6 +1226,10 @@ impl Store { pub fn sweep_meta_once(&self) -> usize { let drained: Vec> = std::mem::take(&mut *self.meta_sweep.lock().unwrap()); + let queued = drained.len(); + let skip_write = META_SWEEP_DISABLE.load(Ordering::Relaxed); + let stats = META_SWEEP_STATS.load(Ordering::Relaxed); + let start = if stats { Some(std::time::Instant::now()) } else { None }; let mut n = 0; for st in drained { // A hard-deleted stream's files are already unlinked — flushing @@ -1195,10 +1241,20 @@ impl Store { .get(&st.path) .is_some_and(|cur| Arc::ptr_eq(cur.value(), &st)); if live && st.meta_dirty.swap(false, Ordering::AcqRel) { - let _ = write_meta_sync(&st, false); + // `skip_write` (dev/bench) drains + clears the dirty flag but + // omits the sidecar write, to isolate the write's cost. + if !skip_write { + let _ = write_meta_sync(&st, false); + } n += 1; } } + if let Some(start) = start { + let us = start.elapsed().as_micros(); + eprintln!( + "META_SWEEP queued={queued} wrote={n} skip_write={skip_write} elapsed_us={us}" + ); + } n } } diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index 765d61dbf1..2bb149bd86 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -379,6 +379,29 @@ pub struct Shard { /// recyclable. const CHECKPOINT_FILE: &str = "checkpoint"; +/// Concurrency for the checkpoint's per-stream `fdatasync` phase (cardinality-cliff +/// H4). At high stream cardinality that phase dominates the checkpoint (~99% of its +/// wall time), and a serial loop pays `latency × N_touched` while the storage +/// device's queue depth (NVMe: many in flight) sits idle. Fanning the syncs across +/// this many OS threads lets the device absorb them concurrently; all still +/// complete before `persist_durable_tails`/recycle, preserving the +/// durability-before-recycle ordering. `1` (the DEFAULT) = serial, i.e. a no-op +/// change unless `--wal-fsync-parallel N` opts in. +/// +/// Default is serial because a fixed high fan-out REGRESSES on a CPU-constrained +/// server or storage that does not do concurrent fsync: measured on a 2-vCPU Linux +/// container (Docker, virtiofs) fan-out=16 was −19% vs serial — the 16 sync threads +/// per shard stole CPU from the runtime, slowing the committer until checkpoints +/// fell behind. The win requires real NVMe (deep device queue) AND spare cores; +/// validate there before raising the default. +static FSYNC_FANOUT: AtomicU64 = AtomicU64::new(1); +pub fn set_fsync_fanout(n: u64) { + FSYNC_FANOUT.store(n.max(1), Ordering::Relaxed); +} +fn fsync_fanout() -> usize { + FSYNC_FANOUT.load(Ordering::Relaxed) as usize +} + /// Name of the per-shard durable-tail map: `/tails` (task 11b). A /// CUMULATIVE `stream_id durable_tail` line map (plain decimal text, one stream /// per line). At checkpoint, each touched stream's current logical `Shared.tail` @@ -833,9 +856,38 @@ impl Shard { let n_touched = touched.len(); let t_capture = t_start.elapsed(); - // 2. fdatasync each touched per-stream file. - for (_, _, f) in &touched { - crate::store::barrier_fsync(f)?; + // 2. fdatasync each touched per-stream file. Fan out across a bounded + // pool of OS threads (H4): the device's queue depth absorbs the + // syncs concurrently instead of paying latency × N_touched serially + // — the checkpoint's dominant cost at high cardinality. ALL syncs + // complete here, before persist_durable_tails/recycle below, so the + // durability-before-recycle ordering is unchanged. `fanout == 1` + // (or a single file) keeps the plain serial loop. + let fanout = fsync_fanout().min(n_touched.max(1)); + if fanout <= 1 { + for (_, _, f) in &touched { + crate::store::barrier_fsync(f)?; + } + } else { + let next = AtomicU64::new(0); + let first_err: Mutex> = Mutex::new(None); + std::thread::scope(|scope| { + for _ in 0..fanout { + scope.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed) as usize; + let Some((_, _, f)) = touched.get(i) else { break }; + if let Err(e) = crate::store::barrier_fsync(f) { + let mut slot = first_err.lock().unwrap(); + if slot.is_none() { + *slot = Some(e); + } + } + }); + } + }); + if let Some(e) = first_err.into_inner().unwrap() { + return Err(e); + } } let t_fsync = t_start.elapsed(); From 649ebda3c167ce946da44f5e044d72981c69a3cb Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 9 Jul 2026 10:41:51 +0100 Subject: [PATCH 02/13] perf(durable-streams-rust): --wal-checkpoint-syncfs (one syncfs barrier vs O(N) per-stream fdatasync) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cardinality-cliff #2. The checkpoint made each touched stream's data file durable via one fdatasync per stream — O(N_touched) device barriers (measured ~29s at 200k streams), which saturate the disk and stall commit fsyncs (the cliff). --wal-checkpoint-syncfs on replaces that loop with a single syncfs() on the data fs: one barrier flushes every touched file. Linux-only (falls back to the per-stream loop elsewhere); default off. Durability-before-recycle ordering unchanged; covered by e2e_checkpoint_syncfs_recovers_acked_records. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/src/main.rs | 13 +++++ packages/durable-streams-rust/src/store.rs | 25 ++++++++++ .../durable-streams-rust/src/wal/e2e_tests.rs | 47 +++++++++++++++++++ .../durable-streams-rust/src/wal/shard.rs | 33 ++++++++++++- 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index 609653029d..3bbb50e45e 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -287,6 +287,19 @@ fn main() { } wal::shard::set_fsync_fanout(n); } + // Checkpoint durability via ONE syncfs() barrier instead of the + // O(N_touched) per-stream fdatasync loop (cardinality-cliff #1). Linux-only. + "--wal-checkpoint-syncfs" => { + let v = val(args.next(), "--wal-checkpoint-syncfs"); + match v.as_str() { + "on" => wal::shard::set_checkpoint_syncfs(true), + "off" => wal::shard::set_checkpoint_syncfs(false), + _ => { + eprintln!("--wal-checkpoint-syncfs must be on|off"); + std::process::exit(2); + } + } + } other => { eprintln!("unknown argument: {other}"); std::process::exit(2); diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index 6ec863a6e2..6bc2fc927e 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -161,6 +161,31 @@ pub(crate) fn barrier_fsync(file: &File) -> std::io::Result<()> { } } +/// A single filesystem-wide durability barrier (Linux `syncfs`). Flushes ALL dirty +/// data+metadata on the filesystem that `file` lives on — used by the checkpoint's +/// `--wal-checkpoint-syncfs` path to make every touched per-stream file durable with +/// ONE syscall instead of `O(N_touched)` `fdatasync`s (cardinality-cliff #1). `file` +/// can be any open fd on the target fs (the checkpoint passes a touched stream file). +/// Linux-only; the caller gates on `cfg!(target_os = "linux")`, so the non-Linux stub +/// (which errors) is never reached in practice — it exists only so the crate compiles +/// on macOS. +#[cfg(target_os = "linux")] +pub(crate) fn syncfs_barrier(file: &File) -> std::io::Result<()> { + // SAFETY: `fd` is a valid open descriptor for the lifetime of `file`. + if unsafe { libc::syncfs(file.as_raw_fd()) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} +#[cfg(not(target_os = "linux"))] +pub(crate) fn syncfs_barrier(_file: &File) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "syncfs is Linux-only", + )) +} + pub struct StreamState { pub id: u64, pub path: String, diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index e2e3819f3e..135313459f 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -952,6 +952,53 @@ async fn e2e_recycled_first_segment_acked_records_survive_crash() { let _ = std::fs::remove_dir_all(&dir); } +/// Cardinality-cliff #1: with `--wal-checkpoint-syncfs on`, the checkpoint makes +/// touched per-stream files durable via ONE `syncfs()` barrier instead of the +/// per-stream `fdatasync` loop. This must preserve the durability-before-recycle +/// guarantee: after a checkpoint recycles the WAL, acked records (both those below +/// the checkpoint floor, made durable by `syncfs`, and those appended after) must +/// still recover byte-identically. On Linux this exercises the real `syncfs` path; +/// on other targets the code falls back to the per-stream loop (still correct). +#[tokio::test] +async fn e2e_checkpoint_syncfs_recovers_acked_records() { + let _guard = DurabilityGuard::wal(); + crate::wal::shard::set_checkpoint_syncfs(true); + const SEG: u64 = 4096; + let dir = tmp("syncfs-ckpt"); + + let h = Harness::boot_with_segment_size(&dir, Some(1), 1, SEG).unwrap(); + create_stream(&h.store, "s", OCTET).await; + + let mut expected = Vec::new(); + // Enough records + small segments to force at least one roll, so the checkpoint + // actually recycles a fully-below-floor segment (relying on the syncfs'd file). + for i in 0..400usize { + let rec = format!("syncfs-{i:04}|").into_bytes(); + append_acked(&h.store, "s", OCTET, &rec).await; + expected.extend_from_slice(&rec); + } + // Force the checkpoint → syncfs barrier → recycle. + h.walset.shards()[0].checkpoint().await.unwrap(); + // More acked appends after the checkpoint (live segment). + for i in 400..500usize { + let rec = format!("syncfs-{i:04}|").into_bytes(); + append_acked(&h.store, "s", OCTET, &rec).await; + expected.extend_from_slice(&rec); + } + + h.crash(); + + let h2 = Harness::boot_with_segment_size(&dir, None, 1, SEG).unwrap(); + let got = stream_file_bytes(&h2.store, "s"); + assert_eq!( + got, expected, + "syncfs-checkpoint acked records recover byte-identical (durability-before-recycle held)" + ); + h2.crash(); + crate::wal::shard::set_checkpoint_syncfs(false); + let _ = std::fs::remove_dir_all(&dir); +} + // =========================================================================== // (7c) WAL-QUIET stream: torn unacked tail truncated via the sidecar proof // =========================================================================== diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index 2bb149bd86..df195225d1 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -33,7 +33,7 @@ use std::cmp::Ordering as CmpOrdering; use std::collections::{BTreeSet, BinaryHeap, HashMap}; use std::io; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use tokio::sync::oneshot; @@ -402,6 +402,27 @@ fn fsync_fanout() -> usize { FSYNC_FANOUT.load(Ordering::Relaxed) as usize } +/// Checkpoint durability strategy (cardinality-cliff #1). The default per-stream +/// `fdatasync` loop (above) issues ONE device durability barrier per *touched* +/// stream — `O(N_touched)` syscalls, measured at ~1.4 s of `fsync` per shard at +/// 200k streams. Those barriers share the device's fixed `fdatasync`/s budget with +/// the commit path, so the checkpoint storm steals throughput from acks (the cliff) +/// AND caps wal throughput regardless of shard count (a shared-device sweep showed +/// s1≈s24). When this is on, step 2 instead issues a SINGLE `syncfs()` on the data +/// filesystem — one barrier that flushes every touched stream file (and everything +/// else dirty on the fs) at once, collapsing the per-stream `O(N_touched)` cost to +/// `O(1)` syscalls. The durability-before-recycle ordering is unchanged: the +/// `syncfs` completes before `persist_durable_tails`/recycle. Linux-only (there is +/// no `syncfs` on macOS); on other targets it falls back to the per-stream loop. +/// Default off — opt in with `--wal-checkpoint-syncfs on`. +static CHECKPOINT_SYNCFS: AtomicBool = AtomicBool::new(false); +pub fn set_checkpoint_syncfs(on: bool) { + CHECKPOINT_SYNCFS.store(on, Ordering::Relaxed); +} +fn checkpoint_syncfs() -> bool { + CHECKPOINT_SYNCFS.load(Ordering::Relaxed) +} + /// Name of the per-shard durable-tail map: `/tails` (task 11b). A /// CUMULATIVE `stream_id durable_tail` line map (plain decimal text, one stream /// per line). At checkpoint, each touched stream's current logical `Shared.tail` @@ -864,7 +885,15 @@ impl Shard { // durability-before-recycle ordering is unchanged. `fanout == 1` // (or a single file) keeps the plain serial loop. let fanout = fsync_fanout().min(n_touched.max(1)); - if fanout <= 1 { + if checkpoint_syncfs() && cfg!(target_os = "linux") && n_touched > 0 { + // #1: ONE filesystem-wide barrier for ALL touched files, replacing + // the O(N_touched) per-stream fdatasync loop. `syncfs` on any fd of + // the data fs (we pass the first touched stream file) flushes every + // dirty page on it — all touched files become durable at once. Still + // strictly before persist_durable_tails/recycle, so the + // durability-before-recycle ordering is unchanged. + crate::store::syncfs_barrier(&touched[0].2)?; + } else if fanout <= 1 { for (_, _, f) in &touched { crate::store::barrier_fsync(f)?; } From db7833f2da6d1df1b9a2648d98a1bce1484ccd5a Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Thu, 9 Jul 2026 12:54:26 +0100 Subject: [PATCH 03/13] chore(durable-streams-rust): drop investigation notes (rationale in PR) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- .../CARDINALITY_CLIFF_CAUSES.md | 190 ------------- .../CARDINALITY_CLIFF_FIX.md | 250 ------------------ .../WRITE_BOTTLENECKS_1M.md | 192 -------------- 3 files changed, 632 deletions(-) delete mode 100644 packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md delete mode 100644 packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md delete mode 100644 packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md diff --git a/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md b/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md deleted file mode 100644 index 9e5f3d8f91..0000000000 --- a/packages/durable-streams-rust/CARDINALITY_CLIFF_CAUSES.md +++ /dev/null @@ -1,190 +0,0 @@ -# The cardinality cliff — code-level cause research - -**Audience:** an engineer/agent picking up the write-cardinality-cliff work with no -prior context. **Date:** 2026-07-08. **Status:** code research only — ranked -hypotheses with file:line evidence and cheap confirmation experiments; no fixes -applied or measured yet. **Companion docs:** -[`CARDINALITY_CLIFF_REPRO.md`](CARDINALITY_CLIFF_REPRO.md) (how to reproduce -locally in ~15-min cycles), [`WRITE_BOTTLENECKS_1M.md`](WRITE_BOTTLENECKS_1M.md) -(background; bottleneck #2 is this cliff). - -## The symptom being explained - -At fixed offered load, append throughput falls steeply and *per-request* latency -(measured below the knee — not queueing) rises as the number of distinct streams -grows. Local kind repro (2-vCPU server): - -| streams | wal thr (rel) | memory thr (rel) | -|---|---|---| -| 1k | 55.4k (100 %) | 112.5k (100 %) | -| 10k | 48.2k (87 %) | 63.7k (57 %) | -| 50k | 27.8k (50 %) | 22.7k (20 %) | - -Two discriminating facts any explanation must fit: - -1. **The cliff exists in BOTH wal and memory mode** → the dominant mechanism is - in the shared append path or shared background machinery, not fsync/WAL - coordination. -2. **Memory mode degrades HARDER than wal locally, ending below it in absolute - terms at 50k** (22.7k vs 27.8k) → whatever memory mode does *instead of* the - WAL checkpoint must be more expensive per distinct stream, not less. - -The WAL contention counters (`--wal-stats`) stay flat across cardinality, ruling -out lock/wakeup coordination — consistent with everything below. - ---- - -## Ranked causes - -### #1 — Per-stream sidecar flush work stops amortizing (both modes; strongest cliff-shaped suspect) - -Every append marks its stream's `.meta` JSON sidecar dirty; a periodic task then -rewrites one sidecar per dirty stream. A rewrite is `Meta::capture` (clones path, -content-type, producers map, manifest) + `serde_json::to_vec` + `File::create(tmp)` -+ `write_all` + `rename` — all into the **single shared `streams/` directory** -(`write_meta_sync`, `store.rs:1122`). The `meta_dirty` CAS dedup only helps while -a stream is appended **more than once per flush interval**. So: - -``` -sidecar writes/sec ≈ min(append rate, N_streams / interval) -``` - -- At 1k streams / 100k ops/s: ≤1k writes/s — <1 % overhead, invisible. -- Once `N_streams > ops/s × interval`: **every append pays a full sidecar - rewrite** — the server's filesystem-metadata work roughly doubles per append, - keyed purely to stream cardinality at fixed load. That is a cliff with exactly - the observed onset shape. - -Mode-specific plumbing (this asymmetry explains why memory ends up *worse*): - -- **Memory mode:** `handle_append_inner` → `store.mark_meta_dirty` - (`handlers.rs:1174` → `store.rs:1170`). The **1 s** sweeper - (`META_SWEEP_INTERVAL`, `main.rs:455`; loop `main.rs:461-476`) drains the queue - and writes *all* dirty sidecars **serially inside one `spawn_blocking` task** - (`store.rs:1184` `sweep_meta_once`). At 50k streams that is up to ~50k - create+rename pairs per second on one blocking thread of a 2-vCPU server, all - contending the `streams/` directory inode rwsem (the pre-batching design's - per-append version of this was measured at ~40 % of server CPU — see the - comment at `handlers.rs:1160-1168`). On top of that, `mark_meta_dirty` takes a - **global** `StdMutex>>` (`Store.meta_sweep`, - `store.rs:389`) on every first-touch-per-cycle — and at high cardinality the - first-touch fraction approaches 100 % of appends, so a global hot-path lock - reappears that wal mode avoids with a plain atomic store (`handlers.rs:1168`). -- **wal mode:** the **3 s** shard checkpoint owns the flush — O(touched streams) - `write_meta_sync` calls (`wal/shard.rs:877-883`) **plus** one `barrier_fsync` - per touched per-stream data file (`wal/shard.rs:837-839`): ~16k fdatasyncs/s - at 50k streams touched per interval. Sharded + concurrent (JoinSet, - `main.rs:437-446`) + 3 s cadence, vs memory's serial global 1 s sweep — hence - the milder wal curve. - -### #2 — wal-only: `persist_durable_tails` is O(total streams ever touched), every checkpoint - -Each checkpoint also rewrites the shard's `tails` file from a **cumulative** -resident map of every stream ever touched on that shard (`tails_cache`, -`wal/shard.rs:354`): collect the whole map → `sort_unstable` → serialize every -entry → write → `sync_all` → rename (`wal/shard.rs:922-957`). Cost is -**O(total distinct streams per shard) regardless of how many were touched this -tick**, grows monotonically (idle streams never leave), and runs on a blocking -thread every 3 s. The field doc estimates ~20 ms/tick at 400k streams. A strong -contributor to the wal curve's steepening and a direct competitor for the -blocking pool / disk bandwidth the committer needs. - -### #3 — Shared per-append working-set physics (both modes; explains the below-knee latency rise) - -- **Registry lookup:** `Store.streams` is `DashMap>` - with the **default SipHash hasher and no capacity hint** (`store.rs:369`, - `:423`); one lookup per append (`store.rs:667`, called at `handlers.rs:899`). - Fixed instruction cost, but at 50k–1M entries the map plus its `Arc` targets - outgrow L2/L3, so each lookup pays several main-memory misses. (The code - already knows: see the comment at `handlers.rs:889-891` about avoiding a - second lookup.) -- **Per-stream cacheline spray:** each append touches the stream's `appender` - AsyncMutex, the `shared` RwLock **six-to-eight separate times** (lookup check; - soft-delete check `handlers.rs:909`; closed check `handlers.rs:958`; - `write_wire`'s write `handlers.rs:741`; the producer/seq write at - `handlers.rs:1087` — taken even when there are no producer headers; - `wal_stream_offset`'s read `handlers.rs:679`; `publish_durable_tail`'s write - `handlers.rs:762`; the final `tail()` read `handlers.rs:1180`), plus the - `last_chunk` RwLock and the `tail_tx` watch. On a hot stream these lines are - cache-resident; at high cardinality every one is a miss. Per-request latency - therefore rises with N with no queueing involved — matching the knee-p50 trend - in both modes, local and remote. -- **Kernel-side one-file-per-stream:** at fixed byte volume, N distinct streams - dirty N distinct inodes per writeback interval — inode writeback, ext4 journal - handles, dentry/page-cache pressure all scale with distinct files touched, - plus ~1 persistent fd per stream (`Shared.file`, `store.rs:72`). - -### #4 — wal-only, secondary: `register_dirty` first-touch transitions - -The hot path is lock-free (`dirty_epoch` compare, `wal/shard.rs:722-726`), but -the first touch per stream per checkpoint interval CASes the epoch, takes the -shard `dirty` mutex and pushes an `Arc` clone (`wal/shard.rs:737-749`). At fixed -load, growing cardinality raises the first-touch fraction toward 100 % of -appends — the same "dedup stops working" arithmetic as #1, but sharded and much -cheaper per event. Real, but secondary. - ---- - -## Ruled out (checked, with evidence) - -- **SSE reactor:** `wake_stream` early-returns when a stream has no subscribers - (`sse_reactor.rs:144-146`); no reactor thread is even spawned without a - `register`. O(1) per append in a write-only benchmark. -- **Telemetry:** default build compiles the recorders to no-ops - (`telemetry.rs:353-398`); with the feature on, label sets are bounded/static — - never keyed per stream. `wal/telemetry.rs` is per-shard, not per-stream. -- **Tiering/blobstore:** fully inert without `--tier` (`maybe_seal_bg` gate, - `handlers.rs:663`; `enabled()` gates inside tier.rs). -- **WAL group commit & durability wakeups:** per-commit work is O(batch) + - O(satisfied waiters) (`wal/shard.rs:1263-1305`, `:1322-1336`); the waiter - registry is one oneshot per in-flight request keyed by LSN — scales with - offered load, not stream count. Matches the flat `*_wait_load` counters. -- **WAL record encoding:** records carry an interned `stream_id: u64` - (`wal/codec.rs:78-84`), never the path string; frame is a fixed 38-byte header - + payload. -- **Full-registry scans:** none exist. `Store.streams` is only ever - point-accessed (get/insert/remove); no background loop iterates it. -- **Segment recycling:** `read_dir` costs scale with byte throughput / segment - count, not stream count. -- **HTTP layer (`engine_raw.rs` / `http1.rs` / `api.rs`):** no per-stream state, - no stream-keyed maps; header access is O(request headers). - ---- - -## Cheap confirmation experiments (in suspect order) - -1. **Memory mode / #1:** temporarily neuter the sweeper's sidecar write (or just - log `sweep_meta_once`'s returned count + wall time per tick, call site - `main.rs:473`). If the 1k→50k memory curve flattens dramatically with writes - disabled, #1 is confirmed for memory mode. (The producer-dedup lag this - introduces is already a documented non-durable window — fine for a bench.) -2. **wal mode / #1+#2:** run with `--wal-stats 1` and read the existing - `WAL_CKPT` line (`wal/shard.rs:887-899`) — `touched=`, `meta_us=`, and the - tails entry count should climb with cardinality while commit-path counters - stay flat. -3. **Shared baseline / #3:** swap the registry hasher (ahash) or key by interned - id, add a capacity hint, and re-measure — isolates the lookup-miss share of - the residual cliff that would remain after #1/#2 are fixed. - -## Fix directions - -> **#1 is DONE and measured — see [`CARDINALITY_CLIFF_FIX.md`](CARDINALITY_CLIFF_FIX.md).** -> A plain non-TTL append no longer writes/queues a sidecar rewrite in either mode -> (the durable tail is carried by the data-file length in memory mode / the -> per-shard `tails` map in wal mode; `last_access` only gates TTL). Result: memory -> cliff essentially eliminated (50k streams **+104%**, 44%→92% of the 1k rate); wal -> a modest, correct win (50k **+11%** throughput, knee p50 4.0→2.5 ms) — the wal -> cliff is fsync/`tails`-bound (#2 below), not the meta write. The items below -> remain. - -- Spread the meta sweep over the interval and shard its queue; make memory-mode - dirty-tracking lock-free like wal's (atomic + per-shard queues). *(Largely moot - now — the common case queues nothing; only producer/seq/TTL appends do.)* -- ~~Skip sidecar rewrites whose captured contents are unchanged (append-only - streams with no producers churn only `durable_tail`/`last_access`).~~ **Done (#1).** -- Incremental/delta `tails` persistence instead of the full-map rewrite per - checkpoint (#2). -- Batch/fan out the checkpoint fdatasyncs; consider syncing only files whose - WAL records are about to be recycled. -- Interned stream ids + faster hasher + sharded registry; pack the per-append - hot fields of `StreamState` into one cacheline (#3). diff --git a/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md b/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md deleted file mode 100644 index 1e8151d059..0000000000 --- a/packages/durable-streams-rust/CARDINALITY_CLIFF_FIX.md +++ /dev/null @@ -1,250 +0,0 @@ -# The cardinality cliff — fix & measurements - -**Audience:** whoever picks up the write-cardinality-cliff work. **Date:** -2026-07-08. **Status:** memory-mode fix implemented, validated, and low-risk; -wal-mode gate implemented, crash-sim-correct, but a modest local win. **Companion -docs:** [`CARDINALITY_CLIFF_CAUSES.md`](CARDINALITY_CLIFF_CAUSES.md) (the ranked -hypotheses this acts on — the fix targets **#1**), -[`WRITE_BOTTLENECKS_1M.md`](WRITE_BOTTLENECKS_1M.md). - -## What was done - -Hypothesis **#1** (per-stream `.meta` sidecar flush work stops amortizing) was -confirmed and fixed. The core change is one predicate in the append path -(`handlers.rs`): - -```rust -// A plain append to a non-TTL stream changes only durable_tail/last_access. -// The durable tail is carried elsewhere (memory: re-derived from the data-file -// length on restart; wal: the checkpoint's per-shard `tails` map), and -// last_access only gates TTL — so a plain non-TTL append needs NO sidecar flush. -let meta_persist_needed = - producer.is_some() || seq_header.is_some() || st.config.ttl_seconds.is_some(); -``` - -- **Memory mode:** only `store.mark_meta_dirty(&st)` when `meta_persist_needed`. - Previously *every* memory-mode append queued a sidecar rewrite for the 1 s - sweeper; at high cardinality the dedup stopped working and the sweep wrote up to - ~N sidecars/s on one blocking thread, all contending the shared `streams/` - directory inode. -- **Wal mode:** only `st.meta_dirty.store(true)` when `meta_persist_needed` (or the - `--wal-meta-gate off` A/B toggle is set). The `~3 s` checkpoint then skips - `write_meta_sync` for plain-append streams. Their `durable_tail` is still fsync'd - and recorded in the per-shard `tails` map (`persist_durable_tails`, independent of - this flag) — the authoritative durable-tail proof recovery reconciles against. - -### Why it is safe - -- **Memory recovery reads neither field.** `boot_meta_durable_tail` / - `meta.durable_tail` is consumed only by `wal/recovery.rs`; memory-mode - `Store::new_with_tier` derives the tail from `file.metadata().len()`. The - pre-existing e2e test `memory_mode_data_survives_restart_via_sidecar` appends 10 - bytes, never flushes the sidecar, reopens, and recovers `tail == 10` — it already - proved this. -- **Wal recovery uses the `tails` map, not the sidecar `durable_tail`.** The map is - persisted every checkpoint regardless of the meta-dirty flag; the sidecar's - `durable_tail` is redundant with it. The randomized crash-recovery simulation - (`crash_recovery_randomized_simulation`, which uses **plain** acked appends) and - `e2e_recycled_first_segment_acked_records_survive_crash` pass unchanged. -- **Compaction / seal / tier / close / delete are unaffected** — they call - `write_meta_sync(.., durable=true)` directly, never via the deferred `meta_dirty` - flag. `last_access` for TTL streams still flushes (the predicate keeps them). - -New test `memory_plain_append_skips_sidecar_flush` locks in the behavior. Full -suite: **104 passed** (0 failed). - -## Measurements (local kind, 2 vCPU, plain 256 B appends, `ds-bench`) - -Normalized to each config's own 1k-stream throughput (the cliff is the *decline*). - -### Memory mode — cliff essentially eliminated - -| streams | baseline | **fixed** | throughput Δ | knee p50 | -|---|---|---|---|---| -| 1k | 123.5k (100%) | 120.8k (100%) | — | 0.5 → 0.4 ms | -| 10k | 86.9k (70%) | 127.6k (106%) | **+47%** | 0.5 → 0.5 ms | -| 50k | 54.6k (44%) | 111.6k (**92%**) | **+104%** | 2.1 → 2.2 ms | - -At 50k streams throughput more than **doubles** (54.6k → 111.6k) and the cliff -flattens from 44% to 92% of the 1k rate. The fixed default build lands on the -`--meta-sweep-disable` neuter ceiling (102k), confirming the fix captures the full -available win. The residual 8% decline is #3 working-set physics (registry / cache -/ page-cache), which this fix does not address. - -### Wal mode — correct, reduces checkpoint work, modest throughput win - -Same-binary A/B (`--wal-meta-gate on|off`, `repeats: 2`, so no cross-image drift): - -| streams | baseline | **gated** | throughput Δ | knee p50 | -|---|---|---|---|---| -| 1k | 54.6k | 66.9k | +23% | 1.7 → 2.4 ms | -| 10k | 57.1k | 54.7k | −4% (noise) | 2.5 → 2.8 ms | -| 50k | 34.1k | 38.0k | **+11%** | 4.0 → **2.5 ms** | - -The wal cliff is fsync-lane bound: the checkpoint's meta write is one of *three* -costs, and the per-stream `fdatasync`s + the O(cumulative) `tails`-map rewrite (#2) -dominate — this gate doesn't touch them. The clean signal is **knee p50 at 50k -dropping 4.0 → 2.5 ms** (the checkpoint does less work); throughput barely moves -because it is disk-bound. The gate is worth keeping — it's free, correct, cuts -write amplification, and may matter more on real NVMe — but **it is not the wal -cliff fix**. The real wal lever is **#2: incremental/delta `tails` persistence** -instead of the full-map rewrite per checkpoint. - -## Dev/bench toggles added (all default-off / gate default-on) - -- `--meta-sweep-stats` — log a `META_SWEEP queued=/wrote=/elapsed_us=` line per tick. -- `--meta-sweep-disable` — drain the sweep queue but skip the sidecar write (the #1 - confirmation neuter; makes the sidecar permanently stale — bench only). -- `--wal-meta-gate on|off` — toggle the wal-mode gate for a same-binary A/B. -- `--mem-meta-gate on|off` — memory counterpart, to A/B the #1 fix on one binary. -- `--wal-fsync-parallel N` — checkpoint fdatasync fan-out (H4; default 1 = serial). -- `--server-stats N` — periodic `SRV_STATS` line (both modes) for bottleneck - analysis: `cpu_cores` (process CPU utilization; ≈ cgroup quota ⇒ CPU-bound), - `inflight` (append queue depth), `svc_us` (handler service time), `applock_us` - (appender-lock wait), `durwait_us` (WAL fsync wait). Linux only for `cpu_cores`. - -Reproduce: `ds-bench` suites `write-cliff-confirm` (memory neuter A/B), -`write-cliff-validate` (fixed default vs wal baseline), `write-cliff-wal-ab` -(same-binary wal gate A/B). - -## Bottleneck analysis — is it CPU-bound or fsync-bound? (measured) - -Added `--server-stats` and profiled both modes in a Linux Docker container (4 CPU, -50k streams, 512 connections, container-to-container networking to avoid the -Docker Desktop port-forward proxy) — plus a real-NVMe cross-check on -`c4d-standard-16-lssd`. - -**Memory mode = CPU-bound.** `SRV_STATS cpu_cores≈3.0–3.5` (of 4), `inflight≈0`, -`svc_us≈8–15`, `durwait=0`, `applock=0`. The server saturates cores with no -queueing; throughput scales with cores (documented 315k@4→526k@8 vCPU). The #1 -sidecar fix helps here precisely because it removes per-append CPU work — measured -**+16% @ 100k, +14% @ 500k on real NVMe** (memory-nogate vs memory-gated: -318k→370k, 237k→270k). - -**WAL mode = fsync/durability-bound, NOT CPU-bound.** `SRV_STATS cpu_cores≈0.9–1.5` -(of 4 — 3 cores idle!), `inflight≈260–500` (deep queue), `svc_us≈8–13ms` of which -`durwait_us` is **~99%**. Hundreds of appends pile up in `wait_durable_lsn` while -the CPU sits idle — the WAL group-commit fsync is the wall. Real-NVMe baseline: -66k @ 100k, 48k @ 500k (4 vCPU / 4 shards). - -**Implications for "should we allocate more CPU?"** -- **Memory: yes** — it is CPU-bound and scales with cores. -- **WAL: no** — it is fsync-bound with 3 of 4 cores idle. More vCPU is wasted spend. - The wal lever is fsync throughput = **parallel fsync lanes = more `--wal-shards`** - (each shard commits/fsyncs independently), NOT CPU and NOT `--wal-fsync-parallel` - (which fans the CHECKPOINT fsync — regressed −11% on NVMe: fsync16 59k vs fsync1 - 66k @ 100k, same CPU-oversubscription failure as local). - -Caveat on local harness: Docker Desktop virtiofs serializes fsync (so wal -shard-scaling is invisible locally) and a single client container caps memory at -~150k (so memory CPU-scaling is invisible locally). The STRUCTURE (which resource -saturates) is valid locally; the SCALING numbers require real NVMe + a client fleet. - -## H2 / H4 investigation — wal checkpoint breakdown (measured, mostly negative) - -Followed up on the two wal-checkpoint hypotheses by capturing the `WAL_CKPT` -phase breakdown under 50k-stream load (`--wal-stats`). **Iterate on a Linux -container, not bare macOS:** macOS `fsync` is `F_FULLFSYNC` (a true drive barrier, -~7 ms, globally serialized), so bare-metal wal throughput is a ~30× artifact. -`DS_BENCH_FAST_FSYNC=1` swaps in plain `fsync` for a faster loop, but the -representative signal comes from running the server in Docker (Linux `fdatasync`). -Local loop: `target/release/durable-streams-server … --wal-stats 1` driven by -`ds-bench multi-stream --api-style durable --streams 50000`. - -Checkpoint phase split at 50k streams (per checkpoint, Linux container): - -``` -touched=21274 fsync_us≈2,000,000 (≈99%) tails_us≈10,000 (0.5%) meta_us≈300 (~0%) -``` - -- **H2 (incremental `tails` persistence): NOT WORTH DOING.** The cumulative - `tails`-map rewrite is **~0.5%** of checkpoint time even as it grows — optimizing - it is optimizing noise. (`meta_us≈0` also confirms the #1 wal gate zeroed the - sidecar phase: `meta=0` on every line.) -- **H4 (checkpoint `fdatasync`): dominant *within* the checkpoint, but NOT the wal - throughput ceiling on Linux.** The commit path batches healthily - (`WAL_CONT batch_avg≈12`, staged/s 8–18k) and the checkpoint runs on separate - threads, so `fdatasync` overlaps commits instead of blocking them. The real - ceiling is the **commit-path group-commit fsync rate** (~1000 fsync/s × batch - ≈ throughput), not the checkpoint. (Bare macOS mis-attributed this to the - checkpoint because `F_FULLFSYNC` serializes *all* fsyncs globally.) -- **Parallelizing the checkpoint `fdatasync` REGRESSES on constrained hardware.** - Added `--wal-fsync-parallel N` (fan the per-stream syncs across N OS threads; all - complete before recycle, crash-ordering preserved, crash-sim green). Measured on - a 2-vCPU Linux container, 50k streams: - - | fanout | ops/s | vs serial | - |---|---|---| - | 1 (serial) | 16.4k | — | - | 2 | 15.8k | −4% | - | 4 | 13.8k | −16% | - | 16 | 10.9k | −19% | - - More fan-out steals CPU from the runtime (slowing the committer until checkpoints - fall behind) and virtiofs doesn't do concurrent fsync. **Default is therefore - serial (`fanout=1`, a no-op);** the parallel path is opt-in, pending validation on - real NVMe (deep device queue + spare cores) where it may pay off. On current - evidence, do not enable it. - -**Net:** neither H2 nor H4 is a wal throughput win on Linux. The next wal lever is -the **commit-path group commit** (raise batch / fsync efficiency), not the -checkpoint — a separate investigation. - -## NVMe shard + fanout sweep — optimal wal config (measured, definitive) - -Controlled sweeps on real Titanium NVMe (`c4d-standard-16-lssd`, **8 vCPU pin**, -200k streams, 256 conns/pod, 256 B; ds-bench `wal-shard-sweep` + `wal-fanout-sweep`). -Both suites are barrier-aligned, per-cell resumable, and ran to completion. - -**`--wal-shards` sweep — flat within 5 %:** - -| shards | 1 | 4 | 8 | 16 | 24 | -|---|---|---|---|---|---| -| peak ops/s | 72k | **75k** | 73k | 73k | 71k | - -**`--wal-fsync-parallel` sweep (at s4) — flat then regresses:** - -| fanout | 1 | 2 | 4 | 8 | -|---|---|---|---|---| -| peak ops/s | 75k | 73k | 75k | **67k** | - -Live telemetry at *every* shard count: SRV_STATS `cpu_cores≈1.1–1.8/8` (idle), -`durwait_us ≈ 97–99 % of svc_us`; WAL_CONT `fsync/s≈900–1000`, `batch_avg` 53→88; -WAL_CKPT `touched≈580`, `fsync_us≈1.4 s` (the per-stream fdatasync storm). - -**Conclusions:** -- **Neither shards nor fsync-parallel is a throughput lever.** wal is bounded by the - disk's `fdatasync`/s rate (~1000), a single shared resource — not per-shard lanes. - This **overturns the old "shards is the knob / s16 ≈ 380k"** number (does not - reproduce). More shards *hurt* at low load (s24 @1 pod = 34k vs s4 = 49k: thinner - per-shard group-commit batches). `--wal-fsync-parallel ≥ 8` regresses (it just adds - concurrent checkpoint fsyncs that steal device budget from commits). -- **Answers the earlier open NVMe question** (follow-up #3 below): no, `--wal-fsync-parallel > 1` - does **not** help on real NVMe either — the deep device queue doesn't rescue it. -- **Optimal wal config:** `--wal-shards 4` (or `min(cores,4)`), `--wal-fsync-parallel 1` - (default), `server_cpus` 4–8 (CPU is idle — don't overspend). **The real throughput - knob is `connections`** (offered load → bigger `batch_avg`), traded against latency - (36 ms p50 at 3072 in-flight). Server default `shards = core count` is too high on - many-core boxes; it should be a small fixed value. - -## Recommended follow-ups (not done) - -Ranked by evidence after the NVMe sweeps above: - -1. **Collapse the checkpoint per-stream durability barrier from O(N_touched) syscalls - to O(1)** — the real cardinality lever. `checkpoint()` (`wal/shard.rs:797`, step 2) - does one `fdatasync` per touched stream (~580/shard/interval → `fsync_us≈1.4 s`), - which steals device flush budget from the commit fsyncs and *is* the cliff. - A single `syncfs()` (or `sync_file_range` batch + one barrier) at checkpoint would - flush all touched files in one shot. No `syncfs` in the tree today; the per-stream - loop is hardcoded (no checkpoint-interval flag). This is the highest-value change. -2. **Wal commit-path group commit** (the throughput ceiling): throughput ≈ - `fsync/s × batch_avg`. `fsync/s` is hardware; lift `batch_avg` without needing - more in-flight (e.g. a small commit-coalescing delay) to raise throughput at lower - latency. NOT the checkpoint. Supersedes the old "#2 tails" idea (disproved: ~0.5%). -3. **#3 — per-append working-set physics** (residual in both modes): interned stream - ids + ahash + capacity hint on `Store.streams`; pack hot `StreamState` fields into - one cacheline. Likely the memory-mode ~8% residual after #1. -4. ~~Validate on remote NVMe / does `--wal-fsync-parallel > 1` help there?~~ — - **done, answered: no** (see the NVMe sweep above; f8 regresses to 67k). -5. ~~#2 incremental `tails` persistence~~ — **dropped.** ~0.5% of checkpoint time. diff --git a/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md b/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md deleted file mode 100644 index 468ff64493..0000000000 --- a/packages/durable-streams-rust/WRITE_BOTTLENECKS_1M.md +++ /dev/null @@ -1,192 +0,0 @@ -# Write bottlenecks at 1M streams — findings brief - -**Audience:** an engineer/agent picking up WAL write-path performance work with no prior -context. **Scope:** single-node write throughput + tail latency of the Rust -`durable-streams` server (`packages/durable-streams-rust`) at high stream cardinality -(200k–1M streams). **Date:** 2026-07-02. **Server under test:** `perf/combined-t1a-t1c-t2a` -(PR: electric-sql/electric#4675), vs the 2026-06-30 baseline `durable-streams:dev`. - ---- - -## TL;DR - -1. **The write ceiling is coordination, not compute or fsync.** Under write saturation the - server plateaus at **~80% CPU** and then *declines* under more load. The cost is the - **group-commit committer scheduling + a durability-wakeup thundering herd**, not lock - contention (at shards ≈ cores) and not fsync. -2. **Fixed the coordination** (`combined` stack): dedicated committer thread (T2a), - coalesced wakeups (T1c), lock-free dirty registration (T1a). Measured **−21× wakeups - per commit**, **~10× more committer activity**, **+60% throughput @ 32 vCPU**, and at - 16 vCPU / 1M streams **p99 80 ms → 32 ms**. -3. **Stream cardinality (200k→1M) adds a second, independent cost** that is *not* - coordination: per-op memory/working-set physics (registry + per-stream state + page - cache + checkpoint fsync fan-out + ~1 fd/stream). This is present in every build and is - the next frontier. -4. **All results are write-only.** The read path (tail-cache, sendfile/splice, live - tailing) is unmeasured. A read+write mix is the recommended next experiment. - ---- - -## Test setup (how to reproduce) - -- **Bench harness:** `ds-bench` (sibling repo). Remote target = GKE; `scripts/bench - suites/.json run`. Golden rule: **always tear clusters down** (`bench` self-tears - on clean completion; arm `scripts/teardown-watchdog.sh`). -- **Server node:** `c4d-standard-16-lssd`, pinned to N vCPU via `SERVER_CPUS=N` (sets - cgroup `cpu.max`; the server reads it through `available_parallelism()`). -- **Server flags:** `--wal-shards N --worker-threads N` (shards = workers = cores). -- **Client:** pool model, `n2d-standard-32` fleet, `CONNS_PER_POD=256`, batch 1, 256 B - payload. **Load knob = total connections = pods × 256.** This is a *fixed-concurrency* - workload, so `throughput ≈ connections ÷ latency` (Little's law) — remember this when - reading the numbers. -- **Telemetry:** `--wal-stats ` makes the server emit a `WAL_CONT` line per interval: - `staged/s`, `fsync/s`, `batch_avg`, `inner_wait_load`, `dirty_wait_load`, - `waiters_woken_avg`. `*_wait_load` = fraction of a core-second spent *parking* on that - lock. Off by default (one relaxed atomic load); do **not** enable it during a latency run - (the hot-path clock reads perturb the tail). - ---- - -## Bottleneck #1 — the coordination ceiling (FIXED by `combined`) - -### Mechanism (per append, in `src/wal/`) - -Every append touches per-shard shared state on the shard its `stream_id` hashes to: - -- `shard.inner` `Mutex` — twice per append (reserve LSN/write-pos, then mark-written). -- `shard.dirty` `Mutex` — a HashMap **insert on every append** (`register_dirty`). -- `durable_tx` `watch` — on each commit, `publish_durable` wakes **every** waiter parked on - the shard (durability barrier), not just the ones whose LSN is now durable. - -Plus the committer itself hopped through the shared Tokio runtime via `spawn_blocking` -per commit — a scheduling round-trip that capped commit *cadence*. - -### Direct evidence (4 vCPU / 4 shards, `--wal-stats`, instrumented baseline vs combined) - -| counter | baseline | combined | note | -|---|---|---|---| -| `waiters_woken` / commit | 340–378 | **16–18** | **~21× fewer** — the herd (T1c) | -| `fsync/s` (commit cadence) | 83–192 | **1021–1178** | **~10× more** — committer freed (T2a) | -| `batch_avg` | 168–189 | 16–18 | baseline commits rarely in huge batches | -| `inner_wait_load` | ~0.000 | ~0.002 | **locks are NOT the gate here** | -| `dirty_wait_load` | ≤0.018 | ~0.000 | ditto | - -**Key insight:** at shards ≈ cores there is ≈1 worker per shard, so the per-shard *locks -barely contend* (`*_wait_load ≈ 0`). The ceiling is the **committer round-trip + wakeup -broadcast**. That is what `combined` removes: - -- **T2a** — dedicated committer OS thread, drop per-commit `spawn_blocking`. *Biggest single - win.* Direct proof: `fsync/s` ~doubles/10×'s — the committer was blocked on runtime - round-trips, not fsync. -- **T1c** — coalesced wakeups (wake only satisfied waiters). Kills the O(waiters) run-queue - storm. At 32 cores the un-coalesced herd is ~415 waiters/commit (≈13.3k in-flight ÷ 32 - shards); T2a's faster committer makes the un-coalesced herd *worse*, so T1c is needed - precisely because T2a helps. -- **T1a** — epoch-gated lock-free `register_dirty` (no per-append `Mutex` insert). - -### Throughput/latency impact - -Pool saturation ladder vs 2026-06-30 baseline: - -| scale | baseline | combined | -|---|---|---| -| 32 vCPU, 200k | 1.48M @ 52 pods, p99 41 ms | **2.37M @ 40 pods (+60%), p99 12 ms** | -| 32 vCPU, 500k | 1.15M @ 52 pods | **≥1.78M (+55%, unsaturated)** | - -Lock-free/committer changes scale with core count: **+11% @ 6 cores → +60% @ 32 cores**, -because the herd and lock pressure only dominate at high core counts. - ---- - -## Bottleneck #2 — stream cardinality (200k → 1M) — NOT yet solved - -This is a **separate axis** from coordination and is present in *every* build. At the same -offered load, more distinct streams cost more per op: - -- **Registry lookups.** `Store.streams: DashMap>` (`store.rs:365`), - hot-path `get(path)` per append (`store.rs:~654`). At 1M entries the map + values exceed - L2/L3 → more cache/TLB misses per append. -- **Per-stream working set.** Each stream is a `StreamState` (`store.rs:164`): an - `AsyncMutex`, a `RwLock`, a `watch::Sender`, a `last_chunk` - cache, and an `Arc`. 1M of these is a large resident set (measured server RSS ~1.16 - GB at 1M — modest, but the *cache-miss* cost is the real tax, not RSS). -- **Page-cache locality.** One data file per stream ⇒ 1M distinct hot tails ⇒ worse - locality, more page-cache churn. -- **Checkpoint fsync fan-out.** The per-shard checkpoint `fdatasync`s *every touched stream - file* before recycling the WAL ⇒ fsync count scales with distinct streams touched per - interval. -- **File descriptors.** ~1 open fd per stream. The server raises `RLIMIT_NOFILE` to the - hard limit on startup (`main.rs:89` `raise_nofile_limit`). On GKE that hard limit is - typically **1,048,576**, so **1M streams sits ~38k below the fd ceiling** once you add - ~8k client sockets + WAL segment/epoll fds. **This is a hard wall just above 1M** — going - materially past 1M streams per node needs a raised limit or fewer fds/stream (e.g. - open-on-demand instead of a persistent `Arc` per stream). - -**Caveat on how cardinality shows up in the metrics:** the WAL contention counters do *not* -attribute the 200k→1M degradation to a lock/wakeup source (they stay flat/low). So this -cost is the memory/IO physics above, not coordination — a different class of fix -(sharded registry, open-on-demand fds, working-set reduction, checkpoint batching). - ---- - -## 1M-stream write scaling (combined, fixed load = 32 pods = 8,192 connections) - -Single fixed rung, 60 s measure, no ladder: - -| server | throughput | p50 | p90 | p99 | p99.9 | max | server CPU | -|---|---|---|---|---|---|---|---| -| **8 vCPU** | 454k ops/s | 2.9 ms | 68 ms | **80 ms** | 94 ms | 3348 ms | **86% of 8** | -| **16 vCPU** | **862k ops/s** | 3.3 ms | 25 ms | **32 ms** | 88 ms | **405 ms** | **77% of 16** | - -Reading it: -- **8 vCPU was capacity-bound at this load.** Doubling cores nearly **doubled throughput - (454k→862k)** at the *same* 8,192 connections — the signature of fixed-concurrency: the - server clears each write faster, so every connection completes more ops/s. (Little's law - checks both ways: ≈8,190 in-flight.) -- **The tail was the story, and it collapsed.** p99 80→32 ms, and the multi-second - outliers vanished (max 3348→405 ms). At 8 vCPU an 86%-CPU redline meant a real slice of - writes queued behind group-commit; 16 vCPU (77%) has slack, so far fewer wait. -- **Median was never the problem** (2.9→3.3 ms). The win is entirely tail + headroom. - -**Implication:** at 1M streams, ~8,000 concurrent writers want **~12–16 vCPU** to keep p99 -in the tens-of-ms range with headroom. 8 vCPU is under-provisioned for that concurrency. - ---- - -## What's fixed vs open - -**Fixed (in `combined`, PR #4675):** the coordination ceiling — dedicated committer (T2a), -coalesced wakeups (T1c), lock-free dirty (T1a), plus `--wal-stats` telemetry and -`--worker-threads`. - -**Deferred, with reasons:** -- **T1b (atomic WAL reserve):** `inner_wait_load ≈ 0` at 4–16 vCPU (shards ≈ cores) ⇒ - lock-restructuring risk for ~no gain at our operating points. Revisit only at - high-core/low-shard configs where the inner lock genuinely contends. -- **T3 (shared-nothing / thread-per-core):** design-only; confirmed to pay off only at - ≥16–32 cores and not needed to capture the current win. The pool client touching all - shards per connection forces a cross-core SPSC handoff that is otherwise net-negative. - -**Open (recommended next work, in priority order):** -1. **Read + write mixed workload.** Everything above is write-only. Reads exercise - tail-cache, sendfile/splice, the `RwLock` read path, and live tailing — none - stressed here. Combined only touches the WAL commit/wakeup path, so read-regression risk - is low and T1c may *help* live tailers (durability wakeups also wake subscribers) — but - verify before calling the write win "done." -2. **Cardinality cost reduction (bottleneck #2).** Sharded/segmented registry, checkpoint - `fdatasync` batching, and reducing per-stream fixed cost (esp. fds — open-on-demand vs - persistent `Arc`) to get past the ~1M fd wall. -3. **Find 16 vCPU's own redline.** These 1M numbers are one fixed rung (77% CPU). A heavier - rung (e.g. 64 pods) pins where 16 vCPU saturates and its p99 there. - ---- - -## Provenance - -- Full investigation + telemetry design: `CONTENTION_INVESTIGATION.md` (same directory). -- Baseline that motivated this: `ds-bench/results/run-durable-pool2/FINDINGS.md`. -- 4-vCPU counter A/B: `ds-bench/suites/run-durable-cpu4-stats-{base,combined}.json`. -- 1M p99 runs: `ds-bench/suites/run-durable-cpu{8,16}-1m-p99-combined.json`. -- Code anchors: `src/wal/shard.rs` (register_dirty / reserve_and_stage / publish_durable), - `src/wal/telemetry.rs` (WAL_CONT), `src/store.rs:164,365` (StreamState / registry), - `src/main.rs:89` (raise_nofile_limit). From 1bbcee1753e934cd073cda35019bf8c493cd940f Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 17:12:47 +0100 Subject: [PATCH 04/13] =?UTF-8?q?perf(durable-streams-rust):=20per-shard?= =?UTF-8?q?=20checkpoint=20triggers=20=E2=80=94=20interval=20knob=20+=20re?= =?UTF-8?q?tained-WAL=20size=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hardcoded 3s all-shards checkpoint wave with a per-shard dual trigger: --wal-checkpoint-interval-ms (time, default 3000 = old behavior) and --wal-checkpoint-wal-bytes (size, 0 = off): checkpoint a shard as soon as its retained WAL exceeds the budget. Rationale (wal-decomp-lane0, 2026-07-13): acks never gate on checkpoint and reads never touch the WAL, so checkpoint frequency is purely a replay-time budget — a size trigger makes that budget explicit and lets shards self-stagger by their own write rates instead of storming the device together. Each shard's clock restarts when its checkpoint finishes, an in-flight guard prevents re-firing a slow shard, and due shards still checkpoint concurrently. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/src/main.rs | 95 ++++++++++++++----- .../durable-streams-rust/src/wal/shard.rs | 27 ++++++ 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index 3bbb50e45e..19b341ffe3 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -300,6 +300,30 @@ fn main() { } } } + // Checkpoint time trigger: per-shard cadence in ms (default 3000). + "--wal-checkpoint-interval-ms" => { + let v = val(args.next(), "--wal-checkpoint-interval-ms"); + match v.parse::() { + Ok(ms) if ms >= 1 => wal::shard::set_checkpoint_interval_ms(ms), + _ => { + eprintln!("--wal-checkpoint-interval-ms must be a positive integer (milliseconds)"); + std::process::exit(2); + } + } + } + // Checkpoint size trigger: checkpoint a shard as soon as its retained + // WAL exceeds this many bytes (0 = disabled). An explicit replay-time + // budget that also self-staggers shards by their own write rates. + "--wal-checkpoint-wal-bytes" => { + let v = val(args.next(), "--wal-checkpoint-wal-bytes"); + match v.parse::() { + Ok(bytes) => wal::shard::set_checkpoint_wal_bytes(bytes), + _ => { + eprintln!("--wal-checkpoint-wal-bytes must be a non-negative integer (bytes)"); + std::process::exit(2); + } + } + } other => { eprintln!("unknown argument: {other}"); std::process::exit(2); @@ -474,43 +498,68 @@ fn main() { }); } -/// How often the checkpoint ticker drives each shard's `checkpoint` (spec §7). -/// A sane v1 constant: frequent enough that the WAL doesn't grow unbounded on a -/// busy server, infrequent enough that the batched per-stream `fdatasync`s stay -/// amortized. Checkpoint is non-blocking w.r.t. acks, so this is purely the -/// WAL-recycle / per-stream-durability cadence (tunable is follow-up #9). -const CHECKPOINT_INTERVAL: std::time::Duration = std::time::Duration::from_secs(3); +/// How often the checkpoint ticker POLLS its triggers. The actual checkpoint +/// cadence is per-shard and knob-driven (see `wal::shard::checkpoint_interval_ms` +/// / `checkpoint_wal_bytes`); this is just the trigger-evaluation resolution. +/// 250 ms keeps the size trigger responsive (a shard writing 1 GB/s overshoots a +/// 1 GiB budget by ≤ 250 MB) at negligible poll cost (two atomic loads/shard). +const CHECKPOINT_POLL: std::time::Duration = std::time::Duration::from_millis(250); -/// Spawn the per-shard checkpoint ticker (spec §7). One `tokio::time::interval` -/// driver that, each tick, runs every shard's `checkpoint` (each: batched -/// `fdatasync` of its touched per-stream files → persist `checkpoint_lsn` → -/// recycle WAL segments below it). A checkpoint error is logged, not fatal — a -/// failed/lagging checkpoint only delays WAL recycling (the disk-bounded safety -/// valve, spec §7), never blocks appends. +/// Spawn the per-shard checkpoint driver (spec §7). Each poll tick, a shard is +/// checkpointed iff (a) its retained WAL exceeds `--wal-checkpoint-wal-bytes` +/// (size trigger, 0 = off), or (b) `--wal-checkpoint-interval-ms` has elapsed +/// since ITS last checkpoint (time trigger, default 3000 = the historical 3 s +/// cadence). Due shards checkpoint CONCURRENTLY (each is one spawn_blocking: +/// capture + fsync/syncfs of touched stream files → persist tails/checkpoint_lsn +/// → recycle); a serial walk would queue every shard's fsync behind one +/// device's. Because each shard's clock restarts when IT finishes, shards drift +/// apart naturally instead of storming in a synchronized wave — and with the +/// size trigger they self-schedule by their own write rates. A checkpoint error +/// is logged, not fatal — a failed/lagging checkpoint only delays WAL recycling +/// (the disk-bounded safety valve, spec §7), never blocks appends. A shard that +/// is still checkpointing is never re-fired (the in-flight set guards it), so a +/// checkpoint that takes longer than the interval degrades to back-to-back +/// checkpoints for that shard only. fn spawn_checkpoint_ticker(walset: Arc) { tokio::spawn(async move { - let mut ticker = tokio::time::interval(CHECKPOINT_INTERVAL); - // Skip the immediate first tick — there is nothing to checkpoint at boot. + let interval = + std::time::Duration::from_millis(wal::shard::checkpoint_interval_ms()); + let wal_bytes = wal::shard::checkpoint_wal_bytes(); + let n = walset.shards().len(); + let mut last_done: Vec = vec![std::time::Instant::now(); n]; + let mut in_flight: Vec = vec![false; n]; + let mut wave: tokio::task::JoinSet = tokio::task::JoinSet::new(); + let mut ticker = tokio::time::interval(CHECKPOINT_POLL.min(interval)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Skip the immediate first tick — there is nothing to checkpoint at boot. ticker.tick().await; loop { ticker.tick().await; - // All shards checkpoint CONCURRENTLY. Each checkpoint is one - // spawn_blocking task (capture + per-stream fdatasyncs + tails/ckpt - // persist + recycle), so a serial walk makes every per-stream fsync - // across the whole server queue behind a single shard's — at high - // stream cardinality that serialization is what stretches the - // checkpoint wave (and on real disks wastes the device's parallelism). - let mut wave = tokio::task::JoinSet::new(); - for shard in walset.shards() { + // Reap finished checkpoints (non-blocking) and restart their clocks. + while let Some(done) = wave.try_join_next() { + if let Ok(i) = done { + in_flight[i] = false; + last_done[i] = std::time::Instant::now(); + } + } + for (i, shard) in walset.shards().iter().enumerate() { + if in_flight[i] { + continue; + } + let size_due = wal_bytes > 0 && shard.wal_size_bytes() >= wal_bytes; + let time_due = last_done[i].elapsed() >= interval; + if !(size_due || time_due) { + continue; + } + in_flight[i] = true; let shard = Arc::clone(shard); wave.spawn(async move { if let Err(e) = shard.checkpoint().await { eprintln!("WAL checkpoint failed for shard {:?}: {e}", shard.dir()); } + i }); } - while wave.join_next().await.is_some() {} } }); } diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index df195225d1..4d94591b8e 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -423,6 +423,33 @@ fn checkpoint_syncfs() -> bool { CHECKPOINT_SYNCFS.load(Ordering::Relaxed) } +/// Checkpoint cadence knobs (cardinality-cliff follow-up). The checkpoint's only +/// job is bounding retained-WAL size (= crash-replay time): acks never gate on it +/// (`shard.rs` "disk-bounded safety valve") and reads never touch the WAL, so +/// firing it less often is free except for disk space and replay budget. Two +/// triggers, whichever comes first per shard: +/// * time: `--wal-checkpoint-interval-ms` (default 3000 — the historical 3 s +/// wave cadence, now per-shard). +/// * size: `--wal-checkpoint-wal-bytes` (default 0 = disabled) — checkpoint a +/// shard as soon as its retained WAL exceeds this many bytes. This turns the +/// hardcoded timer into an explicit replay-time budget (e.g. 1 GiB ≈ <1 s of +/// replay on NVMe) and lets shards self-stagger by their own write rates +/// instead of storming together on a shared tick. +static CHECKPOINT_INTERVAL_MS: AtomicU64 = AtomicU64::new(3_000); +pub fn set_checkpoint_interval_ms(ms: u64) { + CHECKPOINT_INTERVAL_MS.store(ms.max(1), Ordering::Relaxed); +} +pub fn checkpoint_interval_ms() -> u64 { + CHECKPOINT_INTERVAL_MS.load(Ordering::Relaxed) +} +static CHECKPOINT_WAL_BYTES: AtomicU64 = AtomicU64::new(0); +pub fn set_checkpoint_wal_bytes(bytes: u64) { + CHECKPOINT_WAL_BYTES.store(bytes, Ordering::Relaxed); +} +pub fn checkpoint_wal_bytes() -> u64 { + CHECKPOINT_WAL_BYTES.load(Ordering::Relaxed) +} + /// Name of the per-shard durable-tail map: `/tails` (task 11b). A /// CUMULATIVE `stream_id durable_tail` line map (plain decimal text, one stream /// per line). At checkpoint, each touched stream's current logical `Shared.tail` From 0389b45b929874e0f227b0c42beac7f0932a7f42 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 18:49:09 +0100 Subject: [PATCH 05/13] =?UTF-8?q?docs(durable-streams-rust):=20WAL=5FTUNIN?= =?UTF-8?q?G.md=20=E2=80=94=20the=20validated=20ideal=20write-path=20confi?= =?UTF-8?q?guration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/WAL_TUNING.md | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 packages/durable-streams-rust/WAL_TUNING.md diff --git a/packages/durable-streams-rust/WAL_TUNING.md b/packages/durable-streams-rust/WAL_TUNING.md new file mode 100644 index 0000000000..1748e6abc0 --- /dev/null +++ b/packages/durable-streams-rust/WAL_TUNING.md @@ -0,0 +1,104 @@ +# WAL write-path tuning — the ideal configuration + +**Status:** validated 2026-07-13 on GCP `c4d-standard-64-lssd` (6× Titanium local +NVMe, raw block), 256 B appends, 8 vCPU server, up to 100k streams. Campaign: +ds-bench suites `wal-decomp-lane0`, `wal-splitlane`, `wal-sizetrigger`, +`wal-cpubind` (see ds-bench `results//report.md` + `AGENTS.md`). +**Verdict: the write cardinality cliff is eliminated** — WAL-durable throughput +is flat from 10k → 100k streams (−5%) at ~26–32× the pre-fix baseline. + +## The ladder (what each step bought, @100k streams) + +| configuration | ops/s | +|---|---| +| pre-fix: streams on network PD, per-stream checkpoint fdatasync storm | 10.4k | +| + `--wal-checkpoint-syncfs on` (PR #4697) | 13.6k | +| + stream files on local NVMe (not the boot disk) | 46k | +| + split-lane layout (WAL shards on their own NVMe devices) | 272k | +| + size-triggered checkpoint (PR #4704, `--wal-checkpoint-wal-bytes 1GiB`) | 303k | +| + exclusive pinned cores (Guaranteed QoS + static CPU manager) | 328k (measured separately; stacking projects ~360k) | +| reference: memory durability (no fsync anywhere) | 512k | + +The residual ~1.6× gap to memory mode is WAL machinery (staging + double-write), +not fsync — future work: io_uring segment writer (`wal/segment.rs` seam), +batched mark-written. + +## The ideal configuration + +### 1. Hardware / storage layout (the #1 lever) + +Use an instance with **multiple physically attached NVMe devices** (GCP: 4th-gen +`-lssd` types, raw block via `--local-nvme-ssd-block`; do NOT use +`--ephemeral-storage-local-ssd`, which RAID0-stripes every device into one fsync +barrier). Then: + +- **One device for stream data files** — mount it and point `--data-dir` at it. + The per-stream files and the checkpoint's `syncfs` domain live here. +- **One device per WAL shard** — mount device *j* at `/wal/` (the + server opens shard *i* at that path automatically). `--wal-shards` = number of + dedicated WAL devices. +- **Never share a device between WAL and stream data.** Commit `fdatasync` vs + checkpoint writeback contention on one queue was worth 5× by itself + (55k → 272k). On dedicated lanes the commit-fsync cost is ~zero (checkpoint-off + measured *below* an all-lanes-shared no-fsync control). +- **Never leave stream data on the boot disk / network PD.** On Kubernetes + raw-block node pools the default emptyDir sits on the boot PD — this single + mistake mismeasures (and misdeploys) WAL mode by 5–26×. + +Example (6-device box): device 0 → data root, devices 1–5 → 5 WAL shards: + +``` +--data-dir /data/wal/0 --wal-shards 5 +``` + +### 2. Server flags + +``` +--wal-checkpoint-syncfs on # one syncfs barrier per checkpoint instead of + # O(N-touched) per-stream fdatasync (PR #4697) +--wal-checkpoint-wal-bytes 1073741824 # checkpoint a shard when ITS retained WAL + # exceeds 1 GiB (PR #4704) — checkpoint cost ≈ 0, + # crash-replay bounded to ≤1 GiB/shard (<1 s NVMe) +--wal-checkpoint-interval-ms 60000 # fallback timer so an idle shard still recycles +--wal-shards # shards = fsync lanes; on a SINGLE shared + # device keep 2–4 (more only fragments batches) +--worker-threads +``` + +Leave `--wal-fsync-parallel` at its default (1): fanout parallelizes the +per-stream fsync loop that syncfs replaces, and measured as a regression. + +### 3. CPU binding (+21–24%) + +Give the server **exclusive pinned cores**. On Kubernetes: node pool with +`kubeletConfig.cpuManagerPolicy: static` + a **Guaranteed QoS** pod (every +container `requests == limits`, server CPU an integer). Measured 356k @10k / +328k @100k vs 286k/272k on shared cores, same layout and flags. Now that WAL is +no longer fsync-bound, it scales with cores again — don't starve it. + +### 4. What you do NOT need to worry about + +- **Read performance while tuning checkpoints.** Reads never touch the WAL and + are served zero-copy (`sendfile`) from the data file's page cache, which is + written *before* the WAL ack barrier. Checkpoint cadence has zero read-path + cost. +- **Ack latency vs checkpoints.** Acks gate only on the WAL group-commit + `fdatasync`; a checkpoint never blocks appends. +- **Recoverability.** The contract is unchanged by any of these knobs: a WAL + segment is recycled only after its records' stream bytes are fsynced into + their files **and** the durable-tail map is persisted (`wal/shard.rs` + checkpoint ordering; crash-recovery e2e + randomized crash sim cover it). + The size trigger only changes *when* that sequence runs. + +## Caveats / follow-ups + +- Validated to 100k streams, 256 B payloads, single node. 1M-stream behavior of + this exact config is unmeasured (see `CARDINALITY_1M.md` for the older + analysis); the mechanisms that caused the cliff are cardinality-independent + now, but confirm before relying on it. +- The stacked best config (size trigger + pinned cores together) is projected + ~360k @100k, measured only separately — one confirmation run pending. +- `--wal-checkpoint-syncfs` and the size trigger are opt-in; flipping defaults + (syncfs on for Linux) is a candidate after soak. +- Larger retained WAL = longer replay: 1 GiB/shard ≈ sub-second on local NVMe, + but budget it consciously on slower disks. From 586bede1c2ebe9e5dd5a5147c99b05ecd2b8fa05 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 20:08:31 +0100 Subject: [PATCH 06/13] =?UTF-8?q?docs(durable-streams-rust):=20WAL=5FTUNIN?= =?UTF-8?q?G.md=20=E2=80=94=20stacked=20383k=20@100k=20measured;=20note=20?= =?UTF-8?q?the=20separate=20~1M-stream=20wall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/Dockerfile | 28 --------------------- packages/durable-streams-rust/WAL_TUNING.md | 16 ++++++------ 2 files changed, 9 insertions(+), 35 deletions(-) delete mode 100644 packages/durable-streams-rust/Dockerfile diff --git a/packages/durable-streams-rust/Dockerfile b/packages/durable-streams-rust/Dockerfile deleted file mode 100644 index 0fb792ec68..0000000000 --- a/packages/durable-streams-rust/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1 -# -# Built multi-arch (linux/amd64 + linux/arm64) natively per arch by -# .github/workflows/docker_multiarch_image.yml — the build context is the repo -# root and this Dockerfile is `packages/durable-streams-rust/Dockerfile`. - -# ---- build stage: compile the release binary (glibc, matches the runtime) ---- -FROM rust:1-bookworm AS build -WORKDIR /app -# Copy only what the build needs (no target/, no npm/) so we don't depend on a -# .dockerignore at the shared repo root. -COPY packages/durable-streams-rust/Cargo.toml packages/durable-streams-rust/Cargo.lock ./ -COPY packages/durable-streams-rust/src ./src -# Default features only (no `tier`/`telemetry`) — minimal image, matching the -# conformance matrix. To ship S3 tiering, add `--features tier` here AND -# `ca-certificates` to the runtime stage. -RUN cargo build --release --locked - -# ---- runtime stage: distroless (glibc cc), no shell / package manager ---- -FROM gcr.io/distroless/cc-debian12 AS runtime -COPY --from=build /app/target/release/durable-streams-server /usr/local/bin/durable-streams-server -# Protocol default port (PROTOCOL.md §13.1); override with `--port`. -EXPOSE 4437 -ENTRYPOINT ["/usr/local/bin/durable-streams-server"] -# Bind all interfaces by default (the binary defaults to 127.0.0.1, unreachable -# from outside the container). Override by passing your own args. For persistence, -# mount a volume and add `--data-dir /your/path` (default is an ephemeral tmp dir). -CMD ["--host", "0.0.0.0"] diff --git a/packages/durable-streams-rust/WAL_TUNING.md b/packages/durable-streams-rust/WAL_TUNING.md index 1748e6abc0..f442ecff4e 100644 --- a/packages/durable-streams-rust/WAL_TUNING.md +++ b/packages/durable-streams-rust/WAL_TUNING.md @@ -16,7 +16,8 @@ is flat from 10k → 100k streams (−5%) at ~26–32× the pre-fix baseline. | + stream files on local NVMe (not the boot disk) | 46k | | + split-lane layout (WAL shards on their own NVMe devices) | 272k | | + size-triggered checkpoint (PR #4704, `--wal-checkpoint-wal-bytes 1GiB`) | 303k | -| + exclusive pinned cores (Guaranteed QoS + static CPU manager) | 328k (measured separately; stacking projects ~360k) | +| + exclusive pinned cores (Guaranteed QoS + static CPU manager) | 328k | +| **all of the above stacked** (suite `wal-stacked-1m`) | **383k** | | reference: memory durability (no fsync anywhere) | 512k | The residual ~1.6× gap to memory mode is WAL machinery (staging + double-write), @@ -92,12 +93,13 @@ no longer fsync-bound, it scales with cores again — don't starve it. ## Caveats / follow-ups -- Validated to 100k streams, 256 B payloads, single node. 1M-stream behavior of - this exact config is unmeasured (see `CARDINALITY_1M.md` for the older - analysis); the mechanisms that caused the cliff are cardinality-independent - now, but confirm before relying on it. -- The stacked best config (size trigger + pinned cores together) is projected - ~360k @100k, measured only separately — one confirmation run pending. +- Validated to 100k streams, 256 B payloads, single node. The stacked config + measured 383k @100k, 244k @500k, and **56k @1M** — a NEW, different wall + appears near 1M streams (candidates: one open fd per live stream vs the + container nofile ceiling, an ext4 directory with 1M files, stream-map/tails + working set, page-cache pressure from 1M dirty files). Profiling pass pending; + see `CARDINALITY_1M.md` for the older analysis. Below ~500k streams the + configuration above is cliff-free. - `--wal-checkpoint-syncfs` and the size trigger are opt-in; flipping defaults (syncfs on for Linux) is a candidate after soak. - Larger retained WAL = longer replay: 1 GiB/shard ≈ sub-second on local NVMe, From ba5090e73b636dc87a2dd7e3e5e77baa905aca03 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 21:25:11 +0100 Subject: [PATCH 07/13] =?UTF-8?q?perf(durable-streams-rust):=20--stream-la?= =?UTF-8?q?nes=20=E2=80=94=20hash=20stream=20data=20files=20across=20per-d?= =?UTF-8?q?evice=20lane=20dirs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ~1M-stream wall (wal-1m-diag): with every stream file on ONE data device, a checkpoint's syncfs writeback of ~200k dirty files took 60-74s per shard — tiny appends to distinct files amplify to ~8-12KB of data+inode+journal writeback each, saturating the single device and starving append staging (batch_avg collapsed 53->2-6). --stream-lanes N hashes stream files across streams/<0..N>/ subdirs, one per device: N× writeback capacity, N parallel syncfs barriers (one per lane, threaded), and no single ext4 dir holds every stream. Default 1 = byte-identical historical layout. Recovery walks all lanes; e2e covers lanes=3 crash recovery + layout spread. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/src/main.rs | 14 +++ packages/durable-streams-rust/src/store.rs | 106 +++++++++++++++++- .../durable-streams-rust/src/wal/e2e_tests.rs | 62 ++++++++++ .../durable-streams-rust/src/wal/shard.rs | 5 +- 4 files changed, 183 insertions(+), 4 deletions(-) diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index 19b341ffe3..d874aef155 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -311,6 +311,20 @@ fn main() { } } } + // Stream data lanes: hash stream files across streams/<0..N>/ subdirs, + // one per (intended) device, so checkpoint writeback spreads over N + // devices with N parallel syncfs barriers (the ~1M-stream wall fix). + // A LAYOUT choice: must match the on-disk layout across restarts. + "--stream-lanes" => { + let v = val(args.next(), "--stream-lanes"); + match v.parse::() { + Ok(n) if n >= 1 => store::set_stream_lanes(n), + _ => { + eprintln!("--stream-lanes must be a positive integer"); + std::process::exit(2); + } + } + } // Checkpoint size trigger: checkpoint a shard as soon as its retained // WAL exceeds this many bytes (0 = disabled). An explicit replay-time // budget that also self-staggers shards by their own write rates. diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index 6bc2fc927e..fdb0b749e6 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -186,6 +186,89 @@ pub(crate) fn syncfs_barrier(_file: &File) -> std::io::Result<()> { )) } +/// Stream-lane count (`--stream-lanes`, default 1 = the flat `streams/` layout). +/// With N > 1, stream data files are hashed across `streams/<0..N>/` subdirs so +/// each lane can be mounted on its OWN device: the checkpoint's dirty-file +/// writeback (the ~1M-stream wall — one `syncfs` measured at 60–74 s when every +/// stream shared one device, wal-1m-diag 2026-07-13) spreads over N devices and +/// runs N barriers in parallel, and no single ext4 directory holds every stream. +/// Must be set BEFORE `Store::open` and match the on-disk layout across restarts +/// (same N or files won't be found — a layout choice, not a runtime tunable). +static STREAM_LANES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(1); +pub fn set_stream_lanes(n: usize) { + STREAM_LANES.store(n.max(1), Ordering::Relaxed); +} +pub fn stream_lanes() -> usize { + STREAM_LANES.load(Ordering::Relaxed) +} + +/// Stable lane for a stream data-file name (FNV-1a; the fname embeds the stream +/// id, so this is fixed for the stream's lifetime and recomputable anywhere). +fn lane_of(fname: &str) -> usize { + let lanes = stream_lanes(); + if lanes <= 1 { + return 0; + } + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in fname.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x1000_0000_01b3); + } + (h % lanes as u64) as usize +} + +/// Directory of lane `lane`: the flat `streams/` when lanes == 1 (byte-identical +/// to the historical layout), else `streams//`. +fn lane_dir(data_dir: &std::path::Path, lane: usize) -> PathBuf { + let root = data_dir.join("streams"); + if stream_lanes() <= 1 { + root + } else { + root.join(lane.to_string()) + } +} + +/// Open directory fds, one per stream lane, registered at `Store::open` — the +/// checkpoint's syncfs must barrier EVERY lane's filesystem (touched files can +/// live on any lane), and a dir fd is a valid syncfs target. Replaced (not +/// appended) per open so tests that build many stores target the latest layout. +static LANE_SYNC_FDS: StdMutex>>> = StdMutex::new(None); + +/// Checkpoint durability barrier across all stream lanes: one `syncfs` per lane, +/// parallelized (each is a full device writeback and the lanes are independent +/// devices in the intended deployment). Falls back to a single barrier on +/// `fallback`'s fs when no lane registry exists (e.g. shard-only unit tests). +pub(crate) fn syncfs_stream_lanes(fallback: &File) -> std::io::Result<()> { + let fds = LANE_SYNC_FDS.lock().unwrap().clone(); + match fds { + Some(fds) if !fds.is_empty() => { + if fds.len() == 1 { + return syncfs_barrier(&fds[0]); + } + std::thread::scope(|s| { + let handles: Vec<_> = fds + .iter() + .map(|f| s.spawn(move || syncfs_barrier(f))) + .collect(); + let mut first_err = None; + for h in handles { + if let Err(e) = h + .join() + .unwrap_or_else(|_| Err(std::io::Error::other("syncfs thread panicked"))) + { + first_err.get_or_insert(e); + } + } + match first_err { + None => Ok(()), + Some(e) => Err(e), + } + }) + } + _ => syncfs_barrier(fallback), + } +} + pub struct StreamState { pub id: u64, pub path: String, @@ -472,6 +555,17 @@ impl Store { ) -> std::io::Result { let streams_dir = data_dir.join("streams"); std::fs::create_dir_all(&streams_dir)?; + // Create every stream-lane dir and register their dir fds for the + // checkpoint's per-lane syncfs barrier (see `syncfs_stream_lanes`). + { + let mut lane_fds = Vec::with_capacity(stream_lanes()); + for lane in 0..stream_lanes() { + let d = lane_dir(&data_dir, lane); + std::fs::create_dir_all(&d)?; + lane_fds.push(File::open(&d)?); + } + *LANE_SYNC_FDS.lock().unwrap() = Some(Arc::new(lane_fds)); + } // Stream data can be sensitive; keep the data dir owner-only (best-effort). #[cfg(unix)] { @@ -511,10 +605,16 @@ impl Store { /// everything else. Orphan files (crash between create and meta write) are /// discarded. fn recover(&self, streams_dir: &std::path::Path) -> std::io::Result<()> { + let _ = streams_dir; // root; per-lane dirs derived below (lane 0 == root when lanes == 1) let mut metas: HashMap = HashMap::new(); let mut data_files: Vec = Vec::new(); - for entry in std::fs::read_dir(streams_dir)? { - let p = entry?.path(); + let mut entries: Vec = Vec::new(); + for lane in 0..stream_lanes() { + for entry in std::fs::read_dir(lane_dir(&self.data_dir, lane))? { + entries.push(entry?.path()); + } + } + for p in entries { let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); if name.ends_with(".meta.tmp") { let _ = std::fs::remove_file(&p); @@ -859,7 +959,7 @@ impl Store { } let id = self.next_id.fetch_add(1, Ordering::Relaxed); let fname = format!("{}~{}", encode_path(path), id); - let file_path = self.data_dir.join("streams").join(fname); + let file_path = lane_dir(&self.data_dir, lane_of(&fname)).join(fname); let file = Arc::new( OpenOptions::new() .create(true) diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index 135313459f..80606ff1d6 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -952,6 +952,68 @@ async fn e2e_recycled_first_segment_acked_records_survive_crash() { let _ = std::fs::remove_dir_all(&dir); } +/// `--stream-lanes N`: stream data files hash across `streams/<0..N>/` subdirs +/// (one per device in the intended deployment — the ~1M-stream writeback-wall +/// fix). Crash recovery must find every file in its lane dir, and the +/// checkpoint's per-lane syncfs must preserve durability-before-recycle exactly +/// as the single-lane layout does. Guarded by DurabilityGuard (serialized) since +/// stream-lanes is process-global state; reset to 1 before releasing the guard. +#[tokio::test] +async fn e2e_stream_lanes_recover_acked_records() { + let _guard = DurabilityGuard::wal(); + crate::store::set_stream_lanes(3); + crate::wal::shard::set_checkpoint_syncfs(true); + const SEG: u64 = 4096; + let dir = tmp("stream-lanes"); + + let h = Harness::boot_with_segment_size(&dir, Some(1), 1, SEG).unwrap(); + // Enough streams that the FNV lane hash populates more than one lane. + let names: Vec = (0..12).map(|i| format!("lane-s{i}")).collect(); + for n in &names { + create_stream(&h.store, n, OCTET).await; + } + let mut expected: std::collections::HashMap> = Default::default(); + for round in 0..40usize { + for n in &names { + let rec = format!("{n}-r{round:03}|").into_bytes(); + append_acked(&h.store, n, OCTET, &rec).await; + expected.entry(n.clone()).or_default().extend_from_slice(&rec); + } + } + // Checkpoint (per-lane syncfs + recycle), then more acked appends on top. + h.walset.shards()[0].checkpoint().await.unwrap(); + for n in &names { + let rec = format!("{n}-post|").into_bytes(); + append_acked(&h.store, n, OCTET, &rec).await; + expected.entry(n.clone()).or_default().extend_from_slice(&rec); + } + + h.crash(); + + let h2 = Harness::boot_with_segment_size(&dir, None, 1, SEG).unwrap(); + // Layout sanity: files actually spread across lane subdirs. + let lanes_used = (0..3) + .filter(|l| { + std::fs::read_dir(dir.join("streams").join(l.to_string())) + .map(|d| d.flatten().next().is_some()) + .unwrap_or(false) + }) + .count(); + assert!(lanes_used >= 2, "expected streams spread over lanes, got {lanes_used}"); + for n in &names { + let got = stream_file_bytes(&h2.store, n); + assert_eq!( + &got, + expected.get(n).unwrap(), + "stream {n} recovers byte-identical across lanes" + ); + } + h2.crash(); + crate::wal::shard::set_checkpoint_syncfs(false); + crate::store::set_stream_lanes(1); + let _ = std::fs::remove_dir_all(&dir); +} + /// Cardinality-cliff #1: with `--wal-checkpoint-syncfs on`, the checkpoint makes /// touched per-stream files durable via ONE `syncfs()` barrier instead of the /// per-stream `fdatasync` loop. This must preserve the durability-before-recycle diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index 4d94591b8e..909b1f03b8 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -919,7 +919,10 @@ impl Shard { // dirty page on it — all touched files become durable at once. Still // strictly before persist_durable_tails/recycle, so the // durability-before-recycle ordering is unchanged. - crate::store::syncfs_barrier(&touched[0].2)?; + // One syncfs per stream lane (touched files may span lanes on + // multi-device layouts); falls back to touched[0]'s fs when no + // lane registry exists (shard-only unit tests). + crate::store::syncfs_stream_lanes(&touched[0].2)?; } else if fanout <= 1 { for (_, _, f) in &touched { crate::store::barrier_fsync(f)?; From 60a52c318cb8afc27392e9f9c5c039df20a8ecaa Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 21:58:14 +0100 Subject: [PATCH 08/13] =?UTF-8?q?refactor(durable-streams-rust):=20checkpo?= =?UTF-8?q?int=20cleanup=20=E2=80=94=20syncfs=20default-on=20(Linux),=20dr?= =?UTF-8?q?op=20dead=20fsync=20fan-out,=20refresh=20WAL=5FTUNING?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - --wal-checkpoint-syncfs defaults ON for Linux (validated +51% at the cliff, neutral below; off = escape hatch). Non-Linux unchanged (no syncfs). - Remove FSYNC_FANOUT / --wal-fsync-parallel: parallel per-file checkpoint fsync regressed in every controlled test (f8 -11% NVMe, f16 -19% constrained) and the syncfs barrier supersedes it. Flag accepted as a warning no-op so old deploy scripts don't crash. Checkpoint step 2 is now two modes: per-lane syncfs (Linux default) or the serial per-file fallback. - e2e resets restore the platform default instead of hard 'false'. - WAL_TUNING.md: stream-lanes results (1M wall broken), cardinality-based lane-split guidance, updated flag set, fd-ceiling note (#4706). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/WAL_TUNING.md | 45 ++++++--- packages/durable-streams-rust/src/main.rs | 17 ++-- .../durable-streams-rust/src/wal/e2e_tests.rs | 4 +- .../durable-streams-rust/src/wal/shard.rs | 98 ++++++------------- 4 files changed, 69 insertions(+), 95 deletions(-) diff --git a/packages/durable-streams-rust/WAL_TUNING.md b/packages/durable-streams-rust/WAL_TUNING.md index f442ecff4e..61fea8b6fd 100644 --- a/packages/durable-streams-rust/WAL_TUNING.md +++ b/packages/durable-streams-rust/WAL_TUNING.md @@ -20,6 +20,12 @@ is flat from 10k → 100k streams (−5%) at ~26–32× the pre-fix baseline. | **all of the above stacked** (suite `wal-stacked-1m`) | **383k** | | reference: memory durability (no fsync anywhere) | 512k | +At extreme cardinality the stacked config on ONE data lane hits a second wall — +checkpoint writeback (~40× metadata amplification of small appends) saturates +the single data device (`syncfs` = 60–74 s at 1M streams): 244k @500k, 56–68k +@1M. `--stream-lanes 3` (suite `wal-streamlanes-1m`, 3 data lanes + 3 WAL +lanes) breaks it: **374k @100k, 285k @500k, 212k @1M** (`syncfs` 5.7–11 s). + The residual ~1.6× gap to memory mode is WAL machinery (staging + double-write), not fsync — future work: io_uring segment writer (`wal/segment.rs` seam), batched mark-written. @@ -46,28 +52,35 @@ barrier). Then: raw-block node pools the default emptyDir sits on the boot PD — this single mistake mismeasures (and misdeploys) WAL mode by 5–26×. -Example (6-device box): device 0 → data root, devices 1–5 → 5 WAL shards: +Split the box between data lanes and WAL lanes **by cardinality**: -``` ---data-dir /data/wal/0 --wal-shards 5 -``` +- ≤ ~100k streams: 1 data lane is enough — e.g. device 0 → data root, + devices 1–5 → 5 WAL shards: `--data-dir /data/wal/0 --wal-shards 5` +- ≥ ~500k streams: checkpoint writeback dominates; give data more lanes — e.g. + device 0 → data root (stream lane 0), devices 1–2 → stream lanes 1–2 + (mounted at `/streams/1`, `/2`), devices 3–5 → 3 WAL shards: + `--data-dir /data/wal/0 --wal-shards 3 --stream-lanes 3` + (`--stream-lanes` is a LAYOUT choice like the shard count: it must match the + on-disk layout across restarts.) ### 2. Server flags ``` ---wal-checkpoint-syncfs on # one syncfs barrier per checkpoint instead of - # O(N-touched) per-stream fdatasync (PR #4697) --wal-checkpoint-wal-bytes 1073741824 # checkpoint a shard when ITS retained WAL # exceeds 1 GiB (PR #4704) — checkpoint cost ≈ 0, # crash-replay bounded to ≤1 GiB/shard (<1 s NVMe) --wal-checkpoint-interval-ms 60000 # fallback timer so an idle shard still recycles --wal-shards # shards = fsync lanes; on a SINGLE shared # device keep 2–4 (more only fragments batches) +--stream-lanes # hash stream files across per-device dirs + # (PR #4705); layout choice, default 1 --worker-threads ``` -Leave `--wal-fsync-parallel` at its default (1): fanout parallelizes the -per-stream fsync loop that syncfs replaces, and measured as a regression. +The syncfs checkpoint barrier (PR #4697) is **default-on for Linux** — one +`syncfs` per stream lane instead of O(N-touched) per-stream `fdatasync`; +`--wal-checkpoint-syncfs off` is the escape hatch. `--wal-fsync-parallel` is +removed (regressed in every controlled test; accepted as a warning no-op). ### 3. CPU binding (+21–24%) @@ -93,13 +106,15 @@ no longer fsync-bound, it scales with cores again — don't starve it. ## Caveats / follow-ups -- Validated to 100k streams, 256 B payloads, single node. The stacked config - measured 383k @100k, 244k @500k, and **56k @1M** — a NEW, different wall - appears near 1M streams (candidates: one open fd per live stream vs the - container nofile ceiling, an ext4 directory with 1M files, stream-map/tails - working set, page-cache pressure from 1M dirty files). Profiling pass pending; - see `CARDINALITY_1M.md` for the older analysis. Below ~500k streams the - configuration above is cliff-free. +- The 1M-stream writeback wall is diagnosed and broken (`--stream-lanes`, PR + #4705: 68k → 212k @1M). The residual slope (374k @100k → 212k @1M on 3 data + lanes) is per-file writeback amplification against total data-lane + capacity — add data lanes, or see #4695 (log-structured store) for the + structural end-state. +- fd ceiling: the server holds one fd per live stream — 1,005,724 fds at 1M + streams = 96% of the default 1,048,576 limit. Not the throughput wall, but a + hard scale ceiling just above 1M: raise LimitNOFILE, or see #4706 (lazy fd + management). - `--wal-checkpoint-syncfs` and the size trigger are opt-in; flipping defaults (syncfs on for Linux) is a candidate after soak. - Larger retained WAL = longer replay: 1 GiB/shard ≈ sub-second on local NVMe, diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index d874aef155..b0c908ecac 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -278,17 +278,16 @@ fn main() { } } } - // Checkpoint fdatasync fan-out (H4). `1` = serial baseline. + // Removed knob: parallel per-file checkpoint fsync regressed in every + // controlled test; superseded by the (default-on) syncfs barrier. + // Accepted-and-ignored so old deploy scripts don't crash the server. "--wal-fsync-parallel" => { - let n: u64 = parse_val(args.next(), "--wal-fsync-parallel"); - if n == 0 { - eprintln!("--wal-fsync-parallel must be ≥ 1"); - std::process::exit(2); - } - wal::shard::set_fsync_fanout(n); + let _ = val(args.next(), "--wal-fsync-parallel"); + eprintln!("warning: --wal-fsync-parallel is removed (no-op); the syncfs checkpoint barrier supersedes it"); } - // Checkpoint durability via ONE syncfs() barrier instead of the - // O(N_touched) per-stream fdatasync loop (cardinality-cliff #1). Linux-only. + // Checkpoint durability via per-lane syncfs() barriers instead of the + // O(N_touched) per-stream fdatasync loop (cardinality-cliff #1). + // DEFAULT ON for Linux; `off` = escape hatch. No-op elsewhere. "--wal-checkpoint-syncfs" => { let v = val(args.next(), "--wal-checkpoint-syncfs"); match v.as_str() { diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index 80606ff1d6..511bfcf8ca 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -1009,7 +1009,7 @@ async fn e2e_stream_lanes_recover_acked_records() { ); } h2.crash(); - crate::wal::shard::set_checkpoint_syncfs(false); + crate::wal::shard::set_checkpoint_syncfs(cfg!(target_os = "linux")); // restore platform default crate::store::set_stream_lanes(1); let _ = std::fs::remove_dir_all(&dir); } @@ -1057,7 +1057,7 @@ async fn e2e_checkpoint_syncfs_recovers_acked_records() { "syncfs-checkpoint acked records recover byte-identical (durability-before-recycle held)" ); h2.crash(); - crate::wal::shard::set_checkpoint_syncfs(false); + crate::wal::shard::set_checkpoint_syncfs(cfg!(target_os = "linux")); // restore platform default let _ = std::fs::remove_dir_all(&dir); } diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index 909b1f03b8..ccb0576155 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -379,43 +379,29 @@ pub struct Shard { /// recyclable. const CHECKPOINT_FILE: &str = "checkpoint"; -/// Concurrency for the checkpoint's per-stream `fdatasync` phase (cardinality-cliff -/// H4). At high stream cardinality that phase dominates the checkpoint (~99% of its -/// wall time), and a serial loop pays `latency × N_touched` while the storage -/// device's queue depth (NVMe: many in flight) sits idle. Fanning the syncs across -/// this many OS threads lets the device absorb them concurrently; all still -/// complete before `persist_durable_tails`/recycle, preserving the -/// durability-before-recycle ordering. `1` (the DEFAULT) = serial, i.e. a no-op -/// change unless `--wal-fsync-parallel N` opts in. -/// -/// Default is serial because a fixed high fan-out REGRESSES on a CPU-constrained -/// server or storage that does not do concurrent fsync: measured on a 2-vCPU Linux -/// container (Docker, virtiofs) fan-out=16 was −19% vs serial — the 16 sync threads -/// per shard stole CPU from the runtime, slowing the committer until checkpoints -/// fell behind. The win requires real NVMe (deep device queue) AND spare cores; -/// validate there before raising the default. -static FSYNC_FANOUT: AtomicU64 = AtomicU64::new(1); -pub fn set_fsync_fanout(n: u64) { - FSYNC_FANOUT.store(n.max(1), Ordering::Relaxed); -} -fn fsync_fanout() -> usize { - FSYNC_FANOUT.load(Ordering::Relaxed) as usize -} +// (Removed: FSYNC_FANOUT / --wal-fsync-parallel. Parallelizing the per-stream +// fdatasync loop regressed in every controlled test — f8 −11% on NVMe, f16 −19% +// on a CPU-constrained container — because it steals device budget/CPU from the +// commit path. The syncfs barrier (default on Linux, below) replaces the loop +// wholesale; the serial per-file loop remains only as the non-Linux fallback.) -/// Checkpoint durability strategy (cardinality-cliff #1). The default per-stream +/// Checkpoint durability strategy (cardinality-cliff #1). The per-stream /// `fdatasync` loop (above) issues ONE device durability barrier per *touched* /// stream — `O(N_touched)` syscalls, measured at ~1.4 s of `fsync` per shard at /// 200k streams. Those barriers share the device's fixed `fdatasync`/s budget with /// the commit path, so the checkpoint storm steals throughput from acks (the cliff) /// AND caps wal throughput regardless of shard count (a shared-device sweep showed -/// s1≈s24). When this is on, step 2 instead issues a SINGLE `syncfs()` on the data -/// filesystem — one barrier that flushes every touched stream file (and everything -/// else dirty on the fs) at once, collapsing the per-stream `O(N_touched)` cost to -/// `O(1)` syscalls. The durability-before-recycle ordering is unchanged: the -/// `syncfs` completes before `persist_durable_tails`/recycle. Linux-only (there is -/// no `syncfs` on macOS); on other targets it falls back to the per-stream loop. -/// Default off — opt in with `--wal-checkpoint-syncfs on`. -static CHECKPOINT_SYNCFS: AtomicBool = AtomicBool::new(false); +/// s1≈s24). When this is on, step 2 instead issues ONE `syncfs()` per stream lane — +/// a filesystem-wide barrier that flushes every touched stream file (and everything +/// else dirty on that fs) at once, collapsing the per-stream `O(N_touched)` cost to +/// `O(lanes)` syscalls. The durability-before-recycle ordering is unchanged: the +/// `syncfs` completes before `persist_durable_tails`/recycle. +/// +/// **Default ON for Linux** (validated: +51% at the 100k-stream cliff, at worst +/// neutral below it — wal-syncfs/splitlane campaigns 2026-07); `--wal-checkpoint- +/// syncfs off` is the escape hatch. Non-Linux has no `syncfs` and always uses the +/// per-stream loop regardless of this flag. +static CHECKPOINT_SYNCFS: AtomicBool = AtomicBool::new(cfg!(target_os = "linux")); pub fn set_checkpoint_syncfs(on: bool) { CHECKPOINT_SYNCFS.store(on, Ordering::Relaxed); } @@ -904,49 +890,23 @@ impl Shard { let n_touched = touched.len(); let t_capture = t_start.elapsed(); - // 2. fdatasync each touched per-stream file. Fan out across a bounded - // pool of OS threads (H4): the device's queue depth absorbs the - // syncs concurrently instead of paying latency × N_touched serially - // — the checkpoint's dominant cost at high cardinality. ALL syncs - // complete here, before persist_durable_tails/recycle below, so the - // durability-before-recycle ordering is unchanged. `fanout == 1` - // (or a single file) keeps the plain serial loop. - let fanout = fsync_fanout().min(n_touched.max(1)); + // 2. Make every touched per-stream file durable, strictly BEFORE + // persist_durable_tails/recycle (the durability-before-recycle + // ordering). Two modes: + // * syncfs (Linux default): one filesystem-wide barrier per stream + // lane — O(lanes) syscalls instead of O(N_touched) fdatasyncs + // (the cardinality cliff). Lanes are independent devices in the + // intended layout, so the barriers run in parallel. + // * per-file fdatasync loop: the non-Linux (or --wal-checkpoint- + // syncfs off) fallback. Serial on purpose: parallel fan-out + // regressed in every controlled test (it steals device budget + // from the commit path). if checkpoint_syncfs() && cfg!(target_os = "linux") && n_touched > 0 { - // #1: ONE filesystem-wide barrier for ALL touched files, replacing - // the O(N_touched) per-stream fdatasync loop. `syncfs` on any fd of - // the data fs (we pass the first touched stream file) flushes every - // dirty page on it — all touched files become durable at once. Still - // strictly before persist_durable_tails/recycle, so the - // durability-before-recycle ordering is unchanged. - // One syncfs per stream lane (touched files may span lanes on - // multi-device layouts); falls back to touched[0]'s fs when no - // lane registry exists (shard-only unit tests). crate::store::syncfs_stream_lanes(&touched[0].2)?; - } else if fanout <= 1 { + } else { for (_, _, f) in &touched { crate::store::barrier_fsync(f)?; } - } else { - let next = AtomicU64::new(0); - let first_err: Mutex> = Mutex::new(None); - std::thread::scope(|scope| { - for _ in 0..fanout { - scope.spawn(|| loop { - let i = next.fetch_add(1, Ordering::Relaxed) as usize; - let Some((_, _, f)) = touched.get(i) else { break }; - if let Err(e) = crate::store::barrier_fsync(f) { - let mut slot = first_err.lock().unwrap(); - if slot.is_none() { - *slot = Some(e); - } - } - }); - } - }); - if let Some(e) = first_err.into_inner().unwrap() { - return Err(e); - } } let t_fsync = t_start.elapsed(); From f7c5360497beab369f1d588673f98fe43315db75 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 23:06:30 +0100 Subject: [PATCH 09/13] refactor(durable-streams-rust): remove dead knobs and vestigial paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Config-surface audit removals (all confirmed unexercised or A/B residue of already-shipped defaults): - --wal-fsync-parallel (was already a no-op stub) — gone entirely. - --wal-checkpoint-syncfs — gone; the per-lane syncfs barrier is unconditional on Linux, the serial per-file loop is the non-Linux path. - --wal-meta-gate / --mem-meta-gate — gone; the gated (default) behavior is now the only behavior, dead A/B baseline branches deleted. - --meta-sweep-disable / --meta-sweep-stats — gone (bench-only diagnostics; sweep_meta_once simplified). - --tier local + --tier-local-dir CLI — gone (--tier is off|s3); TierKind::Local stays crate-internal for unit tests. - --durability memory + --tier is now rejected at startup (memory-mode acks are not durable; tiering presumes durable segments). - WAL codec: PAYLOAD_CHECKSUMMED is mandatory — a flag-clear header decodes as Torn (the only writer that ever left it clear was the removed splice relay); dropped the legacy decode branch and its pinning test. - Stale zero-copy-era comment on DurabilityMode::Memory corrected. Kept deliberately (audit showed live use + CI coverage): --tail-cache-bytes (fan-out dedup lever), --read-offload (cold-read isolation), --wal-stats / --server-stats (bottleneck telemetry). Docs (README, WAL_TUNING) updated. 105/105 tests pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/README.md | 7 +-- packages/durable-streams-rust/WAL_TUNING.md | 13 ++-- packages/durable-streams-rust/src/handlers.rs | 29 +++------ packages/durable-streams-rust/src/main.rs | 62 +++---------------- packages/durable-streams-rust/src/store.rs | 57 +---------------- .../durable-streams-rust/src/wal/codec.rs | 50 +++------------ .../durable-streams-rust/src/wal/e2e_tests.rs | 4 -- .../durable-streams-rust/src/wal/shard.rs | 33 +++------- 8 files changed, 47 insertions(+), 208 deletions(-) diff --git a/packages/durable-streams-rust/README.md b/packages/durable-streams-rust/README.md index 45b5ee5683..2a2fe0c98d 100644 --- a/packages/durable-streams-rust/README.md +++ b/packages/durable-streams-rust/README.md @@ -89,8 +89,7 @@ durable, single-node server on `127.0.0.1:4437` with its data dir under `$TMPDIR | Flag | Default | Description | | --------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------- | -| `--tier` | `off` | `off` \| `local` (sealed segments to a local dir) \| `s3` (S3-compatible object storage) | -| `--tier-local-dir` | — | (`tier=local`) directory for sealed segments | +| `--tier` | `off` | `off` \| `s3` (S3-compatible object storage) | | `--tier-endpoint` | — | (`tier=s3`) S3 endpoint URL | | `--tier-region` | — | (`tier=s3`) region | | `--tier-bucket` | — | (`tier=s3`) bucket name | @@ -107,7 +106,7 @@ S3 credentials come from the **environment**, never flags: `DS_S3_ACCESS_KEY_ID` | Your situation | Use | | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| **Bounded local disk with long history** | `--tier s3` (or `local`) — seal cold segments to object storage; the recent tail stays local and zero-copy. | +| **Bounded local disk with long history** | `--tier s3` — seal cold segments to object storage; the recent tail stays local and zero-copy. | | **High fan-out p99 across many streams on Linux** | try `--tail-cache-bytes 65536` (off by default there because `sendfile` already covers it). | | **Experimenting on macOS and writes take ~2–10 ms** | expected — see [macOS write latency](#macos-write-latency-f_fullfsync) below. | @@ -179,7 +178,7 @@ Opt-in (`--tier`, off by default). Because streams are append-only and immutable - **S3-compatible, not AWS-only.** The `s3` backend works with any S3-compatible endpoint — Cloudflare R2, Fly/Tigris, MinIO, Backblaze B2 — via a configurable endpoint + path-style addressing. It's built on the `object_store` crate behind the **`tier` Cargo feature** (so a default build pulls no object-storage dependencies): `cargo build --release --features tier`. A `local` backend (sealed segments to a directory) needs no feature/deps and is handy for testing. - **How it stays correct.** Each stream keeps a manifest of its sealed segments. The lifecycle is seal → upload → `head`-verify → durably flip the manifest entry `local → remote` → only then unlink the staged chunk file (safe even under an in-flight read — Unix keeps an open fd readable after unlink). A read resolves each requested offset against the manifest — local ranges (live file or sealed chunk file) keep the zero-copy `sendfile` path; remote ranges are fetched by range-GET and streamed in. JSON seals always land on a value boundary (never inside a string). Durability is unchanged: an append still acks only after the local fsync — offload is strictly post-durability. Fully-sealed ranges are stamped `Cache-Control: immutable` for long-lived CDN caching, and remote objects are GC'd ref-count-aware with forks. The live data file's redundant sealed prefix is reclaimed by **compaction**: once it exceeds `--tier-compact-bytes` (default 64 MiB), the live file is rewritten to hold only the hot tail, under the appender lock and crash-safe via a `pending_compaction` intent. In-flight reads drain off the old fd, so reads stay lock-free — this is why compaction is used rather than `fallocate` hole-punching, which raced those reads. Set `--tier-compact-bytes 0` to disable. -- **Flags:** `--tier {off|local|s3}`, `--tier-segment-bytes`, `--tier-compact-bytes` (live-file compaction threshold; `0` disables), `--tier-key-prefix`, `--tier-local-dir` (local), `--tier-endpoint` / `--tier-region` / `--tier-bucket`, `--tier-path-style` / `--tier-virtual-hosted`, `--tier-allow-http`. S3 credentials come from `DS_S3_ACCESS_KEY_ID` / `DS_S3_SECRET_ACCESS_KEY` (with `AWS_*` fallback), env only. +- **Flags:** `--tier {off|s3}`, `--tier-segment-bytes`, `--tier-compact-bytes` (live-file compaction threshold; `0` disables), `--tier-key-prefix`, `--tier-endpoint` / `--tier-region` / `--tier-bucket`, `--tier-path-style` / `--tier-virtual-hosted`, `--tier-allow-http`. S3 credentials come from `DS_S3_ACCESS_KEY_ID` / `DS_S3_SECRET_ACCESS_KEY` (with `AWS_*` fallback), env only. Conformance passes with tiering on as well as off (verified manually with a small `--tier-segment-bytes` so streams seal and offload mid-suite; catch-up reads, ETag/304, closed-stream EOF, and forks are all served correctly from cold). Note this is a manual check, not a CI gate: the `rust-conformance` matrix in `.github/workflows/ci.yml` builds the default feature set (no `--features tier`) and never passes `--tier`, so the tier code path is not exercised in CI today. ## Observability (OpenTelemetry) diff --git a/packages/durable-streams-rust/WAL_TUNING.md b/packages/durable-streams-rust/WAL_TUNING.md index 61fea8b6fd..e61a73ccca 100644 --- a/packages/durable-streams-rust/WAL_TUNING.md +++ b/packages/durable-streams-rust/WAL_TUNING.md @@ -12,7 +12,7 @@ is flat from 10k → 100k streams (−5%) at ~26–32× the pre-fix baseline. | configuration | ops/s | |---|---| | pre-fix: streams on network PD, per-stream checkpoint fdatasync storm | 10.4k | -| + `--wal-checkpoint-syncfs on` (PR #4697) | 13.6k | +| + syncfs checkpoint barrier (PR #4697) | 13.6k | | + stream files on local NVMe (not the boot disk) | 46k | | + split-lane layout (WAL shards on their own NVMe devices) | 272k | | + size-triggered checkpoint (PR #4704, `--wal-checkpoint-wal-bytes 1GiB`) | 303k | @@ -77,10 +77,9 @@ Split the box between data lanes and WAL lanes **by cardinality**: --worker-threads ``` -The syncfs checkpoint barrier (PR #4697) is **default-on for Linux** — one -`syncfs` per stream lane instead of O(N-touched) per-stream `fdatasync`; -`--wal-checkpoint-syncfs off` is the escape hatch. `--wal-fsync-parallel` is -removed (regressed in every controlled test; accepted as a warning no-op). +The syncfs checkpoint barrier (PR #4697) is unconditional on Linux — one +`syncfs` per stream lane instead of O(N-touched) per-stream `fdatasync`. +Non-Linux uses the serial per-file loop (no `syncfs` there). ### 3. CPU binding (+21–24%) @@ -115,7 +114,7 @@ no longer fsync-bound, it scales with cores again — don't starve it. streams = 96% of the default 1,048,576 limit. Not the throughput wall, but a hard scale ceiling just above 1M: raise LimitNOFILE, or see #4706 (lazy fd management). -- `--wal-checkpoint-syncfs` and the size trigger are opt-in; flipping defaults - (syncfs on for Linux) is a candidate after soak. +- The checkpoint size trigger is opt-in (`--wal-checkpoint-wal-bytes 0` default); + making ~1 GiB the default is a candidate after soak. - Larger retained WAL = longer replay: 1 GiB/shard ≈ sub-second on local NVMe, but budget it consciously on slower disks. diff --git a/packages/durable-streams-rust/src/handlers.rs b/packages/durable-streams-rust/src/handlers.rs index eeed3b9888..74095040b7 100644 --- a/packages/durable-streams-rust/src/handlers.rs +++ b/packages/durable-streams-rust/src/handlers.rs @@ -44,7 +44,7 @@ pub enum DurabilityMode { #[default] Wal, /// No WAL, no fsync: ack on the page-cache write. Durability comes from replication - /// (future). Linux-only (binary appends use zero-copy socket→file). + /// (future). Memory, } @@ -1193,9 +1193,7 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp // `last_access` only gates TTL. So a plain non-TTL append needs no sidecar // rewrite here — dropping it removes the O(touched) `write_meta_sync` calls // that dominate the checkpoint's meta phase at high stream cardinality. - // `wal_meta_gate()` (default on) can be turned off to restore always-mark - // for a same-binary A/B of the gate. - if meta_persist_needed || !crate::store::wal_meta_gate() { + if meta_persist_needed { st.meta_dirty.store(true, std::sync::atomic::Ordering::Release); } } else if meta_persist_needed { @@ -1204,21 +1202,14 @@ async fn handle_append_inner(store: Arc, req: Req, path: String) -> (Resp // batched treatment the wal branch above gets from the checkpoint: no // per-stream timer task, no per-append sidecar rewrite (#4691). // - // GATED (cardinality-cliff fix): only queue when the append actually - // changed state the sidecar must persist — producer/seq idempotency or a - // sliding TTL (see `meta_persist_needed`). A plain append to a non-TTL - // stream changes only `durable_tail`/`last_access`, and memory-mode - // recovery reads NEITHER (the tail is re-derived from the data-file - // length in `Store::new_with_tier`; `last_access` only gates TTL expiry, - // which these streams don't have). Skipping the queue for that common - // case removes the per-append sidecar rewrite whose cost stops amortizing - // at high stream cardinality (CARDINALITY_CLIFF_CAUSES.md #1). - // - // `mem_meta_gate()` (default on) can be turned off to restore always-queue - // for a same-binary A/B of the fix. - store.mark_meta_dirty(&st); - } else if !crate::store::mem_meta_gate() { - // A/B baseline (gate off): old behavior — queue every memory-mode append. + // Only queued when the append actually changed state the sidecar must + // persist — producer/seq idempotency or a sliding TTL. A plain append to + // a non-TTL stream changes only `durable_tail`/`last_access`, and + // memory-mode recovery reads NEITHER (the tail is re-derived from the + // data-file length in `Store::new_with_tier`; `last_access` only gates + // TTL expiry, which these streams don't have). Skipping the queue for + // that common case removes the per-append sidecar rewrite whose cost + // stops amortizing at high stream cardinality. store.mark_meta_dirty(&st); } if !wire.is_empty() { diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index b0c908ecac..4cd69a6c0e 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -168,10 +168,9 @@ fn main() { let v = val(args.next(), "--tier"); tier.kind = match v.as_str() { "off" => tier::TierKind::Off, - "local" => tier::TierKind::Local, "s3" => tier::TierKind::S3, _ => { - eprintln!("--tier must be off|local|s3"); + eprintln!("--tier must be off|s3"); std::process::exit(2); } }; @@ -195,9 +194,6 @@ fn main() { "--tier-allow-http" => { tier.allow_http = true; } - "--tier-local-dir" => { - tier.local_dir = Some(val(args.next(), "--tier-local-dir").into()); - } "--wal-shards" => { let n: usize = parse_val(args.next(), "--wal-shards"); if n == 0 { @@ -240,12 +236,6 @@ fn main() { } } } - // Dev/benchmark toggles for the cardinality-cliff investigation. - // `--meta-sweep-disable` drops the memory-mode sidecar write (makes - // the sidecar permanently stale — bench-only); `--meta-sweep-stats` - // logs a META_SWEEP line per tick. - "--meta-sweep-disable" => store::set_meta_sweep_disable(true), - "--meta-sweep-stats" => store::set_meta_sweep_stats(true), // Periodic SRV_STATS line (both modes): cpu_cores / inflight / service // + appender-lock + durability wait — bottleneck analysis. "--server-stats" => { @@ -256,49 +246,6 @@ fn main() { } server_stats_secs = Some(n); } - "--wal-meta-gate" => { - let v = val(args.next(), "--wal-meta-gate"); - match v.as_str() { - "on" => store::set_wal_meta_gate(true), - "off" => store::set_wal_meta_gate(false), - _ => { - eprintln!("--wal-meta-gate must be on|off"); - std::process::exit(2); - } - } - } - "--mem-meta-gate" => { - let v = val(args.next(), "--mem-meta-gate"); - match v.as_str() { - "on" => store::set_mem_meta_gate(true), - "off" => store::set_mem_meta_gate(false), - _ => { - eprintln!("--mem-meta-gate must be on|off"); - std::process::exit(2); - } - } - } - // Removed knob: parallel per-file checkpoint fsync regressed in every - // controlled test; superseded by the (default-on) syncfs barrier. - // Accepted-and-ignored so old deploy scripts don't crash the server. - "--wal-fsync-parallel" => { - let _ = val(args.next(), "--wal-fsync-parallel"); - eprintln!("warning: --wal-fsync-parallel is removed (no-op); the syncfs checkpoint barrier supersedes it"); - } - // Checkpoint durability via per-lane syncfs() barriers instead of the - // O(N_touched) per-stream fdatasync loop (cardinality-cliff #1). - // DEFAULT ON for Linux; `off` = escape hatch. No-op elsewhere. - "--wal-checkpoint-syncfs" => { - let v = val(args.next(), "--wal-checkpoint-syncfs"); - match v.as_str() { - "on" => wal::shard::set_checkpoint_syncfs(true), - "off" => wal::shard::set_checkpoint_syncfs(false), - _ => { - eprintln!("--wal-checkpoint-syncfs must be on|off"); - std::process::exit(2); - } - } - } // Checkpoint time trigger: per-shard cadence in ms (default 3000). "--wal-checkpoint-interval-ms" => { let v = val(args.next(), "--wal-checkpoint-interval-ms"); @@ -349,6 +296,13 @@ fn main() { // tail-cache-off — those belonged to the removed zero-copy path); the only // gate is refusing to silently ignore a WAL left by a previous wal run. if handlers::durability() == handlers::DurabilityMode::Memory { + // Memory mode acks before anything is fsynced, so pairing it with a cold + // tier would offload un-fsynced (loseable) data as if it were durable — + // a combination with no coherent durability story. Refuse it. + if tier.kind != tier::TierKind::Off { + eprintln!("error: --durability memory cannot be combined with --tier (memory-mode acks are not durable; tiering presumes durable segments)"); + std::process::exit(2); + } // Fail fast on a WAL left by a previous `--durability wal` run: memory mode // never opens/replays it, so starting here would silently ignore those // records (and drop any not yet folded into the per-stream files). Refuse diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index fdb0b749e6..1fc841618a 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -398,47 +398,6 @@ pub fn tail_cache_bytes() -> usize { TAIL_CACHE_BYTES.load(Ordering::Relaxed) } -/// Dev/benchmark toggle: when set, `sweep_meta_once` drains its queue and clears -/// `meta_dirty` but SKIPS the actual sidecar `write_meta_sync`. Used to confirm -/// the cardinality-cliff hypothesis #1 (per-stream sidecar flush cost) in memory -/// mode — if the 1k→50k throughput curve flattens with this on, the sidecar -/// write dominates. NOT for production: it makes the producer-dedup/tail sidecar -/// permanently stale. -static META_SWEEP_DISABLE: AtomicBool = AtomicBool::new(false); -/// Dev/benchmark toggle: when set, emit a `META_SWEEP` line per tick with the -/// number of sidecars written and the wall time spent. -static META_SWEEP_STATS: AtomicBool = AtomicBool::new(false); - -pub fn set_meta_sweep_disable(v: bool) { - META_SWEEP_DISABLE.store(v, Ordering::Relaxed); -} -pub fn set_meta_sweep_stats(v: bool) { - META_SWEEP_STATS.store(v, Ordering::Relaxed); -} - -/// When true (default), a plain non-TTL wal-mode append does NOT mark its stream -/// meta-dirty, so the checkpoint skips the redundant sidecar rewrite (the durable -/// tail is carried by the per-shard `tails` map). Set false to restore the old -/// always-mark behavior — used ONLY to A/B the gate on one binary, since laptop -/// wal throughput is too noisy for a trustworthy cross-image comparison. -static WAL_META_GATE: AtomicBool = AtomicBool::new(true); -pub fn set_wal_meta_gate(v: bool) { - WAL_META_GATE.store(v, Ordering::Relaxed); -} -pub fn wal_meta_gate() -> bool { - WAL_META_GATE.load(Ordering::Relaxed) -} - -/// Memory-mode counterpart of [`wal_meta_gate`]: when true (default), a plain -/// non-TTL memory-mode append does not queue a sidecar sweep. Set false to restore -/// the old always-queue behavior — used only to A/B the #1 fix on one binary. -static MEM_META_GATE: AtomicBool = AtomicBool::new(true); -pub fn set_mem_meta_gate(v: bool) { - MEM_META_GATE.store(v, Ordering::Relaxed); -} -pub fn mem_meta_gate() -> bool { - MEM_META_GATE.load(Ordering::Relaxed) -} impl StreamState { /// Record the just-appended wire chunk as the resident tail. `start` is the @@ -1351,10 +1310,6 @@ impl Store { pub fn sweep_meta_once(&self) -> usize { let drained: Vec> = std::mem::take(&mut *self.meta_sweep.lock().unwrap()); - let queued = drained.len(); - let skip_write = META_SWEEP_DISABLE.load(Ordering::Relaxed); - let stats = META_SWEEP_STATS.load(Ordering::Relaxed); - let start = if stats { Some(std::time::Instant::now()) } else { None }; let mut n = 0; for st in drained { // A hard-deleted stream's files are already unlinked — flushing @@ -1366,20 +1321,10 @@ impl Store { .get(&st.path) .is_some_and(|cur| Arc::ptr_eq(cur.value(), &st)); if live && st.meta_dirty.swap(false, Ordering::AcqRel) { - // `skip_write` (dev/bench) drains + clears the dirty flag but - // omits the sidecar write, to isolate the write's cost. - if !skip_write { - let _ = write_meta_sync(&st, false); - } + let _ = write_meta_sync(&st, false); n += 1; } } - if let Some(start) = start { - let us = start.elapsed().as_micros(); - eprintln!( - "META_SWEEP queued={queued} wrote={n} skip_write={skip_write} elapsed_us={us}" - ); - } n } } diff --git a/packages/durable-streams-rust/src/wal/codec.rs b/packages/durable-streams-rust/src/wal/codec.rs index 0408f29335..6fa8517dc1 100644 --- a/packages/durable-streams-rust/src/wal/codec.rs +++ b/packages/durable-streams-rust/src/wal/codec.rs @@ -24,11 +24,8 @@ //! trivially true even when a crash left a valid header over a zeroed, //! never-fully-written payload. Every WAL record is written by the buffered path, //! which has the payload in userspace and always sets `PAYLOAD_CHECKSUMMED`, so -//! such a torn record now fails decode. The old `--zero-copy` splice relay (the -//! only writer that left the flag clear) has been removed, so Bug #1 is **fully -//! closed** — there is no longer any unchecksummed WAL record in production. The -//! `PAYLOAD_CHECKSUMMED` flag and the flag-clear decode branch are retained only -//! for on-disk format stability. +//! such a torn record now fails decode. Every WAL record is checksummed; a +//! record with the flag clear is treated as torn/corrupt (ends the durable log). //! //! The header CRC also doubles as a torn-header detector: a partially-written //! header almost always fails the CRC, and an all-zero (`fallocate`'d, @@ -41,10 +38,8 @@ pub const HEADER_LEN: usize = 38; /// `flags` bit 0: the record's `payload_crc` field is a valid crc32c over the -/// payload and MUST be verified on decode. Every WAL writer (`encode_into`) sets -/// this — the old zero-copy splice relay that left it clear has been removed, so -/// all production WAL records are checksummed. The flag-clear branch in -/// [`decode_at`] is retained only for on-disk format stability. +/// payload. Every WAL writer (`encode_into`) sets it, and decode REQUIRES it — +/// a flag-clear header is treated as torn/corrupt. pub const PAYLOAD_CHECKSUMMED: u8 = 0b0000_0001; /// Record kind discriminant. The numeric values are the on-disk encoding and @@ -229,12 +224,12 @@ pub fn decode_at(seg: &[u8], off: usize) -> Decoded { return Decoded::Torn; } - // Bug #1 fix: if the writer checksummed the payload (buffered path), verify - // it. A torn/zeroed payload under a valid header (the fallocate'd-tail case) - // fails here and is rejected as Torn. Zero-copy records leave the flag clear - // and keep the "bytes present = complete" behavior (documented residual). - if flags & PAYLOAD_CHECKSUMMED != 0 - && crc32c::crc32c(&seg[payload_off..payload_off + len]) != payload_crc + // Bug #1 fix: verify the payload CRC. A torn/zeroed payload under a valid + // header (the fallocate'd-tail case) fails here and is rejected as Torn. + // Every writer sets PAYLOAD_CHECKSUMMED; a clear flag marks the record + // torn/corrupt (no writer has ever legitimately omitted the checksum here). + if flags & PAYLOAD_CHECKSUMMED == 0 + || crc32c::crc32c(&seg[payload_off..payload_off + len]) != payload_crc { return Decoded::Torn; } @@ -315,31 +310,6 @@ mod tests { } } - // Pure decode-behavior test (NOT a production case): a header with - // PAYLOAD_CHECKSUMMED clear opts out of the payload CRC, so a torn/zeroed - // payload tail decodes as `Record` (the legacy "bytes present = complete" - // branch). No WAL writer emits flag-clear records any more — the zero-copy - // splice relay that used to has been removed — so this branch is unreachable - // in production and Bug #1 is fully closed. The test pins the decode - // semantics only, for on-disk format stability. - #[test] - fn flag_clear_payload_skips_crc_validation_decode_only() { - let len: usize = 8192; - let prefix: usize = 4096; - let mut seg = Vec::new(); - // flags=0, payload_crc=0 — a hand-built flag-clear header (no production - // writer emits this). - encode_header_into(&mut seg, 1, RecordKind::Append, 42, 0, len as u32, 0, 0); - seg.extend(std::iter::repeat(0xAB).take(prefix)); - seg.extend(std::iter::repeat(0u8).take(len - prefix)); - seg.extend(std::iter::repeat(0u8).take(1 << 20)); - - assert!( - matches!(decode_at(&seg, 0), Decoded::Record { .. }), - "a flag-clear (PAYLOAD_CHECKSUMMED unset) header skips payload-CRC \ - validation — decode-only behavior, unreachable in production" - ); - } #[test] fn encode_decode_roundtrip_and_torn() { diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index 511bfcf8ca..a9d5dde6ca 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -962,7 +962,6 @@ async fn e2e_recycled_first_segment_acked_records_survive_crash() { async fn e2e_stream_lanes_recover_acked_records() { let _guard = DurabilityGuard::wal(); crate::store::set_stream_lanes(3); - crate::wal::shard::set_checkpoint_syncfs(true); const SEG: u64 = 4096; let dir = tmp("stream-lanes"); @@ -1009,7 +1008,6 @@ async fn e2e_stream_lanes_recover_acked_records() { ); } h2.crash(); - crate::wal::shard::set_checkpoint_syncfs(cfg!(target_os = "linux")); // restore platform default crate::store::set_stream_lanes(1); let _ = std::fs::remove_dir_all(&dir); } @@ -1024,7 +1022,6 @@ async fn e2e_stream_lanes_recover_acked_records() { #[tokio::test] async fn e2e_checkpoint_syncfs_recovers_acked_records() { let _guard = DurabilityGuard::wal(); - crate::wal::shard::set_checkpoint_syncfs(true); const SEG: u64 = 4096; let dir = tmp("syncfs-ckpt"); @@ -1057,7 +1054,6 @@ async fn e2e_checkpoint_syncfs_recovers_acked_records() { "syncfs-checkpoint acked records recover byte-identical (durability-before-recycle held)" ); h2.crash(); - crate::wal::shard::set_checkpoint_syncfs(cfg!(target_os = "linux")); // restore platform default let _ = std::fs::remove_dir_all(&dir); } diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index ccb0576155..c5758f7fae 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -385,29 +385,14 @@ const CHECKPOINT_FILE: &str = "checkpoint"; // commit path. The syncfs barrier (default on Linux, below) replaces the loop // wholesale; the serial per-file loop remains only as the non-Linux fallback.) -/// Checkpoint durability strategy (cardinality-cliff #1). The per-stream -/// `fdatasync` loop (above) issues ONE device durability barrier per *touched* -/// stream — `O(N_touched)` syscalls, measured at ~1.4 s of `fsync` per shard at -/// 200k streams. Those barriers share the device's fixed `fdatasync`/s budget with -/// the commit path, so the checkpoint storm steals throughput from acks (the cliff) -/// AND caps wal throughput regardless of shard count (a shared-device sweep showed -/// s1≈s24). When this is on, step 2 instead issues ONE `syncfs()` per stream lane — -/// a filesystem-wide barrier that flushes every touched stream file (and everything -/// else dirty on that fs) at once, collapsing the per-stream `O(N_touched)` cost to -/// `O(lanes)` syscalls. The durability-before-recycle ordering is unchanged: the -/// `syncfs` completes before `persist_durable_tails`/recycle. -/// -/// **Default ON for Linux** (validated: +51% at the 100k-stream cliff, at worst -/// neutral below it — wal-syncfs/splitlane campaigns 2026-07); `--wal-checkpoint- -/// syncfs off` is the escape hatch. Non-Linux has no `syncfs` and always uses the -/// per-stream loop regardless of this flag. -static CHECKPOINT_SYNCFS: AtomicBool = AtomicBool::new(cfg!(target_os = "linux")); -pub fn set_checkpoint_syncfs(on: bool) { - CHECKPOINT_SYNCFS.store(on, Ordering::Relaxed); -} -fn checkpoint_syncfs() -> bool { - CHECKPOINT_SYNCFS.load(Ordering::Relaxed) -} +/// Checkpoint durability strategy: on Linux, step 2 issues ONE `syncfs()` per +/// stream lane — a filesystem-wide barrier that flushes every touched stream +/// file at once, `O(lanes)` syscalls instead of `O(N_touched)` per-stream +/// `fdatasync` (the cardinality cliff: the per-file loop measured ~1.4 s of +/// fsync per shard at 200k streams and stole the commit path's device budget). +/// Non-Linux has no `syncfs` and uses the serial per-file loop. The +/// durability-before-recycle ordering is identical in both modes: the barrier +/// completes before `persist_durable_tails`/recycle. /// Checkpoint cadence knobs (cardinality-cliff follow-up). The checkpoint's only /// job is bounding retained-WAL size (= crash-replay time): acks never gate on it @@ -901,7 +886,7 @@ impl Shard { // syncfs off) fallback. Serial on purpose: parallel fan-out // regressed in every controlled test (it steals device budget // from the commit path). - if checkpoint_syncfs() && cfg!(target_os = "linux") && n_touched > 0 { + if cfg!(target_os = "linux") && n_touched > 0 { crate::store::syncfs_stream_lanes(&touched[0].2)?; } else { for (_, _, f) in &touched { From d5589289d8edd1002ff00ab65aac952b754d8b31 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Mon, 13 Jul 2026 23:23:55 +0100 Subject: [PATCH 10/13] =?UTF-8?q?fix(durable-streams-rust):=20review=20fix?= =?UTF-8?q?es=20=E2=80=94=20restore=20Dockerfile,=20lane-count=20guard,=20?= =?UTF-8?q?panic-safe=20checkpoint=20ticker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review findings on #4697: - Restore packages/durable-streams-rust/Dockerfile (deleted collaterally by a bench image-build flow; docker_multiarch_image.yml builds from this path). - Persist the stream-lane count in streams/.lanes and REFUSE a mismatched --stream-lanes at open (mirrors the WAL shard count contract): a silent mismatch made every existing stream invisible. Also refuses lanes>1 over an existing flat layout; recovery skips the marker; e2e asserts the rejection. - Checkpoint ticker: map task-id -> shard so a PANICKED checkpoint task still clears its in-flight guard (previously a panic silenced that shard's checkpoints forever = unbounded retained WAL). - Drop unused AtomicBool import; mark TierKind::Local as the test-only backend (allow(dead_code)); merge the orphaned checkpoint doc comments; drop the removed-knob tombstone comment; whitespace nit. 105/105 tests pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/Dockerfile | 28 +++++++++++ packages/durable-streams-rust/src/main.rs | 18 ++++++- packages/durable-streams-rust/src/store.rs | 49 ++++++++++++++++++- packages/durable-streams-rust/src/tier.rs | 3 ++ .../durable-streams-rust/src/wal/e2e_tests.rs | 11 +++++ .../durable-streams-rust/src/wal/shard.rs | 30 ++++-------- 6 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 packages/durable-streams-rust/Dockerfile diff --git a/packages/durable-streams-rust/Dockerfile b/packages/durable-streams-rust/Dockerfile new file mode 100644 index 0000000000..0fb792ec68 --- /dev/null +++ b/packages/durable-streams-rust/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 +# +# Built multi-arch (linux/amd64 + linux/arm64) natively per arch by +# .github/workflows/docker_multiarch_image.yml — the build context is the repo +# root and this Dockerfile is `packages/durable-streams-rust/Dockerfile`. + +# ---- build stage: compile the release binary (glibc, matches the runtime) ---- +FROM rust:1-bookworm AS build +WORKDIR /app +# Copy only what the build needs (no target/, no npm/) so we don't depend on a +# .dockerignore at the shared repo root. +COPY packages/durable-streams-rust/Cargo.toml packages/durable-streams-rust/Cargo.lock ./ +COPY packages/durable-streams-rust/src ./src +# Default features only (no `tier`/`telemetry`) — minimal image, matching the +# conformance matrix. To ship S3 tiering, add `--features tier` here AND +# `ca-certificates` to the runtime stage. +RUN cargo build --release --locked + +# ---- runtime stage: distroless (glibc cc), no shell / package manager ---- +FROM gcr.io/distroless/cc-debian12 AS runtime +COPY --from=build /app/target/release/durable-streams-server /usr/local/bin/durable-streams-server +# Protocol default port (PROTOCOL.md §13.1); override with `--port`. +EXPOSE 4437 +ENTRYPOINT ["/usr/local/bin/durable-streams-server"] +# Bind all interfaces by default (the binary defaults to 127.0.0.1, unreachable +# from outside the container). Override by passing your own args. For persistence, +# mount a volume and add `--data-dir /your/path` (default is an ephemeral tmp dir). +CMD ["--host", "0.0.0.0"] diff --git a/packages/durable-streams-rust/src/main.rs b/packages/durable-streams-rust/src/main.rs index 4cd69a6c0e..839098308f 100644 --- a/packages/durable-streams-rust/src/main.rs +++ b/packages/durable-streams-rust/src/main.rs @@ -496,6 +496,11 @@ fn spawn_checkpoint_ticker(walset: Arc) { let mut last_done: Vec = vec![std::time::Instant::now(); n]; let mut in_flight: Vec = vec![false; n]; let mut wave: tokio::task::JoinSet = tokio::task::JoinSet::new(); + // task-id → shard index, so a PANICKED checkpoint task (JoinError carries + // no payload) still clears its shard's in-flight guard — otherwise one + // panic would silence that shard's checkpoints forever (unbounded WAL). + let mut task_shard: std::collections::HashMap = + std::collections::HashMap::new(); let mut ticker = tokio::time::interval(CHECKPOINT_POLL.min(interval)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); // Skip the immediate first tick — there is nothing to checkpoint at boot. @@ -504,7 +509,15 @@ fn spawn_checkpoint_ticker(walset: Arc) { ticker.tick().await; // Reap finished checkpoints (non-blocking) and restart their clocks. while let Some(done) = wave.try_join_next() { - if let Ok(i) = done { + let i = match &done { + Ok(i) => Some(*i), + Err(e) => { + eprintln!("WAL checkpoint task failed: {e}"); + task_shard.get(&e.id()).copied() + } + }; + if let Some(i) = i { + task_shard.retain(|_, v| *v != i); in_flight[i] = false; last_done[i] = std::time::Instant::now(); } @@ -520,12 +533,13 @@ fn spawn_checkpoint_ticker(walset: Arc) { } in_flight[i] = true; let shard = Arc::clone(shard); - wave.spawn(async move { + let handle = wave.spawn(async move { if let Err(e) = shard.checkpoint().await { eprintln!("WAL checkpoint failed for shard {:?}: {e}", shard.dir()); } i }); + task_shard.insert(handle.id(), i); } } }); diff --git a/packages/durable-streams-rust/src/store.rs b/packages/durable-streams-rust/src/store.rs index 1fc841618a..70cabde507 100644 --- a/packages/durable-streams-rust/src/store.rs +++ b/packages/durable-streams-rust/src/store.rs @@ -398,7 +398,6 @@ pub fn tail_cache_bytes() -> usize { TAIL_CACHE_BYTES.load(Ordering::Relaxed) } - impl StreamState { /// Record the just-appended wire chunk as the resident tail. `start` is the /// logical offset where `bytes` begins. Chunks larger than the tail-cache cap @@ -514,6 +513,50 @@ impl Store { ) -> std::io::Result { let streams_dir = data_dir.join("streams"); std::fs::create_dir_all(&streams_dir)?; + // Persist + validate the stream-lane count (mirrors the WAL shard count's + // persisted-N contract): opening a laned layout with a different + // `--stream-lanes` would make every existing stream silently invisible + // (recovery walks the wrong dirs). Refuse loudly instead. + { + let marker = streams_dir.join(".lanes"); + match std::fs::read_to_string(&marker) { + Ok(txt) => { + let on_disk: usize = txt.trim().parse().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("corrupt stream-lane marker {}", marker.display()), + ) + })?; + if on_disk != stream_lanes() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "--stream-lanes {} does not match this data dir's on-disk layout ({} lanes, recorded in {}). The lane count is a layout choice and must match across restarts.", + stream_lanes(), + on_disk, + marker.display() + ), + )); + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // Legacy pre-marker dirs are all lanes == 1: refuse enabling + // lanes over an existing flat layout (its files would vanish). + if stream_lanes() > 1 + && std::fs::read_dir(&streams_dir)? + .flatten() + .any(|e| e.path().is_file()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "--stream-lanes > 1 over an existing flat streams/ layout; this data dir was created with 1 lane", + )); + } + std::fs::write(&marker, format!("{}\n", stream_lanes()))?; + } + Err(e) => return Err(e), + } + } // Create every stream-lane dir and register their dir fds for the // checkpoint's per-lane syncfs barrier (see `syncfs_stream_lanes`). { @@ -575,6 +618,10 @@ impl Store { } for p in entries { let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if name == ".lanes" { + // stream-lane layout marker, not stream data + continue; + } if name.ends_with(".meta.tmp") { let _ = std::fs::remove_file(&p); } else if name.ends_with(".compact.tmp") { diff --git a/packages/durable-streams-rust/src/tier.rs b/packages/durable-streams-rust/src/tier.rs index 64328515ae..b7a08f3fd1 100644 --- a/packages/durable-streams-rust/src/tier.rs +++ b/packages/durable-streams-rust/src/tier.rs @@ -173,6 +173,9 @@ pub enum TierKind { Off, /// Local-filesystem BlobStore (a chunk directory) — for testing offload /// without S3. + /// Sealed segments to a local directory. Test-only backend (no CLI + /// surface): exercises the seal/offload/compaction machinery without S3. + #[allow(dead_code)] Local, /// S3-compatible object storage (feature `tier`). S3, diff --git a/packages/durable-streams-rust/src/wal/e2e_tests.rs b/packages/durable-streams-rust/src/wal/e2e_tests.rs index a9d5dde6ca..8b0c1511aa 100644 --- a/packages/durable-streams-rust/src/wal/e2e_tests.rs +++ b/packages/durable-streams-rust/src/wal/e2e_tests.rs @@ -1008,6 +1008,17 @@ async fn e2e_stream_lanes_recover_acked_records() { ); } h2.crash(); + // Layout-mismatch guard: reopening this 3-lane dir with a different lane + // count must be REFUSED (persisted `.lanes` marker) — a silent mismatch + // would make every existing stream invisible. + crate::store::set_stream_lanes(2); + let err = Store::new_with_tier(dir.clone(), TierConfig::default()) + .err() + .expect("opening a 3-lane layout with --stream-lanes 2 must fail"); + assert!( + err.to_string().contains("stream-lanes"), + "mismatch error should name the knob: {err}" + ); crate::store::set_stream_lanes(1); let _ = std::fs::remove_dir_all(&dir); } diff --git a/packages/durable-streams-rust/src/wal/shard.rs b/packages/durable-streams-rust/src/wal/shard.rs index c5758f7fae..eefbf3208d 100644 --- a/packages/durable-streams-rust/src/wal/shard.rs +++ b/packages/durable-streams-rust/src/wal/shard.rs @@ -33,7 +33,7 @@ use std::cmp::Ordering as CmpOrdering; use std::collections::{BTreeSet, BinaryHeap, HashMap}; use std::io; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use tokio::sync::oneshot; @@ -379,26 +379,14 @@ pub struct Shard { /// recyclable. const CHECKPOINT_FILE: &str = "checkpoint"; -// (Removed: FSYNC_FANOUT / --wal-fsync-parallel. Parallelizing the per-stream -// fdatasync loop regressed in every controlled test — f8 −11% on NVMe, f16 −19% -// on a CPU-constrained container — because it steals device budget/CPU from the -// commit path. The syncfs barrier (default on Linux, below) replaces the loop -// wholesale; the serial per-file loop remains only as the non-Linux fallback.) - -/// Checkpoint durability strategy: on Linux, step 2 issues ONE `syncfs()` per -/// stream lane — a filesystem-wide barrier that flushes every touched stream -/// file at once, `O(lanes)` syscalls instead of `O(N_touched)` per-stream -/// `fdatasync` (the cardinality cliff: the per-file loop measured ~1.4 s of -/// fsync per shard at 200k streams and stole the commit path's device budget). -/// Non-Linux has no `syncfs` and uses the serial per-file loop. The -/// durability-before-recycle ordering is identical in both modes: the barrier -/// completes before `persist_durable_tails`/recycle. - -/// Checkpoint cadence knobs (cardinality-cliff follow-up). The checkpoint's only -/// job is bounding retained-WAL size (= crash-replay time): acks never gate on it -/// (`shard.rs` "disk-bounded safety valve") and reads never touch the WAL, so -/// firing it less often is free except for disk space and replay budget. Two -/// triggers, whichever comes first per shard: +/// Checkpoint cadence knobs. The checkpoint's only job is bounding retained-WAL +/// size (= crash-replay time): acks never gate on it (the disk-bounded safety +/// valve) and reads never touch the WAL, so firing it less often is free except +/// for disk space and replay budget. (Durability strategy is not a knob: on +/// Linux, checkpoint step 2 issues one `syncfs` per stream lane; elsewhere it +/// runs the serial per-file `fdatasync` loop. Ordering is identical: the barrier +/// completes before `persist_durable_tails`/recycle.) Two triggers, whichever +/// comes first per shard: /// * time: `--wal-checkpoint-interval-ms` (default 3000 — the historical 3 s /// wave cadence, now per-shard). /// * size: `--wal-checkpoint-wal-bytes` (default 0 = disabled) — checkpoint a From 5b160919b9a4aec8e1e9314efb3a04c1b089006e Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 14 Jul 2026 10:01:28 +0100 Subject: [PATCH 11/13] =?UTF-8?q?fix(durable-streams-rust):=20CI=20green?= =?UTF-8?q?=20=E2=80=94=20clippy=20doc=20indent,=20prettier=20on=20docs,?= =?UTF-8?q?=20changeset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - srvstats module doc: 2-space list continuations (clippy doc_overindented_list_items, -D warnings on 1.97). - README/WAL_TUNING through prettier (repo-wide markdown format check). - .changeset/wal-cardinality-cliff.md: minor bump for @electric-ax/durable-streams-server-rust (the check counts only changesets ADDED vs main). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- .changeset/wal-cardinality-cliff.md | 13 ++++++++ packages/durable-streams-rust/README.md | 30 +++++++++---------- packages/durable-streams-rust/WAL_TUNING.md | 30 +++++++++---------- packages/durable-streams-rust/src/srvstats.rs | 14 ++++----- 4 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 .changeset/wal-cardinality-cliff.md diff --git a/.changeset/wal-cardinality-cliff.md b/.changeset/wal-cardinality-cliff.md new file mode 100644 index 0000000000..535ceb3688 --- /dev/null +++ b/.changeset/wal-cardinality-cliff.md @@ -0,0 +1,13 @@ +--- +"@electric-ax/durable-streams-server-rust": minor +--- + +Eliminate the WAL write cardinality cliff (10.4k → 383k appends/s @100k streams; 212k @1M). + +- Checkpoint durability now uses one `syncfs` barrier per stream lane on Linux (was O(touched-streams) per-file `fdatasync` — the barrier storm that collapsed throughput at high stream counts). +- New `--wal-checkpoint-interval-ms` (per-shard time trigger, default 3000) and `--wal-checkpoint-wal-bytes` (retained-WAL size budget, 0 = off): checkpoint cadence is an explicit crash-replay budget and shards self-stagger instead of storming together. +- New `--stream-lanes N` (default 1 = unchanged layout): hash stream data files across `streams/<0..N>/` dirs, one per device, spreading checkpoint writeback over N devices with N parallel barriers. The lane count is persisted and validated on open. +- New `--server-stats N` telemetry (`SRV_STATS`: cpu / inflight / service / lock / durability-wait per interval) — the dependency-free bottleneck diagnostics used to find all of the above. Memory-mode plain appends no longer queue redundant sidecar flushes. +- Removed dead/diagnostic flags: `--wal-fsync-parallel`, `--wal-meta-gate`, `--mem-meta-gate`, `--meta-sweep-disable`, `--meta-sweep-stats`, `--tier local` / `--tier-local-dir` (tier is `off|s3`). `--durability memory` combined with `--tier` is now rejected at startup. WAL records without `PAYLOAD_CHECKSUMMED` decode as torn (no released writer ever emitted them). + +Deployment guidance (device layout, CPU pinning, checkpoint budgets): `WAL_TUNING.md`. diff --git a/packages/durable-streams-rust/README.md b/packages/durable-streams-rust/README.md index 2a2fe0c98d..49a005ee38 100644 --- a/packages/durable-streams-rust/README.md +++ b/packages/durable-streams-rust/README.md @@ -87,28 +87,28 @@ durable, single-node server on `127.0.0.1:4437` with its data dir under `$TMPDIR **Cold-storage tier** — off by default; see [Tiered storage](#tiered-storage-cold-offload). With `--tier off` the server is byte-identical to a single-file deployment. -| Flag | Default | Description | -| --------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------- | -| `--tier` | `off` | `off` \| `s3` (S3-compatible object storage) | -| `--tier-endpoint` | — | (`tier=s3`) S3 endpoint URL | -| `--tier-region` | — | (`tier=s3`) region | -| `--tier-bucket` | — | (`tier=s3`) bucket name | -| `--tier-key-prefix` | — | object-key prefix for sealed segments | -| `--tier-segment-bytes` | `8 MiB` | sealed-segment size (fixed-size, CDN-friendly) | -| `--tier-compact-bytes` | `64 MiB` | small-segment compaction threshold | -| `--tier-path-style` / `--tier-virtual-hosted` | path-style | S3 addressing style | -| `--tier-allow-http` | off | allow plain HTTP to the S3 endpoint (e.g. a local MinIO) | +| Flag | Default | Description | +| --------------------------------------------- | ---------- | -------------------------------------------------------- | +| `--tier` | `off` | `off` \| `s3` (S3-compatible object storage) | +| `--tier-endpoint` | — | (`tier=s3`) S3 endpoint URL | +| `--tier-region` | — | (`tier=s3`) region | +| `--tier-bucket` | — | (`tier=s3`) bucket name | +| `--tier-key-prefix` | — | object-key prefix for sealed segments | +| `--tier-segment-bytes` | `8 MiB` | sealed-segment size (fixed-size, CDN-friendly) | +| `--tier-compact-bytes` | `64 MiB` | small-segment compaction threshold | +| `--tier-path-style` / `--tier-virtual-hosted` | path-style | S3 addressing style | +| `--tier-allow-http` | off | allow plain HTTP to the S3 endpoint (e.g. a local MinIO) | S3 credentials come from the **environment**, never flags: `DS_S3_ACCESS_KEY_ID` / `DS_S3_SECRET_ACCESS_KEY` (or the standard `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`). ### Choosing a configuration -| Your situation | Use | -| --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Your situation | Use | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------- | | **Bounded local disk with long history** | `--tier s3` — seal cold segments to object storage; the recent tail stays local and zero-copy. | -| **High fan-out p99 across many streams on Linux** | try `--tail-cache-bytes 65536` (off by default there because `sendfile` already covers it). | -| **Experimenting on macOS and writes take ~2–10 ms** | expected — see [macOS write latency](#macos-write-latency-f_fullfsync) below. | +| **High fan-out p99 across many streams on Linux** | try `--tail-cache-bytes 65536` (off by default there because `sendfile` already covers it). | +| **Experimenting on macOS and writes take ~2–10 ms** | expected — see [macOS write latency](#macos-write-latency-f_fullfsync) below. | ### macOS write latency (`F_FULLFSYNC`) diff --git a/packages/durable-streams-rust/WAL_TUNING.md b/packages/durable-streams-rust/WAL_TUNING.md index e61a73ccca..cecd158a42 100644 --- a/packages/durable-streams-rust/WAL_TUNING.md +++ b/packages/durable-streams-rust/WAL_TUNING.md @@ -9,16 +9,16 @@ is flat from 10k → 100k streams (−5%) at ~26–32× the pre-fix baseline. ## The ladder (what each step bought, @100k streams) -| configuration | ops/s | -|---|---| -| pre-fix: streams on network PD, per-stream checkpoint fdatasync storm | 10.4k | -| + syncfs checkpoint barrier (PR #4697) | 13.6k | -| + stream files on local NVMe (not the boot disk) | 46k | -| + split-lane layout (WAL shards on their own NVMe devices) | 272k | -| + size-triggered checkpoint (PR #4704, `--wal-checkpoint-wal-bytes 1GiB`) | 303k | -| + exclusive pinned cores (Guaranteed QoS + static CPU manager) | 328k | -| **all of the above stacked** (suite `wal-stacked-1m`) | **383k** | -| reference: memory durability (no fsync anywhere) | 512k | +| configuration | ops/s | +| ------------------------------------------------------------------------- | -------- | +| pre-fix: streams on network PD, per-stream checkpoint fdatasync storm | 10.4k | +| + syncfs checkpoint barrier (PR #4697) | 13.6k | +| + stream files on local NVMe (not the boot disk) | 46k | +| + split-lane layout (WAL shards on their own NVMe devices) | 272k | +| + size-triggered checkpoint (PR #4704, `--wal-checkpoint-wal-bytes 1GiB`) | 303k | +| + exclusive pinned cores (Guaranteed QoS + static CPU manager) | 328k | +| **all of the above stacked** (suite `wal-stacked-1m`) | **383k** | +| reference: memory durability (no fsync anywhere) | 512k | At extreme cardinality the stacked config on ONE data lane hits a second wall — checkpoint writeback (~40× metadata amplification of small appends) saturates @@ -41,13 +41,13 @@ barrier). Then: - **One device for stream data files** — mount it and point `--data-dir` at it. The per-stream files and the checkpoint's `syncfs` domain live here. -- **One device per WAL shard** — mount device *j* at `/wal/` (the - server opens shard *i* at that path automatically). `--wal-shards` = number of +- **One device per WAL shard** — mount device _j_ at `/wal/` (the + server opens shard _i_ at that path automatically). `--wal-shards` = number of dedicated WAL devices. - **Never share a device between WAL and stream data.** Commit `fdatasync` vs checkpoint writeback contention on one queue was worth 5× by itself (55k → 272k). On dedicated lanes the commit-fsync cost is ~zero (checkpoint-off - measured *below* an all-lanes-shared no-fsync control). + measured _below_ an all-lanes-shared no-fsync control). - **Never leave stream data on the boot disk / network PD.** On Kubernetes raw-block node pools the default emptyDir sits on the boot PD — this single mistake mismeasures (and misdeploys) WAL mode by 5–26×. @@ -93,7 +93,7 @@ no longer fsync-bound, it scales with cores again — don't starve it. - **Read performance while tuning checkpoints.** Reads never touch the WAL and are served zero-copy (`sendfile`) from the data file's page cache, which is - written *before* the WAL ack barrier. Checkpoint cadence has zero read-path + written _before_ the WAL ack barrier. Checkpoint cadence has zero read-path cost. - **Ack latency vs checkpoints.** Acks gate only on the WAL group-commit `fdatasync`; a checkpoint never blocks appends. @@ -101,7 +101,7 @@ no longer fsync-bound, it scales with cores again — don't starve it. segment is recycled only after its records' stream bytes are fsynced into their files **and** the durable-tail map is persisted (`wal/shard.rs` checkpoint ordering; crash-recovery e2e + randomized crash sim cover it). - The size trigger only changes *when* that sequence runs. + The size trigger only changes _when_ that sequence runs. ## Caveats / follow-ups diff --git a/packages/durable-streams-rust/src/srvstats.rs b/packages/durable-streams-rust/src/srvstats.rs index aee1a396e2..7014e8da59 100644 --- a/packages/durable-streams-rust/src/srvstats.rs +++ b/packages/durable-streams-rust/src/srvstats.rs @@ -9,13 +9,13 @@ //! accumulators. //! //! Fields: -//! - `cpu_cores` process CPU utilization in cores (utime+stime delta / wall); -//! ≈ the cgroup cpu quota ⇒ CPU-bound. Linux only (`-1` elsewhere). -//! - `appends_s` acked appends/sec over the interval. -//! - `inflight` in-flight append handlers sampled at tick time (queue depth). -//! - `svc_us` mean append handler wall time (service time). -//! - `applock_us` mean time waiting to acquire the per-stream appender lock. -//! - `durwait_us` mean time in `wait_durable_lsn` (WAL fsync wait; ~0 in memory). +//! - `cpu_cores` process CPU utilization in cores (utime+stime delta / wall); +//! ≈ the cgroup cpu quota ⇒ CPU-bound. Linux only (`-1` elsewhere). +//! - `appends_s` acked appends/sec over the interval. +//! - `inflight` in-flight append handlers sampled at tick time (queue depth). +//! - `svc_us` mean append handler wall time (service time). +//! - `applock_us` mean time waiting to acquire the per-stream appender lock. +//! - `durwait_us` mean time in `wait_durable_lsn` (WAL fsync wait; ~0 in memory). use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::time::Instant; From a445330df4f80f4bdb3368b57cc544f9745860ad Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 14 Jul 2026 10:11:20 +0100 Subject: [PATCH 12/13] chore: changeset bump minor -> patch Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- .changeset/wal-cardinality-cliff.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/wal-cardinality-cliff.md b/.changeset/wal-cardinality-cliff.md index 535ceb3688..d38c7b7956 100644 --- a/.changeset/wal-cardinality-cliff.md +++ b/.changeset/wal-cardinality-cliff.md @@ -1,5 +1,5 @@ --- -"@electric-ax/durable-streams-server-rust": minor +"@electric-ax/durable-streams-server-rust": patch --- Eliminate the WAL write cardinality cliff (10.4k → 383k appends/s @100k streams; 212k @1M). From f672418a776033a7a934abfbcb737e1fbfd75d86 Mon Sep 17 00:00:00 2001 From: Valter Balegas Date: Tue, 14 Jul 2026 11:48:11 +0100 Subject: [PATCH 13/13] fix(durable-streams-rust): long-poll timeout arm must not skip observed data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the flaky conformance test 'should wait for new data with long-poll' (and a latent production data-skip present on main): when a long-poll hits its deadline, the sleep arm read the FRESH tail and returned an empty 204 advertising it as Stream-Next-Offset — without delivering the bytes behind it. An append whose durable-tail publish lands inside the deadline window is then jumped over by the client (which adopts the header offset from every response) and never delivered; with the conformance suite's 500ms sleep aligned to its 500ms --long-poll-timeout-ms, every run of that test sits on this boundary. The wake path itself is race-free (canonical watch subscribe/re-check); only the timeout arm violated the invariant. Fix: the sleep arm now re-checks the tail exactly like the closed-channel arm and serves the data when tail > from. Invariant: a long-poll response never advances the client's offset beyond the bytes it delivered. Regression test drives 200 appends racing the deadline (20ms window) and asserts the invariant on every response; it reproduces the bug in <100 iterations without the fix. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y3x7bcT9vLGiT4tXZeQpnk --- packages/durable-streams-rust/src/handlers.rs | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/packages/durable-streams-rust/src/handlers.rs b/packages/durable-streams-rust/src/handlers.rs index 74095040b7..29d1dc974e 100644 --- a/packages/durable-streams-rust/src/handlers.rs +++ b/packages/durable-streams-rust/src/handlers.rs @@ -1636,7 +1636,19 @@ async fn handle_long_poll( } } _ = tokio::time::sleep(deadline.saturating_duration_since(Instant::now())) => { + // Deadline hit — but re-check the tail EXACTLY like the + // closed-channel arm above. Returning a timeout that advertises + // the fresh tail as `Stream-Next-Offset` while NOT delivering + // the bytes behind it silently SKIPS data: an append whose + // durable-tail publish lands inside the deadline window gets + // jumped over (the client adopts the header offset from every + // response, including empty 204s) and is never delivered. + // Invariant: a long-poll response never advances the client's + // offset beyond the bytes it actually delivered. let t = st.tail(); + if t.bytes > from { + return long_poll_data(&st, from, t, client_cursor, true, cache_hit).await; + } return long_poll_timeout(t.bytes, cursor, t.closed); } } @@ -2381,5 +2393,84 @@ mod memory_mode_tests { let _ = std::fs::remove_dir_all(&dir); } + + /// Long-poll deadline/data race (conformance flake root cause): a long-poll + /// response must NEVER advance `Stream-Next-Offset` past the client's `from` + /// without delivering the bytes in between — otherwise an append whose + /// durable-tail publish lands inside the deadline window is advertised but + /// not sent, and the client skips it forever. This drives many iterations of + /// an append racing a long-poll whose deadline is aligned with the append + /// (the conformance suite's exact shape, tightened) and asserts the + /// invariant on every response. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn long_poll_timeout_never_skips_observed_data() { + let _guard = crate::handlers::test_support::DurabilityGuard::memory(); + crate::handlers::set_long_poll_timeout(20); + let dir = tmp("lp-race"); + let store = Arc::new(Store::new_with_tier(dir.clone(), TierConfig::default()).unwrap()); + + let resp = handle(Arc::clone(&store), put_req("lp/r", "application/octet-stream")).await; + assert!((200..300).contains(&resp.status), "create: {}", resp.status); + + let next_offset = |r: &crate::api::Resp| -> u64 { + r.headers + .iter() + .find(|(k, _)| *k == "stream-next-offset") + .map(|(_, v)| v.rsplit('_').next().unwrap().parse().unwrap()) + .unwrap_or(0) + }; + let mut from: u64 = 0; + for i in 0..200u32 { + // Long-poll from the current offset... + let lp = tokio::spawn(handle( + Arc::clone(&store), + Req { + method: Method::Get, + path: "lp/r".to_string(), + query: Some(format!("live=long-poll&offset={:016}_{:016}", 0, from)), + headers: vec![], + body: Bytes::new(), + }, + )); + // ...and race an append onto the deadline (spread over the window). + tokio::time::sleep(std::time::Duration::from_millis(15 + (i % 10) as u64)) + .await; + let ar = handle( + Arc::clone(&store), + post_req("lp/r", "application/octet-stream", b"x"), + ) + .await; + assert!((200..300).contains(&ar.status), "append: {}", ar.status); + let r = lp.await.unwrap(); + let next = next_offset(&r); + let has_body = !matches!(r.body, crate::api::Body::Empty); + assert!( + has_body || next <= from, + "iteration {i}: empty long-poll response advanced the offset \ + {from} -> {next} without delivering data (status {})", + r.status + ); + // Catch up for the next round. + from = from.max(next); + if !has_body && next == from { + // ensure we don't fall behind the appends + let t_resp = handle( + Arc::clone(&store), + Req { + method: Method::Get, + path: "lp/r".to_string(), + query: Some(format!("offset={:016}_{:016}", 0, from)), + headers: vec![], + body: Bytes::new(), + }, + ) + .await; + from = from.max(next_offset(&t_resp)); + } + } + + crate::handlers::set_long_poll_timeout(30_000); + let _ = std::fs::remove_dir_all(&dir); + } }