diff --git a/Cargo.lock b/Cargo.lock index fc2c0b54..4f40c6f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1308,6 +1308,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1596,6 +1605,29 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.98", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -2469,7 +2501,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02df3085f97750c1f8deb1b56aeb168f242f303e363aee16284821c0a14ff90e" dependencies = [ - "convert_case", + "convert_case 0.6.0", "litrs", ] @@ -2684,6 +2716,22 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "libmdbx" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1da773dcce45661428e2b51ca984eb3c64c08f2aa54865397e4b77d1f61c9f07" +dependencies = [ + "bitflags 2.8.0", + "derive_more", + "indexmap", + "libc", + "mdbx-sys", + "parking_lot", + "sealed", + "thiserror 2.0.12", +] + [[package]] name = "librocksdb-sys" version = "0.17.3+10.4.2" @@ -2810,6 +2858,30 @@ dependencies = [ "digest", ] +[[package]] +name = "mdbx-spike" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "libmdbx", + "lz4_flex", + "rocksdb", + "sqd-storage", + "tempfile", +] + +[[package]] +name = "mdbx-sys" +version = "13.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4002b9ffed3dd01364ab005bca653a99b8f550ca4cd755470a9d3a31a44f466" +dependencies = [ + "bindgen", + "cc", + "libc", +] + [[package]] name = "memchr" version = "2.7.4" @@ -4490,6 +4562,17 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "sealed" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + [[package]] name = "sec1" version = "0.3.0" @@ -4986,7 +5069,7 @@ dependencies = [ "anyhow", "arrow", "bytes", - "convert_case", + "convert_case 0.6.0", "dashmap 6.1.0", "divan", "faster-hex", @@ -5676,6 +5759,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 97d53d0c..138ff970 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/hotblocks", "crates/hotblocks-harness", "crates/hotblocks-retain", + "crates/mdbx-spike", "crates/polars", "crates/primitives", "crates/query", diff --git a/Dockerfile b/Dockerfile index 48870e4f..265ea5b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # syntax=docker/dockerfile:1 -FROM rust:1.89-bookworm AS rust +# rust-toolchain.toml is not copied into the build context, so this base image +# version IS the container toolchain — keep it in lockstep with the pin there. +FROM rust:1.94-bookworm AS rust FROM rust AS builder @@ -81,6 +83,23 @@ COPY --from=flush-bench-builder /out/flush_spill . ENTRYPOINT ["/app/flush_spill"] +FROM builder AS mdbx-spike-builder +ARG TARGETARCH +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETARCH},sharing=locked \ + cargo build -p mdbx-spike --release \ + && mkdir -p /out \ + && cp target/release/mdbx-spike /out/ + + +# ADR 0002 churn bench; run matrix in crates/mdbx-spike/README.md +FROM debian:bookworm-slim AS mdbx-spike +WORKDIR /app +COPY --from=mdbx-spike-builder /out/mdbx-spike . +ENTRYPOINT ["/app/mdbx-spike"] + + FROM builder AS archive-builder ARG TARGETARCH RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ diff --git a/crates/hotblocks/spec/02-data-model.md b/crates/hotblocks/spec/02-data-model.md index a65a8652..067593ba 100644 --- a/crates/hotblocks/spec/02-data-model.md +++ b/crates/hotblocks/spec/02-data-model.md @@ -121,6 +121,11 @@ prefix**; blocks above it are the **volatile suffix**, subject to replacement by | `External` | The lower bound is set at runtime by the retention controller via the SET-RETENTION operation. Until first set: unbounded, and a dataset that is *empty* at activation defers ingestion until the first instruction (WP-5). | | `Unbounded` | Never trim. | +All four policies are additionally subject to the space bounds of RS-13 (09 §1): +self-healing policies (`Window`, `External`) may be trimmed past their bound under +space pressure — observably; promise policies (`Pinned`, `Unbounded`) are never +trimmed by space and pause at their quota instead (FM-STOR-6). + **DEF-10 (Version order).** For one dataset, committed states are totally ordered by `ver`. "Later state" always means greater `ver`, never a comparison of head numbers (a fork can lower the head number while increasing `ver`). diff --git a/crates/hotblocks/spec/03-write-path.md b/crates/hotblocks/spec/03-write-path.md index 89580957..be50b0e3 100644 --- a/crates/hotblocks/spec/03-write-path.md +++ b/crates/hotblocks/spec/03-write-path.md @@ -179,6 +179,11 @@ history cannot be re-acquired through RETAIN. re-acquired in place (there is no downward backfill), so in self-healing modes (`Window`, `External` runtime application) the instruction MUST be executed as `RESET(⟨from − 1, h?⟩)`: the window is discarded and re-ingested from `from` upward. + Exception (RS-13): while a space bound governs the dataset (gap-mode) — and, where a + position cap is configured, unconditionally — the instruction is clamped to `first(D)` + instead — observable, never silent — because the RESET would re-bootstrap in a loop + against the very consumer lag that opened the gap (on a capped dataset a downward + instruction signals that lag even when the cap is not the governing bound). Like every RESET this MUST be observable (OB-9) — a downward `SET-RETENTION` is a destructive re-bootstrap, not a widening, and controllers MUST treat it as such (FM-OP-4). During boot validation of a `Pinned` policy whose `from` lies below the @@ -206,14 +211,18 @@ history cannot be re-acquired through RETAIN. the availability floor RS-3 (at least the last `k` blocks) always holds and the excess bound RS-4 (`first(D) ≥ next(D) − k − P-RETENTION-SLACK`, eventually) is met. Trimming MAY be - batch-granular. + batch-granular. RS-13's effective bound MAY trim beyond `k`'s floor under space + pressure — the sanctioned, alarmed RS-3 exception. - **WP-11 (External application).** A SET-RETENTION instruction accepted through the API MUST be applied (the corresponding RETAIN committed) within `P-RETENTION-APPLY`, and be observable via the retention read afterwards. Acceptance and application are distinct points: the acknowledgement means the instruction is recorded and scheduled, not that the trim has happened. Between the two, GET-RETENTION reports the *instructed* bound — an acknowledged instruction is never silently forgotten (CN-9; violated today, GAP-28) — - and `ver` advances with the RETAIN/RESET commit itself, not at acceptance. What + and `ver` advances with the RETAIN/RESET commit itself, not at acceptance. Application + is subject to RS-13: while a space bound governs — or whenever a position cap is + configured — a downward instruction is clamped: recorded and observable, applied as + `first(D)` (§2.5 exception). What instruction payloads other than a block bound mean for an External dataset (policy-mode changes, e.g. the binding's `"None"`) is unspecified, and the current behavior diverges from the binding's reading (GAP-35). diff --git a/crates/hotblocks/spec/06-invariants.md b/crates/hotblocks/spec/06-invariants.md index 32ff3ed7..8ddc8713 100644 --- a/crates/hotblocks/spec/06-invariants.md +++ b/crates/hotblocks/spec/06-invariants.md @@ -243,7 +243,8 @@ paths). *Check:* CT-5 boot matrix (config × pre-state). **INV-44 — Destructive operations are explicit.** [transition] -Data leaves the store only through: RETAIN per policy, REPLACE of the volatile suffix, +Data leaves the store only through: RETAIN per policy (the policy bound composed with +RS-13's space bound), REPLACE of the volatile suffix, RESET under its defined triggers (WP-6b / WP-9 / downward retention bound, WP §2.5 / operator), or DROP of datasets removed from configuration. Each destructive path is deliberate, documented, and observable (OB-9). There is no other code path by which committed blocks disappear. One non-transition diff --git a/crates/hotblocks/spec/07-liveness.md b/crates/hotblocks/spec/07-liveness.md index a67dcfcb..a9eb1efa 100644 --- a/crates/hotblocks/spec/07-liveness.md +++ b/crates/hotblocks/spec/07-liveness.md @@ -29,7 +29,9 @@ service catches up at a rate ≥ `P-CATCHUP-RATE` until steady lag is reached. Under the LIV-1 preconditions, the interval during which a dataset makes **zero** commit progress while its sources offer new data never exceeds `P-STALL-BUDGET`. This bound covers all internal causes: storage backpressure, maintenance debt, shared-pool -contention, deferred-deletion churn. +contention, deferred-deletion churn. One documented exemption: a dataset paused at its +own disk quota (FM-STOR-6) — alarmed, per-dataset, and excluded from this budget for +that dataset only. *Why this exists:* multi-minute all-dataset freezes have been observed post-deploy; this property is the formal target the stall harness must enforce (GAP-1). *Witness:* OB-3 stall detector. *Tests:* CT-7 soak + CT-6 stress. @@ -62,8 +64,11 @@ alarm (CN-10) without blocking the rest. **LIV-7 — Reclamation liveness.** After logical deletions (retention, forks, drops), physical space converges: within -`P-RECLAIM-LAG`, disk usage returns to within the amplification bound RS-6. Deletion debt -(logically deleted but physically present data) is eventually zero in a quiescent system. +`P-RECLAIM-LAG`, disk usage returns to within the amplification bound RS-6 — in its +ratchet reading (RS-6b): deletion debt converges; a high-water file does not shrink on +its own and is excluded, its return path being the dataset storage reset (RS-14). +Deletion debt (logically deleted but physically present data) is eventually zero in a +quiescent system. *Witness:* OB-6 space accounting. *Tests:* CT-7 (GAP-6). **LIV-8 — No cross-dataset starvation.** diff --git a/crates/hotblocks/spec/08-failure-model.md b/crates/hotblocks/spec/08-failure-model.md index 00d63c91..ba2f3013 100644 --- a/crates/hotblocks/spec/08-failure-model.md +++ b/crates/hotblocks/spec/08-failure-model.md @@ -42,10 +42,11 @@ each. "Required response" uses four verbs: | # | Fault | Required response | |---|---|---| | FM-STOR-1 | Slow storage (I/O saturation, maintenance debt) | degrade: writes throttle; bounded by LIV-2 stall budget; reads keep serving; OB-3 signals pressure | -| FM-STOR-2 | Disk approaching full | alarm at `P-DISK-FLOOR`; degrade per documented policy: reads MUST keep working; writes MAY pause (counts toward stall accounting, exempt from LIV-2 only while below floor); the system MUST NOT corrupt state and MUST provide a documented, bounded recovery path (RS-8 boot maintenance) that does not require scratch space proportional to reclaimable data | +| FM-STOR-2 | Disk approaching full (node level) | alarm at `P-DISK-FLOOR`; degrade per RS-13: self-healing datasets ring past their floors (gap-mode), promise datasets keep data and MAY pause writes; reads MUST keep working; the system MUST NOT corrupt state and MUST provide a documented, bounded recovery path (RS-14 dataset reset; RS-8 at boot) that does not require scratch space proportional to reclaimable data | | FM-STOR-3 | Disk full (hard) | fail-safe: no committed-state corruption (INV-40 holds for the pre-full history); recovery path documented; startup after freeing space restores service | | FM-STOR-4 | Detected corruption of stored state (checksum/decode failure) | fail-safe + alarm per dataset (CN-10): the damaged dataset stops, others serve; corruption MUST be detected (never returned as data), and MUST NOT silently disable global functions (e.g. reclamation for everyone) without alarm (GAP-6 adjacent) | | FM-STOR-5 | Partial write persisted at crash (torn build) | mask: invisible by CN-2/INV-40; residue collected (RS-10) | +| FM-STOR-6 | Per-dataset quota exhausted (`P-DISK-QUOTA`) | degrade per RS-13: self-healing policies trim to the effective bound (gap-mode, alarmed); promise policies — or any dataset at the `P-REORG-KEEP` ring floor, or one whose emergency trim itself cannot commit (deletes need COW pages at a full map) — pause writes (alarmed, LIV-2-exempt for that dataset), reads keep serving, other datasets unaffected (FM-3, INV-35/36); recovery: operator action or RS-14 reset | ## 3. Process faults (FM-PROC) @@ -82,5 +83,5 @@ each. "Required response" uses four verbs: - Crash matrix: FM-PROC-* ⇢ INV-40/42, LIV-5/6, CT-2. - Source misbehavior corpus: FM-SRC-* ⇢ INV-7/12/13/14/23, LIV-1/9, CT-4. - Overload & client abuse: FM-CLI-* ⇢ RP-3/17/18, LIV-3/4/10, CT-6/CT-9. -- Storage pressure: FM-STOR-* ⇢ INV-41, LIV-2/7, RS-6/8, CT-7. +- Storage pressure: FM-STOR-* ⇢ INV-41, LIV-2/7, RS-6/8/13/14, CT-7/CT-8. - Operator: FM-OP-* ⇢ INV-43/44, CT-5 boot matrix. diff --git a/crates/hotblocks/spec/09-retention-and-space.md b/crates/hotblocks/spec/09-retention-and-space.md index d32121a3..a98d0722 100644 --- a/crates/hotblocks/spec/09-retention-and-space.md +++ b/crates/hotblocks/spec/09-retention-and-space.md @@ -10,20 +10,66 @@ between them. | Policy | Guarantee | Trim trigger | |---|---|---| -| `Window(k)` | availability floor RS-3 + excess bound RS-4 | automatic, after commits that advance `next(D)` | -| `Pinned(from, h?)` | everything ≥ `from` kept; anchor asserted at boot (WP-9 refusal on mismatch) | only when `from` is raised by reconfiguration | -| `External` | everything ≥ last instructed bound kept; unbounded until first instruction; a *downward* instruction is a destructive re-bootstrap (RESET, WP §2.5) | SET-RETENTION (WP-11) | -| `Unbounded` | nothing trimmed | never | +| `Window(k)` | availability floor RS-3 + excess bound RS-4 (space exception: RS-13) | automatic, after commits that advance `next(D)` | +| `Pinned(from, h?)` | everything ≥ `from` kept; anchor asserted at boot (WP-9 refusal on mismatch); never trimmed by space — pauses instead (RS-13) | only when `from` is raised by reconfiguration | +| `External` | everything ≥ last instructed bound kept, within the space bounds of RS-13; unbounded until first instruction; a *downward* instruction is a destructive re-bootstrap (RESET, WP §2.5) — clamped instead of executed while a space bound governs or a position cap is configured (RS-13) | SET-RETENTION (WP-11); position cap / space watermark (RS-13) | +| `Unbounded` | nothing trimmed; never trimmed by space — pauses instead (RS-13) | never | - **RS-2 (Retention dominates finality).** Trimming ignores `fin`: finalized blocks below the retention bound are deleted, and `fin` becomes `⊥` when the window passes above it (03 §2.5). Rationale: this is a bounded hot store, not an archive (NG1). Consequence for clients: "finalized" means *irreversible while retained*, not *retained forever* (INV-24 note). +- **RS-13 (Space dominates retention).** RS-2's bounded-store doctrine, one + level up: a dataset's space budget outranks its retention promise — for the + policies that can afford it. Policies split into two classes: + - *Self-healing* (`Window`, `External`): trimmed data is re-acquirable — it + exists (or will) in the archival tier, and clients below the cut re-anchor + upward (RP-4). For these the **effective trim bound** is the maximum of + the policy bound (RS-3/WP-10, or the WP-11 instructed bound), the position + cap `next(D) − P-MAX-BLOCKS` (`External` only, where configured), and the + byte bound derived from the dataset's quota watermark + (`P-DISK-WATERMARK × P-DISK-QUOTA`, compared against *occupied* storage — + allocated minus reusable pages — never against the file, which under a + high-water engine never shrinks (RS-6b) and would pin gap-mode forever + after one peak). A trim to the effective bound is an + ordinary RETAIN — INV-15/18 hold, the anchor hash is carried. While a + space bound governs (effective > policy bound) the dataset is in + **gap-mode**: onset, exit, and the gap width MUST be observable + (OB-6/OB-9). In gap-mode a retention instruction below `first(D)` is + clamped to `first(D)` instead of executing the WP §2.5 downward RESET — + honoring it would re-bootstrap in a loop against the very consumer lag + that opened the gap; the clamp MUST be observable. Where a position cap + is *configured*, the clamp applies even outside gap-mode: the cap is a + standing election of freshness over depth, and the instructed floor + tracks downstream coverage, so an instruction below `first(D)` signals + the same consumer lag whether or not the cap is the bound currently + governing (shipped semantics — PR #77's `clamp_floor` keys on the cap's + presence, not its governance). The byte bound clamps only while it + governs: under a quota-per-dataset regime every dataset carries + `P-DISK-QUOTA`, and quota-configured must not mean the WP §2.5 downward + RESET is unreachable. Instructions at or above `first(D)` apply + normally. + - *Promises* (`Pinned`, `Unbounded`): space never triggers a trim. At the + watermark the dataset alarms (OB-9); at the quota its writes pause + (FM-STOR-6) — reads keep serving, other datasets are unaffected + (INV-35/36), and recovery is an operator action (raise the quota, raise + the policy bound, or DROP). This is WP-9/INV-43's doctrine at runtime: + pinned history is never destroyed silently. + + Ring floor, both classes: the effective bound never rises into the last + `P-REORG-KEEP` positions below `next(D)` — trimming closer would strip the + reorg-absorption depth INV-14 relies on. A self-healing dataset whose quota + cannot hold even that span degrades like a promise (pause + alarm) rather + than trim further; a *configured* quota that cannot hold + `P-REORG-KEEP + P-RETENTION-SLACK` worth of data is a boot-time refusal + (INV-43), not a runtime surprise. - **RS-3 (Availability floor).** For `Window(k)`: at every committed state, all blocks in `[next(D) − k, next(D) − 1] ∩ [window start after initial fill, ∞)` are present and queryable (an interval of *positions* — DEF-9; on a slot-numbered chain it - holds ≤ `k` blocks). Trimming MUST err on the side of keeping more, never less. + holds ≤ `k` blocks). Trimming MUST err on the side of keeping more, never less. The + single sanctioned exception is RS-13's space bound — alarmed and observable, never + silent. - **RS-4 (Excess bound).** For `Window(k)`: eventually (once steady-state is reached and within `P-RETENTION-APPLY` of each trigger), `first(D) ≥ next(D) − k − P-RETENTION-SLACK`. Slack exists because trimming may be batch-granular; it is bounded, not best-effort. @@ -47,10 +93,22 @@ Requirements: immediate and cheap; physical reclamation is asynchronous. Between the two, deleted data is `debt_bytes` — invisible to all reads (INV-41 keeps live readers safe via versioning, not via keeping data visible). -- **RS-6 (Amplification bound).** In steady state, - `disk_bytes ≤ P-SPACE-AMP × live_bytes + P-SPACE-CONST`. This MUST hold under - continuous churn (window datasets trim continuously — churn IS the steady state). - Unbounded `debt_bytes` growth under any configuration is a defect. (GAP-6 history: +- **RS-6 (Amplification bound).** Two bounds, per dataset where the storage layout + permits attribution: + (a) *hard quota* — where `P-DISK-QUOTA` is configured, the dataset's `disk_bytes` + MUST NOT exceed it, ever; this is the bound RS-13 and FM-STOR-6 enforce against. + Σ of configured quotas plus an operating reserve MUST fit the volume — a boot-time + check (INV-43): per-dataset quotas alone do not bound the shared volume; + (b) *amplification ratchet* — in steady state, + `disk_bytes ≤ P-SPACE-AMP × peak_live_bytes + P-SPACE-CONST`, where + `peak_live_bytes` is the maximum `live_bytes` since the dataset's storage was last + reset (RS-14). For an engine that reclaims in place, peak and current live coincide + and this is the familiar `P-SPACE-AMP × live_bytes`; for a copy-on-write engine whose + file is a high-water mark, current live may sit far below the file — that slack is + bounded by the peak term and returned by RS-14, not by background reclaim. + Both MUST hold under continuous churn (window datasets trim continuously — churn IS + the steady state), and unbounded `debt_bytes` growth under any configuration is a + defect in either reading. (GAP-6 history: until 2026-07 the system reclaimed physically only in an optional boot mode and default configurations violated this clause; routine reclaim now runs in default configurations — deferred point deletes swept every `P-CLEANUP-PERIOD` plus engine @@ -109,10 +167,23 @@ Requirements: compaction. Reproduce with the ignored `measure_hash_index_compression_disk_size` release test, then measure the deployment's real hash/key distribution and compaction state. +- **RS-14 (Dataset storage reset).** The store MUST support resetting one dataset's + physical storage online: drop the dataset's physical artifacts and re-acquire its + window per policy — a dataset-scoped RESET (a sanctioned INV-44 path, observable per + OB-9). Bounds: unavailability is confined to that dataset (LIV-5b-scoped readiness; + other datasets' progress and tails unaffected, LIV-8), and the operation MUST be + effective at a full quota without scratch space proportional to the dataset's data + (the drop precedes the re-acquisition — the same clause as RS-8). This is the + sanctioned return path for RS-6's high-water slack and the recovery verb of + FM-STOR-6. + ## 3. Interactions - **Retention × finality:** RS-2 (dominates). FINALIZE below `first(D)` is ignored (WP §2.4). +- **Retention × space:** RS-13 (space dominates — self-healing policies only; promise + policies convert space pressure into a write-pause, FM-STOR-6, never a trim). The + high-water return path is RS-14. - **Retention × forks:** the fork floor is the window start (INV-14): retention determines how deep a reorg can be absorbed in place. Operators choosing `k` MUST size it above the chain's realistic reorg depth; deeper reorgs are RESET events (alarmed). diff --git a/crates/hotblocks/spec/10-performance.md b/crates/hotblocks/spec/10-performance.md index a362834d..6e06ac07 100644 --- a/crates/hotblocks/spec/10-performance.md +++ b/crates/hotblocks/spec/10-performance.md @@ -40,7 +40,7 @@ observability), per dataset unless noted. | SLI-5 **Startup-accept time** | process start → first accepted connection (LIV-5a). | | SLI-6 **Startup-ready time** | process start → dataset readable (per dataset; LIV-5b), and → all ready. | | SLI-7 **Recovery time** | crash → LIV-5 readiness (breakdown by phase via OB-8). | -| SLI-8 **Space amplification** | `disk_bytes / Σ live_bytes` sampled through churn (RS-6). | +| SLI-8 **Space amplification** | `disk_bytes / Σ live_bytes` sampled through churn (RS-6); under a high-water engine the per-dataset ratchet reading is `disk_bytes / peak_live_bytes` since the last RS-14 reset (RS-6b). | | SLI-9 **Stall time** | longest interval with zero commit progress on any dataset whose source offers data (LIV-2); plus count of intervals > 1 s. | | SLI-10 **Error budget** | rate of `INTERNAL` (must be ~0), rate of `OVERLOADED` under nominal load (must be 0), truncation rate (RP-15). | | SLI-11 **Head granularity** | distribution of head advancement step size and inter-commit interval at fixed `W-BLOCK-RATE` (batching-induced freshness quantization, WP-3). | @@ -63,7 +63,7 @@ per row under the named scenario. | SLI-5 | S5 | ≤ 3 s, independent of state size | **~35 s refused-connection window observed** (GAP-7) | | SLI-6 all-ready | S5 | ≤ 60 s for ~50 datasets of nominal size | ~35 s+ (same incident) | | SLI-7 | S5 | SLI-5/6 bounds + no data loss (INV-40) | — | -| SLI-8 | S4 | ≤ 2.0× steady; bounded monotone convergence after churn bursts | unbounded growth in default config until 2026-07; reclaim path since fixed, bound unmeasured (GAP-6) | +| SLI-8 | S4 | ≤ 2.0× steady on the RS-6b basis; debt converges after churn bursts (a high-water file returns only via RS-14) | unbounded growth in default config until 2026-07; reclaim path since fixed, bound unmeasured (GAP-6) | | SLI-10 INTERNAL | all | 0 per 10⁶ requests | unsupported dialects are classified and counted by CT-5; target not load-tested | | SLI-11 | S1 | inter-commit ≤ max(1 s, 1/`W-BLOCK-RATE`); no artificial multi-second quantization at high rates | batch bounds imply stepping (HZ-6) | | SLI-12 p99 | S3 | ≤ 2 s + one batch time | — | diff --git a/crates/hotblocks/spec/11-observability.md b/crates/hotblocks/spec/11-observability.md index b9554835..20c9306b 100644 --- a/crates/hotblocks/spec/11-observability.md +++ b/crates/hotblocks/spec/11-observability.md @@ -35,6 +35,10 @@ surface of the binding (13 §5) with bounded cardinality. `live_bytes` estimate, `debt_bytes` estimate (logically-deleted-not-reclaimed + residue), `disk_bytes`, age of oldest unreclaimed debt, and the current reclamation blocker if any (what pins the watermark — GAP-6 made this a first-class question). + Under a quota regime (RS-13), additionally per dataset: `disk_bytes` against + `P-DISK-QUOTA`, gap-mode as a level *and* onset/exit events, gap width (positions and + bytes), clamped-instruction count, and peak live since the last RS-14 reset (the + RS-6b ratchet term). - **OB-7 (Source health).** Per dataset × source: reachability, current backoff state, last successful delivery, rejected-run counts, arbitration disagreement events (FM-SRC-6), finality-conflict events (FM-SRC-5). @@ -45,7 +49,8 @@ surface of the binding (13 §5) with bounded cardinality. - **OB-9 (Alarm states).** Distinct, queryable, per-dataset alarm conditions with reason codes: integrity conflict (WP-8/FM-SRC-5), RESET occurred (WP §2.6), boot validation refusal (INV-43), dataset stopped (CN-10), dual-writer detected (WP-15), disk floor - breached (FM-STOR-2). Alarms are edge-triggered events *and* level-readable states. + breached (FM-STOR-2), gap-mode entered (RS-13), dataset paused at quota (FM-STOR-6). + Alarms are edge-triggered events *and* level-readable states. - **OB-10 (Bounded cardinality).** All label spaces are bounded by configuration (datasets, sources, classes, outcome enums); unbounded client-derived labels MUST be sanitized (e.g. allowlisted client identities, "other" bucket). diff --git a/crates/hotblocks/spec/12-conformance-tdd.md b/crates/hotblocks/spec/12-conformance-tdd.md index 2c738a46..09d8927c 100644 --- a/crates/hotblocks/spec/12-conformance-tdd.md +++ b/crates/hotblocks/spec/12-conformance-tdd.md @@ -206,8 +206,8 @@ weaken to soundness or it will fail on correct behavior. | CT-4 | **Source-fault corpus** | scripted FM-SRC-1..8 scenarios incl. fork storms, deep forks, finality conflicts, equivocation | INV-12/13/14/23/24; WP-6/8; LIV-9; FM-SRC-*; GAP-3/4/5 | | CT-5 | **Interface conformance** | exhaustive request/response matrix against the binding: error taxonomy, watermark headers, encodings, hash-lookup matrix, boot config matrix | RP-1..16, 19/20; INV-26/43; IB-*; GAP-8/9/39 | | CT-6 | **Performance benchmarks** | reference scenarios S1–S6; SLI capture; SLO gates; saturation knees | SLI-1..12; PF-1..9; LIV-1/3/10; GAP-13 | -| CT-7 | **Soak / endurance** | multi-day S4 churn with fault sprinkling; space, memory, stall, residue tracking | LIV-2/7/11; RS-6/10; INV-16/17; HZ-2/5; GAP-1/6 | -| CT-8 | **Isolation / noisy neighbor** | S6: one dataset saturated/faulted, others measured differentially | INV-35/36; LIV-8; PF-4; GAP-14 | +| CT-7 | **Soak / endurance** | multi-day S4 churn with fault sprinkling; space, memory, stall, residue tracking; retention wave (consumer stall grows the window, then trims back — high-water and post-wave reuse, the harness analog of the ADR 0002 spike run 16) | LIV-2/7/11; RS-6/10/13; INV-16/17; HZ-2/5; GAP-1/6 | +| CT-8 | **Isolation / noisy neighbor** | S6: one dataset saturated/faulted, others measured differentially; stuck consumer on a capped dataset (quota holds, gap-mode alarms, neighbors' tails unaffected); online dataset storage reset (RS-14) with unavailability scoped to the one dataset (LIV-5b) | INV-35/36; LIV-8; RS-13/14; FM-STOR-6; PF-4; GAP-14/42/43 | | CT-9 | **Fuzzing** | source-side payload fuzz (write path) + client-side request fuzz (read path); crash/hang/leak oracles + invariant spot checks | FM-1; WP-18; RP-1/2; GAP-12 | Every test cites the IDs it verifies; CI reports coverage as "properties exercised", not @@ -227,7 +227,7 @@ INV-21/22/23 (checks in parentheses): 6. anchored continuation across responses never breaks parent-hash chains (INV-23); 7. watermark coherence: `first ≤ fin ≤ head` whenever reported together (INV-5/30). -## 5. Traceability matrix (status @ 2026-07-21) +## 5. Traceability matrix (status @ 2026-07-26) Legend: **C** covered, **P** partial (some storage-layer or fixture coverage exists; service-level black-box coverage absent), **U** untested. Rows that changed with Phase 0 name @@ -275,20 +275,22 @@ the same property under forks, crashes and retention is the business of CT-2/CT- | LIV-10 overload recovery | CT-6 | U | | | LIV-11 retention keep-up | CT-7 | U | | | LIV-12 shutdown | CT-2 | **P — known-violated** | `ct2_shutdown` drives the real binary through SIGTERM and pins unready-before-grace, continued serving during grace, the hard drain deadline with an in-flight long-poll, bounded exit and success status. The clean-truncation and ingest-wind-down halves are still open and the deployed grace still forces SIGKILL (GAP-17) | -| RS-3/4 window floor/excess | CT-1/7 | U | | -| RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn | +| RS-3/4 window floor/excess | CT-1/7 | U | RS-13 space exception untested | +| RS-6 amplification | CT-7 | U | reclaim path fixed 2026-07 (GAP-6); bound unmeasured under churn; restated 2026-07-26 as quota (a) + peak-live ratchet (b) — ADR 0002 | | RS-8 boot maintenance | CT-2/7 | P | unlink/orphan-purge behaviors have storage-level tests | +| RS-13 space dominance / gap-mode | CT-7/8 | **U — partially shipped, unobservable** | position cap live since PR #77 (silent clamp, no gap-mode signal — GAP-42); byte bound absent (GAP-43) | +| RS-14 dataset storage reset | CT-2/8 | U | no mechanism exists today (GAP-43-adjacent: shared store) | | RS-10/11 residue/deletion cost | CT-7 | P | GAP-6/13 | | FM-1 robustness | CT-9 | **P — known-violated** | GAP-12 open; unsupported query and query-worker panic classes are closed (§6.1), and the unterminated-record class is pinned by `ct9_source_faults` | | FM-SRC-* corpus | CT-4 | **P** | `ct4_lagging_source`: the multi-endpoint shape production runs — several sources per dataset, one far behind — is covered in both the minority and majority laggard shapes, and a *lagging* source is confirmed harmless to the head, independently of its slot in the fixed poll order. A source answering *wrongly* is not: one endpoint of three signalling a fork above its own tip parks ingestion (GAP-5's shape, GAP-41), pinned `#[ignore]`d. Still no strike/quarantine substrate (GAP-30) | -| FM-STOR-2/3 disk pressure | CT-7 | U | incident-derived; no automated test | +| FM-STOR-2/3/6 disk pressure | CT-7/8 | U | incident-derived; no automated test; FM-STOR-6 has no substrate (GAP-43) | | FM-OP-1..5 | CT-5 | U | | | SLI-1..12 / PF-* | CT-6 | U | no scenario benchmark harness exists; the transaction-index ingest/lookup Criterion microbench is a component baseline only | | OB-1 chain gauges | CT-1 | P | `first_block` / `last_block` / `last_finalized_block` diffed against the model; commit version and retention policy not exported (GAP-34) | | OB-2..11 | all | P | query metrics exist; stall gauges pending on PR #83 (unmerged); OB-2 heartbeat, OB-6 debt accounting, OB-9 alarms, OB-11 forensics absent | | OB-12 index state | CT-1 | **P** | CF-wide estimated keys / live SST bytes are exported for both indexes; per-dataset enabled/count/bytes and lookup hit/miss/latency remain absent (GAP-40) | -## 6. Gap register (dated 2026-07-21, informative) +## 6. Gap register (dated 2026-07-26, informative) Known or strongly suspected divergences between this spec and the current system, from incident history, code-level review, and coverage analysis. Priorities: P0 = active @@ -317,7 +319,7 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-20 | `parent_number` linkage is never validated on any layer (the block trait exposes it; nothing reads it): a hash-linked run can claim an arbitrarily higher number for the next block, storing a false hole on a densely-numbered chain — a silent data gap served as if it were a slot gap. (The originally-filed non-monotonic-numbers scenario is unreachable today: the source-position advance forces ascending numbers.) | WP-2, DEF-4, INV-1 | P2 | CT-4/CT-9: hash-linked run with a number jump on a dense chain; the run MUST be rejected with no state change | | GAP-21 | An anchored query whose `from` sits mid-chunk above a number gap larger than the conflict-check lookback (a hard-coded 100 positions in the plan's base-block check) fails `INTERNAL` instead of evaluating the assertion. A >100-position hole with an anchor landing just above it is probably unrealistic, hence low priority. A correct all-predecessors scan was tried and reverted 2026-07-15 — it regressed `check_parent_block` into an unbounded per-chunk scan+sort; needs a lazy sort-desc + limit(100) | RP-11, INV-26 | P3 | CT-5 `ct5_anchor_is_evaluated_across_a_large_number_hole` (`#[ignore]` until fixed): >100-position hole in one chunk; anchored query just above must yield OK/CONFLICT, never 500 | | GAP-22 | Deep-fork handling can silently replace the finalized prefix: the fork-resolution fallback ignores `fin` (resumes from the window start instead of `⟨fin + 1, fin.hash⟩`), the composed-finality guard admits a REPLACE whose base lies at/below `fin` whenever the pack carries a finality mark ≥ current, and Window trims have dropped the anchor hash (GAP-23) so the replacement attaches unchecked. If the first replacement batch reaches past the old `fin`, the finalized prefix is replaced with no RESET event and no alarm — finalized-only clients observe two hashes at one height (INV-24 broken); otherwise the commit trips the fork-floor check ("can't fork safely") and the epoch parks on the blind 60 s retry loop. Fix: fallback → `fin + 1`; enforce the fork floor at commit unconditionally; carry the anchor hash | WP-6, INV-13/14, INV-24, FM-SRC-5, LIV-9 | **P1** | CT-4: fork with all-mismatching hints on a dataset with `fin` defined; assert REPLACE from `fin + 1` (or alarmed fault) — never a commit whose base ≤ `fin` | -| GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. Disables below-window divergence detection (WP-6b has nothing to contradict) and feeds GAP-22 | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | +| GAP-23 | Window trims drop the anchor hash: the automatic trim passes no hash and the retained state stores `⊥`, though the correct value sits unused in the first batch's `parent_block_hash`. Disables below-window divergence detection (WP-6b has nothing to contradict) and feeds GAP-22. The PR #77 position-cap trim (`trim_floor`) passes no hash either — same fix, second site | INV-18, DEF-7, WP-6b | P1 | CT-1: CONFLICT hints / STATUS at the window edge after a trim; CT-4: below-window fork after a trim must RESET, not absorb silently | | GAP-24 | One dataset's init failure aborts the whole service: startup propagates the first controller error (kind mismatch, retention bail, corrupt state) instead of alarming that dataset and serving the rest | CN-10, FM-OP-1, INV-36, INV-43 | P1 | CT-5 boot matrix: corrupt one dataset's persisted state; assert the others serve and the broken one alarms | | GAP-25 | Downward retention (`from < first(D)`) executes as an *implicit, unobservable* RESET (WP §2.5 as amended 2026-07-12 legalizes the destruction, but requires OB-9 observability) — no event, indistinguishable from a trim; and the boot-time `Pinned` equivalent aborts the entire service (via GAP-24) instead of a dataset-level refusal | WP §2.5, OB-9, INV-43 | P2 | CT-1: SET-RETENTION below `first`; assert a RESET observable + serving continuity; CT-5: boot with lowered `Pinned.from` | | GAP-27 | FINALIZE never checks `e ≥ first(D)`: with `fin = ⊥` (e.g. after a trim passed above it) a lagging source's report below the window commits `fin < first(D)` | WP §2.4, INV-5 | P2 | CT-4: finality below the window on a trimmed dataset; assert the report is ignored | @@ -333,6 +335,8 @@ rare, P3 = polish. **First test** names the cheapest failing-test-first entry po | GAP-39 | A hash-lookup miss and an unknown dataset are both 404 with only a free-text body between them, and IB-7 forbids keying on text. This is worse than the GAP-36 family it belongs to: RP-19 makes "this hash is not indexed" a *deliberately uninformative* answer, so a client that cannot separate it from "this dataset does not exist" cannot tell a misconfiguration from a legitimate miss at all | INV-26, IB-7, RP-19 | P3 | CT-5: unknown dataset vs unknown hash; assert a structured discriminant | | GAP-40 | Hash-index CFs export only engine-wide estimated keys and live SST bytes. OB-12 still lacks per-dataset enabled state / entry count / bytes and hit-vs-miss lookup counts with latency. Because a miss is uninformative by design, an index empty for a structural reason — enabled after the window had filled, wrong kind — remains indistinguishable from one receiving only unknown hashes | OB-12, OB-6 | P3 | CT-1: scrape per-dataset state and exercise hit/miss counters once exported | | GAP-41 | Fork consensus is credulous: `StandardDataSource::poll_next_event` counts fork-signalling endpoints without asking whether a signalling endpoint has any standing at the contested position, and `extract_fork` then adopts the *longest* hint chain among them. RP-5b confines a legitimate signal to `from == tip + 1`, so an in-spec source can only signal where it is; but FM-1 requires surviving one that is not, and a source signalling above its tip is what a source merely *behind* would look like if it answered wrongly. **One** such endpoint out of three suffices, and not by majority: `forks > endpoints.len() / 2` is false at 1-of-3, but the 2 s `fork_consensus_timeout` fires on any poll where every endpoint returned `Pending`, and `accept_new_block` clears that timer only on a commit — so on a chain with block time ≥ 2 s *every inter-block gap is a firing window*. Measured 2026-07-20, 5/5 runs, by instrumenting `poll_next_event`: `forks=1 endpoints=3 active=3 majority=false all_active=false timeout=true`; the adopted hints end at the liar's stale tip, `compute_rollback` rejects them as below `fin`, and ingestion parks per GAP-5 with two healthy sources still offering the chain. The endpoints' own position is known (`Endpoint::last_committed_block`) and unused | FM-1, WP-6, FM-SRC-4/5 | **P1** (was P2 on the premise that it took a majority — measurement overturned it: a lone bad source is enough, on the ordinary inter-block gap rather than a rare coincidence) | `ct4_a_single_source_signalling_a_fork_above_its_tip_does_not_park_ingestion`; the majority path is pinned separately by `ct4_a_fork_signal_majority_above_the_tip_does_not_park_ingestion` so a fix closing only that clause cannot read as green. Both `#[ignore]`d | +| GAP-42 | The shipped position cap (PR #77, `Api.max_blocks`) is invisible in operation: no gap-mode observable, no gap width, and the downward-instruction clamp is silent — a capped dataset sacrificing its tail is indistinguishable from normal retention, and a refused (clamped) instruction from an applied one | RS-13, OB-6/9, FM-OP-4, WP-11 | P2 | CT-1: freeze the instructed floor, ingest past the cap; assert a gap-mode onset event + gap width + a clamp observable | +| GAP-43 | No per-dataset byte bound exists: `P-DISK-QUOTA`/FM-STOR-6 have no substrate, and `P-MAX-BLOCKS` caps positions only (blocks vary in size) and only for `External` — a promise-policy or uncapped dataset still grows to node-disk exhaustion, turning one dataset's stalled consumer into FM-STOR-2/3 for every dataset on the volume | RS-6a, RS-13, FM-STOR-6, LIV-8 | **P1** | CT-8: stuck consumer on a byte-capped dataset; assert the quota holds, gap-mode alarms, and neighbors' commit tails are unaffected | ### 6.1 Closed diff --git a/crates/hotblocks/spec/14-parameters.md b/crates/hotblocks/spec/14-parameters.md index 9cd102d4..e33bb0ee 100644 --- a/crates/hotblocks/spec/14-parameters.md +++ b/crates/hotblocks/spec/14-parameters.md @@ -44,10 +44,14 @@ ran against. | `P-RETENTION-APPLY` | External instruction → committed trim (WP-11, LIV-11) | prompt (unbounded formally) | ≤ 60 s ⚠ | | `P-CLEANUP-PERIOD` | deferred logical-deletion sweep cadence (RS-5) | 10 s | keep | | `P-CLEANUP-BACKOFF` | sweep retry after failure | 30 s | keep | -| `P-SPACE-AMP` | steady-state disk/live amplification bound (RS-6, SLI-8) | bounded since 2026-07 (PR #79: point-delete sweep + compaction); unmeasured (CT-7, GAP-6) | ≤ 2.0× ⚠ | +| `P-SPACE-AMP` | steady-state disk/live amplification bound (RS-6b ratchet vs peak live, SLI-8) | bounded since 2026-07 (PR #79: point-delete sweep + compaction); unmeasured (CT-7, GAP-6) | ≤ 2.0× ⚠ | | `P-SPACE-CONST` | fixed overhead allowance (RS-6) | — | size per deployment ⚠ | | `P-RECLAIM-LAG` | logical delete → physical space convergence (LIV-7) | sweep ≤ 10 s + compaction (typically minutes–hours); ≤ 7 d worst case via periodic compaction; interrupted-build residue: ∞ in default config (GAP-6) | ≤ 24 h ⚠ | -| `P-DISK-FLOOR` | free-disk alarm/degrade threshold (FM-STOR-2) | — | define ⚠ | +| `P-DISK-FLOOR` | free-disk alarm/degrade threshold, node level (FM-STOR-2) | — | define ⚠ | +| `P-MAX-BLOCKS` | optional per-dataset position cap for `External` (RS-13) | shipped 2026-07-17 (PR #77 `Api.max_blocks`): soft, whole-chunk trims; clamp and gap-mode silent (GAP-42) | keep; make observable (GAP-42) | +| `P-DISK-QUOTA` | hard per-dataset disk bound (RS-6a, RS-13, FM-STOR-6) | absent — no byte bound exists (GAP-43) | define per dataset ⚠ | +| `P-DISK-WATERMARK` | quota fraction at which the RS-13 space bound engages (against occupied storage, not the file — RS-13) | — | ~0.9 ⚠ | +| `P-REORG-KEEP` | minimum positions behind `next(D)` that RS-13 never trims (INV-14 interaction) | — | per chain, ≥ realistic reorg depth ⚠ | | `P-BLOCK-INDEX` | block hash index enabled (DEF-17, RS-12) | off by default (`--block-hash-index`); EVM only | keep | | `P-TX-INDEX` | transaction hash index enabled (DEF-17, RS-12) | off by default (`--transaction-hash-index`); EVM only; independent of `P-BLOCK-INDEX` | keep | @@ -64,7 +68,8 @@ ran against. | `P-STARTUP-READY(state)` | per-dataset readable bound (LIV-5b, SLI-6) | — | budget curve vs state size ⚠ | | `P-SHUTDOWN` | drain-and-exit bound (LIV-12) | `--pre-drain-grace-secs` 25 s + `--drain-timeout-secs` 25 s since 2026-07-21 (GAP-17), bounded in-process at last; the deployment's 5 s grace is still below the sum, so the exit remains SIGKILL until the chart is raised | deployment grace ≥ `pre_drain_grace + drain_timeout`; drain deadline ≥ `P-QUERY-TIME` + `P-SCHED-SLACK` ⚠ | | `P-DUR-PROCESS` | commits lost on process crash (CN-6) | 0 | 0 | -| `P-DUR-SYSTEM` | commit-suffix loss window on host/power failure (CN-6b) | bounded, engine-managed (not explicitly configured) | make explicit ⚠ | +| `P-DUR-SYSTEM` | commit-suffix loss window on host/power failure (CN-6b) | bounded, engine-managed (not explicitly configured) | make explicit ⚠; under ADR 0002 = `P-DUR-SYNC-CADENCE` + max write-txn hold | +| `P-DUR-SYNC-CADENCE` | durable-sync period per env under `SafeNoSync` (ADR 0002); also the free-page parking window that sizes the RS-6b burst term | — (engine-managed today) | ~1 s ⚠ | | `P-QUIESCENCE` | harness settling period before model comparison (12 §1) | — | 2× `P-CLEANUP-PERIOD` ⚠ | | `P-RECOVERY-SETTLE` | post-overload return-to-normal bound (LIV-10) | — | ≤ 30 s ⚠ | diff --git a/crates/mdbx-spike/Cargo.toml b/crates/mdbx-spike/Cargo.toml new file mode 100644 index 00000000..8cf41af5 --- /dev/null +++ b/crates/mdbx-spike/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "mdbx-spike" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +libmdbx = "0.6" +lz4_flex = "0.11" +rocksdb = { version = "0.24.0", features = ["jemalloc"] } +sqd-storage = { path = "../storage" } +tempfile = { workspace = true } diff --git a/crates/mdbx-spike/README.md b/crates/mdbx-spike/README.md new file mode 100644 index 00000000..d8e1178c --- /dev/null +++ b/crates/mdbx-spike/README.md @@ -0,0 +1,140 @@ +# mdbx-spike + +Churn bench behind the [ADR 0002](../../docs/adr/0002-storage-engine-libmdbx.md) +decision: drives the hotblocks write shape — chunk commits, chunk merges, +retention trims — against libmdbx and a production-tuned RocksDB reference. + +The `sqd_storage::kv` traits are implemented over libmdbx (`src/kv_mdbx.rs`) +and exercised for real: commits write through `KvWrite`, and a post-run scan +walks every live entry through `KvReadCursor`, decompressing pages. That is the +seam the ADR claims ports verbatim. + +## Decision runs (Linux only) + +Write amplification comes from cgroup `io.stat` (authoritative; `/proc/self/io` +write_bytes counts page dirtying, not proven device traffic). macOS gives +latency and footprint only, and its fsync (`F_FULLFSYNC`) makes sync-cadence +tails unrepresentative. + +### Getting it onto prod-class hardware + +```bash +git push -u origin spike/libmdbx +gh workflow run docker.yaml --ref spike/libmdbx -f target=mdbx-spike -f tag= +# image: subsquid/data-mdbx-spike: +``` + +Run as a Job pinned to an idle prod-class NVMe node (fill the hostname at run +time; keep node names out of this public repo). `emptyDir` lands on the node's +NVMe. Set real resource requests — a zero-request pod is the first eviction +candidate on a full node. + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: { name: mdbx-spike } +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + nodeSelector: { kubernetes.io/hostname: } + containers: + - name: spike + image: subsquid/data-mdbx-spike: + args: ["--engine", "mdbx", "--dir", "/data/spike", + "--datasets", "45", "--chunks", "2000", "--compress"] + resources: + requests: { cpu: "8", memory: 8Gi } + limits: { memory: 16Gi } + volumeMounts: [{ name: data, mountPath: /data }] + volumes: + - name: data + emptyDir: {} +``` + +Collect with `kubectl logs job/mdbx-spike`; record results in +`docs/measurements/` with node/namespace/pod names scrubbed. + +### Run matrix + +```bash +# free-run churn, engine amplification at matched stored volume +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 2000 --compress +mdbx-spike --engine rocks --dir /nvme/spike --datasets 45 --chunks 2000 + +# production pace (~1.3 commits/s/dataset), latency under merge contention +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 600 --compress --paced-ms 750 +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 600 --compress --paced-ms 750 --per-dataset-env + +# stale-reader file growth (GAP-29 shape) +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 8 --chunks 2000 --compress --reader-hold-secs 60 + +# retention wave (downstream-stall shape): file high-water + post-wave reuse; +# single env only — per-dataset envs hide the signal under the 256 MiB +# growth-step floor unless --growth-step-mb is lowered +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 2000 --compress --mdbx-page 4096 --window-wave 640 +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 1200 --compress --mdbx-page 4096 --window-wave 300 --paced-ms 250 +mdbx-spike --engine rocks --dir /nvme/spike --datasets 45 --chunks 2000 --window-wave 640 + +# head commits vs a concurrently running merge (production compaction_loop +# shape; inline merges never contend for the env writer lock) +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 45 --chunks 600 --compress --paced-ms 750 --mdbx-page 4096 --per-dataset-env --concurrent-merges + +# hash-index random-insert stream (ADR 0002 porting note 4): preseed builds +# a realistic-depth tree, then every commit inserts/deletes hash keys in the +# commit/retention txns; --hash-flush-every K prices the write-behind +# batching candidate. Compare against the same run without the stream. +mdbx-spike --engine mdbx --dir /nvme/spike --datasets 16 --chunks 600 --compress --paced-ms 750 --mdbx-page 4096 --per-dataset-env --concurrent-merges --hash-preseed 1000000 --hash-per-commit 300 +``` + +Gate per ADR 0002: proceed only if device writes drop ≥3× vs the rocks +reference without head-freshness (commit tail) regressions. + +## Findings + +Linux decision runs: [2026-07-26 measurements](../../docs/measurements/2026-07-26-mdbx-churn-bench.md). +Headline: the gate outcome is a page-size decision — 64 KiB pages write 1.4× +*more* to the device than the rocks reference, 4 KiB pages write 3.7× less +and pass. Two hard-won constraints: + +- `mdbx_env_sync` from a cadence thread racing `SafeNoSync` commits aborts + the process on the always-on `ENSURE(legal4overwrite)` (libmdbx 0.13.11, + Linux; never reproduced on macOS). Durable sync is now serialized with + write txns per env, so commit tails include the fsync wait — the cost a + port would carry. +- cgroup `io.stat` re-counts bytes on stacked block devices (md/dm); the + bench skips major 9/253 lines and cross-checks against `/proc/self/io`. +- Wave runs (15–17): the mdbx file is a permanent high-water mark — ~1.35× + peak stored live plus growth-step rounding at paced rate (free-run bursts + park up to a sync-cadence window of churn on top: 2.0× on the free-run + wave); tail truncation never fires even with an armed `shrink_threshold` + (80–93% of pages free at end). Reuse + holds: zero post-wave growth at paced rate, exactly one growth step + free-running. The rocks reference reclaims back to ~live for 3.7× the + device writes. +- Hash-index runs (22–26): a global hash-keyed table is a no-go under mdbx — + the random-insert stream costs 27× the data stream (rocks pays 1.3× of + its own baseline for the same), write-behind batching at K=16 recovers + only 26%, and commit/delete latencies degrade 12×/124×. The port uses + per-chunk index tables (ADR 0002 porting note 4). + +Earlier local findings (macOS): + +- Generated pages hit 4.00× LZ4 at `--dup 6` vs the measured 4.03× on prod + `CF_TABLES`. +- `SafeNoSync` parks freed pages until the next durable point: free-running + without a sync cadence ballooned the file to 110× the live data. With the + default 1 s cadence it stayed at one growth step. Cadence and growth-step + tuning is a real porting decision, not a knob to default. + +## Model simplifications + +- Merges are single-level (N fresh chunks → one table, never re-merged); + production `compaction_loop` re-merges up to 200k rows. Understates app-level + churn equally for both engines. +- `--readers N` adds query-shaped reads (random live table: snapshot, prefix + seek, cursor scan, decompress), but the bench's live set fits page cache, + so read numbers compare engine CPU paths, not disk-bound reads. + `--reader-hold-secs` covers only the freelist-pinning effect of long + readers. diff --git a/crates/mdbx-spike/src/data.rs b/crates/mdbx-spike/src/data.rs new file mode 100644 index 00000000..853edbc9 --- /dev/null +++ b/crates/mdbx-spike/src/data.rs @@ -0,0 +1,77 @@ +//! Page generator with controlled LZ4 compressibility + percentile helper. + +pub struct Rng(u64); + +impl Rng { + pub fn new(seed: u64) -> Self { + Self(seed | 1) + } + + pub fn next(&mut self) -> u64 { + // xorshift64* + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545F4914F6CDD1D) + } +} + +/// Deterministic hash-index key for (dataset, counter): a 0xFF sentinel byte +/// (sorts after every 16-byte table prefix, so the hash range shares the mdbx +/// tree without colliding) + 31 uniformly random bytes — the block/tx-hash +/// insert pattern, reproducible so deletes can re-derive their keys. +pub fn hkey(ds: usize, i: u64) -> [u8; 32] { + let seed = ((ds as u64) << 33).wrapping_add(i).wrapping_mul(0x9E3779B97F4A7C15); + let mut rng = Rng::new(seed); + let mut out = [0u8; 32]; + for c in out.chunks_exact_mut(8) { + c.copy_from_slice(&rng.next().to_le_bytes()); + } + out[0] = 0xFF; + out +} + +/// One fresh random word per `dup` words; LZ4 ratio lands near `dup`. +pub fn gen_page(rng: &mut Rng, len: usize, dup: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut word = rng.next(); + let mut i = 0usize; + while out.len() < len { + if i % dup.max(1) == 0 { + word = rng.next(); + } + out.extend_from_slice(&word.to_le_bytes()); + i += 1; + } + out.truncate(len); + out +} + +pub fn percentile(sorted: &[u64], p: f64) -> u64 { + if sorted.is_empty() { + return 0; + } + let idx = ((sorted.len() - 1) as f64 * p).round() as usize; + sorted[idx] +} + +pub struct LatencyReport { + pub count: usize, + pub p50: u64, + pub p90: u64, + pub p99: u64, + pub max: u64 +} + +pub fn latency_report(mut micros: Vec) -> LatencyReport { + micros.sort_unstable(); + LatencyReport { + count: micros.len(), + p50: percentile(µs, 0.50), + p90: percentile(µs, 0.90), + p99: percentile(µs, 0.99), + max: micros.last().copied().unwrap_or(0) + } +} diff --git a/crates/mdbx-spike/src/engine.rs b/crates/mdbx-spike/src/engine.rs new file mode 100644 index 00000000..738b59da --- /dev/null +++ b/crates/mdbx-spike/src/engine.rs @@ -0,0 +1,711 @@ +//! The two engines under test, behind one byte-KV surface. +//! +//! mdbx merge/delete run inside a single write transaction on purpose: that is +//! the worst case ADR 0002 worries about (writer lock held for a whole +//! chunk-merge), so the bench must charge for it. + +use std::{ + fs, + path::Path, + sync::{Mutex, MutexGuard} +}; + +use anyhow::Result; +use libmdbx::{Database, DatabaseOptions, Mode, NoWriteMap, PageSize, ReadWriteOptions, SyncMode, WriteFlags}; +use sqd_storage::kv::{KvRead, KvReadCursor, KvWrite}; + +use crate::kv_mdbx::{MdbxTableReader, MdbxTableWriter}; + +pub const TABLE_PREFIX_LEN: usize = 16; + +pub fn table_prefix(id: u64) -> [u8; TABLE_PREFIX_LEN] { + (id as u128).to_be_bytes() +} + +pub fn meta_key(id: u64) -> Vec { + let mut k = table_prefix(id).to_vec(); + k.push(0); + k +} + +pub fn page_key(id: u64, idx: u32) -> Vec { + let mut k = table_prefix(id).to_vec(); + k.push(3); + k.extend_from_slice(&idx.to_be_bytes()); + k +} + +pub fn is_page_key(key: &[u8]) -> bool { + key.len() == TABLE_PREFIX_LEN + 5 +} + +pub struct DiskUsage { + pub file_bytes: u64, + pub live_bytes: u64 +} + +pub enum Engine { + Mdbx(MdbxEngine), + Rocks(RocksEngine) +} + +impl Engine { + pub fn write_chunk( + &self, + ds: usize, + entries: &[(Vec, Vec)], + hash_inserts: Option> + ) -> Result<()> { + match self { + Engine::Mdbx(e) => e.write_chunk(ds, entries, hash_inserts), + Engine::Rocks(e) => e.write_chunk(ds, entries, hash_inserts) + } + } + + pub fn merge_tables(&self, ds: usize, old: &[u64], new_id: u64) -> Result { + match self { + Engine::Mdbx(e) => e.merge_tables(ds, old, new_id), + Engine::Rocks(e) => e.merge_tables(old, new_id) + } + } + + pub fn delete_tables( + &self, + ds: usize, + ids: &[u64], + hash_deletes: Option> + ) -> Result { + match self { + Engine::Mdbx(e) => e.delete_tables(ds, ids, hash_deletes), + Engine::Rocks(e) => e.delete_tables(ds, ids, hash_deletes) + } + } + + /// Bulk-builds the hash-index tree to `count` entries before the timed + /// run: sorted batches, so the build is append-cheap and the *stream* + /// runs against a tree of realistic depth (a small tree understates + /// random-insert COW by orders of magnitude). + pub fn preseed_hashes(&self, ds: usize, count: u64) -> Result<()> { + match self { + Engine::Mdbx(e) => e.preseed_hashes(ds, count), + Engine::Rocks(e) => e.preseed_hashes(ds, count) + } + } + + /// One query-shaped read: fresh snapshot, seek to the table prefix, cursor + /// over its pages, hand back raw bytes (app LZ4 for mdbx, engine LZ4 for + /// rocks). Returns (pages, raw_bytes); 0 pages = table vanished. + pub fn scan_table(&self, ds: usize, id: u64, compressed: bool) -> Result<(usize, u64)> { + match self { + Engine::Mdbx(e) => e.scan_table(ds, id, compressed), + Engine::Rocks(e) => e.scan_table(id) + } + } + + pub fn hold_reader(&self, secs: u64) -> Result<()> { + match self { + Engine::Mdbx(e) => { + let txn = e.envs[0].begin_ro_txn()?; + std::thread::sleep(std::time::Duration::from_secs(secs)); + drop(txn); + } + Engine::Rocks(e) => { + let snap = e.db.snapshot(); + std::thread::sleep(std::time::Duration::from_secs(secs)); + drop(snap); + } + } + Ok(()) + } + + pub fn sync(&self) -> Result<()> { + match self { + Engine::Mdbx(e) => e.locked_sync()?, + Engine::Rocks(e) => e.db.flush()? + } + Ok(()) + } + + /// SafeNoSync parks freed pages until the next durable point, so without a + /// sync cadence the file balloons; rocks needs nothing (WAL is its cadence). + pub fn periodic_sync(&self) -> Result<()> { + if let Engine::Mdbx(e) = self { + e.locked_sync()?; + } + Ok(()) + } + + pub fn disk_usage(&self) -> Result { + match self { + Engine::Mdbx(e) => e.disk_usage(), + Engine::Rocks(e) => e.disk_usage() + } + } + + /// (write_stopped, num_immutable_memtables, delayed_write_rate) — the three + /// properties that discriminate a flush-backpressure stall from compaction + /// debt. Production stalls on `num_immutable >= max_write_buffer_number` + /// (measured 2026-07-27), so a tuning arm has to show this go to zero. + pub fn stall_probe(&self) -> Option<(u64, u64, u64)> { + match self { + Engine::Rocks(e) => Some(( + e.int_prop("rocksdb.is-write-stopped"), + e.int_prop("rocksdb.num-immutable-mem-table"), + e.int_prop("rocksdb.actual-delayed-write-rate") + )), + Engine::Mdbx(_) => None + } + } + + pub fn describe(&self) -> String { + match self { + Engine::Mdbx(e) => { + let info = e.envs[0].info().map(|i| i.last_pgno()).unwrap_or(0); + let freelist = e.envs[0].freelist().unwrap_or(0); + format!( + "mdbx envs={} page_size={} env0: last_pgno={} freelist_pages={}", + e.envs.len(), + e.page_size, + info, + freelist + ) + } + Engine::Rocks(e) => { + let total = e.int_prop("rocksdb.total-sst-files-size"); + let live = e.int_prop("rocksdb.live-sst-files-size"); + format!("rocks total_sst={total} live_sst={live}") + } + } + } +} + +pub struct MdbxEngine { + envs: Vec>, + /// mdbx_env_sync concurrent with a SafeNoSync commit trips the always-on + /// ENSURE(legal4overwrite) in dxb_sync_locked (libmdbx 0.13.11), so durable + /// sync is serialized with write txns — commits pay the fsync wait, which + /// is the honest cost a port would carry. + write_locks: Vec>, + page_size: usize, + root: std::path::PathBuf +} + +impl MdbxEngine { + pub fn open( + root: &Path, + n_envs: usize, + sync_mode: SyncMode, + page_size: usize, + max_bytes: usize, + growth_step: usize + ) -> Result { + let mut envs = Vec::with_capacity(n_envs); + for i in 0..n_envs { + let path = root.join(format!("env{i}")); + fs::create_dir_all(&path)?; + let options = DatabaseOptions { + page_size: Some(PageSize::Set(page_size)), + mode: Mode::ReadWrite(ReadWriteOptions { + sync_mode, + min_size: Some(0), + max_size: Some(max_bytes as isize), + growth_step: Some(growth_step as isize), + // explicit, so wave runs can answer whether tail truncation + // ever fires under churn (mdbx can shrink only a free tail) + shrink_threshold: Some(2 * growth_step as isize) + }), + ..Default::default() + }; + envs.push(Database::open_with_options(&path, options)?); + } + let write_locks = (0..n_envs).map(|_| Mutex::new(())).collect(); + Ok(Self { + envs, + write_locks, + page_size, + root: root.to_owned() + }) + } + + fn env(&self, ds: usize) -> &Database { + &self.envs[ds % self.envs.len()] + } + + fn write_lock(&self, ds: usize) -> MutexGuard<'_, ()> { + self.write_locks[ds % self.envs.len()] + .lock() + .expect("write lock poisoned") + } + + pub fn locked_sync(&self) -> Result<()> { + for (env, lock) in self.envs.iter().zip(&self.write_locks) { + let _g = lock.lock().expect("write lock poisoned"); + env.sync(true)?; + } + Ok(()) + } + + fn write_chunk( + &self, + ds: usize, + entries: &[(Vec, Vec)], + hash_inserts: Option> + ) -> Result<()> { + let _g = self.write_lock(ds); + let txn = self.env(ds).begin_rw_txn()?; + let table = txn.open_table(None)?; + { + let mut writer = MdbxTableWriter::new(&txn, table); + for (k, v) in entries { + writer.put(k, v)?; + } + } + if let Some(range) = hash_inserts { + // hash keys live in the same tree under the 0xFF sentinel — + // COW-equivalent to a named subtable, disjoint from table prefixes + let mut keys: Vec<[u8; 32]> = range.map(|i| crate::data::hkey(ds, i)).collect(); + keys.sort_unstable(); + let table = txn.open_table(None)?; + for k in &keys { + txn.put(&table, k, [0u8; 12], WriteFlags::UPSERT)?; + } + } + txn.commit()?; + Ok(()) + } + + fn preseed_hashes(&self, ds: usize, count: u64) -> Result<()> { + for start in (0..count).step_by(100_000) { + let end = (start + 100_000).min(count); + let mut keys: Vec<[u8; 32]> = (start..end).map(|i| crate::data::hkey(ds, i)).collect(); + keys.sort_unstable(); + let _g = self.write_lock(ds); + let txn = self.env(ds).begin_rw_txn()?; + let table = txn.open_table(None)?; + for k in &keys { + txn.put(&table, k, [0u8; 12], WriteFlags::UPSERT)?; + } + txn.commit()?; + } + Ok(()) + } + + fn merge_tables(&self, ds: usize, old: &[u64], new_id: u64) -> Result { + let _g = self.write_lock(ds); + let txn = self.env(ds).begin_rw_txn()?; + let table = txn.open_table(None)?; + + let mut moved: Vec<(Vec, Vec)> = Vec::new(); + { + let mut cur = txn.cursor(&table)?; + for id in old { + let prefix = table_prefix(*id); + let mut kv = cur.set_range::, Vec>(&prefix)?; + while let Some((k, v)) = kv { + if !k.starts_with(&prefix) { + break; + } + moved.push((k, v)); + kv = cur.next::, Vec>()?; + } + } + } + + let mut idx = 0u32; + for (k, v) in &moved { + if is_page_key(k) { + txn.put(&table, page_key(new_id, idx), v, WriteFlags::UPSERT)?; + idx += 1; + } + } + txn.put(&table, meta_key(new_id), [0u8; 64], WriteFlags::UPSERT)?; + for (k, _v) in &moved { + txn.del(&table, k, None)?; + } + txn.commit()?; + Ok(moved.len()) + } + + fn delete_tables( + &self, + ds: usize, + ids: &[u64], + hash_deletes: Option> + ) -> Result { + let _g = self.write_lock(ds); + let txn = self.env(ds).begin_rw_txn()?; + let table = txn.open_table(None)?; + if let Some(range) = hash_deletes { + for i in range { + txn.del(&table, crate::data::hkey(ds, i), None)?; + } + } + + let mut keys: Vec> = Vec::new(); + { + let mut cur = txn.cursor(&table)?; + for id in ids { + let prefix = table_prefix(*id); + let mut kv = cur.set_range::, ()>(&prefix)?; + while let Some((k, ())) = kv { + if !k.starts_with(&prefix) { + break; + } + keys.push(k); + kv = cur.next::, ()>()?; + } + } + } + + let deleted = keys.len(); + for k in &keys { + txn.del(&table, k, None)?; + } + txn.commit()?; + Ok(deleted) + } + + fn scan_table(&self, ds: usize, id: u64, compressed: bool) -> Result<(usize, u64)> { + let txn = self.env(ds).begin_ro_txn()?; + let table = txn.open_table(None)?; + let reader = MdbxTableReader::new(&txn, table); + let mut cur = reader.new_cursor(); + let prefix = table_prefix(id); + cur.seek(&prefix)?; + let mut pages = 0usize; + let mut raw = 0u64; + while cur.is_valid() && cur.key().starts_with(&prefix) { + if is_page_key(cur.key()) { + if compressed { + let v = lz4_flex::decompress_size_prepended(cur.value()) + .map_err(|e| anyhow::anyhow!("page decompress failed: {e}"))?; + raw += v.len() as u64; + } else { + raw += cur.value().len() as u64; + } + pages += 1; + } + cur.next()?; + } + Ok((pages, raw)) + } + + /// Walks env0 through the `KvRead` seam and decompresses page values — + /// proves the ADR 0002 read path end to end on real stored data. + pub fn verify_scan(&self, compressed: bool) -> Result<(usize, u64)> { + let txn = self.envs[0].begin_ro_txn()?; + let table = txn.open_table(None)?; + let reader = MdbxTableReader::new(&txn, table); + let mut cur = reader.new_cursor(); + cur.seek_first()?; + let mut entries = 0usize; + let mut raw_bytes = 0u64; + while cur.is_valid() { + if is_page_key(cur.key()) && compressed { + let raw = lz4_flex::decompress_size_prepended(cur.value()) + .map_err(|e| anyhow::anyhow!("page decompress failed: {e}"))?; + raw_bytes += raw.len() as u64; + } else { + raw_bytes += cur.value().len() as u64; + } + entries += 1; + cur.next()?; + } + Ok((entries, raw_bytes)) + } + + fn disk_usage(&self) -> Result { + let mut live = 0u64; + for env in &self.envs { + let info = env.info()?; + let freelist = env.freelist()? as u64; + let used_pages = (info.last_pgno() as u64 + 1).saturating_sub(freelist); + live += used_pages * self.page_size as u64; + } + Ok(DiskUsage { + file_bytes: dir_bytes(&self.root), + live_bytes: live + }) + } +} + +pub struct RocksEngine { + db: rocksdb::DB, + root: std::path::PathBuf, + /// Holds the Statistics object the tickers are read from; dropping it + /// would zero `write_amp`. + db_opts: rocksdb::Options +} + +const CF_TABLES: &str = "tables"; +const CF_HASHES: &str = "hashes"; + +/// The rocks knobs the reference was missing. `direct_io` is production parity +/// (verified 2026-07-27: no stack passes `--rocksdb-disable-direct-io`, and +/// `cli.rs:149` inverts it, so prod runs direct reads + direct flush/compaction); +/// the memtable pair is the tuning arm — prod leaves both at RocksDB defaults +/// and its write stalls are that ceiling being hit. +#[derive(Clone, Copy)] +pub struct RocksOpts { + pub cache_mb: usize, + pub direct_io: bool, + pub write_buffer_mb: usize, + pub max_write_buffers: i32, + /// Universal instead of leveled for the tables CF. Production pays ~6.4× + /// write amplification to leveled (2026-07-28: 8735 GiB of compaction per + /// 1368 GiB flushed, mainnet); universal buys that back with space + /// amplification, which the production volumes have spare — 9% used. + pub universal: bool, + /// Universal's `max_size_amplification_percent`: full compaction fires once + /// the CF exceeds this fraction of its live size. RocksDB's default is 200. + pub size_amp_pct: i32, + /// `level0_file_num_compaction_trigger`; 0 keeps RocksDB's 4. Each L0→base + /// merge rewrites the whole overlapping base level, so the step costs + /// ~1 + base_size/L0_batch — production measures 5.2× at the default + /// (2026-07-28, internal L4). Raising the trigger amortizes one rewrite + /// over proportionally more new data. Slowdown/stop keep their 1:5:9 + /// ratio to the trigger, or the larger L0 would just stall instead. + pub l0_trigger: i32 +} + +impl RocksEngine { + /// Mirrors production `tables_cf_options`: LZ4, dynamic level bytes, + /// compact-on-deletion collector, dedicated block cache — and the + /// db-level options that shape device writes: Zstd WAL compression and + /// max_background_jobs=8 (prod runs cores clamped to 2..=8; the bench + /// container has 8). Runs 01–17 predate the WAL/jobs parity and + /// overstate rocks device writes by the uncompressed-WAL term (≈ raw); + /// runs 01–26 additionally ran buffered against a direct-I/O production. + pub fn open(root: &Path, o: RocksOpts) -> Result { + let mut db_opts = rocksdb::Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + db_opts.set_wal_compression_type(rocksdb::DBCompressionType::Zstd); + db_opts.set_max_background_jobs(8); + db_opts.set_max_subcompactions(4); + // Both arms pay it, so it cannot bias the comparison, and it is the only + // way to split device writes into flush / compaction / WAL the way + // production's LOG does. + db_opts.enable_statistics(); + // Runs are ~10 min; RocksDB's 600 s default would leave a short arm with + // no per-level table in the LOG at all. + db_opts.set_stats_dump_period_sec(120); + if o.direct_io { + db_opts.set_use_direct_reads(true); + db_opts.set_use_direct_io_for_flush_and_compaction(true); + } + + let mut cf_opts = rocksdb::Options::default(); + let mut block = rocksdb::BlockBasedOptions::default(); + block.set_block_cache(&rocksdb::Cache::new_lru_cache(o.cache_mb << 20)); + cf_opts.set_block_based_table_factory(&block); + cf_opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + if o.universal { + cf_opts.set_compaction_style(rocksdb::DBCompactionStyle::Universal); + let mut uco = rocksdb::UniversalCompactOptions::default(); + uco.set_max_size_amplification_percent(o.size_amp_pct); + cf_opts.set_universal_compaction_options(&uco); + } else { + // Leveled-only; universal ignores it, so setting it there would just + // make the run record ambiguous. + cf_opts.set_level_compaction_dynamic_level_bytes(true); + } + cf_opts.add_compact_on_deletion_collector_factory(128 * 1024, 64 * 1024, 0.5); + cf_opts.set_write_buffer_size(o.write_buffer_mb << 20); + cf_opts.set_max_write_buffer_number(o.max_write_buffers); + if o.l0_trigger > 0 { + cf_opts.set_level_zero_file_num_compaction_trigger(o.l0_trigger); + cf_opts.set_level_zero_slowdown_writes_trigger(o.l0_trigger * 5); + cf_opts.set_level_zero_stop_writes_trigger(o.l0_trigger * 9); + } + + // mirrors production `hash_index_cf_options`: bloom, LZ4, deletion + // collector, dynamic level bytes + let mut hash_opts = rocksdb::Options::default(); + let mut hash_block = rocksdb::BlockBasedOptions::default(); + hash_block.set_bloom_filter(10.0, false); + hash_block.set_optimize_filters_for_memory(true); + hash_opts.set_block_based_table_factory(&hash_block); + hash_opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + hash_opts.set_level_compaction_dynamic_level_bytes(true); + hash_opts.add_compact_on_deletion_collector_factory(128 * 1024, 64 * 1024, 0.5); + + let db = rocksdb::DB::open_cf_descriptors( + &db_opts, + root, + vec![ + rocksdb::ColumnFamilyDescriptor::new(CF_TABLES, cf_opts), + rocksdb::ColumnFamilyDescriptor::new(CF_HASHES, hash_opts), + ] + )?; + Ok(Self { + db, + root: root.to_owned(), + db_opts + }) + } + + /// (flush, compaction, WAL) bytes. The same split production reports in its + /// LOG, so a bench arm and a pod arm can be put in the same table. + pub fn write_bytes(&self) -> (u64, u64, u64) { + ( + self.db_opts.get_ticker_count(rocksdb::statistics::Ticker::FlushWriteBytes), + self.db_opts.get_ticker_count(rocksdb::statistics::Ticker::CompactWriteBytes), + self.db_opts.get_ticker_count(rocksdb::statistics::Ticker::WalFileBytes) + ) + } + + fn cf(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(CF_TABLES).expect("tables cf") + } + + fn cf_h(&self) -> &rocksdb::ColumnFamily { + self.db.cf_handle(CF_HASHES).expect("hashes cf") + } + + fn write_chunk( + &self, + ds: usize, + entries: &[(Vec, Vec)], + hash_inserts: Option> + ) -> Result<()> { + let mut batch = rocksdb::WriteBatch::default(); + for (k, v) in entries { + batch.put_cf(self.cf(), k, v); + } + if let Some(range) = hash_inserts { + for i in range { + batch.put_cf(self.cf_h(), crate::data::hkey(ds, i), [0u8; 12]); + } + } + self.db.write(batch)?; + Ok(()) + } + + fn preseed_hashes(&self, ds: usize, count: u64) -> Result<()> { + for start in (0..count).step_by(100_000) { + let end = (start + 100_000).min(count); + let mut batch = rocksdb::WriteBatch::default(); + for i in start..end { + batch.put_cf(self.cf_h(), crate::data::hkey(ds, i), [0u8; 12]); + } + self.db.write(batch)?; + } + Ok(()) + } + + fn merge_tables(&self, old: &[u64], new_id: u64) -> Result { + let mut moved: Vec<(Vec, Vec)> = Vec::new(); + let mut iter = self.db.raw_iterator_cf(self.cf()); + for id in old { + let prefix = table_prefix(*id); + iter.seek(prefix); + while iter.valid() { + let k = iter.key().expect("valid"); + if !k.starts_with(&prefix) { + break; + } + moved.push((k.to_vec(), iter.value().expect("valid").to_vec())); + iter.next(); + } + } + + let mut batch = rocksdb::WriteBatch::default(); + let mut idx = 0u32; + for (k, v) in &moved { + if is_page_key(k) { + batch.put_cf(self.cf(), page_key(new_id, idx), v); + idx += 1; + } + } + batch.put_cf(self.cf(), meta_key(new_id), [0u8; 64]); + // production purges with point deletes (no DeleteRange on the + // transactional db), so the reference does too + for (k, _v) in &moved { + batch.delete_cf(self.cf(), k); + } + self.db.write(batch)?; + Ok(moved.len()) + } + + fn delete_tables( + &self, + ds: usize, + ids: &[u64], + hash_deletes: Option> + ) -> Result { + let mut batch = rocksdb::WriteBatch::default(); + if let Some(range) = hash_deletes { + for i in range { + batch.delete_cf(self.cf_h(), crate::data::hkey(ds, i)); + } + } + let mut deleted = 0usize; + let mut iter = self.db.raw_iterator_cf(self.cf()); + for id in ids { + let prefix = table_prefix(*id); + iter.seek(prefix); + while iter.valid() { + let k = iter.key().expect("valid"); + if !k.starts_with(&prefix) { + break; + } + batch.delete_cf(self.cf(), k); + deleted += 1; + iter.next(); + } + } + self.db.write(batch)?; + Ok(deleted) + } + + fn scan_table(&self, id: u64) -> Result<(usize, u64)> { + let prefix = table_prefix(id); + let mut iter = self.db.raw_iterator_cf(self.cf()); + iter.seek(prefix); + let mut pages = 0usize; + let mut raw = 0u64; + while iter.valid() { + let k = iter.key().expect("valid"); + if !k.starts_with(&prefix) { + break; + } + if is_page_key(k) { + raw += iter.value().expect("valid").len() as u64; + pages += 1; + } + iter.next(); + } + Ok((pages, raw)) + } + + fn int_prop(&self, name: &str) -> u64 { + self.db + .property_int_value_cf(self.cf(), name) + .ok() + .flatten() + .unwrap_or(0) + } + + fn disk_usage(&self) -> Result { + Ok(DiskUsage { + file_bytes: dir_bytes(&self.root), + live_bytes: self.int_prop("rocksdb.live-sst-files-size") + }) + } +} + +pub fn dir_bytes(path: &Path) -> u64 { + let mut total = 0; + if let Ok(entries) = fs::read_dir(path) { + for entry in entries.flatten() { + let p = entry.path(); + if p.is_dir() { + total += dir_bytes(&p); + } else if let Ok(meta) = entry.metadata() { + total += meta.len(); + } + } + } + total +} diff --git a/crates/mdbx-spike/src/kv_mdbx.rs b/crates/mdbx-spike/src/kv_mdbx.rs new file mode 100644 index 00000000..0e8dae4c --- /dev/null +++ b/crates/mdbx-spike/src/kv_mdbx.rs @@ -0,0 +1,162 @@ +//! `sqd_storage::kv` traits over libmdbx — the seam ADR 0002 claims ports verbatim. +//! +//! The cursor copies the current pair out of the mmap; a production impl would +//! borrow (`Cow`) and be strictly faster. Conservative for read benchmarks. + +use libmdbx::{Cursor, NoWriteMap, Table, Transaction, TransactionKind, WriteFlags, RO, RW}; +use sqd_storage::kv::{KvRead, KvReadCursor, KvWrite}; + +pub struct MdbxTableWriter<'db, 'txn> { + txn: &'txn Transaction<'db, RW, NoWriteMap>, + table: Table<'txn> +} + +impl<'db, 'txn> MdbxTableWriter<'db, 'txn> { + pub fn new(txn: &'txn Transaction<'db, RW, NoWriteMap>, table: Table<'txn>) -> Self { + Self { txn, table } + } +} + +impl KvWrite for MdbxTableWriter<'_, '_> { + fn put(&mut self, key: &[u8], value: &[u8]) -> anyhow::Result<()> { + self.txn.put(&self.table, key, value, WriteFlags::UPSERT)?; + Ok(()) + } +} + +pub struct MdbxTableReader<'db, 'txn> { + txn: &'txn Transaction<'db, RO, NoWriteMap>, + table: Table<'txn> +} + +impl<'db, 'txn> MdbxTableReader<'db, 'txn> { + pub fn new(txn: &'txn Transaction<'db, RO, NoWriteMap>, table: Table<'txn>) -> Self { + Self { txn, table } + } +} + +impl<'db, 'txn> KvRead for MdbxTableReader<'db, 'txn> { + type Cursor = MdbxKvCursor<'txn, RO>; + + fn get(&self, key: &[u8]) -> anyhow::Result>> { + Ok(self.txn.get::>(&self.table, key)?) + } + + fn new_cursor(&self) -> Self::Cursor { + MdbxKvCursor { + cur: self.txn.cursor(&self.table).expect("cursor open"), + current: None + } + } +} + +pub struct MdbxKvCursor<'txn, K: TransactionKind> { + cur: Cursor<'txn, K>, + current: Option<(Vec, Vec)> +} + +impl KvReadCursor for MdbxKvCursor<'_, K> { + // rocksdb iterator semantics: next/prev on an invalid cursor keep it invalid + fn seek_first(&mut self) -> anyhow::Result<()> { + self.current = self.cur.first::, Vec>()?; + Ok(()) + } + + fn seek(&mut self, key: &[u8]) -> anyhow::Result<()> { + self.current = self.cur.set_range::, Vec>(key)?; + Ok(()) + } + + fn seek_prev(&mut self, key: &[u8]) -> anyhow::Result<()> { + // last entry <= key: position at first >= key, then step back if overshot + self.current = match self.cur.set_range::, Vec>(key)? { + Some(kv) if kv.0.as_slice() > key => self.cur.prev::, Vec>()?, + Some(kv) => Some(kv), + None => self.cur.last::, Vec>()? + }; + Ok(()) + } + + fn next(&mut self) -> anyhow::Result<()> { + if self.current.is_some() { + self.current = self.cur.next::, Vec>()?; + } + Ok(()) + } + + fn prev(&mut self) -> anyhow::Result<()> { + if self.current.is_some() { + self.current = self.cur.prev::, Vec>()?; + } + Ok(()) + } + + fn is_valid(&self) -> bool { + self.current.is_some() + } + + fn key(&self) -> &[u8] { + &self.current.as_ref().expect("cursor position is not valid").0 + } + + fn value(&self) -> &[u8] { + &self.current.as_ref().expect("cursor position is not valid").1 + } +} + +#[cfg(test)] +mod tests { + use libmdbx::{Database, DatabaseOptions, NoWriteMap}; + use sqd_storage::kv::{KvRead, KvReadCursor, KvWrite}; + + use super::*; + + #[test] + fn cursor_matches_rocksdb_iterator_semantics() { + let dir = tempfile::tempdir().unwrap(); + let db: Database = Database::open_with_options(dir.path(), DatabaseOptions::default()).unwrap(); + + let txn = db.begin_rw_txn().unwrap(); + let table = txn.open_table(None).unwrap(); + let mut writer = MdbxTableWriter::new(&txn, table); + for key in [b"b", b"d", b"f"] { + writer.put(key, key).unwrap(); + } + drop(writer); + txn.commit().unwrap(); + + let txn = db.begin_ro_txn().unwrap(); + let table = txn.open_table(None).unwrap(); + let reader = MdbxTableReader::new(&txn, table); + + assert_eq!(reader.get(b"d").unwrap().as_deref(), Some(b"d".as_slice())); + assert_eq!(reader.get(b"c").unwrap().as_deref(), None); + + let mut cur = reader.new_cursor(); + + cur.seek_first().unwrap(); + assert_eq!(cur.key(), b"b"); + + cur.seek(b"c").unwrap(); + assert_eq!(cur.key(), b"d"); // first >= c + + cur.seek_prev(b"c").unwrap(); + assert_eq!(cur.key(), b"b"); // last <= c + + cur.seek_prev(b"d").unwrap(); + assert_eq!(cur.key(), b"d"); // exact hit stays + + cur.seek_prev(b"z").unwrap(); + assert_eq!(cur.key(), b"f"); // past the end -> last + + cur.next().unwrap(); + assert!(!cur.is_valid()); + cur.next().unwrap(); // stays invalid, no panic + assert!(!cur.is_valid()); + + cur.seek(b"a").unwrap(); + assert_eq!(cur.key(), b"b"); + cur.prev().unwrap(); + assert!(!cur.is_valid()); + } +} diff --git a/crates/mdbx-spike/src/main.rs b/crates/mdbx-spike/src/main.rs new file mode 100644 index 00000000..e18a5783 --- /dev/null +++ b/crates/mdbx-spike/src/main.rs @@ -0,0 +1,813 @@ +//! Churn bench for ADR 0002: hotblocks-shaped write workload (chunk commits, +//! chunk merges, retention trims) against libmdbx and a production-tuned +//! RocksDB reference. Reports latency percentiles, disk footprint and, on +//! Linux, OS-level write amplification from /proc/self/io. + +mod data; +mod engine; +mod kv_mdbx; + +use std::{ + collections::VecDeque, + path::PathBuf, + sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + time::{Duration, Instant} +}; + +use anyhow::Result; +use clap::{Parser, ValueEnum}; + +use crate::{ + data::{gen_page, latency_report, Rng}, + engine::{meta_key, page_key, Engine, MdbxEngine, RocksEngine} +}; + +#[derive(Clone, Copy, ValueEnum)] +enum EngineKind { + Mdbx, + Rocks +} + +#[derive(Clone, Copy, ValueEnum)] +enum SyncOpt { + Durable, + Safe +} + +#[derive(Parser)] +struct Args { + #[arg(long, value_enum, default_value = "mdbx")] + engine: EngineKind, + + /// data directory; wiped on start + #[arg(long)] + dir: PathBuf, + + #[arg(long, default_value_t = 8)] + datasets: usize, + + /// chunk commits per dataset + #[arg(long, default_value_t = 200)] + chunks: usize, + + #[arg(long, default_value_t = 4)] + pages_per_commit: usize, + + /// raw page size before compression (prod avg: 42.6 KiB) + #[arg(long, default_value_t = 43_000)] + page_bytes: usize, + + /// duplication factor of generated pages; 6 lands near the prod 4x LZ4 + /// ratio (match-encoding overhead eats the difference) + #[arg(long, default_value_t = 6)] + dup: usize, + + /// app-level LZ4 before put (the ADR 0002 precondition); rocks compresses + /// in the engine instead, so its reference runs with this off + #[arg(long, default_value_t = false)] + compress: bool, + + #[arg(long, default_value_t = false)] + per_dataset_env: bool, + + #[arg(long, value_enum, default_value = "safe")] + sync: SyncOpt, + + /// merge every N chunks into one table; 0 = off + #[arg(long, default_value_t = 8)] + merge_fanin: usize, + + /// run merges on a background thread (max 1 in flight per dataset), so + /// commits contend with the merge for the env writer lock — the + /// production compaction_loop shape; inline merges (runs 01–17) never + /// exercise that contention + #[arg(long, default_value_t = false)] + concurrent_merges: bool, + + /// pre-build the hash-index tree to this many entries per dataset before + /// the timed run; the random-insert cost only shows against a tree much + /// larger than one batch (ADR 0002 porting note 4). 0 = no hash stream + #[arg(long, default_value_t = 0)] + hash_preseed: u64, + + /// hash-index entries inserted per chunk commit (same txn/batch as the + /// commit — the production shape; ~tx count per chunk) and deleted with + /// the chunk when retention prunes it + #[arg(long, default_value_t = 0)] + hash_per_commit: u64, + + /// accumulate hash inserts and flush every K commits, sorted, in that + /// commit's txn — the write-behind design candidate; 1 = per-commit + #[arg(long, default_value_t = 1)] + hash_flush_every: u64, + + /// retention window in chunks per dataset; 0 = keep everything + #[arg(long, default_value_t = 64)] + window: usize, + + /// grow the window to this many chunks for the middle of the run + /// (commits 20%..50%), then trim back — a downstream-stall wave; 0 = off + #[arg(long, default_value_t = 0)] + window_wave: usize, + + /// per-dataset delay between commits; 0 = free-run + #[arg(long, default_value_t = 0)] + paced_ms: u64, + + /// hold a read txn/snapshot this long in a loop (stale-reader probe); 0 = off + #[arg(long, default_value_t = 0)] + reader_hold_secs: u64, + + /// concurrent readers, each scanning a random live table per iteration + #[arg(long, default_value_t = 0)] + readers: usize, + + /// per-reader delay between scans; 0 = free-run + #[arg(long, default_value_t = 0)] + read_paced_ms: u64, + + #[arg(long, default_value_t = 64 << 30)] + max_db_bytes: usize, + + /// mdbx page size + #[arg(long, default_value_t = 65_536)] + mdbx_page: usize, + + /// mdbx geometry growth step + #[arg(long, default_value_t = 256)] + growth_step_mb: usize, + + /// durable-sync cadence; bounds SafeNoSync file growth ≈ churn × this + /// window (0 = off, file grows until exit) + #[arg(long, default_value_t = 1000)] + sync_every_ms: u64, + + /// rocks block cache + #[arg(long, default_value_t = 256)] + rocks_cache_mb: usize, + + /// rocks: direct reads + direct flush/compaction. **Production parity** — + /// verified 2026-07-27 that no stack passes `--rocksdb-disable-direct-io` + /// and that the flag is inverted (`cli.rs:149`), so prod runs direct I/O. + /// Runs 01–26 were buffered and are NOT parity on this axis. + #[arg(long, default_value_t = false)] + rocks_direct_io: bool, + + /// rocks memtable size; RocksDB's default 64 MB is what production runs + #[arg(long, default_value_t = 64)] + rocks_write_buffer_mb: usize, + + /// rocks memtables before writes stop; the default 2 is the measured + /// production stall condition (`num_immutable` peaks at exactly 2 on every + /// stalled pod and 1 on every healthy one) + #[arg(long, default_value_t = 2)] + rocks_max_write_buffers: i32, + + /// rocks: universal compaction for the tables CF instead of leveled. + /// Production's leveled ladder costs 6.4x write amplification (measured + /// 2026-07-28); this is the arm that prices the alternative. + #[arg(long, default_value_t = false)] + rocks_universal: bool, + + /// rocks universal: `max_size_amplification_percent`. Lower means the DB + /// stays closer to its live size and pays more compaction to hold there. + #[arg(long, default_value_t = 200)] + rocks_size_amp_pct: i32, + + /// rocks leveled: `level0_file_num_compaction_trigger`; 0 keeps RocksDB's 4. + /// The L0→base merge rewrites the whole overlapping base level, so it costs + /// ~1 + base_size/L0_batch — 5.2x in production at the default. Raising this + /// amortizes that rewrite over more new data. + #[arg(long, default_value_t = 0)] + rocks_l0_trigger: i32 +} + +static NEXT_TABLE_ID: AtomicU64 = AtomicU64::new(1); +static LOGICAL_RAW: AtomicU64 = AtomicU64::new(0); +static STORED: AtomicU64 = AtomicU64::new(0); +// logical live stored bytes (workers add on commit, subtract on trim) — the +// race-free reference for what the wave peak *should* occupy on disk +static LIVE_STORED: AtomicU64 = AtomicU64::new(0); +static PEAK_LIVE_STORED: AtomicU64 = AtomicU64::new(0); +// rocks write-stall observability: prod stalls on the memtable ceiling, so a +// tuning arm is only judgeable if the bench reports the same three signals +static STALL_SAMPLES: AtomicU64 = AtomicU64::new(0); +static STALL_STOPPED: AtomicU64 = AtomicU64::new(0); +static STALL_DELAYED: AtomicU64 = AtomicU64::new(0); +static STALL_MAX_IMMUTABLE: AtomicU64 = AtomicU64::new(0); + +struct WorkerOut { + commit_us: Vec, + merge_us: Vec, + delete_us: Vec +} + +/// A live table: id, how many chunks it holds, stored bytes, and the +/// [c_lo, c_hi) chunk-counter span whose hash-index entries it owns. +#[derive(Clone, Copy)] +struct LiveEntry { + id: u64, + chunks: usize, + bytes: u64, + c_lo: u64, + c_hi: u64 +} + +struct PendingMerge<'scope> { + handle: std::thread::ScopedJoinHandle<'scope, Result>, + batch: Vec, + new_id: u64, + n_chunks: usize, + moved_bytes: u64, + c_lo: u64, + c_hi: u64 +} + +/// Joins the in-flight merge and applies its bookkeeping: the merged entry +/// enters `live` and the registry swaps sources for the merged table (readers +/// keep hitting the sources until then — the production read-vs-merge race). +fn finish_merge( + pending: &mut Option>, + live: &mut VecDeque, + reg: &std::sync::Mutex>, + out: &mut WorkerOut +) -> Result<()> { + let Some(p) = pending.take() else { return Ok(()) }; + let us = p.handle.join().expect("merge thread panicked")?; + out.merge_us.push(us); + live.push_back(LiveEntry { + id: p.new_id, + chunks: p.n_chunks, + bytes: p.moved_bytes, + c_lo: p.c_lo, + c_hi: p.c_hi + }); + let mut g = reg.lock().expect("registry"); + g.retain(|t| !p.batch.contains(t)); + g.push(p.new_id); + Ok(()) +} + +fn run_worker<'scope>( + engine: &'scope Engine, + args: &Args, + ds: usize, + reg: &'scope std::sync::Mutex>, + wave_done: &AtomicUsize, + tail_file: &AtomicU64, + scope: &'scope std::thread::Scope<'scope, '_> +) -> Result { + let mut rng = Rng::new(0x5EED ^ ((ds as u64) << 17)); + let mut out = WorkerOut { + commit_us: Vec::with_capacity(args.chunks), + merge_us: Vec::new(), + delete_us: Vec::new() + }; + // oldest first + let mut live: VecDeque = VecDeque::new(); + let mut unmerged: Vec = Vec::new(); + let mut live_chunks = 0usize; + let mut in_tail = false; + let mut pending: Option> = None; + // chunk i owns hash-index counters [P + i·n, P + (i+1)·n) + let hp = args.hash_preseed; + let hn = args.hash_per_commit; + let hk = args.hash_flush_every.max(1); + + for i in 0..args.chunks { + let id = NEXT_TABLE_ID.fetch_add(1, Ordering::Relaxed); + let mut entries: Vec<(Vec, Vec)> = Vec::with_capacity(args.pages_per_commit + 1); + entries.push((meta_key(id), vec![0u8; 64])); + let mut chunk_bytes = 64u64; + for idx in 0..args.pages_per_commit { + let raw = gen_page(&mut rng, args.page_bytes, args.dup); + LOGICAL_RAW.fetch_add(raw.len() as u64, Ordering::Relaxed); + let value = if args.compress { + lz4_flex::compress_prepend_size(&raw) + } else { + raw + }; + STORED.fetch_add(value.len() as u64, Ordering::Relaxed); + chunk_bytes += value.len() as u64; + entries.push((page_key(id, idx as u32), value)); + } + + let hash_inserts = (hn > 0 && (i as u64 + 1) % hk == 0).then(|| { + let hi = i as u64 + 1; + hp + (hi - hk) * hn..hp + hi * hn + }); + let t0 = Instant::now(); + engine.write_chunk(ds, &entries, hash_inserts)?; + out.commit_us.push(t0.elapsed().as_micros() as u64); + + let g = LIVE_STORED.fetch_add(chunk_bytes, Ordering::Relaxed) + chunk_bytes; + PEAK_LIVE_STORED.fetch_max(g, Ordering::Relaxed); + live.push_back(LiveEntry { + id, + chunks: 1, + bytes: chunk_bytes, + c_lo: i as u64, + c_hi: i as u64 + 1 + }); + live_chunks += 1; + unmerged.push(id); + reg.lock().expect("registry").push(id); + + if args.merge_fanin > 0 && unmerged.len() >= args.merge_fanin { + if args.concurrent_merges { + finish_merge(&mut pending, &mut live, reg, &mut out)?; + let batch = std::mem::take(&mut unmerged); + let new_id = NEXT_TABLE_ID.fetch_add(1, Ordering::Relaxed); + // sources leave `live` now so retention can't trim them out + // from under the merge txn; they re-enter as one entry at join + let mut moved_bytes = 0u64; + let mut n_chunks = 0usize; + let mut c_lo = u64::MAX; + let mut c_hi = 0u64; + live.retain(|e| { + let merged = batch.contains(&e.id); + if merged { + moved_bytes += e.bytes; + n_chunks += e.chunks; + c_lo = c_lo.min(e.c_lo); + c_hi = c_hi.max(e.c_hi); + } + !merged + }); + let batch2 = batch.clone(); + let handle = scope.spawn(move || { + let t0 = Instant::now(); + engine.merge_tables(ds, &batch2, new_id)?; + Ok(t0.elapsed().as_micros() as u64) + }); + pending = Some(PendingMerge { + handle, + batch, + new_id, + n_chunks, + moved_bytes, + c_lo, + c_hi + }); + } else { + let new_id = NEXT_TABLE_ID.fetch_add(1, Ordering::Relaxed); + let t0 = Instant::now(); + engine.merge_tables(ds, &unmerged, new_id)?; + out.merge_us.push(t0.elapsed().as_micros() as u64); + let mut moved_bytes = 0u64; + let mut c_lo = u64::MAX; + let mut c_hi = 0u64; + for _ in 0..unmerged.len() { + if let Some(e) = live.pop_back() { + moved_bytes += e.bytes; + c_lo = c_lo.min(e.c_lo); + c_hi = c_hi.max(e.c_hi); + } + } + live.push_back(LiveEntry { + id: new_id, + chunks: unmerged.len(), + bytes: moved_bytes, + c_lo, + c_hi + }); + let mut g = reg.lock().expect("registry"); + g.retain(|t| !unmerged.contains(t)); + g.push(new_id); + drop(g); + unmerged.clear(); + } + } + + let frac = i as f64 / args.chunks as f64; + let window = if args.window_wave > 0 && (0.2..0.5).contains(&frac) { + args.window_wave.max(args.window) + } else { + args.window + }; + while window > 0 && live_chunks > window { + let Some(e) = live.pop_front() else { break }; + live_chunks -= e.chunks; + LIVE_STORED.fetch_sub(e.bytes, Ordering::Relaxed); + unmerged.retain(|t| *t != e.id); + let hash_deletes = (hn > 0).then(|| hp + e.c_lo * hn..hp + e.c_hi * hn); + let t0 = Instant::now(); + engine.delete_tables(ds, &[e.id], hash_deletes)?; + out.delete_us.push(t0.elapsed().as_micros() as u64); + reg.lock().expect("registry").retain(|t| *t != e.id); + } + if args.window_wave > 0 && !in_tail && frac >= 0.5 { + in_tail = true; + // last worker through the trim snapshots the post-wave file size + if wave_done.fetch_add(1, Ordering::Relaxed) + 1 == args.datasets { + if let Ok(u) = engine.disk_usage() { + tail_file.store(u.file_bytes.max(1), Ordering::Relaxed); + } + } + } + + if args.paced_ms > 0 { + std::thread::sleep(Duration::from_millis(args.paced_ms)); + } + } + finish_merge(&mut pending, &mut live, reg, &mut out)?; + Ok(out) +} + +#[cfg(target_os = "linux")] +fn proc_io() -> Option<(u64, u64)> { + let text = std::fs::read_to_string("/proc/self/io").ok()?; + let mut read = None; + let mut write = None; + for line in text.lines() { + if let Some(v) = line.strip_prefix("read_bytes: ") { + read = v.trim().parse().ok(); + } else if let Some(v) = line.strip_prefix("write_bytes: ") { + write = v.trim().parse().ok(); + } + } + Some((read?, write?)) +} + +#[cfg(not(target_os = "linux"))] +fn proc_io() -> Option<(u64, u64)> { + None +} + +/// proc write_bytes counts page dirtying, not proven device traffic +/// (see docs/measurements/2026-07-16-flush-bench); cgroup io.stat is the +/// authoritative number when running in a pod. +fn cgroup_io() -> Option<(u64, u64)> { + let text = std::fs::read_to_string("/sys/fs/cgroup/io.stat").ok()?; + let mut read = 0u64; + let mut write = 0u64; + for line in text.lines() { + // stacked devices (md 9:*, dm 253:*) re-count bytes already charged + // to the physical device below them + if line.starts_with("9:") || line.starts_with("253:") { + continue; + } + for field in line.split_whitespace() { + if let Some(v) = field.strip_prefix("rbytes=") { + read += v.parse::().unwrap_or(0); + } else if let Some(v) = field.strip_prefix("wbytes=") { + write += v.parse::().unwrap_or(0); + } + } + } + Some((read, write)) +} + +fn mb(bytes: u64) -> f64 { + bytes as f64 / 1e6 +} + +fn main() -> Result<()> { + let args = Args::parse(); + + if args.dir.exists() { + std::fs::remove_dir_all(&args.dir)?; + } + std::fs::create_dir_all(&args.dir)?; + + let engine = match args.engine { + EngineKind::Mdbx => { + let n_envs = if args.per_dataset_env { args.datasets } else { 1 }; + let sync_mode = match args.sync { + SyncOpt::Durable => libmdbx::SyncMode::Durable, + SyncOpt::Safe => libmdbx::SyncMode::SafeNoSync + }; + Engine::Mdbx(MdbxEngine::open( + &args.dir, + n_envs, + sync_mode, + args.mdbx_page, + args.max_db_bytes, + args.growth_step_mb << 20 + )?) + } + EngineKind::Rocks => Engine::Rocks(RocksEngine::open( + &args.dir, + engine::RocksOpts { + cache_mb: args.rocks_cache_mb, + direct_io: args.rocks_direct_io, + write_buffer_mb: args.rocks_write_buffer_mb, + max_write_buffers: args.rocks_max_write_buffers, + universal: args.rocks_universal, + size_amp_pct: args.rocks_size_amp_pct, + l0_trigger: args.rocks_l0_trigger + } + )?) + }; + + // verify generated pages actually compress like production data + { + let mut rng = Rng::new(1); + let sample = gen_page(&mut rng, args.page_bytes, args.dup); + let ratio = sample.len() as f64 / lz4_flex::compress_prepend_size(&sample).len() as f64; + println!("sample page lz4 ratio: {ratio:.2}x (prod tables: 4.03x)"); + } + + if args.hash_per_commit > 0 { + println!( + "hash stream: preseed={} per_commit={} flush_every={}", + args.hash_preseed, args.hash_per_commit, args.hash_flush_every + ); + } + if args.hash_preseed > 0 { + let t0 = Instant::now(); + let per_ds = args.hash_preseed; + std::thread::scope(|s| { + let engine = &engine; + let handles: Vec<_> = (0..args.datasets) + .map(|ds| s.spawn(move || engine.preseed_hashes(ds, per_ds))) + .collect(); + handles + .into_iter() + .map(|h| h.join().expect("preseed panicked")) + .collect::>>() + })?; + engine.sync()?; + println!("hash preseed done in {:.1}s", t0.elapsed().as_secs_f64()); + } + + let io_before = proc_io(); + let cg_before = cgroup_io(); + let started = Instant::now(); + let done = AtomicBool::new(false); + let peak_file = AtomicU64::new(0); + let wave_done = AtomicUsize::new(0); + // file size once every worker finished its post-wave trim; growth past + // this at constant live = the freelist failing to serve the churn + let tail_file = AtomicU64::new(0); + let registry: Vec>> = + (0..args.datasets).map(|_| std::sync::Mutex::new(Vec::new())).collect(); + + let (outs, reads, peak) = std::thread::scope(|scope| { + let engine = &engine; + let args = &args; + let done = &done; + let peak_file = &peak_file; + let wave_done = &wave_done; + let tail_file = &tail_file; + let registry = ®istry; + + let sampler = scope.spawn(move || { + while !done.load(Ordering::Relaxed) { + if let Ok(u) = engine.disk_usage() { + peak_file.fetch_max(u.file_bytes, Ordering::Relaxed); + } + std::thread::sleep(Duration::from_millis(500)); + } + }); + // 20 ms, not the sampler's 500 ms: a stall interval on a 20–100 s run is + // short, and prod only sees these at scrape resolution + let staller = engine.stall_probe().is_some().then(|| { + scope.spawn(move || { + while !done.load(Ordering::Relaxed) { + if let Some((stopped, immutable, delayed)) = engine.stall_probe() { + STALL_SAMPLES.fetch_add(1, Ordering::Relaxed); + if stopped > 0 { + STALL_STOPPED.fetch_add(1, Ordering::Relaxed); + } + if delayed > 0 { + STALL_DELAYED.fetch_add(1, Ordering::Relaxed); + } + STALL_MAX_IMMUTABLE.fetch_max(immutable, Ordering::Relaxed); + } + std::thread::sleep(Duration::from_millis(20)); + } + }) + }); + let syncer = (args.sync_every_ms > 0).then(|| { + scope.spawn(move || { + while !done.load(Ordering::Relaxed) { + engine.periodic_sync().expect("periodic durable sync failed"); + std::thread::sleep(Duration::from_millis(args.sync_every_ms)); + } + }) + }); + let reader = (args.reader_hold_secs > 0).then(|| { + scope.spawn(move || { + while !done.load(Ordering::Relaxed) { + let _ = engine.hold_reader(args.reader_hold_secs); + } + }) + }); + + let scanners: Vec<_> = (0..args.readers) + .map(|r| { + scope.spawn(move || { + let mut rng = Rng::new(0x0EAD ^ ((r as u64) << 9)); + let mut scan_us = Vec::new(); + let mut pages = 0usize; + let mut raw = 0u64; + let mut misses = 0usize; + while !done.load(Ordering::Relaxed) { + let ds = (rng.next() as usize) % args.datasets; + let id = { + let g = registry[ds].lock().expect("registry"); + (!g.is_empty()).then(|| g[(rng.next() as usize) % g.len()]) + }; + let Some(id) = id else { + std::thread::sleep(Duration::from_millis(1)); + continue; + }; + let t0 = Instant::now(); + match engine.scan_table(ds, id, args.compress) { + Ok((0, _)) | Err(_) => misses += 1, + Ok((p, r)) => { + scan_us.push(t0.elapsed().as_micros() as u64); + pages += p; + raw += r; + } + } + if args.read_paced_ms > 0 { + std::thread::sleep(Duration::from_millis(args.read_paced_ms)); + } + } + (scan_us, pages, raw, misses) + }) + }) + .collect(); + + let handles: Vec<_> = (0..args.datasets) + .map(|ds| { + scope.spawn(move || run_worker(engine, args, ds, ®istry[ds], wave_done, tail_file, scope)) + }) + .collect(); + let outs = handles + .into_iter() + .map(|h| h.join().expect("worker panicked")) + .collect::>>(); + + done.store(true, Ordering::Relaxed); + sampler.join().expect("sampler panicked"); + if let Some(s) = staller { + s.join().expect("stall sampler panicked"); + } + if let Some(s) = syncer { + s.join().expect("syncer panicked"); + } + if let Some(r) = reader { + r.join().expect("reader panicked"); + } + let reads: Vec<_> = scanners + .into_iter() + .map(|h| h.join().expect("scanner panicked")) + .collect(); + (outs, reads, peak_file.load(Ordering::Relaxed)) + }); + let outs = outs?; + + let elapsed = started.elapsed(); + engine.sync()?; + if args.window_wave > 0 { + // rocks keeps compacting after the last flush; give the wave A/B a + // comparable end state (mdbx files are static by then) + std::thread::sleep(Duration::from_secs(10)); + } + let usage = engine.disk_usage()?; + let io_after = proc_io(); + let cg_after = cgroup_io(); + + let mut commit_us = Vec::new(); + let mut merge_us = Vec::new(); + let mut delete_us = Vec::new(); + for o in outs { + commit_us.extend(o.commit_us); + merge_us.extend(o.merge_us); + delete_us.extend(o.delete_us); + } + + let logical = LOGICAL_RAW.load(Ordering::Relaxed); + let stored = STORED.load(Ordering::Relaxed); + + println!("\n=== result ==="); + println!( + "elapsed: {:.1}s, commits: {} ({:.1}/s)", + elapsed.as_secs_f64(), + commit_us.len(), + commit_us.len() as f64 / elapsed.as_secs_f64() + ); + println!( + "logical raw: {:.1} MB, stored (post-compress): {:.1} MB", + mb(logical), + mb(stored) + ); + for (name, lat) in [("commit", commit_us), ("merge", merge_us), ("delete", delete_us)] { + let r = latency_report(lat); + println!( + "{name}: n={} p50={}us p90={}us p99={}us max={}us", + r.count, r.p50, r.p90, r.p99, r.max + ); + } + if args.readers > 0 { + let mut scan_us = Vec::new(); + let (mut pages, mut raw, mut misses) = (0usize, 0u64, 0usize); + for (us, p, r, m) in reads { + scan_us.extend(us); + pages += p; + raw += r; + misses += m; + } + let rep = latency_report(scan_us); + println!( + "read: n={} p50={}us p90={}us p99={}us max={}us, pages/scan={:.1}, raw {:.1} MB, misses={}", + rep.count, + rep.p50, + rep.p90, + rep.p99, + rep.max, + pages as f64 / rep.count.max(1) as f64, + mb(raw), + misses + ); + } + println!( + "disk: file={:.1} MB (peak {:.1} MB), live={:.1} MB", + mb(usage.file_bytes), + mb(peak.max(usage.file_bytes)), + mb(usage.live_bytes) + ); + if args.window_wave > 0 { + let base = tail_file.load(Ordering::Relaxed); + if base == 0 { + println!("wave: post-wave baseline not captured (run too short?)"); + } else { + println!( + "wave: stored live peak={:.1} MB, file@post-wave={:.1} MB, file@end={:.1} MB, post-wave growth={:.1} MB", + mb(PEAK_LIVE_STORED.load(Ordering::Relaxed)), + mb(base), + mb(usage.file_bytes), + mb(usage.file_bytes.saturating_sub(base)) + ); + } + } + println!("engine: {}", engine.describe()); + let stall_n = STALL_SAMPLES.load(Ordering::Relaxed); + if stall_n > 0 { + let pct = |v: u64| 100.0 * v as f64 / stall_n as f64; + println!( + "rocks stalls: write_stopped {:.2}% of {stall_n} samples, delayed {:.2}%, \ + max_immutable_memtables {} (ceiling {}); wbuf {} MB", + pct(STALL_STOPPED.load(Ordering::Relaxed)), + pct(STALL_DELAYED.load(Ordering::Relaxed)), + STALL_MAX_IMMUTABLE.load(Ordering::Relaxed), + args.rocks_max_write_buffers, + args.rocks_write_buffer_mb + ); + } + if let Engine::Rocks(e) = &engine { + let (flush, compact, wal) = e.write_bytes(); + println!( + "rocks writes: flush={:.1} MB compaction={:.1} MB wal={:.1} MB, \ + lsm_write_amp={:.2}x, style={}, l0_trigger={}", + mb(flush), + mb(compact), + mb(wal), + (flush + compact) as f64 / flush.max(1) as f64, + if args.rocks_universal { "universal" } else { "leveled" }, + if args.rocks_l0_trigger > 0 { + args.rocks_l0_trigger + } else { + 4 + } + ); + } + if let Engine::Mdbx(e) = &engine { + let (entries, raw) = e.verify_scan(args.compress)?; + println!( + "verify: {entries} live entries read back via the KvRead seam, {:.1} MB raw", + mb(raw) + ); + } + match (io_before, io_after) { + (Some((r0, w0)), Some((r1, w1))) => { + let (dr, dw) = (r1 - r0, w1 - w0); + println!( + "os io: read={:.1} MB write={:.1} MB, write_amp vs stored: {:.2}x, vs raw: {:.2}x", + mb(dr), + mb(dw), + dw as f64 / stored.max(1) as f64, + dw as f64 / logical.max(1) as f64 + ); + } + _ => println!("os io: /proc/self/io unavailable (run on Linux for write amplification)") + } + match (cg_before, cg_after) { + (Some((r0, w0)), Some((r1, w1))) => { + let (dr, dw) = (r1.saturating_sub(r0), w1.saturating_sub(w0)); + println!( + "cgroup io (authoritative): read={:.1} MB write={:.1} MB, write_amp vs stored: {:.2}x, vs raw: {:.2}x", + mb(dr), + mb(dw), + dw as f64 / stored.max(1) as f64, + dw as f64 / logical.max(1) as f64 + ); + } + _ => println!("cgroup io: io.stat unavailable (fine outside a cgroup v2 pod)") + } + + Ok(()) +} diff --git a/docs/adr/0002-storage-engine-libmdbx.md b/docs/adr/0002-storage-engine-libmdbx.md new file mode 100644 index 00000000..b9a57a4a --- /dev/null +++ b/docs/adr/0002-storage-engine-libmdbx.md @@ -0,0 +1,731 @@ +# ADR 0002 — Replace RocksDB with libmdbx in hotblocks storage + +Status: proposed — spike done 2026-07-26, write gate passed +([measurements](../measurements/2026-07-26-mdbx-churn-bench.md)); Stage 0 +(spec amendment) landed 2026-07-26; flips to accepted after the Stage 1 +vertical slice — real table build → chunk merge → retention → crash recovery +through the mdbx seam, with the checksummed codec and MAP_FULL / +stale-reader / kill-9 / cold-read-smoke / online-reset tests. The +production-size cold-cache read gate is deliberately *not* an acceptance +gate: it bounds rollout (Stage 4, before Stage 5), and the accept-time read +exposure is bounded instead by the measured production scan bracket +(Measurements). · Date: 2026-07-25 · Scope: +`sqd-storage`, `sqd-hotblocks`, `crates/hotblocks/spec` + +## Context + +The original complaint — "we burn CPU on compaction" — is stale: the CPU cost was +tempfile spill and was removed by ADR 0001 (a production replica now runs at ~2–3 +cores). What remains is the disk bill of the LSM itself, measured on a production +replica serving 45 datasets (2026-07-25): + +| metric | value | +|----------------------------|------------------------------------------------------| +| device writes | ~397 MB/s sustained (≈34 TB/day) | +| device reads | ~426 MB/s sustained | +| live SST size | ~62 GB | +| concurrent compactions | 1.4–2.3, continuously | +| commit rate | ~59 chunk commits/s pod-wide (~72 blocks/s ingested) | +| reader (snapshot) lifetime | avg ~12 s, max 233 s/day, ≤42 concurrent | + +The device figures are **corrected 2026-07-27**; an earlier revision recorded +~640 MB/s write / ~690 MB/s read, ~1.6× too high. `container_fs_*_bytes_total` +carries *two* stacked duplications and the query must exclude both: the same +bytes appear on `/dev/dm-N` and on the underlying `/dev/nvmeXn1` (device-mapper +re-count), **and** on a `container=""` pod-aggregate series beside the +per-container one. Filter `container="hotblocks-db", device=~"/dev/nvme.*"`; +summing by device alone still reads 2×. Cross-validated against in-pod cgroup +`io.stat` (`259:2`, the physical nvme — majors 9/253 are the stacked re-counts), +which matched the corrected metric within burst noise. The correction also +resolves an arithmetic contradiction the old figure carried: 397 MB/s of +compressed output implies ~1.6 GB/s of LZ4 input ⇒ ~2.3–3.2 cores of codec, +which fits the 3.4–3.6 cores the pod actually burns, where 640 MB/s implied +~5 cores and did not. + +Writing ~397 MB/s to rewrite a 62 GB working set is an amplification factor the +data does not ask for: keys are append-mostly (UUIDv7 table prefixes, ascending +chunk keys), values are 16–64 KiB opaque pages, and the whole store is a rolling +retention window. A B+tree engine with in-place page reuse (libmdbx — the engine +under reth and erigon) charges COW page churn (~2–3×) instead of LSM leveling +(~10×+), and has no background compaction to interfere with read tails. + +Write stalls (`is-write-stopped`) fired on two stacks — but **not for the reason +this ADR assumes**, and the measurement is now in (2026-07-27, 7 d): the stalls +are memtable flush backpressure, not compaction debt. `immutable_memtables` +peaks at exactly 2 on all four stalled pods (internal ×2, testnet-dev ×2) and at +1 on all four that never stalled — and RocksDB's default `max_write_buffer_number` +is 2, so that ceiling *is* the stop condition. The alternatives do not fit: +`files_at_level0` peaks at 15 against a default stop trigger of 36 and is the +same order on stalled and healthy pods, and `pending_compaction_bytes` peaks at +16.2 GB, 5.9% of the 256 GB default hard limit. `db.rs` sets no memtable or L0 +option at all, so this is the stock 64 MB × 2. Cost: internal-db-1 spends 1.2% +of wall time write-stopped (~2 h/week). Consequence: the latency half of the +motivation (weight ×3) is not evidence for an engine swap until a larger-memtable +arm is run against the bench reference — see Alternatives. + +One honest caveat: a second, application-level write stream — the chunk +`compaction_loop` that merges small chunks up to 200k rows — rewrites the +retention window log-many times and survives any engine swap. libmdbx removes +only the LSM multiplier stacked on top of it. + +Decision weights, agreed up front: latency ×3, disk amplification ×2, migration +cost ×1. + +- **Latency (×3)**: no compaction ⇒ no stalls, no L0 spikes, flatter read tails. + New structural risk: libmdbx has one writer per environment, and a chunk-merge + transaction holds that lock long enough to stall head freshness of every + dataset in the env. Mitigation is one env per dataset (also restores INV-35 + isolation and demotes HZ-1 from structural to per-dataset). +- **Disk (×2)**: device writes expected to drop severalfold — but only with + application-level page compression in place, because libmdbx does not compress + and the measured LZ4 ratio on `CF_TABLES` is 4× (see Measurements). The seam + for it already exists (`BufferPageWriter::write_page`). The engine-reclaim + machinery (10 s point-delete sweep, deletion-collector, periodic compaction, + startup `DeleteFilesInRange`, the `TableId` watermark, `reclaim-measure`) + becomes unnecessary: pages free at commit. Not the application half: table + builds flush in 8 MiB sub-commits before the chunk metadata publishes + (`TableStorage::put`), so a crash leaves committed orphan pages the engine + cannot recognize — the dirty-table journal and the startup orphan purge + survive the port (cheaper there: one delete txn at boot, no compaction + wait). RS-5/6, LIV-7 and GAP-6 close by construction; RS-10 stays + app-level. +- **Migration (×1)**: no data conversion exists to pay for. The store is fully + re-ingestable from upstream (NG1 "not an archive"; DEF-17 hash indexes are + expendable; GAP-28 Api retention floors are memory-only and re-pushed by + hotblocks-retain within ≤300 s; GAP-15 no format gate). Rollout is blue/green + on an empty volume. The cost is code: the `table::read`/`table::write` codec is + already generic over `kv.rs` (`KvRead`/`KvWrite`/`KvReadCursor`) and ports + nearly verbatim, but `crates/storage/src/db/*` (~1300 lines) threads RocksDB + types throughout — transactions, metrics, CLI, shutdown, tests. Estimate: 3–6 + weeks to port and roll out. + +## Porting notes (not blockers) + +1. **GAP-29 (zombie readers)** costs more under libmdbx — one stale read txn + blocks reuse of every page freed after it started. The containment must be + application-owned: the published `signet-libmdbx` 0.8.3 ships no working + read-timeout (its `timer.rs` is an unwired dead file, no feature flag), + and reth-libmdbx's `MaxReadTransactionDuration` (~5 min) is the wrong + scale next to a 10 s query budget while today's legitimate tail (233 s + streams) exceeds `P-QUERY-TIME` anyway — so the deadline must bound the + *response lifecycle*, owned by the service (or park/re-establish the + snapshot), with `mdbx_env_set_hsr` as the space-pressure backstop. The + backstop *names* the laggard — in-process it cannot safely abort another + thread's txn, so the kill is the service cancelling that response; hsr + detects, the deadline enforces. GAP-29 stays open as its own P2. +2. **Memory regime — production runs DIRECT I/O** (re-verified against the + running pods 2026-07-27; the 2026-07-26 "correction" that claimed + buffered was itself wrong and is withdrawn). The flag is inverted: + `cli.rs:149` computes `.with_direct_io(!self.rocksdb_disable_direct_io)` + from a bare `#[arg(long)]` bool, so *absence* of + `--rocksdb-disable-direct-io` enables direct I/O. All three stacks' + pod args lack the flag and carry no env override, and `db.rs:192-195` + then sets `set_use_direct_reads(true)` **and** + `set_use_direct_io_for_flush_and_compaction(true)`. Verified args also + confirm `--data-cache-size 8192` (the 8 GiB block cache is real, not the + 256 MB default), 12 G request / 64 G limit, 2-CPU request, and + `--block-hash-index --transaction-hash-index` on internal only. + + So the port changes the regime, not merely the cache split, and by more + than an earlier revision assumed. Today: an 8 GiB *uncompressed* block + cache (anon) and **no SST page cache at all** — reads outside the block + cache go to the device, and flush/compaction writes bypass the page + cache, which is why the ~640 MB/s is true device traffic rather than + writeback. Under mdbx: mmap is unconditionally buffered, the anon block + cache disappears, and a ~50 GiB page cache becomes load-bearing for the + first time. Consequences to carry: the freed 8 GiB anon budget is the + natural size bound for the deferred decompressed-page LRU; + `working_set` ≈ limit is a *new* steady state, not a continuation of + today's, so leak detection moves to anon RSS and pressure alerting to + memory PSI + major-fault rate; the read comparison cuts both ways and + must be measured rather than reasoned (today's misses already pay device + latency with no page-cache backstop, so mdbx may read *better* cold + while paying decode); and Stage 4 must run under the production values — + 12 G request / 64 G limit, 2-CPU request — not an unconstrained bench + box. + + **Parity consequence for the bench (open).** The rocks reference ran + buffered and the measurements record asserts that this matches + production — it does not. Runs 18/19 closed the WAL-compression and + background-jobs gaps but not the I/O mode, so the ≥3× gate is still + measured against a reference that differs from production on the axis + that decides both device-write accounting and read latency. Re-run the + reference with direct I/O before Stage 1; note the post-port A/B is + inherently asymmetric (rocks direct writes vs mdbx page-cache + writeback) and must state which side of the cache each number is taken + from. +3. **File high-water (measured, wave runs 15–17).** mdbx recycles freed pages + but returns nothing to the filesystem: across the retention-wave runs tail + truncation never fired even with an armed `shrink_threshold` and 80–93% of + pages free — the file is the high-water mark of the working set, + permanently. Working-set spikes are real (retention is downstream-driven; a + stalled consumer grows the window at ~2.5 MB/s stored pod-wide ≈ 9 GB/h). + Measured shape at production pace: file ≈ 1.35× peak stored live plus + 256 MiB growth-step rounding (paced wave run 16); a free-run burst + additionally parks freed pages up to the sync cadence (≈ device rate × + cadence) and peaked at 2.0× (run 15). Reuse is intact — zero file + growth across 27 k post-wave + commits at constant live, so the bloat does not compound; untouched pages + occupy no RAM and tree depth tracks the live set. The rocks reference + reclaims the same wave for ~3.5× the device writes (parity-corrected; + measurements "Verdict") — that is the trade being made. Consequences and + the reclaim plan: see "Disk policy" below. +4. **Hash indexes must not port as global hash-keyed tables — priced, runs + 22–26.** `CF_BLOCK_HASHES`/`CF_TRANSACTION_HASHES` are hash-keyed; + mainnet-internal runs both today (mainnet runs neither). An LSM absorbs + random inserts in the memtable; a B+tree COWs a leaf path per touched + leaf per txn. Measured at internal's shape (300 entries/commit against a + 1 M-entry tree): the index stream costs mdbx **27× its own data stream** + (+31.2 GB vs a 1.17 GB baseline; rocks pays +4.0 GB for the same) — + with global tables mdbx writes 4.5× *more* than rocks and the whole + Context table inverts. Write-behind batching recovers only 26% at + K=16 and buys 26 ms flush tails. The port therefore restructures: + per-chunk index runs written append-shaped with the chunk (modeled + ~45–80× cheaper — what runs 22–26 measure is the global table's + failure, not the replacement's cost), lookups fan out over runs under + an explicit budget (RP-20 must be amended for this; per-chunk bloom if + the fan-out ever hurts) — or the feature is explicitly kept off + mdbx-backed deployments, which the endpoint-traffic query may decide + on its own. The Stage 4 envelope still includes the internal shape + (both indexes on, per-chunk layout). Full design and prior art: + "Hash-index design" below. + +## Binding (decided 2026-07-26) + +Port on **vorot93 `libmdbx` 0.6.x** (crates.io; `mdbx-sys` exact-pinned at +13.11.0 = C v0.13.11) — the crate the spike measured on. It covers geometry, +SafeNoSync + `sync(force)`, `last_pgno`/freelist; the gaps — `mdbx_env_set_hsr`, +`MDBX_opt_sync_period`/`sync_bytes`, `mdbx_env_copy` — are single FFI calls +shimmed over the pinned sys crate (bindgen already exposes the full +`^(MDBX|mdbx)_.*` surface). Runners-up, for the record: *reth-libmdbx* is +technically strongest (C 0.13.12, wired read-tx timeouts, safe hsr/sync_period) +but is not on crates.io — a git dependency on the reth monorepo, pinned to +tags whose cadence serves reth; its timeout scale is wrong for us anyway (the +read deadline is service-owned, porting note 1). *signet-libmdbx* 0.8.3 is +rejected: its sys crate has never been re-released — production C frozen at +0.13.7, *missing the 0.13.8 SIGBUS-on-full-filesystem fix* that sits directly +on our MAP_FULL path — and its advertised read-timeout support is dead code +(feature undeclared, module unwired). Supply-chain posture: all bindings +vendor the C amalgamation, so a C fix ships only when the binding re-vendors; +vorot93 is single-maintainer with a multi-year ~1–2-month lag record. +Upstream now releases from SourceCraft (RF-hosted; GitHub is a mirror) — +review vendored-C diffs on every bump. Watch item: libmdbx 0.14.x goes +stable 2026-08-08 with native defragmentation and `mdbx_txn_checkpoint` — +re-evaluate at Stage 3, it may hand DP-3 compact-swap a supported primitive. + +## Hash-index design (per-chunk; priced 2026-07-26, runs 22–26) + +**Why the global table fails under mdbx — the mechanism.** A tx hash is a +uniformly random key; in a COW B+tree the unit of write is the page, and in +a large tree each entry of a batch lands on a *distinct* leaf (300 random +keys vs ~21 k leaves collide rarely). One chunk commit therefore rewrites +~300 leaves for inserts plus ~300 for the pruned chunk's deletes ≈ 600 × +4 KiB ≈ 2.4 MB of device writes for ~26 KB of payload — measured 27× the +entire data stream (245× on the index churn itself), inverting the engine +decision. The data stream is immune for the same reason in reverse: chunk +keys are append-shaped (fresh `table_id` range), so pages are created +adjacent and written once. An LSM absorbs the same random inserts in its +memtable and sorts them on flush — rocks paid +4.0 GB where mdbx paid ++31.2 GB. + +**Design: the index is a sidecar run of the chunk.** Stop maintaining one +sorted-by-hash order over the whole window; keep many small sorted runs, +one per chunk, that are only ever created and dropped — never edited. +The write-side failure of the global table is *measured* (runs 22–26); +everything below about the per-chunk replacement is *modeled* and must +pass a Stage 2 prototype gate before it is load-bearing. + +- *Layout.* Key `table_id ‖ tag_hash ‖ hash` → value 8 B (block number) / + 12 B (block + tx index). The hash is stored as the **exact bytes the + dataset uses**, 1..`P-HASH-MAXLEN` — not a decoded 32 B digest: IB-10 + binds `{hash}` to a byte-for-byte comparison with "no normalization, no + case folding, no `0x` handling", and the corpus is not all hex (the + fork tests key on `fork_{n}`). Decoding to 32 B would be an API change + needing ingest validation, lookup normalization and an IB-10/CT rewrite; + it is not free and is not taken here. Cost of the contract: an EVM entry + is 16 + 1 + 66 + 12 = 95 B of payload against 61 B for a decoded digest, + so the index is ~1.6× larger than a binary layout would be — material + against RS-12, which warns `tidx` can be the largest single consumer in + the store. +- *Write cost (modeled, not measured).* A fresh `table_id` puts the whole + chunk index into a contiguous fresh key range. At ~105 B/entry including + libmdbx node overhead, ~300 entries ≈ 31.5 KB ⇒ **~8 leaves plus spine, + ~36–40 KB/commit with `MDBX_APPEND`**; without append mode split-fill + drops to 50–60% and it is 60–72 KB. Entries are sorted by construction, + so append mode is available and is a Stage 1 requirement, not an + optimization — it is a factor of 2. Against the measured global-table + cost of ~3.25 MB/commit that is **~45–80× cheaper**, not the ~270× an + earlier revision claimed from raw payload alone. In absolutes on the + run-22 shape: +0.38…0.69 GB against a 1.17 GB data baseline (+32…59%), + not the +10% that 270× implies. The ≥3× gate survives — 1.55–1.86 GB + against rocks' 7.13 GB is 3.8–4.6×, since rocks pays more for the same + index — but the margin is stated, not assumed. +- *Integrity.* The run MUST NOT be an exception to the page codec. mdbx has + no data-page checksums and INV-45 makes a hit authoritative — a client + cannot cheaply re-verify one — so a silently corrupted entry is exactly + the failure INV-45 calls a defect. Two admissible shapes, decided at + Stage 1: (a) per-entry envelope `version ‖ position ‖ checksum(table_id ‖ + tag ‖ hash ‖ position)` — cheapest to read, but ~33% value overhead and + it must cover the key; (b) store the run as **checksummed codec pages** + (no LZ4 — `spec/09` measured 0.49%/0.36% on random hashes, there is + nothing to compress) and binary-search within a run. (b) keeps FM-STOR-4 + uniform, removes the raw-KV exception entirely and gives the bloom a + natural home, at 1–2 page reads per chunk instead of one point-get. + Bloom values carry the same version + checksum either way. +- *Write path.* The source stays `for_each_*_hash` at chunk build + (`tx.rs:311–328`); sort in memory, write through the normal table-build + path with its 8 MiB sub-commits. The optimistic staging with scans and + retries goes away. +- *Chunk metadata.* The run is **not** one more entry of `Chunk::tables`: + `compaction.rs:260-263` treats table-count equality as part of schema + compatibility (a toggled flag would split merge runs mid-window) and + then opens every entry with `create_table_reader` for `.schema()` / + `.num_rows()`, which a raw run cannot answer. It needs its own slot — + `Chunk::V2 { tables, block_hash_run: Option, + transaction_hash_run: Option }` — plus explicit rules: the run + is built in sub-commits and stays dirty until the metadata commit + publishes chunk and run IDs together and clears the dirty markers; + REPLACE/RETAIN removes chunk metadata and queues the run IDs for + physical GC in one transaction; compaction merges only runs that + already exist and never scans base tables to backfill; behavior does not + depend on the current enable flag, so an on→off transition keeps + existing entries until they age out naturally; and on a duplicate hash, + newest chunk wins. Without these, a mixed window (off→on, on→off, or a + merge across the boundary) either backfills unexpectedly or drops an + index that was there. +- *Deletion.* Retention drops the chunk's key ranges; the index dies with + them. Today's delete path re-reads the dying chunk to re-derive keys for + the global CF (`tx.rs:361–393`) — deleted code. Physical reclamation of + a dropped run is a bounded debt queue / range-GC, sized so a trim + transaction still fits the remaining freelist (DP-2's headroom + guarantee); the mechanism and the minimum libmdbx version it needs are a + Stage 1 deliverable, not an implicit "range-delete is free". +- *Reorg/REPLACE.* Consistency for free: the index is atomic with its + chunk, so the global-CF obligation to purge the forked branch's hashes + (dangling positions otherwise) vanishes structurally. +- *Merge.* `compaction_loop` carries the sources' runs into the merged + chunk. The values are positions (block number / block + tx index) that a + merge does not change, so the merged chunk SHOULD **inherit** its + sources' runs rather than re-sort them — the merge then writes zero + index bytes by construction and fan-out runs over runs rather than + chunks. If inheritance is rejected, the re-sort cost must be priced per + generation: an earlier revision's "~0.3% of chunk bytes" is wrong by two + orders of magnitude — it divided the per-chunk write cost by the + *global table's* per-commit cost. The real space ratio is ~12 KB of + index against a ~43 KB stored chunk ≈ 28%, consistent with RS-12. +- *Lookup — bounded fan-out, and it changes the contract.* Snapshot → + live chunks (`CF_CHUNKS` enumerates) → probe per run, newest first, + early exit. This is θ(runs), and RP-20 says lookup "cost MUST NOT scale + with the window" — a hard MUST this design violates, not a formality: + an uncapped dataset or schema-incompatible chunks can hold far more than + the nominal 50–150 runs, and a lookup flood would compete for page cache + and storage workers that RP-20 separately requires to stay isolated from + range queries. The contract must therefore change explicitly (Stage 0 + item 8): permit bounded fan-out, introduce `P-HASH-MAX-RUNS` plus a + wall-time / read-byte budget, and give lookups their own `P-HASH-SLOTS` + admission budget disjoint from `P-EXEC-SLOTS`. A probe abandoned on + budget returns a miss — consistent with RP-19, under which a miss + already proves nothing — but that behavior must be normative, not + incidental. Estimated ~0.1–1 ms hot / ~10–15 ms cold at 50–150 runs is + modeled; a miss pays the full fan-out, which is why the Tier-0 endpoint + miss-rate query decides whether this design is the right shape at all. + The no-backfill property is unchanged (chunks without a run are + skipped). A cold fan-out touches ~0.5–1.8 MB of page cache (spine + one + leaf per run); "spines stay resident" is an assumption to be measured, + not a property. +- *Escalation, behind a trigger like the read LRU:* per-chunk bloom + (~10 bits/entry, stored beside the run, versioned + checksummed, lazily + RAM-cached) → expected cost = in-RAM filter checks + 1 real get. + Ship fan-out first; add blooms only if the endpoint p99 or miss + read-amp misbehaves. +- *Acceptance.* Beyond DEF-17 (indexes expendable, which stands): a CT + hitting newest and oldest run plus a miss, at 50 / 150 / 500 runs, hot + and cold, concurrent with range-query load — the test that decides + whether the bounded-fan-out contract holds under the flood RP-20 was + written to prevent. Mixed-window semantics (off→on, on→off, merge across + the boundary, duplicate hash) get their own CT. + +**Prior art — reth/erigon run the same global table and survive it; their +mitigations do not transfer.** reth's `TransactionHashNumbers` is one +global mdbx table, viable because (a) the bulk volume bypasses random +inserts entirely — the `TransactionLookup` stage accumulates into ETL +collectors (500 MB sorted spills, 5 M-tx batches) and writes mdbx in +sorted order, in append mode when the table is empty; batch ≫ tree width +is where batching actually amortizes (our K=16 recovered 26% — a rolling +tip cannot accumulate for hours); (b) the live tip is ~25 random +inserts/s on L1 with *zero deletes* — an archive index, kept whole by +full nodes for `eth_getTransactionByHash` — vs our thousands/s plus equal +deletes across 45 datasets, permanently; (c) reth moved *data* to static +files but kept the *index* global because their lookup is a hot O(1) RPC +over thousands of segments — fan-out is unaffordable there and affordable +here (rare endpoint, 50–150 chunks). The symmetry: reth pays the +global-table write tax because it cannot afford fan-out reads; we take +fan-out reads because we cannot afford the write tax. Per-chunk index +runs are the static-file idea applied to the index itself — the same +split the sealed-tier escalation below already cites. + +## Disk policy (per-dataset env) + +Decided on the wave runs and one hard requirement: **one stuck dataset must +never stop the rest**. The product half of this policy already shipped: +PR #77 (NET-896, merged 2026-07-17) lets `Api { max_blocks }` datasets +sacrifice the tail past the portal floor to keep the head fresh — a cap in +block *positions*, opt-in per dataset, soft (whole-chunk trims), with +downward portal instructions clamped to `first(D)` so gap mode does not +resync-loop. What remains unbounded is bytes (blocks vary in size), +uncapped datasets, and the shared volume itself — and none of it is +observable yet. The port closes the class by construction; DP-2/DP-4 +generalize NET-896 rather than replace it. + +- **DP-1 Quota.** Geometry `max_size` = the dataset's disk budget, enforced + by the engine. Sizing from the waves: intended peak window ×1.4, rounded up + to the growth step — the paced-churn shape (run 16). Catch-up and + reset-reingest bursts add the SafeNoSync parking term on top (≈ per-env + device rate × sync cadence; the free-run wave peaked at 2.0×, run 15): + leave that headroom, or accept a gap-mode blip while the burst runs. + Deployment check: Σ quotas plus an operating reserve MUST fit the volume + (INV-43) — per-env quotas alone do not bound the shared volume. The + refusal is dataset-scoped: the dataset(s) whose addition or quota raise + created the overcommit are refused and alarmed while the admitted set + boots (N2/GAP-24 containment — a config mistake on dataset 46 must not + take down 45); whole-pod startup failure is reserved for an overcommit + with no identifiable marginal dataset. +- **DP-2 Ring past the floor.** The effective floor is the max of every + governing bound: the instructed floor, NET-896's position cap + (`next(D) − max_blocks`), and a byte watermark (~90% of quota). The + watermark needs **two signals**, because "free" and "reusable" diverge + exactly when it matters: the *retention* signal is occupied pages + (allocated minus freelist — never the file, which is a permanent + high-water and would pin gap-mode forever after the first peak); the + *availability* signal is reusable headroom = growth headroom + (`max_size − file`) plus the freelist share actually recyclable now — + excluding pages parked behind the oldest reader txn and pages freed + since the last durable sync (SafeNoSync parks them until the next sync; + the stale-reader run held 8.3 GB of file against 33 MB live with nearly + the whole file "free"). Occupied low + headroom low means parking, and + trimming more data does not help — the escalation ladder on a + low-headroom signal is: force a durable sync (unparks SafeNoSync frees) + → cancel laggard readers (service deadline; hsr names them, porting + note 1) → trim → pause + alarm (FM-STOR-6). Trimming + past the downstream floor keeps the head fresh and opens a temporary + coverage gap between the archive's top and our tail: queries in the hole + get `RANGE_UNAVAILABLE`, every other dataset is untouched, and the gap + closes from below as the archive catches up from its own source. Shipped + semantics are kept — soft whole-chunk trims, downward instructions + clamped on any position-capped dataset (the byte bound clamps only while + it governs — no resync loops), but the clamp becomes observable instead + of silent. `MDBX_MAP_FULL` is the backstop, not the plan: deletes + themselves need COW pages and can return `MAP_FULL` on an exhausted map, + so the watermark must trim early enough that the emergency-trim txn still + fits the remaining freelist (headroom guarantee, `TXN_FULL`-bounded + batches); if even the trim cannot commit, the dataset degrades to + pause + alarm (FM-STOR-6), never a crash loop. After a successful + emergency trim, retry the commit. Precondition (must hold): no archive ingests + through hotblocks (NG1) — otherwise trimming past the floor would hole + the archive permanently. +- **DP-3 Defrag, env-granular.** The reclaim unit is the env file, not pages + — in-file defragmentation is compaction under another name and is exactly + the write stream this ADR removes. Two mechanisms: + - *reset-reingest* (required): drop the env dir, re-ingest the window from + upstream — the boot path narrowed to one dataset. Minutes, pod stays up, + the sibling replica covers. Sufficient for every reclaim case (slack + return, quota shrink, decommission). + - *compact-swap* (optional tooling, later): take the env's write lock (the + sync-serialization mutex the port already carries), `mdbx_env_copy` with + `MDBX_CP_COMPACT` (sequential, ~seconds per live-GB, transient ~live of + extra space), swap files, reopen. Freshness dip of seconds, no upstream + traffic. Neither `libmdbx` 0.6.6 nor `signet-libmdbx` 0.8.3 wraps + env-copy — needs a one-call FFI shim over `mdbx-sys` or an upstream PR. + Trigger on pressure, not cadence: `file − live > max(abs, k × live)` + sustained *and* node headroom low → worst offender first, one env at a + time, never both replicas of a dataset at once. `max_size` can only be + lowered onto a file that already fits — quota shrink implies defrag first. +- **DP-4 Observability & acceptance.** Per-dataset gauges for file/live/quota + bytes, gap-mode flag + gap width, slack alert. Port acceptance: resetting a + single env runs online with other datasets' commit tails unaffected + (CT-covered in the harness), and the portal serves `RANGE_UNAVAILABLE` + across the hole instead of failing (prior art: the portal crash-looped on + an unexpected hotblocks response). + +Escalation if env-granular defrag becomes routine: sealed immutable chunk +files from the terminal `compaction_loop` merge, retention by `unlink` — +reth's mdbx + static-files split; its own ADR. + +## Decision + +*(Written 2026-07-25; the spike it commissioned is done. Kept as the record — +with one premise overturned by the runs: 64 KiB pages fail the write gate, +4 KiB pass. The port plan carries the corrected geometry.)* + +Do not port yet. Run a 1–2 week spike and decide on numbers: + +1. Implement `KvRead`/`KvWrite`/`KvReadCursor` over libmdbx (64 KiB pages match + the table page size; geometry auto-grow instead of a fixed map size), with + LZ4 at the `BufferPageWriter::write_page` seam from the start — the 4× ratio + makes an uncompressed spike unrepresentative. +2. Drive the S4 churn-soak envelope (`spec/10-performance.md`) through + `crates/hotblocks-harness`: device writes, file growth under retention churn, + head-flush latency under ingest × chunk-merge × retention contention — one + shared env vs env-per-dataset. +3. Compare against the production baseline in this document. + +Proceed to the port only if the spike shows device writes dropping ≥3× without +head-freshness regressions. + +The write gate passed (Measurements). Reads did not get a spike gate: +app-LZ4 scans measured 18× p50 over rocks on page-cache-hot runs (the C +decoder cuts it ~6×; engine paths are comparable and tails tighter). The +accept-time exposure is bounded by production numbers instead: at the +measured query mix the decode ceiling is ~0.15 core/pod (Measurements, +scan bracket) — decode CPU cannot invert the ADR 0001 win even with no +decompressed-page cache. What production cannot bound today is cold-cache +behavior at full live size, so Stage 4 carries the read gate as a +*rollout* gate: cold-cache S1/S2 query mix at production live size under +the production memory values (porting note 2) — SLI-2/3 within agreed +bounds of the rocks baseline, major faults and PSI captured. Abort +criterion: if SLI-2/3 breach and a decompressed-page LRU sized within the +freed ~8 GiB block-cache budget does not restore them, the port stops +before Stage 5 — rocks stays, the seam and codec remain as the +measurement record. + +## Measurements + +- `CF_TABLES` LZ4 ratio, production replica 2026-07-25 + (`rocksdb.aggregated-table-properties`): raw value size 238.0 GB → data + blocks 59.1 GB = **4.03×** (5.58 M entries, avg raw value 42.6 KiB → ~10.6 KiB + compressed; keys negligible at 169 MB). +- Consequence: application-level page compression is a **precondition**, not a + contingency. Uncompressed pages inflate both the file (~62 → ~250 GB) and the + COW write stream by 4×, which cancels the expected write-amplification win + (LSM ~10× on compressed bytes ≈ COW ~2.5× on 4×-inflated bytes). The spike + must therefore run with LZ4 at the `BufferPageWriter::write_page` seam from + day one. +- **Production read-scan bracket** (2026-07-26, VictoriaMetrics, 30 m rates). + Mainnet: ~6.3 k queried blocks/s per pod over a ~2.8 M-block window holding + ~68 GB live SST → ~98 KB raw per block → **≤ ~620 MB/s raw scanned per pod** + as the upper bound (assumes every queried block reads all its tables; the + response-byte lower bound is ~4.2 MB/s). At in-service C-`lz4` decode + (~0.25 ms/MB, decoder shoot-out) the ceiling is **~0.15 core/pod**; + morpho ≈ 0.06, internal ≈ 0.02. The deferred-LRU trigger (~2 cores) is + unreachable at today's mix by an order of magnitude. The bracket collapses + to a real number once the raw-scanned-bytes counter ships (Stage 2 + observability) — the same counter is the post-port LRU-trigger SLI. + +## Spec compliance (checked 2026-07-26) + +The disk policy was audited against `crates/hotblocks/spec/`. The *doctrine* +already points this way: RS-2 lets retention trim finalized data because this +is "a bounded hot store, not an archive" (NG1), and DP-2 extends the same +dominance one level up — space bounds retention. The *range* read path needs +no changes at all: a query below the new `first` is already specified +(`RANGE_UNAVAILABLE`, RP-4) with the client rule "re-anchor upward", and the +ring trim is mechanically an ordinary `RETAIN` (INV-15/18 and WP §2.5 apply +unchanged). The hash-lookup path is the exception, and a hard one — RP-20, +item 8 below. The isolation doctrine (INV-35/36, LIV-8, FM-3) is what DP-1 +finally makes enforceable — today LIV-8 is marked known-violated (GAP-1). + +The *letter* conflicts in nine places; amending them is Stage 0 below and +the gate for flipping this ADR to accepted. Half of DP-2 is not a proposal +at all: PR #77 (NET-896) shipped the position cap on 2026-07-17 — five days +*after* the spec audit — so part of Stage 0 is reconciling the spec with +behavior already in production: + +1. **RS-1/RS-3.** `External` guarantees "everything ≥ instructed bound + kept" — factually stale since PR #77; `Window` floors "err on the side + of keeping more, never less". DP-2 trims past both under pressure. Fix: + new **RS-13 (space dominates retention)** — the effective bound is the + max of the instructed bound, the position cap (`max_blocks`, shipped), + and the byte bound (quota watermark, the port); while any space bound + governs, the dataset is in an alarmed, observable gap-mode. +2. **INV-44.** Data may leave only via "RETAIN *per policy*". With RS-13 the + space bound becomes part of policy semantics, so INV-44 needs a pointer, + not a new destructive path. +3. **RS-6 + SLI-8.** `disk ≤ P-SPACE-AMP × live + const` (target ≤ 2.0×) is + violated by the measured mdbx high-water (5.6–16.6× vs *current* live + after a wave). Restate per dataset: hard bound `env ≤ P-DISK-QUOTA` + always; the amplification bound holds against *peak* live since the last + env reset (ratchet), restored tight by DP-3. +4. **LIV-7.** "Disk returns to within RS-6": deletion *debt* converges + faster than today (pages free at the next durable point — no sweep, no + compaction wait), but the *file* converges only through DP-3; restate + against the RS-6 ratchet. +5. **FM-STOR-2.** The documented degrade is "writes MAY pause". Replace with + ring-mode: writes continue, floor overridden, gap alarmed; + `MDBX_MAP_FULL` backstop = emergency trim + retry. The recovery-path + clause ("no scratch space proportional to reclaimable data") is satisfied + by reset-reingest and NOT by compact-swap — the latter stays routine + tooling, never the FM-STOR-2/3 recovery path. +6. **Parameters.** New `P-DISK-QUOTA(D)`, `P-DISK-WATERMARK` (soft + fraction), `P-REORG-KEEP` (minimum span the ring may never trim into — + the INV-14 interaction: under pressure the window must still absorb a + realistic reorg, else RESET+alarm), plus registering the shipped + `max_blocks` (positions, DEF-9). `P-DISK-FLOOR` stays as the node-level + threshold. A quota below the minimum viable span is an INV-43 boot + refusal, not a runtime surprise. +7. **WP §2.5 / WP-11 (downward instructions).** The audit's N3 resolution + made a downward SET-RETENTION an explicit alarmed RESET. PR #77's + `clamp_floor` deliberately breaks that for capped datasets: a portal + floor below `first(D)` is clamped up, silently — necessarily, because a + dataset in gap mode would otherwise resync-loop against a stuck archive + (the portal keeps re-asking for the pre-gap floor). RS-13 carves the + exception to match the ship: on a position-capped dataset downward + instructions always clamp (`clamp_floor` keys on the cap's presence, not + its governance), while the byte bound clamps only while it governs — so + the WP §2.5 RESET stays reachable once every dataset carries a quota — + and the clamp is observable (today it is silent). +8. **RP-20 (lookup cost).** "Lookups are point reads: their cost MUST NOT + scale with the window" is a hard MUST that the accepted per-chunk + hash-index design violates by construction — fan-out is θ(runs). The + claim above that "the read path needs no changes at all" holds for + range queries only. Amend RP-20 to permit *bounded* fan-out and add the + bound: `P-HASH-MAX-RUNS` plus a wall-time / read-byte budget, an + abandoned probe returning a miss (RP-19-compatible, but normative), and + `P-HASH-SLOTS` as an admission budget disjoint from `P-EXEC-SLOTS` so a + lookup flood cannot starve range queries and vice versa (PF-4). Without + this item the ADR would flip to accepted carrying a knowingly violated + MUST. Drops out entirely if the indexes are descoped from the port. +9. **IB-10 / INV-45 (hash keys and index soundness).** IB-10's exact-bytes + contract is preserved — the index key stores the hash as given, not a + decoded 32 B digest — so no amendment is needed there; what needs + recording is the *consequence*, that the per-chunk run is ~1.6× larger + than a binary layout and RS-12's "tidx can be the largest single + consumer" applies with that multiplier. INV-45 does need a pointer: a + hit is authoritative, mdbx does not checksum data pages, so the run + carries the integrity envelope specified in "Hash-index design" rather + than being a raw-KV exception to the page codec. + +Plus observability and conformance rows: OB-6 gains env file/live/quota +gauges and the gap-mode flag + gap width; CT-7 gains the wave scenario (the +harness analog of spike run 16); CT-8 gains stuck-consumer (one dataset's +floor frozen forever — assert the quota ceiling, the gap alarm, untouched +neighbors); a CT covers online env reset (LIV-5b scoped to one dataset). + +Four pre-existing audit findings become port prerequisites because DP leans +on them: **N5/GAP-2** (anchor hash dropped on Window trims — INV-18; PR #77's +`trim_floor` passes `None` for the hash too, so the new path has the same +one-line bug to fix), **N6** (External retention bound not durable — persists +in the label the port rewrites anyway), **N2/GAP-24** (one dataset's boot +failure kills the whole service — per-env containment is the natural fix and +DP-4 acceptance depends on it), and **GAP-7** (readiness is all-or-nothing: +`/ready` gates on full init and models no per-dataset readability — RS-14's +online reset is inoperable without per-dataset readiness plus snapshot-handle +draining and dataset-aware portal retry). + +## Port plan + +- **Stage 0 — spec amendment (docs, ~2 d).** The nine items above (the + parameter registry is item 6; items 8–9 are the hash-lookup contract and + drop out if the indexes are descoped) + OB/CT rows; cross-references + checked by hand — no spec CI gate exists yet. The accepted flip + additionally waits on the Stage 1 vertical slice (Status). +- **Stage 1 — engine seam (~1 wk).** `kv_mdbx` from the spike into + `crates/storage` as a real backend: env-per-dataset, 4 KiB pages, + SafeNoSync with per-env serialized durable sync (the abort constraint), + geometry from config (`P-DISK-QUOTA` as `max_size`, tuned growth step, + armed shrink threshold), LZ4 at `BufferPageWriter::write_page` behind a + versioned page codec — header with raw length and checksum, because mdbx + has no data-page checksums (meta `validator_id` is reserved-zero) while + RocksDB verifies blocks on every read: FM-STOR-4 regresses without it — + decoded by C `lz4` into pooled buffers (measured 5.8× over `lz4_flex` + safe-decode on the decision-runs Xeon, spike page shape; flex without + safe-decode is only 2.0× — not worth the unsafe path); read deadline + owned by the service, bounding the response lifecycle (porting note 1), + with `mdbx_env_set_hsr` via the Binding FFI shim as the space-pressure + backstop. **Bounded write txns as a stated invariant**: table builds keep + the 8 MiB sub-commit granularity, the writer lock is released between + sub-commits, and head flushes interleave — a merge holds the writer one + sub-commit at a time, never for the whole rebuild. Differential tests + against the rocks backend, corruption-injection tests on the codec, and + a CT-2 kill matrix at the seam. kill-9 verifies integrity and recovery + only — under SafeNoSync dirty pages survive process death in the page + cache, so the loss window it observes is ~zero; the real loss window is + a *system*-crash property (CN-6b) resting on mdbx's steady-sync-point + guarantee under `NoWriteMap` (dm-log-writes power-cut injection stays + optional hardening). Pin `P-DUR-SYSTEM`, today "make explicit ⚠", to + **sync cadence + max write-txn hold** — durable sync serializes behind + the writer lock, so the hold bound is part of the durability claim, and + the cadence itself joins the parameter registry. Exit: the vertical + slice the accepted flip waits on (Status). +- **Stage 2 — db layer (~2–3 wk).** The ~1300 lines of + `crates/storage/src/db/*`: transactions, snapshots, label/meta (persist + the retention bound — N6), chunk index, metrics, CLI, shutdown. Delete the + engine-reclaim machinery (10 s sweep, deletion collector, periodic + compaction, startup `DeleteFilesInRange`, `TableId` watermark, + `reclaim-measure`); keep the dirty-table journal and the startup orphan + purge — multi-commit table builds still leave committed orphans at a + crash (RS-10). Fix N5 anchor carry-over in the trim path it touches. + Hash-index layout: the *rejection* is decided by measurement (porting + note 4, runs 22–26 — global hash-keyed tables cost 27× the data stream); + the replacement is not. Per-chunk runs written with the chunk enter + Stage 2 behind a **prototype gate** — build the run in the bench at the + real key shape and measure device writes, then the fan-out probe at + 50 / 150 / 500 runs — before the `hashes/{hash}` endpoints are + re-implemented against it. Carries the `Chunk::V2` sidecar slots, the + integrity envelope, the bounded-fan-out budget from Stage 0 items 8–9, + and the physical-GC debt queue for dropped runs. Whole item drops if the + endpoint-traffic query descopes the feature. Ship the raw-scanned-bytes + counter (the Measurements bracket's replacement and the LRU-trigger SLI). +- **Stage 3 — disk policy (~1 wk).** DP-1 quota config; DP-2 = extend + NET-896's trim with the byte watermark (effective floor = max of + instructed, position cap, byte bound) + gap-mode flag/alarm + observable + clamp + `MAP_FULL` backstop + `P-REORG-KEEP` guard + INV-43 boot check; + DP-3 reset-reingest as a dataset-scoped observable RESET; DP-4 gauges and + alerts. +- **Stage 4 — conformance & perf (~1 wk, overlaps 3).** CT-7 wave, CT-8 + stuck-consumer and online-reset CT in `crates/hotblocks-harness`; S4 + churn-soak on a prod-class node against the rocks baseline (the Decision + step 2 envelope: device writes, head-flush tails under ingest × merge × + retention, wave high-water) — merges run *concurrently* with ingest (the + bench's `--concurrent-merges` shape; inline merges measure no writer-lock + queueing) and the matrix includes the internal shape with both hash + indexes on (porting note 4). Run the Decision read gate: cold-cache S1/S2 + at production live size under the production memory values (porting + note 2), SLI-2/3 against the rocks baseline with major-fault and PSI + capture (the spike's read runs were page-cache-hot by construction). + Confirm the decode-CPU share against the Measurements bracket via the + Stage 2 raw-scan counter; the decompressed-page LRU (coherence is + trivial — chunk tables are immutable) stays deferred behind its trigger: + decode CPU > ~2 cores/pod or read p99 against SLO. Portal contract + check: a gap range must pass through as `RANGE_UNAVAILABLE`, not break + the portal (cross-repo). +- **Stage 5 — rollout (~1 wk).** Blue/green on an empty volume: + internal → morpho → mainnet; watch device writes (expect ≥3× drop), commit + tails, env file/live/quota, gap-mode. mmap makes `working_set` read as + page cache — dashboards must not mistake it for a leak. Rollback = + previous image on the old volume (the store is re-ingestable). + +Sums to ~5–7 weeks against the 3–6 week code estimate above. Deliberately +out of scope, recorded: the compact-swap FFI shim (DP-3 optional tooling), +the sealed-tier ADR, and portal-side gap handling if the Stage 4 contract +check fails. + +## Alternatives considered + +- **Tune RocksDB harder** (universal compaction, larger memtables, relaxed + leveling): reduces but keeps the multiplier and the reclaim machinery; the + workload shape (rolling window, append keys, big values) is the textbook + non-LSM case. **But the "keeps stalls" half of this dismissal is now + disproven** (Context, 2026-07-27): the stalls are flush backpressure on the + stock 64 MB × 2 memtable, which is exactly what this alternative would change, + and neither production nor the bench reference sets a single memtable, L0, + file-size or compaction-style option — the reference is untuned by omission, + not by design. Before Stage 1, run one arm (512 MiB `write_buffer_size`, + `max_write_buffer_number` 4) against the prod-parity reference. If it removes + the stall events and recovers a material share of the write gap, the port must + be re-scored against tuning-plus-status-quo, with the remaining buy being + reclaim-machinery deletion and read tails rather than the headline ratio. +- **Vanilla LMDB**: fixed map size must be pre-declared — no growth/shrink + geometry to hang `P-DISK-QUOTA` on — and there is no HSR hook for the + stale-reader backstop. (The original draft also held its 4 KiB pages + against it; the spike made that moot — 4 KiB won the write gate and the + 3-contiguous-page overflow runs allocate fine under churn.) +- **Reduce the application chunk-merge churn instead**: engine-independent and + worth doing regardless, but it cannot remove LSM leveling amplification on the + ingest stream itself. +- **An engine that reclaims disk online** (staying on RocksDB, fjall, SQLite + `auto_vacuum`, redb `compact()`): online reclaim is compaction under another + name — every candidate pays the rewrite multiplier this ADR removes (the wave + A/B prices it: rocks returns the file to ~live for ~3.5× the device writes). + With deletes already partitioned by age, unlink-based reclaim (DP-3, and the + sealed-tier escalation) is the structural answer if high-water hoarding ever + bites. diff --git a/docs/measurements/2026-07-26-mdbx-churn-bench.md b/docs/measurements/2026-07-26-mdbx-churn-bench.md new file mode 100644 index 00000000..08e18963 --- /dev/null +++ b/docs/measurements/2026-07-26-mdbx-churn-bench.md @@ -0,0 +1,315 @@ +# 2026-07-26 — mdbx vs rocks churn bench (ADR 0002 decision runs) + +Supporting measurements for [ADR 0002](../adr/0002-storage-engine-libmdbx.md). +Bench: `crates/mdbx-spike` (branch `spike/libmdbx`, with the sync-serialization +and io.stat fixes made during these runs — see "incidents" below). + +**Environment:** a bare-metal Debian 12 / Linux 6.1 node — Xeon E-2136 +(6c/12t), 62 GiB RAM, 2× NVMe (WDC SN720) in md RAID0, ext4. Bench ran in a +docker container (`--cpus 8 --memory 16g`, private cgroupns), so cgroup +`io.stat` isolates bench IO from the box's light resident workload (~10% CPU). +Device-write numbers below are cgroup `io.stat` after excluding stacked md/dm +devices (they re-count bytes already charged to the physical NVMe); on every +run the corrected figure matched `/proc/self/io` write_bytes to <1%. + +Two reference caveats: the rocks reference runs buffered I/O, which +**does not match production** — the 2026-07-26 note claiming it does is +withdrawn. Verified against the running pods 2026-07-27: no stack passes +`--rocksdb-disable-direct-io`, and the flag is inverted +(`cli.rs:149` = `.with_direct_io(!disable)`), so production runs +`set_use_direct_reads(true)` + `set_use_direct_io_for_flush_and_compaction(true)` +(`db.rs:192-195`). Runs 18/19 closed WAL compression and background jobs +but not the I/O mode, so this parity gap is still open on the axis that +decides both device-write accounting and read latency; the read runs stay +scoped to CPU cost anyway, since the bench cache geometry differs from +prod's 8 GiB block cache (`--data-cache-size 8192`, confirmed). Peak-file +is sampled at 500 ms — exact for mdbx (the file only grows), able to +undershoot a rocks transient between compactions. + +Workload shape per the run matrix in `crates/mdbx-spike/README.md`: 45 +datasets, 4 pages × ~43 KiB per commit, merge fan-in 8, retention window 64, +`SafeNoSync` + 1 s durable-sync cadence, app-level LZ4 for mdbx (4.00× on +generated pages vs 4.03× prod), engine-level LZ4 for the rocks reference. + +## Free-run churn — engine write amplification at matched volume + +45 datasets × 2000 chunks = 15.48 GB raw churned (mdbx stores 3.87 GB +post-LZ4; rocks compresses internally). + +| run | engine | mdbx page | elapsed | commits/s | device writes | amp vs raw | commit p50/p90/p99/max | +|---|---|---|---|---|---|---|---| +| 02 | rocks (pre-parity) | — | 64.0 s | 1406 | 49.0 GB | 3.16× | 18.8 / 31.7 / 212 / 286 ms | +| 18 | rocks (**prod parity**) | — | 98.0 s | 918 | 47.1 GB | 3.04× | 37.5 / 52.7 / 126 / 239 ms | +| 01 | mdbx | 64 KiB | 53.0 s | 1697 | 69.5 GB | 4.49× | 14.5 / 31.4 / 438 / 1360 ms | +| 06 | mdbx | 16 KiB | 25.5 s | 3529 | 24.3 GB | 1.57× | 6.5 / 13.7 / 32.6 / 2042 ms | +| 07 | mdbx | 4 KiB | 20.9 s | 4305 | 13.3 GB | 0.86× | 6.0 / 12.6 / 29.4 / 758 ms | + +Page size is the whole story. At 64 KiB every COW leaf write costs a full +page for a ~10 KiB value, and mdbx writes **1.4× more** than the rocks +reference. At 4 KiB it writes **3.5× less** — the ADR 0002 gate (≥3×) passes, +and throughput more than doubles on top. Free-run max-latency tails (0.7–2 s) +are growth-step/sync stalls at 25–70× production pace; they do not appear at +production pace (below). + +Run 18 re-runs the reference at production parity after external review +caught the gap (runs 01–17 rocks lacked Zstd WAL compression and ran the +default 2 background jobs; prod sets both — `db.rs` `db_options`). Free-run +the correction nets out to −4%: the ~11 GB the compressed WAL saves is eaten +by the extra in-window compaction eight jobs complete (elapsed 64→98 s — +free-running 45 writers now share 8 CPUs with 8 compaction jobs, which is +also why its commit p50 doubles). Gate ratio 49.0→47.1 GB: **3.54×**, robust. + +## Production pace — latency under merge contention + +45 datasets × 600 chunks, `--paced-ms 750` → ~60 commits/s aggregate +(≈ prod 1.3/s/dataset × 45). 4.64 GB raw churned. + +| run | engine | config | device writes | amp vs raw | commit p50/p90/p99/max | file / live | +|---|---|---|---|---|---|---| +| 08 | rocks (pre-parity) | — | 20.8 GB | 4.48× | 275 µs / 388 µs / 2.4 ms / 37.5 ms | 183 MB / 123 MB | +| 19 | rocks (**prod parity**) | — | 12.5 GB | 2.69× | 524 µs / 669 µs / 3.5 ms / 42.8 ms | 135 MB / 123 MB | +| 03 | mdbx | 64 KiB, 1 env | 13.6 GB | 2.93× | 215 µs / 697 µs / 7.9 ms / 55 ms | 268 MB / 142 MB | +| 04 | mdbx | 64 KiB, env/dataset | 9.7 GB | 2.09× | 174 µs / 248 µs / 463 µs / 12.8 ms | 12.1 GB / 189 MB | +| 09 | mdbx | 4 KiB, 1 env | 3.8 GB | 0.81× | 174 µs / 486 µs / 4.0 ms / 20.6 ms | 268 MB / 143 MB | +| 10 | mdbx | 4 KiB, env/dataset | 3.3 GB | 0.71× | 116 µs / 198 µs / 1.1 ms / 11.0 ms | 12.1 GB / 144 MB | + +Notes: + +- **Parity matters most exactly here.** Free-run the WAL correction nets out + (run 18); at production pace the uncompressed WAL was ~40% of the old + reference's device writes — run 19 lands at 12.5 GB (amp 2.69× vs 4.48×), + and the mdbx advantage honestly reads **3.3× (single env) / 3.8× + (env-per-dataset)**, not the 5.5–6.3× the pre-parity reference suggested. + The gate still passes; the Zstd WAL work moves rocks' commit p50 to + ~2× mdbx's and its tails stay compaction-shaped (p99 3.5 ms vs 1.1 ms). +- The pre-parity claim "rocks amplification is worse at production pace than + free-running" (4.48× vs 3.16×) was mostly the uncompressed-WAL artifact: + at trickle pace the WAL is the dominant write stream, free-running it is + compaction. On the parity reference the ordering inverts — paced 2.69× + vs free-run 3.04×. +- Per-dataset envs are the latency winner (no cross-dataset writer-lock + contention, syncs naturally staggered) but cost a file-footprint floor: + 45 envs × ~268 MB = 12.1 GB for <200 MB live, driven by the 256 MiB + `growth_step` — page size does not move it (runs 04 and 10 land on the + same file size). Growth-step tuning is a real porting decision. + +## Concurrent merges — head commits vs the writer lock (runs 20–21) + +External review caught that runs 01–17 merged *inline* in the worker loop: +with env-per-dataset, commits never waited on a writer lock held by a +running merge, so those latency tables measure no head-behind-merge +queueing — while production's `compaction_loop` merges concurrently with +ingest. `--concurrent-merges` moves each merge to a background thread +(max 1 in flight per dataset) and re-runs the paced shape: + +| run | engine | config | device writes | commit p50/p90/p99/max | merge p50/p99/max | +|---|---|---|---|---|---| +| 20 | mdbx | 4 KiB, env/dataset, concurrent | 3.28 GB (0.71×) | 128 µs / 204 µs / **323 µs** / 14.1 ms | 0.6 / 9.7 / 15.1 ms | +| 21 | rocks | prod parity, concurrent | 13.0 GB (2.81×) | 577 µs / 987 µs / 3.9 ms / 60.2 ms | 4.8 / 57.1 / 113.5 ms | +| 10 | mdbx | same as 20, inline (reference) | 3.3 GB (0.71×) | 116 µs / 198 µs / 1.1 ms / 11.0 ms | — | + +Device writes are unchanged for mdbx and the commit p99 stays +sub-millisecond under contention; the commit *max* is exactly one merge-txn +hold (14.1 ms commit vs 15.1 ms merge max) — a queued head flush waits out +the running merge txn, nothing more. That bound is the load-bearing fact: +it scales with merge-txn size, which is why the port carries the +bounded-write-txn invariant (8 MiB sub-commits, lock released between — +ADR Stage 1) and the S4 envelope re-asserts it at production merge sizes. +The feared trade also inverts: rocks has no writer lock to queue on, yet +its head tails under the same concurrent-merge load are ~12× worse at p99 +and ~4× at max (compaction jitter) — the lock mdbx pays for is cheaper +than the compaction rocks pays with. Write ratio in this shape: 3.97×. + +## Hash-index random-insert stream (runs 22–26, ADR porting note 4) + +`CF_BLOCK_HASHES`/`CF_TRANSACTION_HASHES` are hash-keyed — uniformly random +inserts, one per transaction, enabled on mainnet-internal today. An LSM +absorbs them in the memtable; a B+tree COWs a leaf path per touched leaf per +txn. Priced here: 16 datasets at production pace with concurrent merges, +plus a hash stream of 300 entries/commit (~internal's per-chunk tx count, +32 B keys / 12 B values) inserted in the commit txn and deleted with the +chunk at retention, against a **preseeded 1 M-entry tree per dataset** +(~21 k leaves — a small tree understates the cost by orders of magnitude; +prod tidx is 10–100× deeper). mdbx keeps hash keys in the same tree under a +sentinel prefix (COW-equivalent to a named subtable); rocks gets a dedicated +CF mirroring production (`hash_index_cf_options`: bloom, LZ4, deletion +collector). + +| run | engine | hash stream | device writes | Δ index cost | commit p50/p99/max | delete p50 | +|---|---|---|---|---|---|---| +| 25 | mdbx | — | 1.17 GB | — | 132 µs / 418 µs / 5.9 ms | 86 µs | +| 22 | mdbx | per-commit (K=1) | 32.34 GB | **+31.2 GB** | 1.58 ms / 2.7 ms / 125 ms | 10.7 ms | +| 23 | mdbx | write-behind K=16 | 23.97 GB | +22.8 GB | 213 µs / 26 ms / 64 ms | 8.8 ms | +| 26 | rocks | — | 3.13 GB | — | 583 µs / 3.1 ms / 22.8 ms | 515 µs | +| 24 | rocks | per-commit (K=1) | 7.13 GB | **+4.0 GB** | 1.28 ms / 4.5 ms / 16.7 ms | 4.6 ms | + +- **The index stream inverts the engine decision on this shape.** The same + stream costs mdbx +31.2 GB (27× its own data stream; ~3.25 MB per commit = + ~600 random leaf COWs × 4 KiB, matching the model) vs rocks +4.0 GB — + total 32.3 vs 7.1 GB, i.e. with global hash tables mdbx writes **4.5× + more** than rocks. The raw index churn is ~127 MB — mdbx pays ~245× on it. +- **Write-behind batching does not rescue it**: K=16 saves only 26% (4 800 + sorted keys still touch mostly-distinct leaves of a 21 k-leaf tree) and + moves the cost into flush-commit tails (p99 26 ms). Amortization arrives + only when a batch approaches the tree width — hours of accumulation at + this rate. +- Latency degrades across the board under K=1: commit p50 12× (132 µs → + 1.58 ms), delete p50 124× (86 µs → 10.7 ms — 300 random-key deletes COW + as many leaves as inserts). +- Conclusion: under mdbx the hash indexes MUST NOT port as global + hash-keyed tables. **This is the only half these runs measure** — the + replacement was never executed: the spike writes hashes globally under a + `0xFF` sentinel prefix (`engine.rs:248–256`), so no per-chunk layout has + been run at all. The append-shaped restructure — per-chunk index runs + written with the chunk, and lookups fanning out over runs under an + explicit budget — is *modeled* at ~36–40 KB/commit with `MDBX_APPEND` + (~8 leaves plus spine at ~105 B/entry including node overhead, exact + IB-10 key bytes), 60–72 KB without append: **~45–80× cheaper** than the + measured 3.25 MB/commit, not the ~270× an earlier revision derived by + dividing raw payload alone. In absolutes that is +0.38…0.69 GB on this + shape against the 1.17 GB baseline (+32…59%), still clearing the ≥3× + gate at 3.8–4.6× since rocks pays +4.0 GB for the same stream. The + alternative remains an explicit decision to keep the feature + rocks-side/elsewhere. ADR porting note 4 and the Stage 2 prototype gate + carry the verdict. + +## Read latency under production write load + +8 readers, each scanning one random live table per 10 ms (~700–780 scans/s +aggregate; a scan = fresh snapshot/ro-txn, seek to the table prefix, cursor +over ~23 pages ≈ 1 MB raw, ending with raw bytes in hand — mdbx decompresses +app-side through the `KvRead` seam, rocks serves uncompressed block-cache +blocks). Write load = the paced production shape. The live set fits page +cache, so this compares engine read-path CPU cost, not disk-bound reads. + +| run | engine | config | scan p50 | p90 | p99 | max | +|---|---|---|---|---|---|---| +| 11 | rocks | engine LZ4 | 84 µs | 461 µs | 734 µs | 40.1 ms | +| 12 | mdbx | 4 KiB, 1 env, app LZ4 | 1.51 ms | 1.79 ms | 2.59 ms | 5.0 ms | +| 13 | mdbx | 4 KiB, env/dataset, app LZ4 | 1.51 ms | 1.79 ms | 2.60 ms | 8.2 ms | +| 14 | mdbx | 4 KiB, 1 env, **no compress** | 136 µs | 256 µs | 396 µs | 4.6 ms | + +- The engine read paths are comparable: mdbx 136 µs vs rocks 84 µs at p50, + and mdbx tails are *tighter* (p99 396 µs vs 734 µs, max 4.6 ms vs 40 ms — + no compaction jitter). +- The gap in runs 12/13 is entirely the app-compression precondition: + ~1.4 ms per MB scanned in `lz4_flex` safe decode (~700 MB/s single-thread). + RocksDB hides the same work inside its uncompressed block cache. Runs 12 + and 13 are identical, so the cost is per-scan CPU, not env contention. + Decoder shoot-out on the same generated page shape, hot single-thread on + this node (2026-07-26; an M2 Max agrees on the ratios within ~10%): + C `lz4` into a reused buffer 7.7 GB/s = **5.8×** over the `lz4_flex` + safe-decode-with-alloc used here (1.3 GB/s); `lz4_flex` without + safe-decode only 2.0×. The hot-loop safe figure (0.76 ms/MB) vs the + in-situ ~1.4 ms/MB above shows ~1.9× of mmap-cold + cursor overhead that + any decoder keeps — in service expect C `lz4` at ~0.25 ms/MB, not 0.13. + A small decompressed-page cache stays the fallback if per-request read + volume still makes decode matter. +- Readers do not perturb writes on either engine (commit percentiles match + the reader-less runs), and reads never miss the write path's mutex: max + scan stayed ≤8 ms while commits, merges and 1 s durable syncs ran. + +## Stale-reader file growth (GAP-29 shape) + +Run 05: 8 datasets free-running 60 s with a read txn held 60 s in a loop +(64 KiB pages): file ballooned to **8.3 GB against 33 MB live** (freelist +126 248 of 126 757 pages), commit max 1.24 s on file-extension stalls. Growth +≈ device-write rate × reader-hold duration, confirming the ADR 0002 estimate: +long readers park the entire churn window. At production pace a 60 s reader +parks ~0.5 GB in a shared 4 KiB env (8.4 MB/s device, run 09), ~10 MB per +env in the recommended per-dataset layout (7.3 MB/s pod-wide across 45 +envs, run 10), and ~1.8 GB only in the 64 KiB single-env shape (30 MB/s, +run 03). + +## Retention wave — file high-water and post-wave reuse + +Runs 15–17 answer the reclaim question ADR 0002 left open: the working set +grows when the downstream consumer stalls (retention is downstream-driven) +and shrinks back — what does the file do? `--window-wave` holds the window at +N chunks for the middle of the run (commits 20%..50%), then trims back to 64 +and keeps churning; the post-wave tail measures whether a fragmented freelist +still serves the churn (at 4 KiB pages every ~10.6 KiB value needs a +3-contiguous-page overflow run). mdbx runs with an explicit +`shrink_threshold` (2× growth step), so tail truncation was armed. + +| run | engine | config | peak stored live | peak file | file@end | Δfile after wave | live@end | +|---|---|---|---|---|---|---|---| +| 15 | mdbx | 4 KiB, free-run, wave 64→640→64 | 1203 MB | 2416 MB | 2416 MB | +268 MB (= 1 growth step) | 146 MB | +| 16 | mdbx | 4 KiB, paced 250 ms, wave 64→300→64 | 582 MB | 805 MB | 805 MB | **0.0 MB** | 143 MB | +| 17 | rocks | free-run, wave 64→640→64 | 4944 MB (raw; engine LZ4) | 1975 MB | **152 MB** | −1507 MB (reclaimed) | 124 MB | + +- **The mdbx file is a permanent high-water mark.** Tail truncation never + fired in either run despite the armed `shrink_threshold` and 80–93% of + pages sitting in the freelist at the end — churn keeps the file tail + occupied, so the only shrink path mdbx has is unreachable in practice. + Run 15 ended at 16.6× live; the wave's footprint (Δ vs the no-wave run 07: + 2416 − 1342 ≈ 1.07 GB) matches the wave's live delta almost exactly. +- **But it does not creep: reuse is intact.** The paced run churned 27 000 + commits after the trim at constant live and grew the file by **zero + bytes** — freelist fragmentation does not block overflow-run allocation at + production-like cadence. Free-running overshot by exactly one growth step + (the burst outran the 1 s durable cadence — the known SafeNoSync parking, + bounded by cadence), then held. +- **Sizing rule (paced)**: file ≈ 1.35× peak-ever stored live, rounded up to + the 256 MiB growth step (run 16: 805.3 MB = exactly 3 steps for 582 MB + peak live). Free-run bursts add the SafeNoSync parking term — roughly + device-write rate × sync cadence — on top: run 15 peaked at 2.0× peak + live, and the no-wave free-run file (run 07, 1342 MB over ~145 MB live) + is almost entirely parking. Volumes must budget the paced figure plus + catch-up-burst parking per env, permanently; `max_size` geometry turns + the budget into a hard quota (`MDBX_MAP_FULL` as backpressure). +- **What rocks charges for its reclaim**: same logical churn (15.48 GB raw), + rocks wrote 49.2 GB to the device vs mdbx's 13.2 GB (3.7×) and its + free-run tails under the wave were brutal (commit p99 200 ms, merge p99 + 220 ms, delete p99 114 ms) — but the file came back to ~live within ~10 s + of the last flush. That is the trade in one table. + +## Incidents hit during the runs (both are ADR-relevant findings) + +1. **Concurrent `mdbx_env_sync` aborts the process.** `mdbx_env_sync(force)` + from a cadence thread racing `SafeNoSync` commits reliably trips the + always-on `ENSURE(legal4overwrite)` in `dxb_sync_locked` (libmdbx 0.13.11 + via crates.io `libmdbx` 0.6.6; reproduced 2/2 on Linux within a minute, + never seen on macOS). No upstream fix found in the 0.13.x changelog; the + crate does not expose the built-in `MDBX_opt_sync_period` (which has its + own assert history, upstream issue #248). A port must serialize durable + sync with the write path; the bench now does exactly that (per-env mutex), + so commit tails honestly include fsync waits — and at production pace they + stayed sub-millisecond (run 04). +2. **cgroup `io.stat` double-counts on stacked block devices.** md/dm layers + re-count bytes charged to the physical device; the bench now skips + major 9/253 lines. Runs 01–05 predate the fix and were halved (validated + against `/proc/self/io`, which matched to <1% on this workload — unlike + the tempfile-spill case in the 2026-07-16 flush bench, there is no + dirty-page rewrite gap here). + +## Verdict vs the ADR 0002 gate + +With 4 KiB pages, app-level LZ4 and a 1 s sync cadence, libmdbx clears the +gate on the churn shape against the **production-parity** reference (runs +18–19: Zstd WAL + 8 background jobs): **3.5× fewer device writes +free-running and 3.3–3.8× fewer at production pace**, with better commit +tails in every paced config. The pre-parity reference (runs 02/08, no WAL +compression) overstated the paced advantage as 5.5–6.3× — the free-run +ratio barely moved. 64 KiB pages — the initial default — *fail* the gate +by writing 1.4× more than rocks; the recommendation is page-size 4 KiB (or +16 KiB if read-scan cost proves dominant, at 2× the device writes). + +One scale caveat on the gate: bench live is ~150 MB against production's +~68 GB. LSM leveling amplification grows with live size (more levels, more +rewrite generations) while mdbx COW spine depth grows logarithmically — the +error is in mdbx's favor, but the production-size magnitude is confirmed +only by the S4 soak (ADR Stage 4), not by this bench. The bench also models +neither metadata CFs nor the hash indexes (ADR porting note 4 — a +random-insert stream that an LSM absorbs and a B+tree pays leaf-path COW +for; internal runs both indexes). + +The wave runs close the reclaim question: the mdbx file is a permanent +high-water mark (~1.35× peak stored live at paced rate, plus a sync-cadence +parking window on free-run bursts; tail truncation never fires) but does +not creep at constant live, while rocks buys its reclaim for ~3.5× the +device writes (wave run 17 ran the pre-parity reference; the free-run +parity correction is −4%). Consequences are codified as ADR 0002 "Disk policy": per-env +`max_size` quota, ring-past-the-floor trimming under quota pressure (coverage +gap instead of unbounded growth), and env-granular defrag (reset-reingest +required, compact-swap optional). diff --git a/docs/measurements/2026-07-28-lsm-depth-and-compaction-style.md b/docs/measurements/2026-07-28-lsm-depth-and-compaction-style.md new file mode 100644 index 00000000..4cdc367e --- /dev/null +++ b/docs/measurements/2026-07-28-lsm-depth-and-compaction-style.md @@ -0,0 +1,142 @@ +# LSM depth, compaction style, and the ADR 0002 gate at production scale + +2026-07-28. Bare-metal Xeon E-2136 / 2× NVMe md RAID0, docker `--cpus 8 --memory +16g --cgroupns private`. Device writes from cgroup `io.stat`, stacked md/dm +majors excluded. + +## Why these runs exist + +Every earlier bench arm (runs 01–39) used `--window 64`, which leaves **119 MB +live**. Production `CF_TABLES` is 63 GB (internal) to 75 GB (mainnet). At 119 MB +RocksDB has a single populated level: no ladder, no compaction cascade, and an +`lsm_write_amp` of ~1×. The engine comparison the ADR gate rests on was +therefore run against a RocksDB that never did the thing that costs it money. + +`--window 6000` puts 11.8 GB live and three populated levels on the bench. +That is not production's four levels at 75 GB, but it is the same regime. + +| | bench, 119 MB | bench, 11.8 GB | prod mainnet, 75 GB | +|---|---|---|---| +| populated levels | 1 | 3 | 4 | +| `lsm_write_amp` | ~1× | 4.52× | 6.39× | +| device writes / raw ingest | 1.05× | 1.61× | 1.81× | + +## Production attribution (for reference) + +Read from `/run/db/LOG`, which RocksDB dumps every `stats_dump_period_sec` +(600 s, unset in our options). `--rocksdb-stats` was **not** needed and would not +have helped: it only calls `enable_statistics()`, and `RocksDbCollector` reads +integer properties, never tickers. + +Per-pod, cumulative over 9.3 h, `mainnet-internal-db-0`: + +| source | MiB/s | share | +|---|---|---| +| compaction L0→L4 | 169.9 | 69.8 % | +| flush memtable→L0 | 32.9 | 13.5 % | +| WAL (post-Zstd, by residual) | 20.3 | 8.3 % | +| compaction L4→L5 | 10.9 | 4.5 % | +| compaction L5→L6 | 7.4 | 3.0 % | +| hash indexes (both) | 1.9 | 0.8 % | +| **device, measured** | **243.3** | | + +The L4 concentration is **not** the natural shape: internal was running +`--rocksdb-level-base-mb 2048` at the time, the setting measured at +16 % device +writes and reverted. `mainnet`, on the 256 MB default, spreads the same total +across the ladder — L3 21.8 %, L4 23.6 %, L5 23.4 %, L6 8.5 %. The +source split (compaction ~77 %, flush ~14 %, WAL ~8 %) holds on both. + +WAL compression cross-checks: the bench's `rocksdb.wal.bytes` ticker reports +232.5 GB against 187.0 GB of measured device writes, so Zstd compresses the WAL +~5.8×. The production residual gives ~6.3× by an independent route. + +Absolute device-write figures reported before 2026-07-28 (401 / 465 MiB/s) were +~1.9× high: the PromQL summed a duplicated device dimension. `io.stat` shows it +directly — `259:0` and `253:4` carry identical `wbytes`. + +## Compaction style + +45 datasets × 15000 chunks, window 6000, 512 MB × 4 memtable, direct I/O. + +| run | style | `lsm_write_amp` | compaction | device | file / live | commit p99 / max | +|---|---|---|---|---|---|---| +| 40 | leveled | **4.52×** | 114.6 GB | **187.0 GB** | 11.9 / 11.8 GB | **57.7 / 185 ms** | +| 41 | universal, amp 200 | 7.21× | 201.6 GB | 274.0 GB | 15.4 / 14.2 GB | 68.2 / 661 ms | +| 42 | universal, amp 400 | 7.05× | 196.0 GB | 268.4 GB | 41.2 / 17.7 GB | 68.2 / 713 ms | + +Universal loses on every axis simultaneously — it is not a space-for-writes +trade here. `max_size_amplification_percent` triggers a **full** rewrite of the +CF whenever accumulated garbage doubles the live size, and retention produces +that garbage continuously. Raising the bound to 400 % barely moves the write +volume (7.21 → 7.05) and inflates the file to 41 GB against 17.7 GB live. + +The reasoning that motivated the arm — "universal has lower write amplification" +— holds for insert-dominated workloads. This one is inserts plus a dense delete +stream. + +## L0 compaction trigger + +Same workload, leveled, `level0_slowdown/stop` held at the stock 1:5:9 ratio. + +| run | `level0_file_num_compaction_trigger` | `lsm_write_amp` | device | +|---|---|---|---| +| 40 | 4 (default) | 4.52× | 187.0 GB | +| 43 | 8 | **4.24×** | **177.7 GB** | +| 44 | 16 | 4.31× | 180.2 GB | +| 45 | 32 | 4.29× | 179.7 GB | + +A one-off 5 % at 4→8, then a plateau; 8/16/32 sit within 2 % of each other. No +stalls in any arm (`write_stopped` 0.00 %). + +The model that motivated this — each L0→base merge rewrites the whole +overlapping base level, so the step costs `1 + base_size/L0_batch` — is right +about production (internal's L4 measures W-Amp 5.2 with `Rnp1/Rn` = 4.9) but +does not transfer to the bench, where the base level is *smaller* than one L0 +batch: + +| level | size | Write | W-Amp | +|---|---|---|---| +| L0 | 71 MB | 29.0 GB | 1.0 | +| L4 (base) | 71 MB | 32.7 GB | 2.2 | +| L5 | 998 MB | 19.7 GB | 10.2 | +| L6 | 9.99 GB | 40.1 GB | 2.8 | + +At 11.8 GB live the ladder puts the base at 71 MB against a 568 MB L0 batch, so +the base rewrite was already cheap and there was no denominator to grow. + +## Engine comparison at depth + +Same workload and depth; mdbx at 4 KiB pages, `--compress`, single env, default +1 s sync cadence. + +| | rocks leveled (40) | mdbx 4 KiB (46) | ratio | +|---|---|---|---| +| device writes | 187.0 GB | **112.0 GB** | **1.67×** | +| device / raw ingest | 1.61× | 0.97× | | +| throughput | 1073 commit/s | **2816 commit/s** | 2.6× | +| commit p50 | 33.9 ms | **7.8 ms** | 4.3× | +| commit p99 | 57.7 ms | 61.9 ms | parity | +| commit max | **185 ms** | 1965 ms | 10.6× worse | +| merge p50 | 41.4 ms | **0.49 ms** | 85× | +| file / live | 11.9 / 11.8 GB | 14.5 / 13.4 GB | | + +Depth widens mdbx's write advantage over the 1.23× / 1.43× that runs 27–34 +measured at 119 MB live, but only to **1.67×**. The ADR 0002 gate of ≥3× fails +at production depth as well — the objection that the tuned-RocksDB verdict was +measured at the wrong scale is now closed. + +The commit tail is the honest cost: 1.96 s max against RocksDB's 185 ms, from +the durable-sync cadence serialized against write txns. For a service whose SLI +is head freshness that is a separate problem to solve, tracked as P-DUR-SYSTEM. + +## What tuning has and has not bought + +| change | effect | status | +|---|---|---| +| memtable 512 MB × 4 | stalls 1.206 % → 0, device writes −40 % (internal) | shipped | +| `max_bytes_for_level_base` 2048 | +16 % device writes | reverted (infra#634) | +| universal compaction | +46 % device writes, +18 % commit p99 | rejected | +| `level0_file_num_compaction_trigger` | −5 %, plateaus at 8 | not worth a flag | + +The residual ~6.4× is the level count, not one bad step, and no knob removes a +level without paying more per merge than it saves. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f018685e..c49e94a1 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.89" +channel = "1.94" components = [ "rustfmt", "clippy" ]