From 0d423eda0463979e9c2a0c9fd2789ab6adc2ca68 Mon Sep 17 00:00:00 2001 From: Kai Aldag Date: Fri, 3 Jul 2026 21:27:40 +0100 Subject: [PATCH] =?UTF-8?q?evm-fork-cache=200.2.0=20=E2=80=94=20liveness,?= =?UTF-8?q?=20bulk-storage=20loading,=20code=20seeding,=20reactive=20lifec?= =?UTF-8?q?ycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The liveness & hardening release. Squashes the 0.2.0 development line into one commit. All gates green across the feature matrix (default, --all-features, reactive-polling-only, no-reactive core); pre-1.0, so breaking changes are allowed in this minor and are recorded in CHANGELOG.md. Phase 8 — liveness & state invalidation as a first-class engine concern: - storageHash root gate with per-account TrackingPolicy and CoverageGap detection; RootGateCadence keeps eth_getProof off the per-block path. - CacheHealth / CacheMetrics foundation; journaled deep-reorg self-heal with missed-range detection; strict block context with advance_block env refresh. - Honest freshness verdict taxonomy, Validity stamping of reactive writes, BLOCKHASH fail-closed, and a snapshot-generation guard. - Cold-start root baseline (roots.bin) and the trace-backed resync source. Bulk storage extraction is the default loader — Dedaub-style eth_call state extraction with Multicall3 dispatch, gzip, and automatic point-read fallback; provider-neutral trace resync execution. Verified code seeding — CodeSeedState marks, seed/etch primitives, persisted code_seeds.bin, the AccountFieldsFetchFn seam, verify_code_seeds (fail-closed against on-chain EXTCODEHASH), and a cold-start verify_code phase. Proof economics: batched eth_getProof fan-out with max_concurrent_proofs. Mid-lifecycle adapter register/unregister — ReactiveEngine binds a ReactiveRuntime to an EventSubscriber and drives handler lifecycle as one operation: register_handler auto-anchors a new handler's log backfill to the runtime's last canonical block (register_handler_with_backfill / _live_only variants; sync_handler_interests bootstraps a pre-populated runtime), and owner-scoped interest growth is continuity-safe (the changed subscription inherits the prior delivery anchor and self-heals the gap). unregister_handler plus untrack_account and cancel_pending_resyncs are the teardown recipe. Backed by the InterestOwnerSubscriber trait (implemented by AlloySubscriber) and runtime last_canonical_block() / pending_resyncs() accessors. Release hardening: - #![warn(missing_docs)] enforced; full public-item doc coverage. - CI runs the whole feature matrix (--all-features + polling-only + no-reactive core), not just default features. - New examples/reactive_engine_lifecycle.rs (offline, end-to-end lifecycle) and tests/reactive_subscriber_ingest.rs (real AlloySubscriber batch → runtime ingest, closing the documented integration gap). - Docs reconciled (README, ROADMAP, KNOWN_ISSUES, CHANGELOG) and package hygiene: internal specs/plans excluded from the published crate. --- .github/workflows/ci.yml | 27 +- CHANGELOG.md | 255 +- Cargo.lock | 78 +- Cargo.toml | 23 +- README.md | 189 +- benches/fanout.rs | 8 +- benches/freshness.rs | 14 +- benches/simulation.rs | 34 +- docs/INTERNALS.md | 10 +- docs/KNOWN_ISSUES.md | 138 +- docs/ROADMAP.md | 162 +- docs/bulk-storage-extraction.md | 331 +++ docs/phase-8-liveness-spec.md | 355 +++ docs/trace-resync-benchmarks.md | 170 ++ docs/v0.2.0-release-plan.md | 202 ++ docs/verified-code-seeding-spec.md | 498 ++++ examples/bulk_storage_bench.rs | 986 ++++++++ examples/bundle_simulation.rs | 4 +- examples/call_tracer.rs | 4 +- examples/cold_start.rs | 7 +- examples/fetch_minimization_counted.rs | 18 +- examples/freshness_multi_sim.rs | 29 +- examples/freshness_optimistic.rs | 30 +- examples/parallel_overlays.rs | 4 +- examples/reactive_cache.rs | 7 +- examples/reactive_engine_lifecycle.rs | 318 +++ examples/reactive_runtime.rs | 7 +- examples/snapshot_and_restore.rs | 12 +- fixtures/README.md | 7 + src/access_list.rs | 32 +- src/bulk_storage.rs | 1359 +++++++++++ src/bundle.rs | 20 +- src/cache/binary_state.rs | 13 +- src/cache/bytecode.rs | 16 +- src/cache/code_seeds.rs | 260 ++ src/cache/metadata.rs | 17 +- src/cache/mod.rs | 2482 ++++++++++++++++--- src/cache/overlay.rs | 129 +- src/cache/slot_observations.rs | 9 +- src/cache/snapshot.rs | 6 +- src/cache/versioned.rs | 6 +- src/cold_start/config.rs | 18 +- src/cold_start/driver.rs | 147 +- src/cold_start/error.rs | 14 +- src/cold_start/mod.rs | 6 +- src/cold_start/plan.rs | 12 +- src/cold_start/results.rs | 10 + src/cold_start/roots.rs | 284 +++ src/deploy.rs | 63 +- src/errors.rs | 697 +++++- src/events/mod.rs | 66 +- src/freshness.rs | 225 +- src/lib.rs | 58 +- src/multicall.rs | 31 +- src/prefetch_registry.rs | 16 +- src/reactive/mod.rs | 3048 ++++++++++++++++++++++-- src/tracing.rs | 2 +- tests/block_context.rs | 303 +++ tests/builder_chain_id.rs | 29 +- tests/bulk_storage.rs | 795 ++++++ tests/bundle_simulation.rs | 87 +- tests/bundle_simulation_extra.rs | 8 +- tests/cache_state.rs | 9 +- tests/call_tracer.rs | 8 +- tests/code_seeding.rs | 662 +++++ tests/cold_start.rs | 176 +- tests/common/mod.rs | 98 +- tests/cow_snapshot.rs | 48 +- tests/errors.rs | 15 +- tests/event_pipeline.rs | 66 +- tests/fetch_minimization.rs | 18 +- tests/freshness.rs | 272 ++- tests/liveness_cold_start.rs | 388 +++ tests/liveness_root_gate.rs | 796 +++++++ tests/public_release_surface.rs | 111 +- tests/reactive_alloy_subscriber.rs | 206 +- tests/reactive_engine.rs | 627 +++++ tests/reactive_freshness.rs | 313 +++ tests/reactive_health.rs | 657 +++++ tests/reactive_registry.rs | 99 + tests/reactive_resync.rs | 427 +++- tests/reactive_runtime.rs | 168 ++ tests/reactive_subscriber_ingest.rs | 152 ++ tests/reactive_trace_resync.rs | 394 +++ tests/serialization_roundtrip.rs | 7 +- tests/snapshot_overlay.rs | 18 +- 86 files changed, 18772 insertions(+), 1158 deletions(-) create mode 100644 docs/bulk-storage-extraction.md create mode 100644 docs/phase-8-liveness-spec.md create mode 100644 docs/trace-resync-benchmarks.md create mode 100644 docs/v0.2.0-release-plan.md create mode 100644 docs/verified-code-seeding-spec.md create mode 100644 examples/bulk_storage_bench.rs create mode 100644 examples/reactive_engine_lifecycle.rs create mode 100644 src/bulk_storage.rs create mode 100644 src/cache/code_seeds.rs create mode 100644 src/cold_start/roots.rs create mode 100644 tests/block_context.rs create mode 100644 tests/bulk_storage.rs create mode 100644 tests/code_seeding.rs create mode 100644 tests/liveness_cold_start.rs create mode 100644 tests/liveness_root_gate.rs create mode 100644 tests/reactive_engine.rs create mode 100644 tests/reactive_freshness.rs create mode 100644 tests/reactive_health.rs create mode 100644 tests/reactive_subscriber_ingest.rs create mode 100644 tests/reactive_trace_resync.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01fc583..cbd1575 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,22 +26,31 @@ jobs: - name: Formatting run: cargo fmt --all --check - - name: Clippy - run: cargo clippy --all-targets --no-deps -- -D warnings + - name: Clippy (all features) + run: cargo clippy --all-targets --all-features --no-deps -- -D warnings - - name: Tests (all targets) - run: cargo test --all-targets + - name: Tests (all targets, all features) + run: cargo test --all-targets --all-features - - name: Doc tests - run: cargo test --doc + - name: Doc tests (all features) + run: cargo test --doc --all-features - - name: Docs - run: cargo doc --no-deps + - name: Docs (all features) + run: cargo doc --no-deps --all-features env: RUSTDOCFLAGS: "-D warnings" + # --all-features enables reactive-ws, which shadows the polling-only and + # no-reactive cfg branches. Exercise those distinct compile paths too so a + # feature-gated regression cannot slip through. + - name: Feature matrix (polling-only + no-reactive core) + run: | + cargo clippy --all-targets --no-default-features --features reactive-polling --no-deps -- -D warnings + cargo test --no-default-features --features reactive-polling + cargo build --no-default-features + - name: Benchmarks compile - run: cargo bench --no-run + run: cargo bench --no-run --all-features - name: Package run: cargo package --locked diff --git a/CHANGELOG.md b/CHANGELOG.md index ef86827..2d71191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,247 @@ surface freezes at 1.0. ## [Unreleased] -No unreleased changes. +Remaining from the 0.2.0 plan, tracked as fast-follow: the docs/examples +polish remainder (a toy constant-product AMM example and the end-to-end +reactive integration tests enumerated in +[`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md)). + +## [0.2.0] - 2026-07-05 + +Closes the top publication-review gaps (account-level freshness, strict +block-context, bounded `derived_slots`, conservative deep-reorg behavior) and +lands the full Phase-8 liveness plan (spec steps 1–6). Breaking changes are +grouped under **Changed**; they are permitted pre-1.0 (see the policy above). + +### Performance + +The round's RPC-economics wins (each documented with numbers and a reproduction +path; see the README "Performance & honest trade-offs" section for the full, +hedged picture): + +- **Bulk `eth_call` storage extraction is now the default loader.** One + state-override call reads thousands of slots at a flat ~26 CU instead of one + point read each — e.g. 10,000 slots in 1 call vs 200,000 CU, and a full + Uniswap V3 tick range (7,674 slots) in 2 calls / 52 CU (**~2,952× cheaper**), + with automatic point-read fallback. Methodology + tables: + [`docs/bulk-storage-extraction.md`](docs/bulk-storage-extraction.md); + reproduce with `RPC_URL=… cargo run --release --example bulk_storage_bench`. +- **Verified code-seed cold starts.** Materializing a known contract set's code + and balances is one bulk call settled against on-chain `EXTCODEHASH` rather + than three reads per account: **~46× cheaper, ~25.6× faster** for 20 contracts + (48 ms vs ~1.2 s, 26 CU vs 1,200). Same doc + bench (scenario 11). +- **Proof economics.** `eth_getProof` (no bulk substitute) is kept off the + per-block path: root-gate probes fire on a cadence (default every 16 blocks → + **16× fewer probes** by construction) and batch every gated account into one + bounded fan-out (**≈4.7–7.3× over serial** across runs, live-measured for a + 50-account sweep — bounded by the concurrency cap, larger when per-proof + latency is higher; reproduce with `E2E_RPC_URL=… cargo test --test + liveness_root_gate -- --ignored`). + +### Added + +- **Mid-lifecycle adapter register/unregister (`ReactiveEngine`).** A new + `ReactiveEngine` binds a `ReactiveRuntime` to an `EventSubscriber` and drives + handler lifecycle as one operation, for consumers that add and drop adapters + while the cache runs (e.g. register an AMM on a `PoolCreated` event, drop it + later). `register_handler` updates runtime routing and subscriber interests + together and, once ingestion has journaled a canonical block, backfills the + new handler from that block automatically — closing the discovery→subscription + gap with no caller bookkeeping (`register_handler_with_backfill` for deeper + history, `register_handler_live_only` to opt out; `sync_handler_interests` + bootstraps a pre-populated runtime). `unregister_handler` removes routing and + transport for that handler only; the runtime adds `last_canonical_block()`, + `pending_resyncs()`, `cancel_pending_resyncs(address)`, `handler_ids()`, + `contains_handler`, `handler_interests`, and `unregister_handler` for the full + teardown recipe (unregister + `untrack_account` + `cancel_pending_resyncs`; + cache eviction stays explicit). Under the hood the new + `InterestOwnerSubscriber` trait lets a subscriber add/remove interests keyed by + a stable per-adapter `HandlerId` without disturbing unrelated owners; growing + an owner's filter set carries its delivery anchor across the change and + self-heals the gap, identical filters across owners share one subscription, and + retired filters release their bookkeeping. `SubscriberBackfill` describes the + anchor for owner-scoped `get_logs` backfill (`ReactiveEngineError` / + `ReactiveEngineRegisterError` are the engine's typed errors). All existing + runtime/subscriber APIs are unchanged; `register_interests` remains the + full-replacement setup path. +- **Queryable cache health + metrics.** `ReactiveRuntime::health() -> CacheHealth` + (`Healthy` / `Degraded` / `Unhealthy`) and `metrics() -> CacheMetricsSnapshot` + (atomic counters: `deep_reorgs`, `reorgs_recovered`, `resync_requests`/`_failures`, + `missed_ranges`, `coverage_gaps`, `pending_contamination`). `ReactiveReport::Health` + transition reports and a caller-invoked `reset_health()`. +- **Conservative deep-reorg self-heal + missed-range detection.** A forward gap in + the canonical block sequence (block *N* → *N+k*) is no longer silently accepted: + it emits `ReactiveReport::MissedBlockRange` (+ `ResyncReason::MissedBlockRange`) + and escalates health, while still applying the arriving block. Repeated trust-loss + events (deep reorg beyond the journal, or a missed range) escalate + `Healthy → Degraded → Unhealthy` (a "stop until rebuilt" signal). +- **Account/root fetcher seam.** `EvmCache::account_proof_fetcher` / + `set_account_proof_fetcher` (`AccountProofFetchFn` over `eth_getProof`, returning + `AccountProof`); `ResyncTarget::Account` resyncs now resolve through it + (materializing so cold accounts are not silently skipped). +- **Bulk storage extraction (`bulk_storage` module) — now the default storage + loader.** Every provider-backed cache ships a `StorageBatchFetchFn` that + loads thousands of slots per `eth_call` by overriding target code with + [Dedaub's 23-byte extractor](https://dedaub.com/blog/bulk-storage-extraction/) + (storage survives a code override), plus a Multicall3-dispatch path that + spans many contracts in one call — with the classic point-read fetcher as + automatic fallback (tiny requests, providers without override support — + which also latch the fetcher to point reads after consecutive failures — + and precompile targets). Public surface: + `bulk_call_storage_fetcher[_with_fallback]`, the async core + `fetch_slots_bulk`, `BulkCallConfig` (chunking, concurrency, + `CallDispatch::CallMany` for Erigon-lineage endpoints where one 20-CU + `eth_callMany` request carries the whole batch), `planned_call_count`, + `StorageFetchStrategy` + `EvmCacheBuilder::{storage_fetch_strategy, + bulk_call_config}` (opt-out / tuning), `point_read_storage_fetcher` (the + extracted classic path), and both Shanghai (`PUSH0`) and pre-Shanghai + extractor variants — all executed against revm in the offline test suite. + Measured on Alchemy: 10,000 slots in one 26-CU call (vs 200,000 CU as point + reads); a full Uniswap V3 pool tick range — 7,674 slots — in 2 calls / + 52 CU; 100 contracts × 30 slots in one call + ([benchmarks + limitations](docs/bulk-storage-extraction.md)). + `StorageFetchError` and `RuntimeError` now derive `Clone` so one + chunk-level failure can be reported per affected slot. +- **Custom storage programs + companion extractors + prewarm.** + `StorageProgram` / `run_storage_program[s]` inject caller-supplied bytecode + via the same override transport, enabling data-*dependent* extraction + derived in-EVM (a worked one-shot Uniswap V3 observation-ring loader — zero + calldata — ships in the benchmark example and revm tests); + `fetch_account_fields_bulk` (`BALANCE` + `EXTCODEHASH` for many accounts in + one call) and `fetch_block_context` (seven env words in one call) ride the + same mechanism; `EvmCache::prewarm_slots` bulk-loads a declared working set + into layer 2 through the installed fetcher, returning a `PrewarmReport`. +- **Verified code seeding & local etch.** Adapters can push runtime bytecode + into the cache instead of paying an `eth_getCode` (plus the lazy backend's + balance/nonce round trips) per address, with per-address trust marks + (`CodeSeedState`; absence of a mark = RPC-origin) persisted to + `code_seeds.bin` — saved *before* `bytecodes.bin` and pruned on load, so an + unverified claim can never masquerade as chain-fetched across restarts. + `seed_account_code[_with]` records a *canonical claim* (`Pending`); + `verify_code_seeds()` settles the whole pending set against on-chain + `EXTCODEHASH` in **one** `eth_call` through the new `AccountFieldsFetchFn` + seam (default-wired to `fetch_account_fields_bulk`), marking matches + `Verified` durably (real balance patched from the same response) and + purging contradicted claims for refetch — `CodeVerifyReport` buckets + `mismatched` / `not_deployed` / `codeless` / `unverifiable` (the last is + fail-safe: transport failures keep seeds `Pending`). Chain-fetched code + beats templates: an equal-hash seed over RPC-origin code verifies + instantly with zero RPC; a conflicting one errors + (`CacheError::CodeSeedConflict`) without touching the cache. + `etch_account_code` is the raw-bytes *deliberate divergence* sibling + (`Etched`, never verified); `override_account_code*` and `deploy_contract` + targets now also record `Etched`, making `etched_accounts()` the single + local-divergence health surface. The cold-start driver gained a + `verify_code` phase that runs **first** — no discover sim executes over an + unverified claim — recording `ColdStartResults.code_verifications` and + guarded by `ColdStartError::NoAccountFieldsFetcher` for pending-bearing + rounds. Spec: `docs/verified-code-seeding-spec.md`. +- **Strict block context + engine-driven env refresh.** `BlockContextRequirements` + (`strict`/`lenient`/per-field) + `BlockContextError`; + `EvmCacheBuilder::strict_block_context` / `block_context_requirements` and a + fallible `try_build` (`EvmCache::new` stays infallible + lenient); + `EvmCache::advance_block(header)` refreshes the full block env + (number/basefee/coinbase/prevrandao/gas-limit/timestamp) with the same strict + validation, driven from the reactive canonical-header path; new + `coinbase()` / `prevrandao()` / `block_gas_limit()` getters. +- **`Validity` stamping (opt-in).** `ReactiveRuntime::enable_freshness_stamping()` + stamps canonical event-derived writes `Validity::ValidThrough(N)` so + event-maintained slots stop being needlessly re-verified. +- **storageHash root gate + `TrackingPolicy` + `RootGateCadence`.** + `TrackingPolicy` (`Slots` / `WholeAccount` / `Scalars`) + `track_account`; + the root gate emits `ReactiveReport::CoverageGap` (+ + `ResyncReason::RootMoved`) when a tracked account's storage root moves with + no covering decoder, and `Scalars` tracks native balance/nonce/code (which + do not move the storage root). The gate fires per `RootGateCadence` + (`set_root_gate_cadence`), **default every 16 canonical blocks, never + per-block**: `eth_getProof` is the slowest read the crate issues, and the + gate diffs against its persisted baseline (never block-over-block), so + skipping blocks trades bounded detection lag for a 16× probe-cost cut + without losing detection. The decoder-touched set accumulates across + skipped blocks (drained per firing) so covered writes never false-positive + as gaps; `every_n_blocks(1)` restores per-block probing and `Disabled` + turns the gate off. All root-gate and account-resync probes now go through + **one seam invocation per firing** (grouped by resync block), and the + default proof fetcher fans them out with bounded, order-preserving + concurrency — `EvmCacheBuilder::max_concurrent_proofs` (default 8) — so a + fleet probe costs ~`ceil(N / cap)` round trips instead of `N`. +- **Cold-start root baseline (`roots.bin`).** `RootBaseline` persists each + tracked account's `storageHash` alongside the state file (versioned binary, + magic `EFCROOT`); on restart, `RootBaselinePlanner` root-probes the baseline + via `eth_getProof` and converts moved roots into targeted resyncs instead of + trusting stale disk state (restart-drift detection, Phase-8 step 5). +- **Tier-3 trace-backed resync.** The reactive runtime resolves matching + resync targets from one `debug_traceBlockByNumber` (`prestateTracer`, + `diffMode`) block diff before falling back to storage/account point reads — + `BlockStateDiffFetchFn` (+ `set_block_state_diff_fetcher`), + `BlockStateDiff`/`BlockStateAccountDiff`/`BlockStateStorageDiff`, and + `ReactiveRuntime::ingest_batch_with_resync` (Phase-8 step 6). Measured + economics/latency in + [`docs/trace-resync-benchmarks.md`](docs/trace-resync-benchmarks.md). +- **Bundle cost accounting.** `BundleResult::successful_tx_gas` and + `reverted_tx_gas` — net searcher cost (`coinbase_payment + reverted_tx_gas`) is now + direct under `AllowReverts`. +- **Snapshot-consistency generation guard.** `EvmCache::snapshot_generation()` + — an opaque monotonic counter bumped by targeted state writes + (`apply_update`/`apply_updates`/`modify_slot` and everything built on them) + and block re-pins (`set_block`/`advance_block`), but not by cold prefetch. + Read it around `snapshot()` to detect (and retry) a snapshot taken + mid-block during continuous ingestion, closing the ROADMAP-listed + consistency gap (G6). + +### Changed (breaking) + +- **Freshness verdict taxonomy is honest about scope.** `Validation::Confirmed` is + renamed `ConfirmedStorage` (it only ever meant "no volatile storage slot changed", + not account-level state); a new `ConfirmedFull` covers storage **and** verified + account fields; `Validation::Corrected`'s `changed` field is renamed + `changed_slots` and gains `changed_accounts: Vec`. +- **`ResyncFailureKind::UnsupportedAccountTarget` removed**, replaced by + `MissingAccountFetcher` / `AccountFetchFailed` / `AccountFetchOmitted` now that + account resync is supported. +- **Growth-bound enums are now `#[non_exhaustive]`.** `ReactiveReport`, + `ResyncReason`, `ResyncFailureKind`, `TrackingPolicy`, `CacheHealth`, and the + `CacheMetricsSnapshot` struct are marked `#[non_exhaustive]` (matching the + `StateUpdate`/`PurgeScope` precedent): downstream `match`es need a wildcard arm, + and future variants/counters will no longer be breaking changes. +- **Snapshot API names are now role-based.** The low-level in-place rollback copy + is `EvmCache::checkpoint()` and the immutable fan-out view is + `EvmCache::snapshot() -> Arc`. The pre-0.2.0 `snapshot()` / + `create_snapshot()` public names were removed rather than aliased; the hidden + benchmark/reference helper is now `snapshot_deep_clone()`. +- **Storage batch tuning is per instance.** `StorageBatchConfig` exposes + `slots_per_batch` and `max_concurrent_batches` for the provider-backed + storage fetcher. `EvmCacheBuilder::storage_batch_config(...)` accepts exact + values, `CacheSpeedMode` remains as presets via `From`, and + `EvmCacheBuilder::speed_mode(...)` is shorthand. The process-global + `set_cache_speed_mode` / `cache_speed_mode` functions and static are removed. +- **`SpeculativeSim::validate()` returns `Result`.** Validator-owned + uncertainty still returns `Validation::Unverified`, while a failed background + task (for example a panic) now surfaces as an error instead of being folded into + a verdict or panicking on the handle. +- **Crate-owned fallible APIs now return typed errors instead of `anyhow`.** + Public helpers expose domain errors such as `CacheError`, `StorageFetchError`, + `RpcError`, `FreshnessError`, `DeployError`, `MulticallError`, and + `AccessListError`; `SimError::Other` now carries `SimHostError`. `anyhow` is no + longer a normal library dependency, though examples/tests may still use it as + harness glue. + +### Fixed + +- **`EventPipeline::derived_slots` no longer grows unbounded.** It is now a + block-horizon ring bounded to `ReorgConfig::depth` (mirroring the `touched` ring), + so steady-state ingestion does not leak memory. +- **`BLOCKHASH`-reading sims fail closed in freshness validation.** Validator + overlays carry no block hashes, so an in-lookback-range `BLOCKHASH` read + resolves to ZERO; such sims were previously eligible for a silent + `ConfirmedStorage`. The read is now recorded + (`EvmOverlay::blockhash_zero_fallback`) and the verdict is + `Validation::Unverified` — on the optimistic pass and on corrected re-runs. + Out-of-lookback reads return the spec-mandated ZERO and are not flagged. +- **Block-diff traces now surface full account deletions.** A SELFDESTRUCTed + account (present in the trace's `pre`, absent from `post`) gets explicit + zeroed balance/nonce/code in `BlockStateDiff`, so account-target resyncs + resolve from the trace instead of falling back to point reads. ## [0.1.0] - 2026-06-30 @@ -202,8 +442,9 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). a pubsub stream terminates, plus an adapter from legacy `EventDecoder`s to reactive handlers. HTTP polling `watch_logs` / `watch_pending_transactions` support is still exported behind the opt-in `reactive-polling` feature. Full - block bodies, full pending transaction hydration, and arbitrary historical - backfill remain follow-up transport work. Generic core. + block bodies and full pending transaction hydration remain follow-up transport + work; owner-scoped log backfill can fetch from an explicit block anchor when + adding new interests. Generic core. - **Reactive storage resync execution** — `ReactiveRuntime::ingest_batch_with_resync` preserves the direct-effect behavior of `ingest_batch`, then executes surfaced storage resync requests through `EvmCache`'s provider-neutral @@ -211,8 +452,9 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). updates, and reports requested targets, applied updates, the resulting `StateDiff`, and per-target failures in `ResyncReport`. `ResyncFailureKind` gives downstream retry policy and metrics a stable failure classification. - Account-field resyncs remain explicitly unsupported until a provider-neutral - account fetch callback exists. Generic core. + Account targets resolve through the provider-neutral account proof fetcher; + missing, failed, or omitted account fetches surface as typed failures. Generic + core. - **Reactive block journaling and reorg recovery** — canonical block inputs and applied handler reports are retained in a depth-bounded runtime journal. Removed logs, explicit `ChainStatus::Reorged` inputs, and parent-hash @@ -440,5 +682,6 @@ pre-release development phases (see [`docs/ROADMAP.md`](docs/ROADMAP.md)). - `EvmCache` requires a multi-thread tokio runtime for any RPC-touching path. - See [`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md) for current limitations. -[Unreleased]: https://github.com/KaiCode2/evm-fork-cache/compare/v0.1.0...HEAD +[Unreleased]: https://github.com/KaiCode2/evm-fork-cache/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/KaiCode2/evm-fork-cache/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/KaiCode2/evm-fork-cache/releases/tag/v0.1.0 diff --git a/Cargo.lock b/Cargo.lock index fb1be26..f4f9e0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -1052,6 +1058,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -1433,6 +1451,23 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "const-hex" version = "1.19.1" @@ -1530,6 +1565,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -1930,7 +1974,7 @@ dependencies = [ [[package]] name = "evm-fork-cache" -version = "0.1.0" +version = "0.2.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -1950,6 +1994,7 @@ dependencies = [ "criterion", "foundry-fork-db", "futures", + "reqwest", "revm", "rustls", "serde", @@ -2025,6 +2070,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2854,6 +2909,16 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -4344,6 +4409,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -4756,12 +4827,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", diff --git a/Cargo.toml b/Cargo.toml index 0bb3b68..9001670 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "evm-fork-cache" -version = "0.1.0" +version = "0.2.0" edition = "2024" rust-version = "1.88" license = "MIT OR Apache-2.0" @@ -11,11 +11,11 @@ readme = "README.md" repository = "https://github.com/KaiCode2/evm-fork-cache" documentation = "https://docs.rs/evm-fork-cache" -# Keep the published crate lean: the per-phase development specs are internal -# planning artifacts (they reference an adapter surface extracted before release), -# and the CI workflow has no consumer value. The consumer-facing ROADMAP.md and -# KNOWN_ISSUES.md under docs/ are still shipped. -exclude = ["docs/phase-*-spec.md", ".github/"] +# Keep the published crate lean: development specs and release plans (per-phase +# build orders with internal line references) are planning artifacts, and the CI +# workflow has no consumer value. The consumer-facing ROADMAP.md, KNOWN_ISSUES.md, +# INTERNALS.md, and benchmark notes under docs/ are still shipped. +exclude = ["docs/*-spec.md", "docs/*-plan.md", ".github/"] # Build the docs.rs page with every feature enabled so the full surface — the # reactive runtime, the default WebSocket subscriber, the opt-in polling @@ -49,7 +49,6 @@ alloy-rpc-types-eth = "1.0.38" alloy-sol-types = "1.4" futures = "0.3" -anyhow = "1.0.98" bincode = "1.3" foundry-fork-db = "0.22" revm = { version = "34.0", features = ["std", "serde", "optional_eip3607", "optional_no_base_fee", "optional_balance_check"] } @@ -67,11 +66,17 @@ tokio = { version = "1.48.0", features = ["rt-multi-thread", "time"] } tracing = "0.1.41" [dev-dependencies] +anyhow = "1.0.98" alloy-node-bindings = "1.1.2" alloy-rpc-client = { version = "1.0.38", features = ["reqwest"] } alloy-transport = "1.0.38" alloy-transport-http = "1.0.38" criterion = "0.5" +# Gzip-capable HTTP client for the bulk-storage benchmark example: enabling the +# `gzip` feature makes reqwest advertise `Accept-Encoding: gzip` and +# transparently decompress responses — a large win for multi-hundred-KB +# `eth_call` payloads (see docs/bulk-storage-extraction.md). +reqwest = { version = "0.12", default-features = false, features = ["gzip"] } # `macros` powers `#[tokio::main]`/`#[tokio::test]` in the examples and tests. # `time` lets live RPC examples bound their subscription windows. tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "time"] } @@ -116,6 +121,10 @@ required-features = ["reactive"] name = "reactive_runtime" required-features = ["reactive"] +[[example]] +name = "reactive_engine_lifecycle" +required-features = ["reactive"] + [[example]] name = "cold_start" required-features = ["reactive"] diff --git a/README.md b/README.md index 278b9d7..d7cdebc 100644 --- a/README.md +++ b/README.md @@ -45,15 +45,16 @@ around three capabilities that target exactly this workload: > `AlloySubscriber` (WebSocket `subscribe_logs`/`subscribe_blocks`/`subscribe_pending_transactions`, > exponential-backoff reconnect, and `get_logs` backfill) plus protocol-neutral > cold-start (2); and the optimistic verify-and-rerun loop (3). Honest remaining -> transport work: full block bodies, full pending-transaction hydration, and -> arbitrary historical backfill are follow-ups. The public API still changes +> transport work: full block bodies and full pending-transaction hydration are +> follow-ups; owner-scoped log backfill is available for newly added reactive +> interests. The public API still changes > between minor versions — see [Stability](#stability). ## What it provides today - **Forked EVM cache** backed by `foundry-fork-db` with lazy RPC loading and on-disk persistence for accounts, storage, bytecode, and immutable metadata. -- **Snapshots and overlays** — `create_snapshot()` produces an immutable, +- **Snapshots and overlays** — `snapshot()` produces an immutable, `Send + Sync` point-in-time view; each `EvmOverlay` is a cheap clone that simulates in isolation, ideal for parallel candidate evaluation. - **Bundle simulation** — `simulate_bundle` applies an ordered sequence of @@ -89,7 +90,13 @@ around three capabilities that target exactly this workload: resyncs. The `ReactiveRegistry` exposes consolidated Alloy log filters for provider subscription setup and exact local log routing with optional route keys. The - provider-agnostic `EventSubscriber` trait and `AlloySubscriber` are included; + registry and runtime support incremental handler lifecycle: + `register_handler` remains append-only and duplicate-checked, while + `unregister_handler(&HandlerId)` removes only that handler's future decode + routes and interests. It does **not** purge `EvmCache`, reset health/metrics, + clear the reorg journal, or drop root/freshness tracking; cache eviction stays + an explicit caller action. The provider-agnostic `EventSubscriber` trait and + `AlloySubscriber` are included; the Alloy subscriber uses WebSocket/pubsub `subscribe_logs`, `subscribe_blocks`, and `subscribe_pending_transactions` by default for live log, block-header, and pending-transaction-hash inputs. If an established @@ -99,9 +106,24 @@ around three capabilities that target exactly this workload: through `get_logs`, marking recovered records as `InputSource::Backfill` while suppressing recent duplicate canonical inputs. HTTP polling `watch_logs` / `watch_pending_transactions` remains available behind the opt-in - `reactive-polling` feature. Full block bodies, full pending transaction - hydration, and arbitrary historical backfill remain explicit follow-up - transport work. + `reactive-polling` feature. For pool/feed churn (register a new AMM on a + `PoolCreated` event; drop one that is no longer of interest), the recommended + binding is `ReactiveEngine`, which owns a `ReactiveRuntime` plus an + `EventSubscriber` and drives handler lifecycle as one operation: + `engine.register_handler(handler)` updates runtime routing and subscriber + interests together and — once ingestion has journaled a canonical block — + **backfills the new handler from that block automatically**, so a pool + discovered mid-stream misses none of its own logs between discovery and live + subscription (`register_handler_with_backfill` for deeper history, + `register_handler_live_only` to opt out). Growing an existing handler's filter + set is continuity-safe too: the changed subscription inherits the old delivery + anchor and self-heals the gap. Use stable per-pool or per-adapter `HandlerId` + values. Dropping an adapter is `engine.unregister_handler(&id)` for + routing/transport, plus — for a pool that will not return — + `runtime.untrack_account` (stop root-gate `eth_getProof` probes) and + `runtime.cancel_pending_resyncs` (drop its queued repairs); cache eviction + stays an explicit caller action. Full block bodies and full pending + transaction hydration remain explicit follow-up transport work. - **Cold-start** — declaratively warm a working set of accounts and storage slots into the cache in one batched pass via `EvmCache::run_cold_start` and a `ColdStartPlanner` (discover slots via a view-call, then verify them), returning @@ -119,8 +141,32 @@ around three capabilities that target exactly this workload: touch set; helpers build an EIP-2930 access list and estimate whether attaching one is profitable on an L2. - **Multicall3 batching** for running many view calls inside the fork in one pass. -- **Deployment & etching** — deploy from creation code, or etch locally compiled - Foundry runtime bytecode over a forked contract while preserving its storage. +- **Bulk storage extraction — the default storage loader** — thousands of + storage slots (across many contracts) in a *single* `eth_call` via + state-override code injection + ([Dedaub's technique](https://dedaub.com/blog/bulk-storage-extraction/)), + with automatic point-read fallback for providers without override support. + One 26-CU call replaces up to 10,000 20-CU `eth_getStorageAt`s on Alchemy; a + full Uniswap V3 pool tick range (7,674 slots) loads in 2 calls / ~220 ms, + and `eth_callMany` dispatch drops the whole batch to 20 CU. Also ships + custom *storage programs* (derive what to read in-EVM — e.g. a one-shot V3 + observation-ring loader with zero calldata), bulk account-field and + block-context extractors, and `EvmCache::prewarm_slots` for declared + working sets — see + [`docs/bulk-storage-extraction.md`](docs/bulk-storage-extraction.md). +- **Verified code seeding** — skip `eth_getCode` (and the lazy backend's + per-account round trips) for contracts whose bytecode the adapter already + knows: `seed_account_code` writes the claim, `verify_code_seeds` settles + the entire pending set against on-chain `EXTCODEHASH` in **one** `eth_call`, + and a confirmed seed is `Verified` durably (persisted across restarts, never + re-checked). A wrong template degrades to one refetch — never to wrong + sims — and a newly detected contract materializes fully verified in ~1 + round trip. The cold-start driver verifies pending seeds before anything + simulates. +- **Deployment & etching** — deploy from creation code, or etch runtime + bytecode (`etch_account_code` for raw bytes, no source account needed) over + a forked contract while preserving its storage; every locally-divergent + code site is tracked and queryable via `etched_accounts()`. - **CREATE3 address derivation** utilities. - **An extensible revert decoder** — the two Solidity built-ins (`Error(string)` and `Panic(uint256)`) decode natively; register your own contract-defined @@ -143,7 +189,7 @@ sol! { function balanceOf(address account) external view returns (uint256); } -# async fn example() -> anyhow::Result<()> { +# async fn example() -> Result<(), Box> { let provider = ProviderBuilder::new() .network::() .connect_http("https://example-rpc.invalid".parse()?); @@ -193,7 +239,7 @@ reactive-sync control plane): flowchart BT RPC["RPC provider"] -->|"lazy fetch · once"| CACHE LOGS["on-chain event logs"] -.->|"decode → write · 0 RPC"| CACHE - CACHE["EvmCache · !Send
fetch · cache · targeted writes/purge"] -->|"create_snapshot()"| SNAP + CACHE["EvmCache · !Send
fetch · cache · targeted writes/purge"] -->|"snapshot()"| SNAP SNAP["EvmSnapshot · Send + Sync
immutable · Arc · point-in-time"] -->|"cheap Arc clone × N"| OV OV["EvmOverlay × N · Send
isolated parallel simulations"] classDef hot fill:#102a17,stroke:#3fb950,color:#e6edf3; @@ -232,7 +278,7 @@ sequenceDiagram V->>R: verify volatile read-set (~L ms) R-->>V: fresh values alt nothing the sims read changed - V-->>S: validate().await → Confirmed + V-->>S: validate().await? → Confirmed else a read slot changed V->>V: re-run only the affected sims V-->>S: Corrected { results, changed } @@ -254,8 +300,8 @@ and inject all state directly: | `create3_addresses` | Basic | Derive CREATE3 deployment addresses off-chain. | | `storage_access_list` | Basic | Merge touch sets, estimate EIP-2929 savings, build an EIP-2930 list. | | `erc20_balance_override` | Basic | Set an ERC20 balance by scanning for its storage slot. | -| `snapshot_and_restore` | Intermediate | In-place `snapshot()`/`restore()` rollback on one cache. | -| `parallel_overlays` | Intermediate | Fan one `create_snapshot()` out to many isolated `EvmOverlay` simulations. | +| `snapshot_and_restore` | Intermediate | In-place `checkpoint()`/`restore()` rollback on one cache. | +| `parallel_overlays` | Intermediate | Fan one `snapshot()` out to many isolated `EvmOverlay` simulations. | | `transfer_inspector` | Intermediate | Report per-token balance deltas from a simulation. | | `deploy_and_override` | Intermediate | Deploy from creation code and etch it over another address. | | `foundry_artifact_etching` | Intermediate | Etch a locally compiled Foundry artifact (from a JSON file) over a fork. | @@ -278,6 +324,7 @@ endpoint (they print instructions and exit if it is unset): | `fork_token_balance` | Basic | Lazy RPC loading and warm-cache reuse (cold vs. warm read). | | `multicall_batch` | Intermediate | Batch many view calls through Multicall3 in one pass. | | `multicall_with_error_handling` | Intermediate | Batch with `allowFailure`; read partial results when a call reverts. | +| `bulk_storage_bench` | Advanced | Benchmark bulk `eth_call` storage extraction vs point reads: scaling, multicall dispatch, a full Uniswap V3 tick-range load, gzip, verified code-seed cold starts, and the provider's chunk ceiling. | | `fork_override_balance` | Intermediate | Discover a real token's balance slot and override it. | | `reactive_alloy_amm_live_probe` | Advanced | Subscribe to live mainnet AMM logs through the WebSocket-backed `AlloySubscriber`. | @@ -304,7 +351,7 @@ runnable [`foundry_artifact_etching`](examples/foundry_artifact_etching.rs) exam use alloy_primitives::Address; use evm_fork_cache::deploy::{encode_constructor_args, etch_foundry_artifact_or_create}; -# fn example(cache: &mut evm_fork_cache::cache::EvmCache) -> anyhow::Result<()> { +# fn example(cache: &mut evm_fork_cache::cache::EvmCache) -> Result<(), Box> { let target = Address::repeat_byte(0x42); let constructor_args = encode_constructor_args((Address::ZERO,)); @@ -344,9 +391,11 @@ primitives don't give you: `foundry-fork-db`'s cache is **not block-keyed**: re-pinning to a new block does not invalidate cached slots, so a refresh-by-refetch loop must re-read every slot that changed *each block* to stay correct. Decoding the block's logs into targeted -writes keeps that hot state correct with **0 RPC fetches/block** — pinned in -[`tests/event_pipeline.rs`](tests/event_pipeline.rs). Sampled `reconcile()` -re-reads a fraction to catch drift (the honesty backstop). See +writes keeps that hot state correct with **0 RPC fetches/block** — the log→write +path runs fully offline in [`tests/event_pipeline.rs`](tests/event_pipeline.rs), +and the zero-extra-fetch integer is tallied by a real fetch counter in +[`tests/fetch_minimization.rs`](tests/fetch_minimization.rs). Sampled +`reconcile()` re-reads a fraction to catch drift (the honesty backstop). See [`reactive_cache`](examples/reactive_cache.rs). ![Cumulative RPC slot reads over 8 blocks: refresh-by-refetch climbs linearly to 64 while event-driven writes stay flat at 8 (warmed once, then 0 per block)](assets/cross_block_freshness.svg) @@ -357,7 +406,7 @@ re-reads a fraction to catch drift (the honesty backstop). See > not an unreachable number. **② Parallel fan-out — available, modest, workload-dependent.** -`create_snapshot()` is an immutable `Send + Sync` view; cloning the `Arc` hands +`snapshot()` is an immutable `Send + Sync` view; cloning the `Arc` hands each thread its own overlay, so candidates fan out across cores — which a single mutable fork cannot do. The measured speedup is honest and modest: **~1.2×** across the 64–1,024-candidate sweep (`cargo bench --bench fanout`) on a 10-core M1 Pro, @@ -380,17 +429,111 @@ searcher who simply acts on warm state is equally fast; the value is the **safe, selective re-run**, not a latency multiplier. See [`freshness_optimistic`](examples/freshness_optimistic.rs). +**⑤ Cold-load economics — the second quantitative win (live-measured).** +Since 0.2.0 the **default** batch storage fetcher packs slot reads into single +`eth_call`s whose target code is overridden with a 23-byte extractor +([Dedaub's bulk storage extraction](https://dedaub.com/blog/bulk-storage-extraction/) — +full credit to their write-up and +[reference implementation](https://github.com/Dedaub/storage-extractor)), so a +cold working set loads in a handful of calls instead of one billed read per +slot. Live-measured on Alchemy mainnet (`RPC_URL=… cargo run --release +--example bulk_storage_bench`, medians of 3): + +| Workload | Bulk (the default) | Same load as point reads | +|---|---|---| +| 10,000 slots, one contract | 1 call · 26 CU · 148 ms | 200,000 CU | +| 3,000 slots across 100 contracts | 1 call · 26 CU · 77 ms | 60,000 CU | +| Full Uniswap V3 pool tick range (7,674 slots) | 2 calls · 52 CU · ~220 ms | 153,480 CU (**2,952× cheaper**) | +| 20 known contracts: runtime code + balances (verified code seeding) | 1 call · 26 CU · 48 ms | 60 per-account reads · 1,200 CU · ~1.2 s · ~211 KB bytecode (**46× cheaper, 25.6× faster**) | + +`CallDispatch::CallMany` drops any batch to a flat 20 CU on Erigon-lineage +endpoints (Alchemy included); custom *storage programs* go further and derive +what to read in-EVM (a one-shot V3 observation-ring loader ships as a worked +example); the code-seeding row rides the same transport's account-fields +extractor — one call settles every pending bytecode claim against on-chain +`EXTCODEHASH` and materializes real balances, with zero code bytes on the +wire. Providers without state-override support degrade automatically to +the classic point-read path. Methodology, latency tables, gzip measurements, +and every limitation found: +[`docs/bulk-storage-extraction.md`](docs/bulk-storage-extraction.md). + +The **CU costs are deterministic and exact** — re-verified live three times +through 2026-07-05, identical every run. The **wall-clock latencies above are a +conservative floor**: they were captured across constrained, variable networks, +so well-provisioned connectivity should meet or beat them (repeat runs measured +the same loads up to several× slower purely from network load, never faster CU). + > [!NOTE] > **Methodology & candor.** Offline (mocked provider, state injected — no > network). We deliberately do **not** lead with the headline multipliers a naive > baseline would produce (~500× fewer reads, ~545× throughput, ~3,800× latency): > all three collapse toward ~1× against a competent `SharedBackend` + > `checkpoint`/`revert` loop — which is the very primitive this crate wraps. The -> "0 fetches/block" integer is real and CI-pinned (`cargo test --test -> event_pipeline`); the parallel-fan-out ratio is a Criterion median on an Apple -> M1 Pro, read as a ratio not an absolute. Live-RPC checks live behind the +> zero-extra-fetch integer is real and CI-pinned (`cargo test --test +> fetch_minimization`, with the log→write path exercised offline by `cargo test +> --test event_pipeline`); the parallel-fan-out ratio is a Criterion median on an +> Apple M1 Pro, read as a ratio not an absolute. Live-RPC checks live behind the > `RPC_URL` gate. +Phase 8's trace-backed resync path has separate live-RPC measurements in +[`docs/trace-resync-benchmarks.md`](docs/trace-resync-benchmarks.md): on Alchemy +CU pricing, one block trace breaks even with two storage reads and wins at three +or more; in latency tests, batched storage stayed faster for small known slot +sets, while gzip materially reduced large Alchemy trace response latency. +`eth_getProof` — the one call with no bulk substitute (storage roots and nonces +are not EVM-visible) — is kept off the per-block path entirely: root-gate probes +fire on a cadence (default every 16 blocks, `RootGateCadence` — 16× fewer probes +than per-block, by construction) and each firing batches every gated account into +one fan-out (`EvmCacheBuilder::max_concurrent_proofs`, default 8 — live-measured +≈4.7–7.3× over serial for a 50-account sweep across runs, bounded by the cap and +larger when per-proof latency is higher). Reproduce the fan-out measurement with +the RPC-gated test `E2E_RPC_URL=… cargo test --test liveness_root_gate -- --ignored` +(`default_proof_fetcher_fans_out_concurrently`). + +## Production safety checklist + +The defaults favor doing something reasonable over failing; production +deployments should opt into the strict/observable variants deliberately: + +- [ ] **Multi-thread tokio runtime.** RPC-backed calls bridge sync→async via + `block_in_place`; on a current-thread runtime they degrade to typed + `RuntimeError`s. Use `#[tokio::main(flavor = "multi_thread")]`. +- [ ] **Pin a concrete block** (`EvmCacheBuilder::block` / `EvmCache::at_block`) + for anything that must be reproducible; `latest` pins are for exploration. +- [ ] **Strict block context.** `builder.strict_block_context(true)` + + `try_build()` so a missing `basefee`/`prevrandao` fails loudly instead of + silently defaulting the EVM env (per-field knobs via + `BlockContextRequirements` for pre-London/pre-merge chains). +- [ ] **Set the chain id explicitly** (`EvmCacheBuilder::chain_id`) rather than + relying on `eth_chainId` inference with its mainnet fallback. +- [ ] **Watch the health surface.** Poll `ReactiveRuntime::health()` and treat + `Degraded`/`Unhealthy` as "stop trading until resynced"; export + `metrics()` counters (`deep_reorgs`, `resync_failures`, `coverage_gaps`, + `missed_ranges`) to your dashboards. +- [ ] **Track balance/nonce-sensitive accounts.** `track_account` with + `TrackingPolicy::WholeAccount`/`Scalars` — storage-only freshness cannot see + a native-balance move, and only tracked accounts upgrade verdicts to + `ConfirmedFull` (see the verdict taxonomy in `freshness`). +- [ ] **Gate on code-seed verification.** If you seed bytecode + (`seed_account_code`), require the round's `CodeVerifyReport.unverifiable` + bucket to be empty and `pending_code_seeds()` drained before serving sims; + audit deliberate divergence via `etched_accounts()`. +- [ ] **Size reorg horizons deliberately.** `ReorgConfig::depth` and + `ReactiveConfig::journal_depth` bound purge/rollback reach; deeper reorgs + degrade health and purge conservatively rather than replaying. +- [ ] **Know your provider.** The default bulk storage loader needs `eth_call` + state-override support (major providers have it; the fetcher latches to + point reads after two fully-failed batches — a `warn!` you should alert on); + the trace resync accelerator needs the `debug` namespace and falls back to + point reads without it. Enable gzip on the HTTP client for both + ([`docs/bulk-storage-extraction.md`](docs/bulk-storage-extraction.md)). +- [ ] **Persisted state is trust-gated, not trusted.** Load disk state with a + `RootBaseline` (`roots.bin`) so restart drift is detected via root probes + instead of silently simulating on stale slots. +- [ ] **Read [`docs/KNOWN_ISSUES.md`](docs/KNOWN_ISSUES.md)** — the accepted + limitations (BLOCKHASH-in-overlays, decoder assumptions, deep-reorg bounds) + are documented there rather than discoverable by surprise. + ## Benchmarks Criterion benchmarks live in [`benches/`](benches) and run fully offline (mocked @@ -402,7 +545,7 @@ provider) so they are reproducible: | `freshness` | **Act-then-validate (④).** The optimistic loop CPU cost, selective re-run, and the latency-hiding shape (vs a baseline that elects to block). `verify_slots` at scale; multi-sim fan-out. | | `event_pipeline` | **Cross-block freshness (①).** `ingest_logs` decode+apply throughput (1 → 1000 logs), `reorg_to` purge; the 0-fetch/block property is pinned in `tests/event_pipeline.rs`. | | `state_update` | `apply_updates` throughput across batch sizes (1 → 1000 `Slot`s) and per-variant apply cost (`Slot` vs `Account` vs `Purge`). | -| `simulation` | Hot-path micro-benches and snapshot-implementation regression guards (`create_snapshot` vs the deep-clone reference — an internal cost model, see [`docs/INTERNALS.md`](docs/INTERNALS.md)). | +| `simulation` | Hot-path micro-benches and snapshot-implementation regression guards (`snapshot` vs the deep-clone reference — an internal cost model, see [`docs/INTERNALS.md`](docs/INTERNALS.md)). | | `access_list` | Touch-set merge and EIP-2930 list construction. | | `revert_decoding` | Built-in (`Error`/`Panic`) and custom-error revert decoding, and decoder dispatch over a registered custom error. | | `create3` | CREATE3 address derivation. | diff --git a/benches/fanout.rs b/benches/fanout.rs index a99acad..4cdca94 100644 --- a/benches/fanout.rs +++ b/benches/fanout.rs @@ -3,7 +3,7 @@ //! The honest, defensible win of the snapshot/overlay model is **parallelism**. //! A live mutable fork (one `revm` EVM isolated with `checkpoint`/`revert`) can //! only evaluate candidates *sequentially* — it cannot be shared mutably across -//! threads. `create_snapshot()` produces an immutable `Send + Sync` view; cloning +//! threads. `snapshot()` produces an immutable `Send + Sync` view; cloning //! the `Arc` hands each worker thread its own cheap `EvmOverlay`, so the same N //! candidates fan out across cores. //! @@ -85,7 +85,7 @@ fn warm_cache(rt: &Runtime) -> (EvmCache, Address, Address, Bytes) { fn bench_candidate_fanout(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let (mut cache, token, owner, calldata) = warm_cache(&rt); - black_box(cache.create_snapshot()); // warm the memoized base once + black_box(cache.snapshot()); // warm the memoized base once let workers = thread::available_parallelism() .map(|n| n.get()) @@ -114,7 +114,7 @@ fn bench_candidate_fanout(c: &mut Criterion) { // here too — single-threaded the snapshot model is ~1×, not a win. group.bench_with_input(BenchmarkId::new("sequential", n), &n, |b, &n| { b.iter(|| { - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); for _ in 0..n { let mut overlay = EvmOverlay::new(snapshot.clone(), None); run_candidate(&mut overlay); @@ -127,7 +127,7 @@ fn bench_candidate_fanout(c: &mut Criterion) { // which a single mutable fork cannot do. group.bench_with_input(BenchmarkId::new("parallel", n), &n, |b, &n| { b.iter(|| { - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let per = n.div_ceil(workers); thread::scope(|s| { for start in (0..n).step_by(per) { diff --git a/benches/freshness.rs b/benches/freshness.rs index d4cbd82..c1a0c5a 100644 --- a/benches/freshness.rs +++ b/benches/freshness.rs @@ -114,7 +114,7 @@ fn stub_fetcher( values: HashMap<(Address, U256), U256>, delay: Option, ) -> StorageBatchFetchFn { - Arc::new(move |reqs: Vec<(Address, U256)>, _block: Option| { + Arc::new(move |reqs: Vec<(Address, U256)>, _block: BlockId| { if let Some(d) = delay { std::thread::sleep(d); } @@ -191,7 +191,7 @@ fn bench_phase2_cpu(c: &mut Criterion) { vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], ) .unwrap(); - black_box(sim.validate().await); + black_box(sim.validate().await.unwrap()); }); }, BatchSize::SmallInput, @@ -214,7 +214,7 @@ fn bench_phase2_cpu(c: &mut Criterion) { vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], ) .unwrap(); - let v = sim.validate().await; + let v = sim.validate().await.unwrap(); debug_assert!(matches!(v, Validation::Corrected { .. })); black_box(v); }); @@ -250,7 +250,7 @@ fn bench_phase2_latency(c: &mut Criterion) { cache .verify_slots(&[(TOKEN, balance_slot(SENDER))]) .unwrap(); // pays L - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); black_box(overlay.call_raw(SENDER, TOKEN, calldata.clone()).unwrap()); }, @@ -299,7 +299,7 @@ fn bench_phase2_latency(c: &mut Criterion) { vec![SimRequest::new(SENDER, TOKEN, calldata.clone())], ) .unwrap(); - black_box(sim.validate().await); + black_box(sim.validate().await.unwrap()); }); }, BatchSize::SmallInput, @@ -376,8 +376,8 @@ fn bench_multi_sim(c: &mut Criterion) { |(mut cache, mut ctrl, reqs)| { rt.block_on(async { let sim = ctrl.run(&mut cache, reqs).unwrap(); - let v = sim.validate().await; - debug_assert!(matches!(v, Validation::Confirmed)); + let v = sim.validate().await.unwrap(); + debug_assert!(matches!(v, Validation::ConfirmedStorage)); black_box(v); }); }, diff --git a/benches/simulation.rs b/benches/simulation.rs index 658de52..05e1ccc 100644 --- a/benches/simulation.rs +++ b/benches/simulation.rs @@ -6,17 +6,17 @@ //! quantify the Pillar A (copy-on-write snapshot) win. //! //! Expected shapes: -//! - **`create_snapshot` group (A/B).** The cold index is seeded into **layer 2** +//! - **`snapshot` group (A/B).** The cold index is seeded into **layer 2** //! via `inject_storage_batch` (`populated_cache_layer2`), the way a fork cache -//! actually holds it. For each size it benches both the COW `create_snapshot` -//! and the retained `create_snapshot_deep_clone`. The deep clone is an O(total +//! actually holds it. For each size it benches both the COW `snapshot` +//! and the retained `snapshot_deep_clone`. The deep clone is an O(total //! state) copy, so its cost slopes up with the index size; the COW path shares //! the memoized base and avoids cloning total storage slots. It still scans //! accounts and new layer-1 entries, so it should be much flatter than the deep //! clone, especially as slots/account grows, but not strictly flat by account //! count. //! - **`resnapshot_hot_loop`.** Warms the base with one snapshot, applies a small -//! `apply_updates` layer-1 mutation, then measures `create_snapshot`. This is +//! `apply_updates` layer-1 mutation, then measures `snapshot`. This is //! the memoization win: the COW path avoids cloning cold storage slots but //! remains sensitive to account scans and new layer-1 entries. //! - **`overlay_fanout`.** Measures fanning one frozen snapshot out into many @@ -62,7 +62,7 @@ fn offline_cache(rt: &Runtime) -> EvmCache { /// A cache whose cold index lives in **layer 2** — seeded via /// `inject_storage_batch`, the path a fork cache actually uses to bulk-load its -/// cold state. This is what the COW `create_snapshot` memoizes into its base. +/// cold state. This is what the COW `snapshot` memoizes into its base. fn populated_cache_layer2(rt: &Runtime, accounts: usize, slots_per: usize) -> EvmCache { let mut cache = offline_cache(rt); let mut batch: Vec<(Address, U256, U256)> = Vec::with_capacity(accounts * slots_per); @@ -80,15 +80,15 @@ fn populated_cache_layer2(rt: &Runtime, accounts: usize, slots_per: usize) -> Ev cache } -/// A/B snapshot creation across cold-index sizes: the COW `create_snapshot` vs. -/// the retained `create_snapshot_deep_clone`, both over a layer-2-seeded index. +/// A/B snapshot creation across cold-index sizes: the COW `snapshot` vs. +/// the retained `snapshot_deep_clone`, both over a layer-2-seeded index. /// /// The deep clone slopes up with total slots; the COW path, after a warm-up /// snapshot has memoized the base, avoids cloning those slots but still pays the /// O(accounts) growth scan. -fn bench_create_snapshot(c: &mut Criterion) { +fn bench_snapshot(c: &mut Criterion) { let rt = Runtime::new().unwrap(); - let mut group = c.benchmark_group("create_snapshot"); + let mut group = c.benchmark_group("snapshot"); for &(accounts, slots) in &[ (100usize, 8usize), (1_000, 8), @@ -99,24 +99,24 @@ fn bench_create_snapshot(c: &mut Criterion) { let mut cache = populated_cache_layer2(&rt, accounts, slots); // Warm the memoized base once so the COW measurement reflects the // steady-state (reuse) cost, not the first full build. - black_box(cache.create_snapshot()); + black_box(cache.snapshot()); group.throughput(criterion::Throughput::Elements((accounts * slots) as u64)); group.bench_with_input( BenchmarkId::new("cow", format!("{accounts}acct_x{slots}slot")), &accounts, - |b, _| b.iter(|| black_box(cache.create_snapshot())), + |b, _| b.iter(|| black_box(cache.snapshot())), ); group.bench_with_input( BenchmarkId::new("deep_clone", format!("{accounts}acct_x{slots}slot")), &accounts, - |b, _| b.iter(|| black_box(cache.create_snapshot_deep_clone())), + |b, _| b.iter(|| black_box(cache.snapshot_deep_clone())), ); } group.finish(); } /// The memoization win: a hot re-snapshot loop. Warm the base once, apply a -/// *small* layer-1 mutation, then measure `create_snapshot`. Cost should track +/// *small* layer-1 mutation, then measure `snapshot`. Cost should track /// account scanning plus new layer-1 entries, staying much flatter than the deep /// clone as cold storage grows. fn bench_resnapshot_hot_loop(c: &mut Criterion) { @@ -125,7 +125,7 @@ fn bench_resnapshot_hot_loop(c: &mut Criterion) { for &(accounts, slots) in &[(1_000usize, 8usize), (5_000, 16), (10_000, 16)] { let mut cache = populated_cache_layer2(&rt, accounts, slots); // Warm the base. - black_box(cache.create_snapshot()); + black_box(cache.snapshot()); // A handful of layer-1 writes (does not dirty the memoized base). let target = addr(0); group.throughput(criterion::Throughput::Elements((accounts * slots) as u64)); @@ -137,7 +137,7 @@ fn bench_resnapshot_hot_loop(c: &mut Criterion) { cache .db_mut() .insert_account_info(target, AccountInfo::default()); - black_box(cache.create_snapshot()); + black_box(cache.snapshot()); }) }, ); @@ -170,7 +170,7 @@ fn bench_overlay_fanout(c: &mut Criterion) { .insert_mapping_storage_slot(token, U256::from(BALANCE_SLOT), owner, U256::from(1_000u64)) .unwrap(); - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let calldata = Bytes::from(MockERC20::balanceOfCall { account: owner }.abi_encode()); let mut group = c.benchmark_group("overlay_fanout"); @@ -331,7 +331,7 @@ fn bench_inject_storage_batch(c: &mut Criterion) { criterion_group!( benches, - bench_create_snapshot, + bench_snapshot, bench_resnapshot_hot_loop, bench_overlay_fanout, bench_cache_call_raw, diff --git a/docs/INTERNALS.md b/docs/INTERNALS.md index 661b1a2..2baddec 100644 --- a/docs/INTERNALS.md +++ b/docs/INTERNALS.md @@ -4,7 +4,7 @@ > consumer cares about (fetch minimization, candidate throughput, reactive sync, > optimistic latency) see the **Performance** section of the [README](../README.md). -`create_snapshot()` is the operation a search loop runs once per block before +`snapshot()` is the operation a search loop runs once per block before fanning candidates out. Phase 5 replaced its original O(total state) deep clone with a two-tier copy-on-write view: the cold `BlockchainDb` index (layer 2) is flattened once into an immutable, `Arc`-shared base (per-account storage shared by @@ -13,15 +13,15 @@ addresses; each snapshot then folds just the hot CacheDB delta (layer 1). Reads stay O(1) and bit-for-bit identical to the deep clone (pinned by the differential gate in `tests/cow_snapshot.rs`). -The retained `create_snapshot_deep_clone()` is kept as (a) the read-equivalence +The retained `snapshot_deep_clone()` is kept as (a) the read-equivalence reference and (b) the A/B regression baseline, so a future change can't silently -regress `create_snapshot` back toward O(total state). The `create_snapshot` group +regress `snapshot` back toward O(total state). The `snapshot` group in `benches/simulation.rs` measures both. Indicative A/B (Apple M1 Pro, `aarch64-apple-darwin`, Criterion medians; offline, state injected directly — read the ratio, not the absolute): -| Cache size (accounts × slots) | Deep clone | COW `create_snapshot` | Ratio | +| Cache size (accounts × slots) | Deep clone | COW `snapshot` | Ratio | |:-----------------------------:|:----------:|:---------------------:|:-----:| | 100 × 8 | 53 µs | 2.1 µs | ~25× | | 1,000 × 8 | 791 µs | 19 µs | ~41× | @@ -42,5 +42,5 @@ still O(total state). The decisions and cost model are summarized in [`ROADMAP.md`](ROADMAP.md), and the residual is recorded in [`KNOWN_ISSUES.md`](KNOWN_ISSUES.md). -Reproduce: `cargo bench --bench simulation` (the `create_snapshot` and +Reproduce: `cargo bench --bench simulation` (the `snapshot` and `resnapshot_hot_loop` groups). diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md index 6036506..580f2c9 100644 --- a/docs/KNOWN_ISSUES.md +++ b/docs/KNOWN_ISSUES.md @@ -35,8 +35,8 @@ Confidence legend: **[V]** verified against the source during review; 4. **[FIXED] Explicit persistence failures are observable.** `cache::save_binary_state`, `PrefetchRegistry::save`, and - `EvmCache::flush` now return `anyhow::Result<()>`. `Drop` remains best-effort - and logs flush errors. + `EvmCache::flush` now return typed errors (`PersistenceError` / + `CacheError`). `Drop` remains best-effort and logs flush errors. 5. **[FIXED] Access-list profitability uses exact EIP-2930 RLP bytes.** Arbitrum profitability now centralizes data-gas accounting in @@ -86,38 +86,48 @@ surface was moved out of this crate. ## API ergonomics -1. **[R] `snapshot()` vs `create_snapshot()`.** `snapshot()` returns a low-level - `revm::database::Cache` for in-place `restore()`; `create_snapshot()` returns - an `Arc` for cross-thread fan-out. The names don't convey the - difference. Docs now cross-reference them (see the rustdoc), but a rename - could be considered pre-1.0. +1. **[R][Closed in 0.2.0] Snapshot API names.** The low-level + `revm::database::Cache` restore point is now `checkpoint()`; the + cross-thread `Arc` fan-out view is now `snapshot()`. The old + `snapshot()` / `create_snapshot()` public names were removed rather than + kept as aliases. -2. **[R] Process-global cache speed mode.** `set_cache_speed_mode` / - `cache_speed_mode` are a process-wide `static`, so two caches in one process - cannot tune concurrency independently. Phase 1 moved configuration toward - per-instance (`EvmCacheBuilder::cache_config`); the global setter remains. +2. **[R][Closed in 0.2.0] Per-instance storage batch tuning.** + `StorageBatchConfig` exposes exact `slots_per_batch` and + `max_concurrent_batches` knobs for one cache's provider-backed storage + fetcher. `CacheSpeedMode` remains as preset shorthand via + `From` / `EvmCacheBuilder::speed_mode`. The old + process-global setter/getter/static are removed. -3. **[V] `SpeculativeSim` consumption contract.** Both `validate()` and - `into_optimistic()` take `self` by value, so double-consumption is unreachable - under normal ownership. Internally `validate()` uses `.expect("validation - handle taken twice")` (defensive) while `into_optimistic()` no-ops if the - handle was already taken; this is now documented with a `# Panics` note on - `validate`. A `Result`-returning variant could remove the residual foot-gun. +3. **[V][Closed in 0.2.0] `SpeculativeSim` consumption contract.** + `SpeculativeSim::validate()` now returns `Result`: validator + uncertainty remains `Validation::Unverified`, while a failed background task + surfaces as an error instead of panicking on a defensive `.expect(...)`. ## Limitations by design / roadmap -- **Freshness reconciles storage slots only, not account-level state.** The - optimistic verify-and-rerun loop (`FreshnessController::run`) builds its verify - set exclusively from the volatile storage *slots* in each sim's read set; it - never re-fetches or diffs an account's native balance, nonce, or bytecode. - Consequently `Validation::Confirmed` guarantees only that *no volatile storage - slot the sims read had changed* — a sim whose result depends on a - `BALANCE`/`SELFBALANCE` (or nonce/code) that moved on-chain without a - co-changing storage slot can still be reported `Confirmed`. If account-level - state matters to a sim, classify the account `Pinned` and keep it fresh via - event-driven writes (`apply_update`/`apply_updates`), or reconcile it out of - band. The verdict types and the `freshness` module docs state this scope; a - future revision may extend reconcile to touched accounts. +- **Storage-only freshness verification is the default; account-level + verification is opt-in.** The optimistic verify-and-rerun loop builds its + verify set from the volatile storage *slots* in each sim's read set, and the + default verdict says exactly that: `Validation::ConfirmedStorage` guarantees + only that no volatile storage slot the sims read had changed — a sim whose + result depends on a `BALANCE`/`SELFBALANCE` (or nonce/code) that moved + on-chain without a co-changing storage slot can still be + `ConfirmedStorage`. Since 0.2.0, opting accounts into account-field + verification (`TrackingPolicy::WholeAccount`/`Scalars` + the `eth_getProof` + fetcher seam) upgrades the verdict to `ConfirmedFull` (storage **and** + verified account fields). The residual limitation is that account + verification remains per-tracked-account opt-in rather than derived + automatically from each sim's `BALANCE`/`CODE` read set. +- **`Verified` code seeds are never re-verified (EIP-6780 assumption).** A + canonical code seed confirmed once against on-chain `EXTCODEHASH` is marked + `CodeSeedState::Verified` durably, including across restarts. Post-Cancun + (EIP-6780) deployed code is immutable — `SELFDESTRUCT` only works in the + creating transaction — so one confirmation is sound on mainnet and current + major L2s. On a chain without 6780, a `SELFDESTRUCT`-then-redeploy can + change code under a `Verified` mark; the escape hatch is + `EvmCache::purge_account` (clears the mark; the next touch refetches + authoritative chain state) before re-seeding. - **Solidity `Panic(uint256)` codes above `u64::MAX` decode as `Unknown`.** `decode_solidity_panic` drops out-of-range codes rather than exposing a lossy `u64`. Real compiler-emitted panic codes are single-byte constants, so this is @@ -131,17 +141,25 @@ surface was moved out of this crate. token emitting such a value would corrupt the reconstructed delta. Real ERC-20 supplies are far below 2^255, so this is unreachable for honest tokens; a malicious token can misreport balances by other means regardless. -- **`BLOCKHASH` resolves to ZERO in ext-db-less overlays.** Snapshots do not track - block hashes (`block_hashes` is always empty — the live cache does not track - them either), so an `EvmOverlay` built without an `ext_db` (e.g. the freshness - validator's internal overlays) returns `B256::ZERO` for the `BLOCKHASH` opcode. - A simulation of a contract that reads `BLOCKHASH` through such an overlay sees - ZERO rather than the real historical hash. -- **`EventPipeline::derived_slots` grows unbounded without reorgs.** Every - event-derived `(address, slot)` is retained for sampled reconciliation and is - only pruned by `reorg_to`. In long-running steady-state ingestion (no reorgs) - the set grows monotonically (~52 bytes/slot). The field doc says "seen so far"; - a future revision may bound it to the reorg ring's horizon. +- **[Hardened in 0.2.0] `BLOCKHASH` resolves to ZERO in ext-db-less overlays — + and the freshness validator now fails closed on it.** Snapshots do not track + block hashes (the live cache does not track them either), so an `EvmOverlay` + built without an `ext_db` returns `B256::ZERO` for in-lookback-range + `BLOCKHASH` reads. Since 0.2.0 the freshness pipeline records such reads + (`EvmOverlay::blockhash_zero_fallback`) and reports the batch + `Validation::Unverified` — on the optimistic pass **and** on corrected + re-runs — instead of silently confirming a result whose control flow may + depend on the real hash. Out-of-range reads return the spec-mandated ZERO + without a database call and are deliberately not flagged (they are correct + on-chain too). Direct, non-validator simulations over ext-db-less overlays + still observe ZERO; supply an `ext_db` or snapshot-provided hashes when + `BLOCKHASH` accuracy matters to such a sim. +- **[Closed in 0.2.0] `EventPipeline::derived_slots` is bounded.** The + event-derived `(address, slot)` set is now a block-horizon ring bounded to + `ReorgConfig::depth` (mirroring the `touched` ring), so steady-state + ingestion no longer grows it monotonically. Slots aged past the horizon + cannot be reorg-invalidated anyway, so eviction loses nothing actionable; + sampled reconciliation now draws from the resident window. - **Reorgs deeper than `ReorgConfig::depth` cannot fully purge.** The touched- address ring is bounded to `depth` blocks; `reorg_to(n)` only purges addresses still in the ring, so state touched solely in blocks that have already aged out @@ -167,7 +185,7 @@ surface was moved out of this crate. reverted txs' gas yourself (it is in `per_tx[i].gas_used`). Only relevant under `AllowReverts` with a tx that actually reverts; the `Atomic` path is unaffected. A future revision may surface reverted-tx gas cost directly. -- **Copy-on-write snapshots (Phase 5, Pillar A) — done.** `create_snapshot()` is +- **Copy-on-write snapshots (Phase 5, Pillar A) — done.** `snapshot()` is no longer an O(total state) deep clone. The cold `BlockchainDb` index (layer 2) is flattened once into an internal, immutable, `Arc`-shared base (per-account storage shared by `Arc`), memoized across snapshots and rebuilt copy-on-write @@ -179,10 +197,10 @@ surface was moved out of this crate. since `foundry-fork-db` cannot be hooked) plus an **O(layer-1) fold** of the hot delta — so the cost tracks `accounts + changed state`, not total slots. A full rebuild (first snapshot, or after `set_block`/re-pin) is still O(total - state). `create_snapshot` is now `&mut self` (it memoizes the base, Decision - D5). The retained `create_snapshot_deep_clone()` (the legacy full flatten) is + state). `snapshot` is `&mut self` (it memoizes the base, Decision D5). The + retained `snapshot_deep_clone()` (the legacy full flatten) is kept as the A/B benchmark baseline and the read-equivalence reference; the - `create_snapshot` group in `benches/simulation.rs` measures both. Decisions and + `snapshot` group in `benches/simulation.rs` measures both. Decisions and the cost model are summarized in `ROADMAP.md` and `docs/INTERNALS.md`. - **Layer-2 unchecked accessors remain an explicit contract boundary (Phase 5).** The snapshot base's growth scan is count/absence-based, which is sufficient for the @@ -221,20 +239,30 @@ surface was moved out of this crate. (WebSocket `subscribe_logs`/`subscribe_blocks`/`subscribe_pending_transactions` with exponential-backoff reconnect and `get_logs` backfill) all ship. Honest remaining transport limits: **full block bodies**, **full pending-transaction - hydration**, and **arbitrary historical backfill** are not implemented (the + hydration**, and non-log historical backfill are not implemented (the subscriber returns a typed `SubscriberError::Unsupported` for non-hash pending - interests); reconnect backfills inclusively from the last-seen block and relies - on the bounded `dedupe_window` to suppress overlap, so a reconnect after more - than `dedupe_window` matching logs can re-emit already-processed logs as - `Backfill` records (the runtime's canonical dedup catches most). Account-field - resync is unsupported until a provider-neutral account fetch callback exists. + interests). Mid-lifecycle handler registration through `ReactiveEngine` + backfills a new handler's logs from the runtime's last canonical block + automatically and catches the newly connected stream up from that anchor after + it subscribes, so the discovery→subscription window is closed without caller + bookkeeping; the bounded `dedupe_window` suppresses the overlap. The residual + limit is a genuinely live-only registration (no anchor and no backfill + requested): logs between the registration call and the live subscription start + are not fetched. A reconnect after more than `dedupe_window` matching logs can + re-emit already-processed logs as `Backfill` records (the runtime's canonical + dedup catches most). Account-field resync requires an account proof fetcher; + missing, failed, or omitted fetches surface as typed failures. - **Reactive integration-test coverage is partial.** The reactive runtime, the subscriber, and reorg recovery are well covered individually (reorg rollback is pinned by state-equivalence assertions; reconnect/backfill/dedup by inline unit - tests), but several public paths lack dedicated/integration coverage: composing - `AlloySubscriber` output into `ReactiveRuntime::ingest_batch` end-to-end, the - block-header ingest path, the `ReactiveReport::Decoded` shape, the - `EventDecoderHandler` adapter, and custom pending-tx matcher/route-key routing. - These are tracked follow-ups, not known defects. + tests). Composing `AlloySubscriber` output into `ReactiveRuntime::ingest_batch` + end-to-end is now covered offline in `tests/reactive_subscriber_ingest.rs` (a + real subscriber batch, produced via the mockable `get_logs` backfill path, + drives a real runtime ingest and asserts the cache write). The remaining paths + without dedicated integration coverage are the block-header ingest path, the + `ReactiveReport::Decoded` shape, the `EventDecoderHandler` adapter, and custom + pending-tx matcher/route-key routing; the live WebSocket transport plumbing is + covered by reconnect/termination unit tests but not by a networked end-to-end + test. These are tracked follow-ups, not known defects. - **Recent toolchain.** MSRV 1.88 and edition 2024 are intentional and CI-enforced; consumers on older toolchains are not supported. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 33ded65..d5185b5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,8 +1,10 @@ # evm-fork-cache — engineering roadmap -> Status: living document. Last updated after Phase 6 — the provider-neutral -> reactive runtime, the live `AlloySubscriber` transport, and cold-start — landed -> on top of the Phase 0-5 core. +> Status: living document. Phases 0-8 have landed: through bundle simulation + +> call tracing, plus Phase 8 — storageHash liveness & state invalidation — +> which shipped in **0.2.0** (all six steps of +> [`phase-8-liveness-spec.md`](phase-8-liveness-spec.md), including the +> cold-start root baseline and the Tier-3 trace-backed resync source). ## Vision @@ -20,11 +22,15 @@ Today the crate implements all three pillars end-to-end: copy-on-write snapshots and overlays, the freshness control plane, targeted state-update writers, the event-to-state reader pipeline, and — as of Phase 6 — a provider-neutral reactive runtime with journaled reorg recovery, a live WebSocket `AlloySubscriber` -transport, declarative cold-start warming, and — as of Phase 7 — multi-transaction -bundle simulation with coinbase accounting and a call-frame tracer. The remaining -work toward 1.0 is breadth on those primitives (`Create`-kind bundle txs, opcode +transport, declarative cold-start warming; as of Phase 7 — multi-transaction +bundle simulation with coinbase accounting and a call-frame tracer; and — as of +Phase 8 — liveness as a first-class engine responsibility via `storageHash`-based +state invalidation (the root gate, tracking policies, and the trace-backed resync +source), plus mid-lifecycle adapter register/unregister. The remaining work +toward 1.0 is breadth on those primitives (`Create`-kind bundle txs, opcode tracing) and transport depth (full block bodies, full pending-tx hydration, -arbitrary historical backfill), not the core control plane. +non-log historical backfill; log interests can request owner-scoped `get_logs` +backfill from a block anchor). ## Target architecture @@ -34,7 +40,7 @@ Four layers, bottom to top: Parallel overlays ×N (isolated Send simulations) ▲ clone ×N (cheap) Snapshot · Arc · COW (rapidly clonable, point-in-time) - ▲ create_snapshot + ▲ snapshot Fork DB (foundry-fork-db) (lazy RPC fetch + local state cache) ▲ lazy fetch ▲ targeted writes / purge (no RPC) RPC node Event-driven sync ← WS logs · new block @@ -50,7 +56,7 @@ RPC node Event-driven sync ← WS logs · new block ### The three pillars -- **Pillar A — COW snapshots.** Replace the deep-clone `create_snapshot` with +- **Pillar A — COW snapshots.** Replace the deep-clone `snapshot` with structurally-shared, copy-on-write state so cloning is O(changed), not O(total). This is the performance payoff for parallel fan-out. - **Pillar B — Event → state pipeline.** Decode protocol events into targeted @@ -86,6 +92,7 @@ RPC node Event-driven sync ← WS logs · new block | **5** | COW snapshots (Pillar A): structural sharing; overlay buffer reuse. | **Done** (`phase-5-cow-snapshots`) | | **6** | Reactive runtime + live transport: provider-neutral `ReactiveRuntime` / `ReactiveHandler`, journaled depth-bounded reorg recovery, the WebSocket `AlloySubscriber`, and declarative `cold_start` warming. | **Done** (`cold-start-sync`) | | **7** | Bundle simulation + call tracing: `EvmOverlay::simulate_bundle` (ordered cumulative-state txs, `RevertPolicy`, coinbase-payment accounting) and a `CallTracer` (call-frame tree) + composable `InspectorStack`. | **Done** (`phase-6-bundle-sim`) | +| **8** | storageHash liveness & state invalidation (Pillar C): account/root fetcher seam (`eth_getProof`); per-block root gate + complement resync (`ResyncReason::RootMoved`) + coverage alarm; per-contract `TrackingPolicy` (`Slots`/`WholeAccount`/`Scalars`); event-write `Validity` stamping; `advance_block` block-env refresh; cold-start root baseline (`roots.bin`); Tier-3 trace-backed resync. | **Done in 0.2.0** ([spec](phase-8-liveness-spec.md)) | Cross-cutting remaining work: `Create`-kind / state-override bundles, opcode-level tracing, and a full no-provider build split. @@ -104,11 +111,11 @@ a 1.0. - **Change:** derive the simulation error with `thiserror`; add a first-class `Halt { reason, gas_used }` variant instead of folding halts into the generic - `Other(anyhow::Error)`. Keep `SimulationError` (the decoded revert) as the - `Revert` payload. + host failure arm. Keep `SimulationError` (the decoded revert) as the + `Revert` payload, and route host failures through `SimHostError`. - **Files:** `src/errors.rs` (+ `thiserror` dep), call sites in `src/cache/mod.rs` / `src/cache/overlay.rs`. -- **API:** `enum SimError { Revert(Box), Halt { .. }, Host(anyhow::Error) }`; +- **API:** `enum SimError { Revert(Box), Halt { .. }, Other(SimHostError) }`; `type SimulationResult = Result`. `SimulationErrorKind` retained as a deprecated alias. - **Done when:** halts surface typed; `cargo test` + clippy green. @@ -132,7 +139,7 @@ a 1.0. ### 1c — Hot-path benchmarks -- **Change:** add offline criterion benches for the real hot paths — `create_snapshot` +- **Change:** add offline criterion benches for the real hot paths — `snapshot` across cache sizes (N accounts × M slots), an M-way overlay fan-out (clone + simulate), and `inject_storage_batch`. Build the cache once inside a runtime; benchmark the sync hot paths. @@ -142,14 +149,14 @@ a 1.0. ### 1d — Builder - **Change:** add `EvmCacheBuilder` (fluent: block, spec, cache config, - shared-memory capacity) as the preferred constructor over positional + shared-memory capacity, storage batch config) as the preferred constructor over positional `with_cache` / `from_backend`. - **Files:** `src/cache/mod.rs` (+ a `builder` submodule). - **API:** `EvmCache::builder(provider).block(..).spec(..).build().await`. Existing constructors retained (possibly deprecated). -- **Done:** builder constructs an equivalent cache. The legacy process-global - speed-mode setter remains as accepted API ergonomics debt (tracked in - `docs/KNOWN_ISSUES.md`). +- **Done:** builder constructs an equivalent cache. Storage batch tuning is now + per cache via `EvmCacheBuilder::storage_batch_config`, with + `EvmCacheBuilder::speed_mode` as preset shorthand. ### 1e — Protocol adapter extraction @@ -240,22 +247,27 @@ pub struct FreshnessController { /* regis `run` returns a `SpeculativeSim { optimistic, validation }` **as soon as the optimistic sims finish** — it does *not* await RPC. The caller computes against -`optimistic()` immediately and `validate().await`s the verdict when ready. +`optimistic()` immediately and `validate().await?`s the verdict when ready. ```rust pub struct SpeculativeSim { /* optimistic results + JoinHandle */ } impl SpeculativeSim { pub fn optimistic(&self) -> &[SimOutcome]; - pub async fn validate(self) -> Validation; + pub async fn validate(self) -> Result; } pub enum Validation { - Confirmed, - Corrected { results: Vec, changed: Vec }, + ConfirmedStorage, + ConfirmedFull, + Corrected { + results: Vec, + changed_slots: Vec, + changed_accounts: Vec, + }, Unverified { reason: String }, } ``` -Main thread (`run`): drain pending corrections into the cache → `create_snapshot()` +Main thread (`run`): drain pending corrections into the cache → `snapshot()` → run optimistic sims (capturing read-sets) → **spawn** the validator with `Send` data only (`Arc`, the `Arc` `StorageBatchFetchFn`, requests, read-sets) → return `SpeculativeSim`. @@ -407,7 +419,7 @@ is recorded in `KNOWN_ISSUES.md`. ## Phase 5 — copy-on-write snapshots (detailed, decisions locked) -Builds **Pillar A**: replace the O(total state) deep-clone `create_snapshot` with +Builds **Pillar A**: replace the O(total state) deep-clone `snapshot` with a two-tier copy-on-write view whose cost tracks *changed* state, not *total* state. The cold `BlockchainDb` index (layer 2) is flattened once into an internal, immutable, `Arc`-shared base — both the base as a whole and each @@ -425,10 +437,10 @@ below. 2. **Base memoized as immutable; over-invalidation is acceptable, silent staleness is not** (D2). The write-through funnel marks an address dirty unconditionally; the differential-equivalence test is the hard backstop. -3. **Keep the deep clone** as `create_snapshot_deep_clone` (D3) — the A/B +3. **Keep the deep clone** as `snapshot_deep_clone` (D3) — the A/B benchmark baseline and the read-equivalence reference. 4. **Overlay reuse: buffer reuse *and* `reset()` recycle** (D4) — both in scope. -5. **`create_snapshot` becomes `&mut self`** (D5) — the memoization cost; the +5. **`snapshot` becomes `&mut self`** (D5) — the memoization cost; the freshness controller and all callers are updated. ### Acceptance — met @@ -441,8 +453,8 @@ snapshot/overlay/freshness tests pass unchanged. Landed on `phase-5-cow-snapshots`: the memoized two-tier base (`BaseState` + the rewritten two-tier `EvmSnapshot` with `account_info`/`storage_value`/`code` accessors, `src/cache/snapshot.rs`); `EvmCache::refresh_base`/`build_base_full`, -the COW `create_snapshot` (now `&mut self`), the retained -`create_snapshot_deep_clone`, and the `mark_base_dirty`/`invalidate_base` +the COW `snapshot` (now `&mut self`), the retained +`snapshot_deep_clone`, and the `mark_base_dirty`/`invalidate_base` invalidation wired into `write_slot_through`/`apply_slot_run`/ `write_account_info_through`/`inject_storage_batch`/the `purge_*` paths and `set_block` (`src/cache/mod.rs`); `EvmOverlay::reset` plus the reusable @@ -501,10 +513,11 @@ WebSocket transport, and a declarative cold-start warmer. Landed on the backstop. `journal_depth` must exceed the deepest reorg you intend to recover from. - The subscriber surfaces pending-transaction **hashes** only (no full pending-tx - hydration), no full block bodies, and backfill is anchored at the last-seen - block (no arbitrary historical backfill). Account-field resync targets return a - typed `Unsupported` error. Hook dispatch is synchronous on the ingest thread; - `hook_backpressure` is reserved for a future async dispatcher. + hydration), no full block bodies, and non-log historical backfill is not + implemented. Log interests can request owner-scoped `get_logs` backfill from a + block anchor; reconnect backfill is still anchored at the last-seen block. Hook + dispatch is synchronous on the ingest thread; `hook_backpressure` is reserved + for a future async dispatcher. ### Acceptance — met @@ -517,6 +530,80 @@ WebSocket transport, and a declarative cold-start warmer. Landed on --- +## Phase 8 — storageHash liveness & state invalidation (shipped in 0.2.0) + +Deepens **Pillar C** by making liveness a first-class engine responsibility +instead of a consumer chore. Today freshness can only re-check slots a sim +actually read or the caller explicitly sampled, and the event pipeline only keeps +state fresh for protocols with a decoder — so state changed by any uncovered path +(proxy `sstore`, admin writes, an undecoded token, a `SELFDESTRUCT`/`CREATE2` +redeploy) goes silently stale. An account's storage-trie root (`storageHash`, from +`eth_getProof`) is a commitment over *all* its storage, so `root_unchanged ⟹ +provably nothing under the account changed` — a sound, per-account change oracle +with zero false negatives, obtainable from any standard RPC in one call, with **no +local trie and no proof verification**. This phase uses it to detect and repair +the staleness the current footprint-bounded model cannot see. + +Full design, type sketches, tests, and the cold-start correctness argument live in +[`phase-8-liveness-spec.md`](phase-8-liveness-spec.md). The build set (in order): + +1. **Account/root fetcher seam** on `EvmCache` (`AccountProofFetchFn` over + `eth_getProof`, mirroring `StorageBatchFetchFn`). The linchpin — it also + resolves the tracked `ResyncTarget::Account` `Unsupported` gap + (`reactive/mod.rs`) and the account-field freshness gap. +2. **`advance_block(header)`** — engine-driven block-env refresh from the + canonical header stream (the runtime does not refresh scalars per block today). +3. **`Validity` stamping** of reactive/event-derived writes — the first (minimal, + intentional) coupling of the reactive runtime to `FreshnessRegistry` + (`valid_through_slot(N)` on touched slots, aged by `on_new_block`). +4. **The centerpiece:** per-contract `TrackingPolicy` (`Slots` / `WholeAccount` / + `Scalars`) + a per-block **root gate** that, on a root move no decoder + explained, emits a `ResyncRequest` (new `ResyncReason::RootMoved`) through the + runtime's existing resync channel and raises a `CoverageGap` report. +5. **Cold-start root baseline** (`roots.bin`) + restart drift diff in the + `ColdStartPlanner` — "if the observed root matches the persisted baseline, + we're already synced; skip re-reading." +6. A Tier-3 state-diff trace source (`debug_traceBlock` / + `trace_replayBlockTransactions`) that resolves matching resync targets with + one block-level diff where available, then falls back to portable point reads + for unresolved targets. + [`trace-resync-benchmarks.md`](trace-resync-benchmarks.md) records the live + RPC measurements behind the policy: trace wins CU economics quickly, gzip + materially helps large trace payloads, and latency-sensitive small slot sets + should continue to use batched point reads. + +### Locked decisions + +1. **No local storage trie; no cryptographic proof verification.** The root is + *observed* via `eth_getProof`, never reconstructed locally, and never verified + against a `stateRoot` fetched from the same (already-trusted) RPC — that would + be circular. `alloy-trie` stays out. No new production dependency. +2. **`storageHash` gates `WholeAccount` only**, never `Slots` (a sparse-interest + contract's root churns every block → noisy, wasteful). The policy is a pure + cost knob: a false-positive resync is never *incorrect*, only redundant. +3. **Compose, don't merge** — a `TrackingRegistry` sits beside + `FreshnessRegistry`; the freshness API stays verbatim. +4. **Reuse the existing resync channel** — the root gate emits `ResyncRequest`s; + it does not build a parallel repair loop. +5. **Cold-start gate is currency, not completeness** — the across-time root diff + proves tracked slots are current, not that the cache is complete; a + `FullMirror` completeness mode is explicitly deferred. + +### Closes (on landing, convert in `KNOWN_ISSUES.md`) + +Account-field resync `Unsupported` (`reactive/mod.rs`), "freshness reconciles +storage slots only" (the balance/nonce/code gap), and the account-field-resync +transport-depth item below. + +### Acceptance (target) + +Branch `phase-8-liveness`. Green bar at every commit: `cargo fmt --all --check`, +`cargo clippy --all-targets --all-features -- -D warnings`, `cargo test`, +`RUSTDOCFLAGS=-D warnings cargo doc --no-deps`, `cargo bench --no-run`. Offline +acceptance contract in the spec (`tests/liveness_*`). + +--- + ## Remaining work toward 1.0 1. **Bundle-simulation breadth (Phase 7 — core shipped).** `EvmOverlay::simulate_bundle` @@ -531,12 +618,13 @@ WebSocket transport, and a declarative cold-start warmer. Landed on subscriptions, exponential-backoff reconnect, `get_logs` backfill, and journaled parent-hash reorg recovery. The remaining transport gaps are full block bodies, full pending-transaction hydration (today only pending-tx - hashes), arbitrary historical backfill beyond the last-seen-block anchor, and - account-field (balance/nonce/code) resync — all tracked in - `docs/KNOWN_ISSUES.md`. -3. **Snapshot consistency point in continuous ingestion.** Applications that run - a live event loop should snapshot at block boundaries or behind their own - generation guard so simulations do not observe a partially applied block. + hashes), and non-log historical backfill. Log interests can request + owner-scoped `get_logs` backfill from a block anchor. Remaining gaps are + tracked in `docs/KNOWN_ISSUES.md`. +3. **Snapshot consistency point in continuous ingestion.** Closed in 0.2.0: + `EvmCache::snapshot_generation()` is the crate-provided generation guard — + read it around `snapshot()` and re-snapshot when it moved, so simulations + never observe a partially applied block (G6). 4. **Full no-provider build split.** The dependency graph still includes provider/RPC crates. A later `rpc` feature can make those optional for pure offline users. diff --git a/docs/bulk-storage-extraction.md b/docs/bulk-storage-extraction.md new file mode 100644 index 0000000..5ca334e --- /dev/null +++ b/docs/bulk-storage-extraction.md @@ -0,0 +1,331 @@ +# Bulk storage extraction (`eth_call` state overrides) + +Captured on July 2, 2026 against an Alchemy Ethereum-mainnet HTTPS endpoint +(pinned block `25445270`, 3 samples per measurement, gzip-enabled transport). +This note records the design, the measured before/after behavior, and every +limitation hit while building the `bulk_storage` module — **the crate's +default storage loader since 0.2.0**. + +> **Re-verified live twice more** (July 4 block `25459219`, July 5 block +> `25466494`, same harness: `RPC_URL=… cargo run --release --example +> bulk_storage_bench`). The **CU costs and ratios below reproduced exactly all +> three times** — they are deterministic (a fixed price per call), so they are +> the durable claim. The wall-clock latencies here are the fastest of the three +> captures and are a **conservative floor**: the re-verify sessions ran on +> constrained networks and measured the same loads *slower* (up to several×), +> never with different CU, so well-provisioned connectivity should meet or beat +> these millisecond figures. Read the CU column and the ratios, not the absolute +> latencies. One chain-state-dependent figure moved with pool activity: both +> re-verify blocks held **7,654** initialized-tick slots (**153,080 CU as point +> reads → 2,944× cheaper**) versus 7,674 / 2,952× here — the count drifts as the +> pool trades; see that section. The verified-code-seeding table below uses the +> July-4 capture. + +The core technique is Dedaub's "bulk storage extraction", used here with full +credit: + +- blog: +- reference implementation: + +## Mechanism in one paragraph + +`eth_call` accepts a *state-override set* that can replace the **code** at any +address while leaving its **storage** intact. The fetcher overrides each target +contract with Dedaub's 23-byte extractor +(`0x5f5b80361460135780355481526020016001565b365ff3`), which treats calldata as +a raw array of 32-byte slot keys, `SLOAD`s each one, and returns the packed +values — no selector, no ABI, ~2,664 gas per slot. For multi-contract batches +the canonical Multicall3 runtime is *also* injected (at +`0xcA11bde05977b3631167028862bE2a173976CA11`) and one `aggregate3` call fans +out to every overridden target, so the dispatcher exists on every chain and at +every historical block. See the `bulk_storage` module docs for the annotated +disassembly and the full API. + +## RPC economics (Alchemy CU table) + +| Method | CU | Covers | +| --- | ---: | --- | +| `eth_getStorageAt` | 20 | one slot | +| `eth_call` | 26 | up to `max_slots_per_call` slots (default 10,000) | +| `eth_callMany` | 20 | **all** chunks of a batch, in one request | +| `debug_traceBlockByNumber` | 40 | one block's *changed* slots | +| `eth_simulateV1` | 40 | works with overrides too, but twice `eth_callMany`'s price — not used | + +Break-even against point reads is **two slots**. The bulk call is also cheaper +than the Tier-3 trace path (see +[`trace-resync-benchmarks.md`](trace-resync-benchmarks.md)) whenever the slots +are *known* rather than "whatever changed in this block" — the two are +complementary: traces repair event-driven drift, bulk calls load known working +sets. + +Alchemy prices `eth_call`/`eth_callMany` flat, regardless of gas used. +Providers that meter by gas or execution time will price this differently — +re-check per provider. + +## Default-on integration + +Since 0.2.0 every provider-backed `EvmCache` installs the bulk fetcher as its +default `StorageBatchFetchFn`, wrapping the classic point-read fetcher as +fallback. Every batch consumer — freshness verification, cold-start +verify/probe, reactive resync point reads, `prefetch_registry`, and the new +`EvmCache::prewarm_slots` — flows through it automatically. Degradation rules: + +- requests below `BulkCallConfig::point_read_threshold` (default 2) go + straight to point reads (20 CU < 26 CU for a single slot); +- any slot the bulk path fails is repaired through the point-read fallback; +- two *consecutive* fully-failed batches with provider-level errors (the + signature of an endpoint without state-override support) **latch** the + fetcher to the fallback for its lifetime, so such providers pay at most two + wasted calls total; +- EVM-lazy single-slot misses during simulation (the `SharedBackend` path) + are unchanged — they are inherently serial single reads. + +Opt-out and tuning via the builder: + +```rust +// Tune the bulk path (chunk size, concurrency, dispatch mode): +EvmCache::builder(provider.clone()) + .bulk_call_config(BulkCallConfig { max_slots_per_call: 15_000, ..Default::default() }) + .build().await; + +// Or restore the classic pre-0.2.0 point-read behavior: +EvmCache::builder(provider) + .storage_fetch_strategy(StorageFetchStrategy::PointRead) + .build().await; +``` + +Async callers (e.g. cold-starting an AMM pool) can use the core directly: +`bulk_storage::fetch_slots_bulk(&provider, requests, block, config).await`, +then inject via `EvmCache::inject_storage_batch` — or in one step, +`EvmCache::prewarm_slots(&requests)`. + +**Gzip:** enable the `gzip` feature on `reqwest` and build the provider over a +`reqwest::Client::builder().gzip(true)` client (see +`examples/bulk_storage_bench.rs::make_provider`). + +## Measured results + +All numbers from `examples/bulk_storage_bench.rs` (medians of 3, correctness +spot-checked against `eth_getStorageAt` ground truth at the same pinned block — +16/16 slots identical, plus per-scenario spot checks). The point-read baseline +is the classic fetcher at its default `Slow` preset (75 slots per JSON-RPC +batch, 4 in flight). Latencies vary meaningfully between runs on a hosted +endpoint; CU counts do not. + +### Single-target scaling (WETH, pseudo-random slots) + +| Slots | Bulk calls | Bulk median | Bulk CU | Point median | Point CU | CU ratio | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 10 | 1 | 25 ms | 26 | 27 ms | 200 | 8x | +| 100 | 1 | 27 ms | 26 | 28 ms | 2,000 | 77x | +| 1,000 | 1 | 45 ms | 26 | 39 ms | 20,000 | 769x | +| 5,000 | 1 | 89 ms | 26 | — (skipped) | 100,000 | 3,846x | +| 10,000 | 1 | 148 ms | 26 | — (skipped) | 200,000 | 7,692x | +| 15,000 | 2 | 282 ms | 52 | — (skipped) | 300,000 | 5,769x | + +Latency is comparable-to-better at small sizes and the only game in town at +large ones (a 10k-slot point-read run would need 134 HTTP batches and 200k CU). +The CU column is the headline: cost is *flat* per call. + +### Multi-contract dispatch + +| Workload | Calls | Median | CU | Point-read CU | +| --- | ---: | ---: | ---: | ---: | +| 20 real tokens × 25 slots (500) | 1 | 35 ms | 26 | 10,000 | +| 100 contracts × 30 slots (3,000) | 1 | 77 ms | 26 | 60,000 | + +The 100-contract fleet uses synthetic (empty) addresses — dispatch cost is +identical whatever the slots hold; the 20-token row carries real nonzero data. + +### Uniswap V3 USDC/WETH 0.05% — full tick-range load + +The `evm-amm-state` cold-start shape, at block 25445270: + +| Phase | Slots | Calls | Median | +| --- | ---: | ---: | ---: | +| 1. statics (slot0..8) + all 694 tickBitmap words | 703 | 1 | 33 ms | +| 2. 1,562 initialized ticks × 4 slots + 723 observations | 6,971 | 1 | 183 ms | +| **Total** | **7,674** | **2** | **~220 ms** | + +**52 CU versus 153,480 CU as point reads — 2,952× cheaper**, and the whole +pool (every initialized tick over the full range plus the entire observation +ring) is resident after two round trips. Sanity: all 1,562 ticks decoded from +the bitmap had nonzero `liquidityGross`; spot samples matched +`eth_getStorageAt` exactly. + +The exact slot count tracks live pool activity: at the July-4 re-verify block +(`25459219`) the same load was **7,654** slots (1,557 initialized ticks) in 2 +calls — **52 CU versus 153,080 CU, 2,944× cheaper**. The mechanism (whole pool +in two flat-priced calls) is invariant; only the tick population drifts. + +### `eth_callMany` dispatch (`CallDispatch::CallMany`) + +Verified live on Alchemy (Erigon-lineage method; geth proper does not serve +it — the standardized `eth_simulateV1` also works but costs 40 CU): + +| Payload | `eth_call` (per-call) | `eth_callMany` | +| --- | --- | --- | +| 6,971 tick slots (1 chunk) | 129 ms / 26 CU | **107 ms / 20 CU** | +| 25,000 slots (3 chunks) | **292 ms** / 78 CU | 497 ms / **20 CU** | + +The tradeoff is clean: for payloads that fit one chunk, `CallMany` is both +faster and cheaper. For multi-chunk jobs it is ~3.9× cheaper but slower — the +bundled transactions execute *sequentially* server-side, while per-call chunks +run in parallel. Defaults stay `PerCall` (universal support, parallel); +CU-sensitive Alchemy/Erigon deployments should opt into `CallMany`. A provider +without the method transparently re-dispatches that request per-call inside +the same fetch. Hash-pinned blocks always dispatch per-call (`eth_callMany` +takes a number/tag block context). + +### Custom storage programs + +`StorageProgram` / `run_storage_program[s]` inject *caller-supplied* bytecode +through the same override transport — removing the "client must know every +slot key" constraint, because the program derives what to read in-EVM. The +worked example is a 40-byte one-shot Uniswap V3 observation-ring loader: it +reads `observationCardinality` out of `slot0` inside the EVM, then returns the +whole ring — **723 slots in one 48 ms call with zero calldata**, values +verified against slot-list extraction. The same idea extends to a full +tick-walker (bitmap scan + tick loads in one call), the natural next step +inside `evm-amm-state` where the pool layout is owned. + +`run_storage_programs` batches programs with distinct targets into one +Multicall3 dispatch (programs sharing a target run individually — one code +override per address per call). + +### Companion extractors + +| Extractor | One call returns | Measured | +| --- | --- | --- | +| `fetch_account_fields_bulk` | `BALANCE` + `EXTCODEHASH` per address | 20 contracts in 25 ms / 26 CU (vs 800 CU via `eth_getBalance` + `eth_getCode`) | +| `fetch_block_context` | `NUMBER`, `TIMESTAMP`, `BASEFEE`, `COINBASE`, `PREVRANDAO`, `GASLIMIT`, `CHAINID` | one call, 32 ms; `number` matched the pin | + +Caveats measured live: **nonces and storage roots are not EVM-visible** — the +`eth_getProof` path (`AccountProofFetchFn`) remains the tool for those; and +**`BASEFEE` reads 0 through `eth_call`** when no gas price is attached (geth +zeroes it so unfunded calls succeed), so treat the basefee word as unreliable +unless the call carries explicit gas pricing. `EXTCODEHASH` follows EIP-1052: +zero for non-existent accounts, `keccak256("")` for code-less ones. + +### Verified code seeding (cold-start account materialization) + +Since 0.2.0 the account-fields extractor above also powers +`EvmCache::verify_code_seeds`: an adapter that already embeds a contract's +deployed bytecode writes it locally (`seed_account_code`) and one bulk call +settles the **entire pending set** against on-chain `EXTCODEHASH`, +materializing each account's real native balance from the same response. +Nothing is fetched per account and no code bytes travel. Captured July 4, +2026 at block 25459219 (scenario 11, 20 mainnet tokens, medians of 3): + +| Path | Latency | RPCs | CU | Code on the wire | +| --- | ---: | ---: | ---: | ---: | +| `ensure_account` × 20 (balance + nonce + code each) | 1,218 ms | 60 | 1,200 | ~211 KB | +| `seed_account_code` × 20 + one `verify_code_seeds` | **48 ms** | **1** | **26** | **0** | + +**46× cheaper, 25.6× faster** for a 20-contract working set (108,495 bytes of +runtime code that never travels), and the CU gap widens linearly with fleet +size — verification costs ~5.3k gas per address inside one flat-priced call. +The same run exercised the fail-closed path live: a deliberately wrong +template came back `mismatched` (both hashes reported: expected +`0xd0a06b12…`, actual `0xd80d4b7c…`) and was purged for honest refetch, and +an undeployed address was classified `not_deployed` — one call classified +both alongside the valid claims. Caveat shared with the extractor: nonces are +not EVM-visible, so seeded contracts default to nonce 1 +(`seed_account_code_with` overrides) — the right value for any contract that +has never `CREATE`d. + +### Gzip vs identity (tick-range payload, 6,971 slots) + +| Mode | Wire bytes | Wire time | +| --- | ---: | ---: | +| identity | 446,182 | 364 ms | +| gzip | 138,821 | 121 ms | + +A consistent **68.9% payload reduction**. End-to-end medians flip between runs +(99 ms gzip vs 250 ms identity in one run; 151 vs 132 in another) — network +variance dominates at this payload size, matching the trace-benchmarks +finding: treat compression as a guaranteed payload win and an empirical +latency win. Keep it on. + +### Chunk-ceiling probe (single `eth_call`) + +| Slots/call | Est. gas | Result | +| ---: | ---: | --- | +| 15,000 | ~40M | ok, 279 ms | +| 20,000 | ~53M | ok, 336 ms | +| 25,000 | ~67M | ok, 377 ms | +| 30,000 | ~80M | ok, 543 ms | +| 40,000 | ~107M | **HTTP 413 Payload Too Large** | + +Two findings, stable across runs: + +1. **Alchemy's `eth_call` gas allowance exceeds Geth's 50M default** — 30k + slots (~80M estimated gas) executed fine. +2. **The binding constraint on Alchemy is the request body, not gas.** + 40,000 slot keys ≈ 2.56 MB of hex calldata → HTTP 413. Practical per-call + ceiling ≈ 30k slots (~1.9 MB). The defaults + (`max_slots_per_call = 10_000`, `max_slots_per_request = 25_000` for + `CallMany`) stay well inside both this and Geth-default gas caps. + +## Limitations and issues encountered + +1. **Request-payload cap (HTTP 413).** Slot keys travel as incompressible hex + calldata (~64 bytes each in JSON); Alchemy rejects bodies somewhere between + 1.9 MB and 2.6 MB. Request bodies are not compressed (`Accept-Encoding` + negotiation is response-only), so chunking is the only mitigation — handled + by the planner. +2. **Provider support required.** State overrides are Geth-lineage/Reth/Erigon + standard and verified on Alchemy, but not guaranteed everywhere. The + default fetcher repairs failures through point reads and **latches** to + them after two consecutive fully-failed batches. +3. **Precompiles cannot be overridden.** Geth dispatches precompiles + (`0x01..=0x11` post-Pectra) by address before consulting code; the + exact-length response check turns that into a per-slot error (repaired by + the fallback) rather than silent garbage. +4. **`PUSH0` needs Shanghai.** All three shipped extractors use `PUSH0`; the + slot extractor has a `PUSH1 0x00` variant + (`STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI`, `BulkCallConfig::pre_shanghai_extractor`) + for older chains. Every shipped bytecode is executed against revm in the + offline test suite. +5. **Zero vs absent is indistinguishable** — exactly like `eth_getStorageAt`. + No semantic change for the cache. +6. **Dispatcher-address collision.** A request targeting `MULTICALL3_ADDRESS` + itself gets a dedicated call (its extractor override cannot share an + override map with the dispatcher override) — in both dispatch modes. +7. **Flat CU pricing is provider-specific.** Alchemy bills `eth_call` at 26 CU + and `eth_callMany` at 20 CU regardless of gas; gas-metered providers erode + (not erase) the win. The round-trip and latency win is provider-independent. +8. **`eth_callMany` bundles execute sequentially server-side** — cheaper but + slower than parallel per-call chunks for multi-chunk jobs (measured above). +9. **Tiny requests are cheaper as point reads** — handled by + `point_read_threshold` (default 2). +10. **In-EVM account sampling is partial.** No nonce/storage-root opcodes + (use `eth_getProof`); `BASEFEE` reads 0 without explicit gas pricing; + querying the extractor-host address itself reports the overridden code + hash. +11. **CU accounting for planners:** `bulk_storage::planned_call_count` exposes + exactly how many calls a request set will produce. + +## Follow-ups + +- Wire `evm-amm-state` cold start through `fetch_slots_bulk` / + `prewarm_slots` using the two-phase bitmap → ticks pattern (benchmark + scenario 4 is the reference), then collapse it to one call with a custom + tick-walker `StorageProgram` (the observation-ring program is the template). +- Consider `CallDispatch::CallMany` as the configured default in + Alchemy-pinned deployments once per-tx gas behavior has soaked in + production. + +## Reproduction + +```sh +RPC_URL=https://eth-mainnet.g.alchemy.com/v2/ \ + cargo run --release --example bulk_storage_bench +``` + +Knobs: `BULK_BENCH_SAMPLES` (default 3), `BULK_BENCH_BASELINE_MAX` (default +1000 — caps the point-read baseline, which costs 20 CU *per slot*), +`BULK_BENCH_PROBE=0` to skip the ceiling probe, and +`BULK_BENCH_SCENARIOS=4,11` to re-run selected scenarios without paying for +the rest. A full default run costs ~130k CU, dominated by the point-read +baselines. diff --git a/docs/phase-8-liveness-spec.md b/docs/phase-8-liveness-spec.md new file mode 100644 index 0000000..190fb27 --- /dev/null +++ b/docs/phase-8-liveness-spec.md @@ -0,0 +1,355 @@ +# Phase 8 — storageHash liveness & state invalidation (spec) + +> Status: **planned / design distillation.** This document captures a design +> thread on making liveness a first-class engine responsibility rather than a +> consumer chore. It is written against the post-Phase-7 tree (reactive runtime, +> cold-start, bundle sim already landed) and supersedes the informal "P0–P4" +> numbering used while sketching. Read it **with** [`ROADMAP.md`](ROADMAP.md) +> (Pillar B/C) and [`KNOWN_ISSUES.md`](KNOWN_ISSUES.md) (the liveness/staleness +> limitations this closes). + +## Motivation + +The engine is excellent at *simulation* but offloads most *liveness* complexity +to the consumer. Freshness (`src/freshness.rs`) can only re-check slots a sim +actually read or the caller explicitly sampled; the event pipeline only keeps +state fresh for protocols someone wrote a decoder for; and nothing detects state +that changed via a path no decoder covers. The result: **state can go silently +stale outside the active read/decoder footprint, with no signal.** + +An account's storage-trie root (`storageHash`, from `eth_getProof`) is a +collision-resistant commitment over *all* of that account's storage. So +`root_unchanged ⟹ provably nothing under the account changed` — a **sound, +per-account change oracle with zero false negatives**, obtainable from any +standard RPC in one call, with no local trie and no proof verification. That is +the centerpiece of this phase: use it to detect and repair the staleness the +current pull-based, footprint-bounded model cannot see. + +## 0. Ground rules (non-negotiable) + +- Branch: `phase-8-liveness`. Green bar at every commit: + `cargo fmt --all --check`, `cargo clippy --all-targets --all-features -- -D warnings`, + `cargo test`, `RUSTDOCFLAGS=-D warnings cargo doc --no-deps`, `cargo bench --no-run`. +- MSRV 1.88, edition 2024. **No new production dependencies** — `eth_getProof` + is already reachable via the existing `alloy-provider` dep + (`Provider::get_proof(address, keys) -> RpcWithBlock<_, EIP1186AccountProofResponse>`, + `keys: Vec` accepts `vec![]`, block-pinnable via `.number(n)`). +- Feature-gate to match the tree: the runtime/cold-start live behind the + default-on `reactive` feature; `src/freshness.rs` is the generic (ungated) + core. New liveness wiring that touches the runtime is `reactive`-gated; the + probe seam on `EvmCache` and any `Validity` coupling stay in the generic core. +- Unsigned commits, `Co-Authored-By` trailer, no force-push to `main`. + +## 1. Objective & scope + +**In scope** + +1. A provider-neutral **account/root fetcher seam** on `EvmCache` (mirrors the + existing `StorageBatchFetchFn`). This single seam unblocks *three* things: the + storageHash probe, the account-field (`Scalars`) resync, and the tracked-but- + `Unsupported` `ResyncTarget::Account` path (`reactive/mod.rs:2000`). +2. **`StorageHashProbe`** + a per-block **root gate**: probe tracked accounts' + `storageHash` each block, and on a move that no decoder explained, emit a + `ResyncRequest` through the runtime's *existing* resync channel + raise a + coverage alarm. +3. A per-contract **`TrackingPolicy`** (`Slots` / `WholeAccount` / `Scalars`) + that decides *how* an account is kept live. +4. **`Validity` stamping** of event/reactive-derived writes (couple the + reactive apply path to `FreshnessRegistry`), closing the "event writes carry + no freshness stamp" gap. +5. Engine-driven **block-env refresh** (`advance_block`) from the canonical + header stream, so a long-running reactive cache stops simulating against a + stale block env. +6. A **cold-start root baseline** (`roots.bin`) so a process restarting after + downtime can cheaply detect which tracked accounts changed while it was down. + +**Out of scope (documented as follow-ups, not built here)** + +- A local storage MPT / trie maintenance. **Decision locked** (§Decisions): the + root is *observed*, never *reconstructed*. +- Cryptographic proof verification of storage/account proofs against a + `stateRoot`. **Decision locked**: circular trust against the same RPC; no + independent root source exists. `alloy-trie` stays out. +- A trace-only runtime that depends on `debug`/`trace` RPC availability. The core + path can use block state diffs when available, but must degrade to existing + storage/account point reads without changing handler semantics. +- A `FullMirror` completeness-verified policy (would require full-storage + enumeration via `debug_storageRangeAt` + local root reconstruction). Explicitly + deferred; §Decisions explains why. + +## 2. What already shipped — build on, do not rebuild + +The earlier sketch predates Phases 6–7. Re-grounded against the current tree: + +| Sketch item | Status in tree today | +| --- | --- | +| Engine-driven per-block pipeline ("drive") | **Shipped** as `ReactiveRuntime::ingest_batch{,_with_resync}` (`reactive/mod.rs:1341/1362`). | +| Engine-detected parent-hash reorg + recovery | **Shipped, stronger than sketched.** Journaled parent-hash detection + LIFO rollback + conservative purge (`recover_for_canonical_input` 1518, `recover_dropped_journals` 1692), bounded by `ReactiveConfig::journal_depth` (default 64). | +| Slot resync mechanism | **Shipped** — `execute_resync_requests` (1973) batches `ResyncTarget::StorageSlot{,s}` through `cache.storage_batch_fetcher()`. `ResyncTarget::Account` returns `UnsupportedAccountTarget` (2000) — the seam this phase adds. | +| Resync/invalidation *vocabulary* | **Shipped** — `ReactiveEffect::{Resync,Invalidate}`, `ResyncRequest`/`ResyncTarget`/`ResyncReason`/`ResyncPriority` (662–770). This phase *emits into* it, adding a `RootMoved` reason. | +| Declarative cold-start warming | **Shipped** — `EvmCache::run_cold_start` + `ColdStartPlanner` (`cold_start/`). Pure warming; no root baseline, no restart diff. | +| Validity classifier | **Shipped but unwired** — `FreshnessRegistry`/`Validity{Pinned,Volatile,ValidThrough}` (`freshness.rs:163,184`). Nothing auto-stamps it; the runtime never touches it. | +| storageHash / `get_proof` / account root | **Entirely greenfield.** Zero occurrences in `src/`. | + +The consequence: this phase is *smaller* than the original sketch. It adds one +RPC seam, one probe, one policy type, and wires two already-built subsystems +(reactive runtime ↔ freshness) together. It does **not** build a new controller, +a reorg engine, or a drive loop — those exist. + +## 3. The liveness signal hierarchy + +Three tiers, increasing completeness / decreasing availability. They compose; +they do not compete. + +1. **Event decoders (Tier 1 — shipped).** Free (logs already ingested), + value-carrying, but only covers protocols with a decoder. The baseline via + reactive handlers / `EventPipeline`. +2. **`storageHash` probe (Tier 2 — this phase).** `get_proof(addr, []).number(N)` + → the account's storage root. A sound per-account change oracle: *no values, + no slot detail*, but zero false negatives, universal on any RPC at `latest`. + Catches everything Tier 1 misses. +3. **State-diff trace (Tier 3 — trace-first when available, §7).** `debug`/`trace` + per-block diff: decoder-free, exact changed slots + values, one call per + block. Subsumes point storage reads for already-tracked accounts *where the + namespace is enabled*; unresolved cold targets still fall back to the existing + storage/account seams. + +The soundness of Tier 2 is the whole point: today a change on a path no decoder +covers (proxy `sstore`, admin writes, a token without a decoder, +`SELFDESTRUCT`/`CREATE2` redeploy) is invisible until the optimistic validator or +a sampled `reconcile` happens to re-read the slot. A per-block root gate turns +that into an explicit, cheap, complete signal. + +## 4. `TrackingPolicy` — the WETH vs. Uniswap-V2 nuance + +The root gate behaves *oppositely* for two contract shapes, so liveness strategy +must be per-contract: + +```rust +pub enum TrackingPolicy { + /// Sparse interest (e.g. WETH: a few balance slots). The root churns on + /// nearly every block, so it is a noisy gate — do NOT root-gate. Keep the + /// enumerated slots fresh via decoders + cadence reconcile. + Slots { slots: Vec }, + /// Whole economic state (e.g. a V2 pool). root_moved ≈ my_state_changed, + /// so the root is a tight, cheap gate: probe each block; on a move re-read + /// the tracked slot set. + WholeAccount, + /// balance / nonce / code_hash only — resolved from the same get_proof + /// response's account fields; no storage interest. + Scalars, +} +``` + +Per-block engine behavior: + +| State after decode | `WholeAccount` | `Slots` | `Scalars` | +| --- | --- | --- | --- | +| root `!moved` | stamp tracked set `ValidThrough(N)` — **0 reads** | (not root-gated) cadence reconcile | stamp `ValidThrough(N)` | +| root `moved`, addr ∈ touched | **skip** — decoder already applied authoritative values | decoder already applied | diff account fields | +| root `moved`, addr ∉ touched | **resync** tracked slots + **coverage alarm** | n/a | resync scalars + alarm | + +A false-positive resync is never *incorrect* — it costs one batched read — so the +policy is a **pure cost knob**, not a correctness lever. `Slots` opts out of the +noisy gate; `WholeAccount` opts in because for it the gate is tight. + +## 5. Core types & behavior (proposed) + +### 5.1 Account/root fetcher seam (`src/cache/mod.rs`) + +Mirror `StorageBatchFetchFn`. One new callback the builder can install; drives +`eth_getProof` through the same `block_in_place_handle` bridge used by +`ensure_account_blocking` / the storage batch fetcher. + +```rust +/// Fetches account headers + (optionally) storage proofs, one entry per address. +/// keys == &[] ⇒ root-only probe (no storage-proof payload). +pub type AccountProofFetchFn = Arc< + dyn Fn(Vec<(Address, Vec)>, BlockId) + -> Vec<(Address, Result)> + Send + Sync, +>; + +pub struct AccountProof { + pub storage_hash: B256, + pub balance: U256, + pub nonce: u64, + pub code_hash: B256, + pub slots: Vec<(U256, U256)>, // populated only for requested keys +} +``` + +This seam is the linchpin: it also directly resolves +`ResyncTarget::Account` (`UnsupportedAccountTarget`, `reactive/mod.rs:2000`) and +the "freshness reconciles storage slots only" account-field gap tracked in +`KNOWN_ISSUES.md`. + +### 5.2 `StorageHashProbe` + per-block root gate (`src/liveness/`, `reactive`-gated) + +- A small `TrackingRegistry` mapping `Address -> TrackingPolicy` (see §6 on + composition with `FreshnessRegistry`), plus per-account baseline + `HashMap`. +- Hook the per-block boundary in `ReactiveRuntime::ingest_batch_direct` + (`reactive/mod.rs:1418`, right after the canonical block input is journaled) — + or the `SubscriberEvent::BlockHeader` path (`mod.rs:3105`). At that point the + runtime holds `&mut EvmCache`, the canonical `BlockRef`, and the batch's + touched-address set. +- For each `WholeAccount`/`Scalars` target: probe the root; compare to baseline; + apply the §4 table. A move with `addr ∉ touched_addrs` synthesizes a + `ResyncRequest{ targets: tracked slots, block: ResyncBlock::Hash{..}, + reason: ResyncReason::RootMoved }` fed through the existing + `ingest_batch_with_resync` → `execute_resync_requests` path. + +### 5.3 Coverage alarm + +`ResyncReason::RootMoved` (new variant) + a new `ReactiveReport` variant (e.g. +`CoverageGap { address, block }`) dispatched via the existing `dispatch_reports` +(`mod.rs:1510`) so `ReactiveHook::on_report` observers can see "a tracked +account's root moved with no covering event." This is the high-value new signal +— the decoder-coverage blind spot made observable. + +### 5.4 `Validity` stamping of event/reactive writes + +Hand the runtime a `&mut FreshnessRegistry` (or a thin trait it can call). After +`cache.apply_updates(&execution.state_updates)` (`mod.rs:1442`), stamp each +touched `(address, slot)` from the returned `StateDiff` as `ValidThrough(N)` via +`FreshnessRegistry::valid_through_slot` (`freshness.rs:246`). On a new canonical +block, `FreshnessController::on_new_block(N)` (`freshness.rs:837`) already ages +`ValidThrough(m)` into `Volatile` once `N > m`. This makes event-maintained slots +stop being needlessly re-verified while staying honest. + +### 5.5 Cold-start root baseline (`roots.bin`) + +- Extend `ColdStartPlan` with `probe_roots: Vec
` (mirrors the existing + slot `probe` phase) and `ColdStartResults` with observed roots. Persist a + versioned `roots.bin` next to `evm_state.bin` (same magic+`u32`+bincode + envelope as `binary_state.rs`; unknown magic ⇒ cache miss). +- On restart, feed loaded baselines into `ColdStartPlanner::initial_plan`: probe + each tracked account's root now; where it equals the persisted baseline, the + cached tracked slots are still valid — **skip re-reading** (this is the "if no + divergence, we're already synced" case). Where it diverges (or no baseline), + re-read the tracked slots and adopt the new root. The multi-round + `Continue/Done` protocol already supports "root changed → re-read next round." + +## 6. Cold-start gate — what it proves, and the trap to avoid + +The tempting version — *reconstruct* the storage root from our local slots and +compare to chain — is a **trap** and is explicitly out of scope: + +- `storageHash` commits over the account's **entire** storage. Reproducing it + requires holding **every** slot; for the contracts worth tracking (a V2 pool + carries its full LP-token `balanceOf` map; WETH carries every balance) the full + storage is large/unbounded, and there is **no portable enumeration** (alloy + exposes no `debug_storageRangeAt`; it is archive-gated where it exists). +- Reconstruction also needs the trie machinery (`alloy-trie` `HashBuilder`) — the + exact infra Decision D2 rules out. + +So we compare the on-chain root **across time**, never local-vs-chain: persist +the *observed* on-chain root as a baseline and diff `root_now` vs baseline. This +is a **currency** gate, not a **completeness** gate: + +> `root_unchanged ⟹ nothing under the account changed ⟹ the tracked subset is +> unchanged ⟹ if it was correct at the baseline block, it is correct now` +> (and it was, because the baseline values came from RPC at that block). + +It cannot tell you that you were *missing* a slot you should have tracked — a +missing slot is not a "change." Completeness is a separate property only full +enumeration/reconstruction gives, which is why `FullMirror` is deferred. + +## 7. Tier 3: state-diff trace acceleration + +Where an archive/trace node is available, a single per-block +`debug_traceBlockByNumber` (prestate `diffMode`) or +`trace_replayBlockTransactions().state_diff()` returns every changed +`(account, slot, value)` — a decoder-free, value-carrying superset of Tier 1/2 +for tracked accounts. Model it as a block-scoped state-diff source that runs +before per-slot/per-account resync fetches and **degrades honestly** when the +namespace is unavailable (`-32601`) or historical trie data is missing. One trace +call/block (all accounts) replaces N point reads when the requested targets are +present in the diff; unmatched cold targets still go through the portable fetch +seams so the core never depends on trace availability. + +Live RPC benchmarking and gzip transport measurements are documented in +[`trace-resync-benchmarks.md`](trace-resync-benchmarks.md). The important design +constraint from those measurements is that trace is an excellent CU/completeness +accelerator, but small known-slot repairs can still be faster through batched +point reads; production policy should stay adaptive. + +## 8. Tests (offline, no network) — the acceptance contract + +Mock the `AccountProofFetchFn` (and `StorageBatchFetchFn`) with in-memory tables; +no live RPC. Grouped by file: + +- `tests/liveness_root_gate.rs`: unchanged root ⇒ 0 slot reads + tracked set + stamped `ValidThrough(N)`; `WholeAccount` root moved + addr ∉ touched ⇒ resync + emitted + `CoverageGap` report; root moved + addr ∈ touched ⇒ skip; `Slots` + policy is never root-gated; native balance/nonce move detected via account + fields. +- `tests/liveness_validity_stamp.rs`: a reactive/event write stamps + `ValidThrough(N)`; `on_new_block(N+1)` ages it to `Volatile`. +- `tests/liveness_cold_start.rs`: baseline round-trip through `roots.bin` + (versioned envelope; legacy/unknown magic ⇒ miss); equal baseline ⇒ no + re-read; diverged baseline ⇒ re-read + adopt. +- `tests/liveness_account_resync.rs`: `ResyncTarget::Account` now succeeds via + the new seam (previously `UnsupportedAccountTarget`). + +## 9. Decisions (LOCKED by the design thread) + +1. **No local storage trie.** The root is observed via `eth_getProof`, never + reconstructed locally. +2. **No cryptographic proof verification.** Verifying a storage proof against a + `stateRoot` fetched from the same RPC is circular; the crate has no + independent root source. `alloy-trie` stays out. (Revisit only if the engine + ever ingests state from an untrusted peer or must prove correctness to a third + party.) +3. **`storageHash` gates `WholeAccount` only**, never `Slots` (root churns every + block for sparse-interest contracts → noisy, wasteful). +4. **Compose, don't merge.** A `TrackingRegistry` wraps / sits beside + `FreshnessRegistry`; the entire freshness API stays verbatim. (This introduces + the *first* coupling between the reactive runtime and `freshness.rs` — an + intentional, minimal one, for §5.4.) +5. **Reuse the existing resync channel.** The root gate emits `ResyncRequest`s; + it does not build a parallel repair loop. +6. **Cold-start gate is currency, not completeness** (§6). `FullMirror` deferred. +7. **Tier 3 (trace) is a trace-first accelerator when configured/available**; the + existing point-fetch seams remain the portable floor. + +## 10. Open decisions (confirm before building) + +- **Default `TrackingPolicy`** for a touched-but-unregistered address — + recommend *untracked / best-effort* (lazy semantics) so liveness cost is + strictly opt-in. +- **Coverage-alarm action** — surface-only in the report (recommended; caller + sets policy) vs. auto-`mark_volatile` the emitter so it self-heals next cycle. +- **`Scalars` gating** — cheap empty-key probe every block (recommended; account + fields are free in the response) vs. treat as `Pinned`. +- **Where `advance_block` is driven** — inside the runtime's canonical-block + path (automatic, opinionated) vs. a `FreshnessController::on_new_block` + extension the caller wires (explicit). + +## 11. Build order (commit per step, green each time) + +1. `AccountProofFetchFn` seam on `EvmCache` + builder installer (unblocks + `ResyncTarget::Account`; land with a test flipping the `Unsupported` path). +2. `advance_block(header)` block-env refresh + tracked-set dirty/reverify; drive + from the reactive canonical-block path. +3. `Validity` stamping of reactive/event writes (§5.4) + aging test. +4. `TrackingPolicy` / `TrackingRegistry` + `StorageHashProbe` + per-block root + gate + `ResyncReason::RootMoved` + `CoverageGap` report (the centerpiece). +5. Cold-start root baseline (`roots.bin`) + restart diff in the planner. +6. Tier-3 block-state-diff fetcher that resolves matching resync targets before + falling back to storage/account point reads. + +## 12. ROADMAP integration + +Add to `ROADMAP.md`: + +- **Phase-table row** (once landed): `| **8** | storageHash liveness & + invalidation: account/root fetcher seam; per-block root gate + complement + resync (`RootMoved`); `TrackingPolicy`; event-write `Validity` stamping; + `advance_block` env refresh; cold-start root baseline | **Done** (`phase-8-liveness`) |`. +- A `## Phase 8 — storageHash liveness (detailed, decisions locked)` section + after the Phase 7 detailed section, and a "Remaining work toward 1.0" bullet + until it lands. +- On landing, convert the `KNOWN_ISSUES.md` items this closes: account-field + resync `Unsupported` (`reactive/mod.rs:2000`), "freshness reconciles storage + slots only," and the account-field-resync transport-depth item. diff --git a/docs/trace-resync-benchmarks.md b/docs/trace-resync-benchmarks.md new file mode 100644 index 0000000..45d6048 --- /dev/null +++ b/docs/trace-resync-benchmarks.md @@ -0,0 +1,170 @@ +# Trace-backed resync benchmarks + +Captured on July 2, 2026 while evaluating Phase 8 Tier-3 state-diff resync. +This note records the measured behavior of: + +- `debug_traceBlockByNumber` with Geth `prestateTracer` `diffMode: true` +- batched `eth_getStorageAt` point reads for known slots +- gzip-compressed HTTPS responses for large trace payloads + +The core takeaway is deliberately split into three axes: + +| Axis | Result | +| --- | --- | +| Alchemy compute units | Trace wins once it replaces 3 or more point storage reads in the same block. | +| Latency without compression | Storage batches stayed faster through the clean measurable range on the public endpoint tested. | +| Latency with gzip on Alchemy | Gzip substantially reduced one large trace response's transfer time. | + +## RPC economics + +Alchemy's published CU table lists: + +- `eth_getStorageAt`: 20 CU +- `eth_getProof`: 20 CU +- `debug_traceBlockByNumber`: 40 CU + +That means the CU break-even is two point reads: + +| Storage slots repaired from one block | Point-read CU | Trace CU | Trace CU result | +| ---: | ---: | ---: | --- | +| 1 | 20 | 40 | 2x more expensive | +| 2 | 40 | 40 | break-even | +| 3 | 60 | 40 | 33% cheaper | +| 10 | 200 | 40 | 5x cheaper | +| 100 | 2000 | 40 | 50x cheaper | + +This is a billing/throughput result, not a latency result. Trace responses are +large JSON payloads and can still arrive later than a small storage batch. + +Sources: + +- Alchemy CU table: https://www.alchemy.com/docs/reference/compute-unit-costs +- Alchemy debug endpoint: https://www.alchemy.com/docs/node/debug-api/debug-api-endpoints/debug-trace-block-by-number +- Geth tracer behavior: https://geth.ethereum.org/docs/developers/evm-tracing/built-in-tracers + +## Latency benchmark: trace vs batched storage + +### Setup + +- Pool: Uniswap V3 USDC/WETH 5 bps + `0x88e6A0c2dDD26FEEb64F039a2c41296FcB3f5640` +- Trigger: live `Swap(address,address,int256,int256,uint160,uint128,int24)` + subscription plus historical blocks containing Swap logs. +- Trace call: + `debug_traceBlockByNumber(block, {"tracer":"prestateTracer","tracerConfig":{"diffMode":true}})` +- Storage calls: + one JSON-RPC batch of `eth_getStorageAt(pool, slot, block)` for slots + `0..N-1`. +- Endpoint for the scaling run: MEW public Ethereum RPC, which supports both + `debug_traceBlockByNumber` and JSON-RPC storage batches. + +This benchmark intentionally measures end-to-end RPC response latency from one +client process. It does not isolate node execution time from network transfer +time. + +### Live Swap-triggered samples + +| Slots | Trace latency | Storage batch latency | Winner | +| ---: | ---: | ---: | --- | +| 1 | 455 ms | 111 ms | storage | +| 8 | 418 ms | 73 ms | storage | +| 32 | 282 ms | 35 ms | storage | +| 128 | 610 ms | invalid | storage batch had 29 rate-limit errors | + +### Historical Swap-block scaling + +Clean successful median latencies: + +| Slots | Trace median | Storage median | Trace wins | Storage wins | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 480 ms | 34 ms | 0 | 4 | +| 2 | 352 ms | 55 ms | 1 | 3 | +| 4 | 476 ms | 34 ms | 0 | 4 | +| 8 | 305 ms | 48 ms | 0 | 4 | +| 16 | 645 ms | 31 ms | 0 | 4 | +| 32 | 277 ms | 47 ms | 0 | 4 | +| 64 | 332 ms | 92 ms | 1 | 3 | +| 80 | 182 ms | 42 ms | 0 | 4 | +| 96 | 486 ms | 233 ms | 0 | 2 | + +No reliable latency crossover was observed before the endpoint's clean batch +limit. The same public endpoint started returning per-item rate-limit errors near +100 `eth_getStorageAt` requests per second, even when sent as one JSON-RPC +batch. A noisy linear interpolation suggested a possible crossover around 260 +slots, but that is outside the clean measurable region and should not be treated +as a production threshold. + +Interpretation: + +- For small and medium known slot sets, batched storage reads are latency-faster. +- Trace can still be preferable when CU budget, completeness, or avoiding many + point reads matters more than wall-clock latency. +- Provider behavior matters. Public RPC endpoints differ materially in debug + namespace support, batch limits, response caching, and rate limits. + +## Alchemy gzip test + +### Setup + +- Endpoint: Alchemy Ethereum mainnet HTTPS. +- Block: `25444665` (`0x1844139`). +- Payload: same `debug_traceBlockByNumber` prestate diff call. +- Client: raw HTTPS with automatic decompression disabled, so wire bytes, + decompression time, and JSON parse time were measured separately. +- Comparison: + - Standard: `Accept-Encoding: identity` + - Gzip: `Accept-Encoding: gzip` + +### Result + +| Mode | Wire bytes | Full response received | Parsed and ready | +| --- | ---: | ---: | ---: | +| Standard HTTPS (`identity`) | 4.48 MB | 664 ms | 671 ms | +| Gzip HTTPS (`gzip`) | 1.03 MB | 268 ms | 286 ms | + +Observed improvement: + +- Wire payload reduction: 76.9% +- Receive-time improvement: 397 ms +- Parsed-and-ready improvement: 385 ms +- End-to-end parsed latency improvement: about 57% +- Local gzip decompression cost: 11 ms + +A separate concurrent sanity check on another block also reduced wire bytes by +78.7%, but gzip arrived slower in that single paired sample. Treat compression as +a strong payload-size win and a likely latency win for large traces, but keep +latency thresholds empirical per provider. + +## Runtime policy guidance + +1. Always request gzip for debug/trace RPC over HTTPS when the provider supports + it. The payload is large, repetitive JSON and compresses well. +2. Do not replace small known-slot repairs with trace solely for latency. Storage + batches were faster up to the clean measured range. +3. Keep the trace source as an accelerator with fallback, not a hard dependency. + `debug`/`trace` namespaces are commonly gated or disabled. +4. Make the trace decision adaptive: + - Use point reads for low slot counts on latency-sensitive hot paths. + - Use trace when expected unresolved targets per block are high enough, when + CU budget dominates, or when whole-block completeness is valuable. + - Continue falling back to point reads for cold targets absent from the trace + diff. +5. Separate thresholds by objective: + - CU threshold on Alchemy: trace breaks even at two point reads and wins at + three or more. + - Latency threshold: not established by the public-endpoint run; after gzip, + it should be measured directly on the target Alchemy plan and region. + +## Follow-up benchmark to run before hard-coding defaults + +Run the same scaling harness directly against Alchemy with: + +- gzip enabled for trace +- the same Alchemy endpoint for storage batches +- slot counts past 100 if the plan's rate limits allow it +- at least 20 samples per slot count bucket +- separate reporting for transfer time, decompression time, JSON parse time, and + provider errors + +Until that exists, the implementation should expose configuration rather than +hard-code a latency threshold. diff --git a/docs/v0.2.0-release-plan.md b/docs/v0.2.0-release-plan.md new file mode 100644 index 0000000..184d667 --- /dev/null +++ b/docs/v0.2.0-release-plan.md @@ -0,0 +1,202 @@ +# evm-fork-cache — 0.2.0 release plan + +> Status: **scope locked; ready to build.** Turns the publication-readiness review +> + the maintainer's four committed improvements into an ordered, green-bar build +> plan, integrated with the **full** [`phase-8-liveness-spec.md`](phase-8-liveness-spec.md) +> and [`KNOWN_ISSUES.md`](KNOWN_ISSUES.md). Line references verified against branch +> `phase-8-liveness`. Cut as **0.2.0** (a minor, not a patch) so misleading APIs +> are renamed cleanly rather than papered over with deprecation aliases — permitted +> by the pre-1.0 policy in `CHANGELOG.md`. + +## Locked decisions + +- **D1 — 0.2.0, rename cleanly.** Breaking renames land outright (no deprecation + aliases): the verdict (`Confirmed → ConfirmedStorage`, add `ConfirmedFull`) and + the snapshot methods. The process-global cache speed mode is removed in favor of + a per-instance builder option. +- **D2 — full Phase 8 (spec steps 1–6).** Includes the cold-start root baseline + (`roots.bin`) and the Tier-3 state-diff trace path that resolves matching + resync targets before falling back to point reads. +- **D3 — Tier 1 + Tier 2 + WS-8/9/10.** Only the cold-start error taxonomy (G8) is + deferred to a follow-up. + +## TL;DR — three framing insights + +1. **Item #1 (account-level freshness) is a three-part change, not one — and Phase + 8 covers only the first.** The `AccountProofFetchFn` seam gives the *fetch/resync* + half. But the verify loop still builds its set from storage slots only + (`run_validator`, `freshness.rs:1036`), and the verdict still says + `Validation::Confirmed` (`freshness.rs:538`) while ignoring balance/nonce/code. + Closing #1 = seam [spec] **+** account-field wiring in the verify loop [net-new] + **+** a verdict that stops over-promising [net-new]. +2. **Item #4 has a critical sibling the review missed: silent forward block-gaps.** + `recover_for_canonical_input` (`reactive/mod.rs:1539-1541`) does `if block.number + > latest.number + 1 { return None }` — a jump from *N* to *N+k* is silently + accepted as "not a reorg," the skipped blocks' state is never applied, and **no + signal fires.** Must drain into the same conservative recovery path as #4. +3. **Items #2 and #4 both need one missing primitive: a queryable cache-trust + surface.** Today "something's wrong" is only a `tracing::warn!` + (`reactive/mod.rs:1608`). Build `CacheHealth` + lightweight atomic-counter + metrics **first**, and four separate hardening tasks become one coherent trust + model — the difference between "a log line" and "the engine knows to stop + trading." + +## Phase 8 mapping (all steps in scope) + +| Phase-8 spec step | Role | +| --- | --- | +| 1. `AccountProofFetchFn` seam | Foundation for item #1 + account resync (`ResyncTarget::Account`, today `Unsupported` at `reactive/mod.rs:2000`). | +| 2. `advance_block(header)` env refresh | Steady-state half of item #2. | +| 3. `Validity` stamping of reactive writes | Event-maintained slots stop being needlessly re-verified. | +| 4. `TrackingPolicy` + root gate + `RootMoved` + `CoverageGap` | Complements item #4 (detects post-reorg divergence for tracked accounts; upgrades `Degraded → Unhealthy`). | +| 5. Cold-start root baseline (`roots.bin`) | Restart-drift detection. **In scope (D2).** | +| 6. Tier-3 trace resync source | Trace-first block-diff accelerator with point-read fallback. **In scope (D2), built last.** | + +--- + +## Workstreams + +### Tier 1 — correctness must-haves + +#### WS-1 · Account-level freshness + verdict truth *(item #1; G1)* +- **(a) Seam** [spec 1] — `AccountProofFetchFn` on `EvmCache` (`eth_getProof`, + mirrors `StorageBatchFetchFn`); flip `ResyncTarget::Account` from + `UnsupportedAccountTarget` (`reactive/mod.rs:2000`) to a real fetch. +- **(b) Verify-loop wiring** [net-new] — extend `run_validator` (`freshness.rs:1036`) + to reconcile touched accounts' balance/nonce/code-hash vs. the snapshot, gated by + `TrackingPolicy`/`Pinned` so it stays opt-in and cheap. Add + `override_balance`/`override_nonce`/`override_code` to `EvmOverlay` for + account-aware re-runs. +- **(c) Verdict rename** [net-new, breaking] — `Confirmed → ConfirmedStorage`; add + `ConfirmedFull` (storage **and** account fields verified); add `changed_accounts` + to `Corrected`. Update all call sites/tests/docs (no aliases, per D1). +- **Tests:** balance-moved-without-slot is caught (`KNOWN_ISSUES.md:109-120`); + `ConfirmedFull` only when accounts verified; `ResyncTarget::Account` round-trips. + +#### WS-2 · Strict block-context mode *(item #2)* +Block-env fields silently default via `if let Some` guards in `build_evm` +(`cache/mod.rs` ~4200); a construction-time fetch failure degrades all to `None`. +- `EvmCacheBuilder::block_context_requirements(BlockContextRequirements)` + + `.strict_block_context(true)` convenience. Per-chain configurable + (`basefee`/`prevrandao` legitimately absent pre-London/pre-merge). +- **Construction:** strict + missing required field / failed fetch ⇒ typed + `BlockContextError`. +- **Steady state:** enforce the same in the Phase-8 `advance_block(header)` path + [spec 2] — WS-2 and spec step 2 land together. +- **Tests:** strict + missing basefee ⇒ `Err`; lenient ⇒ today's behavior; pre-merge + + prevrandao-off ⇒ `Ok`. + +#### WS-3 · Bounded `derived_slots` *(item #3)* +`EventPipeline::derived_slots` (`events/mod.rs:228`) is unbounded, pruned only by +`reorg_to` (`KNOWN_ISSUES.md:140-144`). +- Replace with a block-horizon ring mirroring `touched` (`events/mod.rs:225`): + `VecDeque<(u64, HashSet<(Address, U256)>)>` bounded to `ReorgConfig::depth`; trim + with `trim_ring`; drop per-block sets on `reorg_to`; `derived_slots()` flattens + the window. +- **Correctness:** a slot aged past `depth` can't be reorg-invalidated anyway + (`KNOWN_ISSUES.md:145-149`), so eviction loses nothing actionable. One tunable, no + new config. +- **Tests:** steady-state ingest over `> depth` blocks holds size ≤ window. + +#### WS-4 · Conservative deep-reorg self-heal + missed-range + health surface *(item #4; G2, G3, G7)* +- **Health machine (connective primitive):** `CacheHealth { Healthy, Degraded { + since_block }, Unhealthy }` on `ReactiveRuntime`, queryable via `health()`; emit a + `ReactiveReport` per transition. +- **Deep reorg:** on `warn_under_recovery` (`reactive/mod.rs:1606`), transition to + `Degraded`, conservatively purge the affected working set + emit `ResyncRequest`s. + Cascading ⇒ `Unhealthy` (refuse to serve until explicit resync/`set_block`). +- **Missed range [G2]:** replace the silent `return None` at + `reactive/mod.rs:1539-1541` with `ResyncReason::MissedBlockRange { from, to }` + + report into the same conservative path. Optional `strict_continuity` flag for + hard-error. +- **Purge/write ordering [G7]:** in `recover_dropped_journals`, apply purges last + when a `(address, slot)` is both rolled back and purge-scoped, so recorded + `StateDiff` can't diverge from the cache. Confirm frequency with a test first. +- **Composition:** Phase-8 root gate [spec 4] upgrades `Degraded → Unhealthy` on + divergence. +- **Tests:** reorg > `journal_depth` ⇒ `Degraded` + purge + resync; N→N+k ⇒ + `MissedBlockRange`; cascading ⇒ `Unhealthy`; purge-wins ordering. + +### Tier 2 — production hardening + +#### WS-5 · Observability / metrics *(G4, G9)* +Arc-shared `CacheMetrics` (atomic `u64`, **no new deps**): `resync_requests`, +`resync_failures`, `coverage_gaps`, `deep_reorgs`, `missed_ranges`, `stale_verdicts`, +`pending_contamination_attempts`. Expose `cache.metrics()`. Pairs with WS-4; gives +the `InvalidPendingEffect` path (`reactive/mod.rs:~1121`) a counter. + +#### WS-6 · API footgun cleanup *(review #4/#5; breaking, allowed in 0.2.0)* +- **Snapshot rename:** `snapshot()` (layer-1 `Cache`) → `checkpoint()`; + `create_snapshot()` (`Arc` fan-out) → `snapshot()`. No aliases. +- **Storage batch tuning:** remove the process-global `CACHE_SPEED_MODE` static; + add `StorageBatchConfig { slots_per_batch, max_concurrent_batches }` plus + `EvmCacheBuilder::storage_batch_config(...)`; keep `CacheSpeedMode` as preset + shorthand via `From` / `EvmCacheBuilder::speed_mode(...)`. +- **`SpeculativeSim`:** `Result`-returning `validate` retiring the `.expect(...)` + (`freshness.rs:702`). + +#### WS-7 · Bundle cost accounting *(review #9; additive)* +Add `successful_tx_gas` + `reverted_tx_gas` to `BundleResult` (`bundle.rs:125`) via a +one-pass fold over `per_tx`, so `net_searcher_cost = coinbase_payment + +reverted_tx_gas` is direct (`KNOWN_ISSUES.md:160-169`). Clarify the `gas_used` +docstring. + +### Tier 3 — lower-priority correctness + polish + +- **WS-8 · Snapshot generation guard [G6]** — `EvmCache::snapshot_generation() -> + u64` bumped on `apply_updates`; lets a fan-out consumer detect a mid-block + snapshot and re-snapshot. Closes the ROADMAP-listed consistency gap. +- **WS-9 · `BLOCKHASH` fail-closed in validator overlays [G5]** — thread the cache's + `block_hashes` into validator overlays, or flag a `BLOCKHASH`-reading sim as + `Unverified` instead of silently confirming (`KNOWN_ISSUES.md:134-139`). +- **WS-10 · Docs & tests** — README "Production Safety Checklist"; architecture + diagram; toy constant-product AMM example (decode → cold-start → `StateUpdate` → + overlay sim → reconcile); benchmark artifact notes including + [`trace-resync-benchmarks.md`](trace-resync-benchmarks.md); the missing e2e + reactive integration tests (`KNOWN_ISSUES.md:231-238`); ROADMAP update (Phase 8 + → Done + the hardening workstreams); convert the `KNOWN_ISSUES` items this + release closes. + +--- + +## Additional gaps found (beyond the four named) + +| # | Gap | Severity | Fold into | +| --- | --- | --- | --- | +| G1 | Verdict `Confirmed` over-promises | **Critical** | WS-1(c) | +| G2 | Silent forward block-gap (`reactive/mod.rs:1539`) | **Critical** | WS-4 | +| G3 | No queryable cache-health surface | **High** | WS-4 | +| G4 | No metrics (resync/coverage/reorg/stale rates) | **High** | WS-5 | +| G5 | `BLOCKHASH` → ZERO in validator overlays | **Medium-High** | WS-9 | +| G6 | Snapshot mid-block inconsistency in ingestion | **Medium** | WS-8 | +| G7 | purge/write ordering under deep-reorg recovery | **Medium** | WS-4 | +| G8 | Cold-start transient-vs-fatal RPC failure taxonomy | **Medium** | *deferred (D3)* | +| G9 | Pending→canonical contamination has no counter | **Medium** | WS-5 | + +--- + +## Build order (green bar at every commit) + +Each step compiles + passes `cargo fmt --all --check`, `cargo clippy --all-targets +--all-features -- -D warnings`, `cargo test`, `RUSTDOCFLAGS=-D warnings cargo doc +--no-deps`, `cargo bench --no-run` (matching the Phase-8 spec's acceptance bar). +Foundational primitives first. + +1. **WS-4 health primitive + WS-5 metrics scaffold** — `CacheHealth`, `metrics()`, + report variants. Everything downstream reports into these. +2. **Phase-8 step 1 — `AccountProofFetchFn` seam** (unblocks WS-1a + account resync). +3. **WS-4 deep-reorg + missed-range self-heal** — wire `warn_under_recovery` and the + `:1539` gap into purge/resync + health transitions. +4. **WS-2 strict block context (construction) + Phase-8 step 2 `advance_block` + (steady-state).** +5. **Phase-8 step 3 — `Validity` stamping of reactive writes.** +6. **WS-1(b)/(c) — account-field verify loop + verdict rename.** +7. **WS-3 — bound `derived_slots`.** +8. **Phase-8 step 4 — `TrackingPolicy` + root gate + `CoverageGap`** (upgrades + `Degraded → Unhealthy`). +9. **Phase-8 step 5 — cold-start root baseline (`roots.bin`) + restart-drift diff.** +10. **Phase-8 step 6 — Tier-3 trace resync source** (block diff first, then + storage/account point-read fallback for unresolved targets). +11. **WS-6 API footgun renames · WS-7 bundle accounting.** +12. **WS-8 snapshot generation guard · WS-9 `BLOCKHASH` fail-closed.** +13. **WS-10 docs, e2e tests, ROADMAP/KNOWN_ISSUES updates.** diff --git a/docs/verified-code-seeding-spec.md b/docs/verified-code-seeding-spec.md new file mode 100644 index 0000000..7f5e767 --- /dev/null +++ b/docs/verified-code-seeding-spec.md @@ -0,0 +1,498 @@ +# Verified code seeding & local etch (spec) + +Status: **implemented in 0.2.0** — C1–C5 landed (`3c2e46a`, `9c0e80d`, +`396ecad`, `976dd56`, `bab4caa`); all locked decisions below are as-built. +Consumers: `evm-amm-state` ships its initial release on the canonical path. + +Also carries a companion workstream (§6): batched `eth_getProof` fan-out and +a root-gate cadence. Review call: `eth_getProof` is the slowest read the +crate issues, and anything issued per block gets expensive quick — per-block +proofing is retired as a default. + +The feature adds a first-class answer to "how does code get into the cache +without an `eth_getCode`?" — with two deliberately distinct trust classes: + +1. **Canonical seed** — the adapter pushes bytecode it *claims* is the + on-chain code (a patched Uniswap V3 pool template, a shared V2 pair blob). + The cache verifies the claim once against the on-chain code hash, then + marks it verified forever. Mismatch ⇒ purge + resync from RPC. +2. **Local etch** — the adapter pushes bytecode that is *deliberately not* + on-chain (an unreleased searcher contract, a test harness). Never + verified, never mistaken for chain state, visible as divergence. + +## Motivation + +- Cold start today materializes each account lazily via `SharedBackend:: + basic_ref` → `eth_getBalance` + `eth_getTransactionCount` + `eth_getCode` + per address. For N pools that is 3N round trips and megabytes of hex code + payload; the code bytes are data the adapter already possesses. +- `bytecodes.bin` removes the cost on *warm* restarts only. Fresh machines, + wiped caches, and — critically — **newly deployed pools detected while the + cache is live** all pay the full fetch. +- The verification read is nearly free: `EXTCODEHASH` for thousands of + addresses fits in **one** `eth_call` via the already-shipped + `ACCOUNT_FIELDS_EXTRACTOR_CODE` (~5.3k gas/address), and `eth_getProof` + responses already carry `codeHash` through `AccountProof` unused. +- The existing divergence primitives (`override_account_code*`, + `deploy_contract`) copy code between accounts or run creation code; there + is no raw-bytes write, and nothing records *which* accounts diverge from + chain — the health surface cannot report it. + +Target outcome for the live-registration path: a newly detected V3 pool goes +from "factory event seen" to "fully materialized, code-verified, storage +synced" in **~1 round trip and zero `eth_getCode`**. + +## 0. Ground rules + +- **Fail-closed on trust, fail-safe on transport.** A *successful* hash + comparison that mismatches purges the seed. A *failed* verification call + (transport error, missing fetcher) leaves the seed `Pending` and reports it + — it never silently promotes to `Verified` and never destroys the seed. +- **`Pending` never masquerades as chain-fetched.** Marks persist across + save/load, including `Pending`. An unverified seed that survives a restart + is still unverified. +- **Discover/sims never run over unverified canonical seeds** in the + cold-start driver: code verification is the *first* phase of a round. +- **Etched state is always distinguishable.** Every non-RPC, non-canonical + code write records an `Etched` mark; the set is queryable in one place. +- All new behavior is offline-testable with stub fetchers, per house rules. + +## 1. Objective & scope + +In scope: +- Per-address code-seed marks (`Pending` / `Verified` / `Etched`) with + persistence (`code_seeds.bin`) and conflict rules. +- `seed_account_code` / `etch_account_code` write primitives. +- An `AccountFieldsFetchFn` seam (sync, type-erased, default-wired to + `fetch_account_fields_bulk`) + `verify_code_seeds()`. +- A `verify_code` cold-start driver phase (runs before `accounts`). +- Snapshot-generation and health-surface semantics for both classes. +- Companion (§6): single-invocation proof batching at both reactive call + sites, a bounded-concurrency default proof fetcher, and `RootGateCadence` + (per-block proofing stops being the default). + +Out of scope (explicit non-goals): +- Content-addressing / deduplicating `bytecodes.bin` (N identical V2 blobs + are stored N times today; orthogonal disk optimization). +- Verifying nonce/balance beyond what the fields sample provides (exact + nonce needs the proof path; only matters for contracts that `CREATE`). +- EIP-7702 delegated EOAs (`EXTCODEHASH` returns the hash of the delegation + designator; not an AMM shape — documented, not handled). +- Pre-Cancun `SELFDESTRUCT`-then-redeploy code mutation. Post-EIP-6780, + deployed code is immutable, so `Verified` is durable. On chains without + 6780 the escape hatch is `purge_account` (clears the mark). + +## 2. What already exists — build on, do not rebuild + +| Piece | Where | Role here | +|---|---|---| +| `ACCOUNT_FIELDS_EXTRACTOR_CODE` + `fetch_account_fields_bulk` | `src/bulk_storage.rs:1035,1057` | one `eth_call` returns `(balance, EXTCODEHASH)` for every seeded address | +| `AccountFieldsSample` | `src/bulk_storage.rs:1041` | `code_hash` zero ⇒ nonexistent; `keccak256("")` ⇒ codeless (EOA) | +| `AccountProof.code_hash` | `src/cache/mod.rs:110` | free future piggyback wherever `probe_roots` already proofs an address | +| `purge_account` | `src/cache/mod.rs:3138` | the mismatch resync primitive (both layers; bumps generation via `apply_update`) | +| `ensure_account` early-return | `src/cache/mod.rs:3953` | a seeded (present) account never triggers the `basic_ref` RPC triple | +| Cold-start phase order | `src/cold_start/driver.rs:87` | `accounts → verify → probe → probe_roots → discover`; `verify_code` slots in front | +| Versioned envelope persistence | `src/cache/versioned.rs`, `roots.bin`/`bytecodes.bin` pattern | `code_seeds.bin` reuses it; load at `mod.rs:1643`, save at `mod.rs:1988`, path from `CacheConfig` (`metadata.rs`) | +| Sync fetcher bridging | `point_read_storage_fetcher`, `block_in_place_handle` | `AccountFieldsFetchFn` default wiring is the same shape (multi-thread runtime requirement inherited) | +| `run_root_gate` / account-resync proof loops | `src/reactive/mod.rs:1993,2938` | per-address seam calls today — §6.1 collapses each into one batched invocation | +| Default account-proof fetcher | `src/cache/mod.rs:1717` | serial await loop — §6.1 makes it a bounded-concurrency fan-out | +| Cold-start `probe_roots` batching | `src/cold_start/driver.rs:176` | already one seam call for the whole list — the call shape §6.1 mirrors | + +## 3. Core types & behavior + +### 3.1 Marks (`CodeSeedState`) + +```rust +/// Provenance + trust state of an address's cached bytecode, for code that +/// did NOT arrive via the lazy RPC backend. Absence of a mark means +/// RPC-origin (fetched from the provider, trusted as chain state). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CodeSeedState { + /// Canonical claim awaiting on-chain code-hash verification. + Pending { code_hash: B256 }, + /// Canonical claim confirmed against the chain. Never re-verified. + Verified { code_hash: B256, verified_at_block: u64 }, + /// Deliberate local divergence. Never verified, excluded from all + /// canonical machinery, reported on the health surface. + Etched { code_hash: B256 }, +} +``` + +Stored as `HashMap` on `EvmCache`. Getters: +`code_seed_state(&addr)`, `pending_code_seeds() -> Vec
`, +`etched_accounts() -> Vec
` (health surface). + +### 3.2 Write primitives + +```rust +/// Seed canonical runtime code for `address` without fetching it. +/// Marks the seed `Pending` until `verify_code_seeds` confirms it. +/// Defaults: nonce 1 (EIP-161 contract minimum; exact only for contracts +/// that never `CREATE`), balance ZERO until verification patches the real +/// value from the same response. Returns the recorded keccak code hash. +pub fn seed_account_code(&mut self, address: Address, code: Bytes) -> Result; +pub fn seed_account_code_with( + &mut self, address: Address, code: Bytes, nonce: u64, balance: U256, +) -> Result; + +/// Etch deliberately-local runtime code at `address` (raw-bytes sibling of +/// `override_or_create_account_code`, no source account needed). Marks +/// `Etched`; preserves existing balance/nonce/storage when present. +pub fn etch_account_code(&mut self, address: Address, code: Bytes) -> Result; +``` + +Write mechanics (both): build `Bytecode::new_raw` + `hash_slow`, insert into +the CacheDB overlay **and** BlockchainDb accounts (the +`override_account_code_with_missing_target` dual-layer pattern, +`mod.rs:5135`), `mark_base_dirty`. Because the account is then present in +both layers, `basic_ref` never fires for it — that alone deletes the RPC +triple for seeded accounts. + +**Conflict rules for `seed_account_code`** (chain-fetched state is +authoritative over templates): + +| Existing state at `address` | Seed behavior | +|---|---| +| Absent | insert, mark `Pending` | +| Present, RPC-origin, same code hash | keep, mark `Verified` immediately (warm caches verify for free, zero RPC) | +| Present, RPC-origin, different hash (incl. codeless EOA) | `Err(CacheError::CodeSeedConflict { address, cached, seeded })` — never clobber known chain code with a template; adapter purges first if it believes the chain moved | +| Present, `Pending`/`Verified`/`Etched` | overwrite, mark `Pending` (re-seed restarts the claim) | + +`etch_account_code` never conflicts: it overwrites anything and marks +`Etched` (divergence is the caller's explicit intent). + +**Unification (locked):** `override_account_code_with_missing_target` and +`deploy_contract` also record `Etched` for their target/created address, so +*every* locally-divergent code site is visible through `etched_accounts()`. +No other behavior of those APIs changes. + +### 3.3 Verification (`AccountFieldsFetchFn` seam + `verify_code_seeds`) + +```rust +/// Callback fetching `(balance, EXTCODEHASH)` samples for many addresses at +/// a pinned block — one bulk `eth_call` by default. Sync, type-erased, same +/// bridging rules as `StorageBatchFetchFn`. +pub type AccountFieldsFetchFn = Arc< + dyn Fn(Vec
, BlockId) -> StorageFetchResult> + + Send + Sync, +>; + +pub fn set_account_fields_fetcher(&mut self, f: AccountFieldsFetchFn); +pub fn account_fields_fetcher(&self) -> Option<&AccountFieldsFetchFn>; + +/// Verify every `Pending` seed against the chain at the pinned block, in one +/// fields call. Match ⇒ `Verified` + real balance injected. Mismatch ⇒ +/// `purge_account` + reported (re-fetch is the caller's/driver's next step). +/// Transport failure ⇒ everything stays `Pending`, reported unverifiable. +pub fn verify_code_seeds(&mut self) -> Result; + +pub struct CodeVerifyReport { + pub verified: Vec
, + pub mismatched: Vec, // { address, expected: B256, actual: B256 } — purged + pub not_deployed: Vec
, // EXTCODEHASH == 0 at pinned block — purged + pub codeless: Vec
, // EXTCODEHASH == keccak("") (EOA) — purged + pub unverifiable: Vec<(Address, String)>, // fetch failed — still Pending +} +``` + +Per-outcome semantics: +- **Match** ⇒ mark `Verified { verified_at_block }`; patch the account's + balance from the sample (materialization of pinned-block truth — see §3.6; + nonce keeps the seeded value). +- **Mismatch / not-deployed / codeless** ⇒ `purge_account` (clears both + layers *and* the mark; generation bumps via the purge path, so downstream + fan-out sees the change). The next touch — driver `accounts` phase or lazy + `basic_ref` — refetches real chain state. +- **`not_deployed`** deserves its own bucket: it is the live-registration + race (factory event from block N+k, cache pinned at N). The adapter's + correct response is retry-after-re-pin, not template debugging. + +Constructor wiring: the provider-backed constructor installs the default +fetcher (bridging `fetch_account_fields_bulk`); `from_backend` leaves it +`None` (symmetric with the other three seams at `mod.rs:1957`). One fields +call covers ~10k addresses inside an 80M-gas envelope; chunking is deferred +until a consumer approaches that (documented ceiling). + +Host-address caveat (documented, inherited from the extractor): a seeded +address equal to `MULTICALL3_ADDRESS` cannot be verified by the fields path +(the extractor is hosted there under the override) — reported +`unverifiable`; use the proof path for it. + +### 3.4 Persistence (`code_seeds.bin`) + +- Versioned envelope (`EFCSEED\0`, v1), bincode `HashMap`; `Option`-tolerant load like `roots.bin` (missing/legacy + file ⇒ empty map, never an error — old caches upgrade cleanly). +- Path from `CacheConfig` beside `bytecodes.bin` (`metadata.rs`). +- Load in the constructor next to the bytecode seeding (`mod.rs:1643`) — + marks must be restored *whenever* `bytecodes.bin` code is restored. +- Save in the persistence path (`mod.rs:1988`) **before** `bytecodes.bin`: + if the process dies between the two writes, marks-without-code is harmless + (a mark for absent code is ignored/pruned on load); code-without-marks + would let a `Pending` seed masquerade as RPC-origin. Fail-closed ordering. +- `Pending` persists as `Pending`; `Etched` persists as `Etched` (a reloaded + searcher contract is still divergence); `Verified` persists and is never + re-verified. + +### 3.5 Cold-start driver: `verify_code` phase + +New phase order: **verify_code → accounts → verify → probe → probe_roots → +discover** (`driver.rs:87`). + +- Implicit work set: `pending_code_seeds()` — no new `ColdStartPlan` field. + Seeds are cache state the adapter just wrote; duplicating them in the plan + invites drift. +- Guard (mirrors `NoBatchFetcher`/`NoAccountProofFetcher`): a round with + pending seeds and no fields fetcher short-circuits with + `ColdStartError::NoAccountFieldsFetcher` before issuing any read. +- Outcome recorded as `ColdStartResults.code_verifications: + Option`; it runs first, so an `accounts`-phase hard error + preserves it (never `NotAttempted`). +- Mismatched/purged addresses that also appear in `plan.accounts` are + refetched by the very next phase — resync falls out of existing machinery. +- `unverifiable` (transport) is **not** a round hard error (matches + probe_roots' per-address-failure stance), but it is visible in the report; + adapters that require verified code before serving gate on the report. +- Ordering rationale: `discover` executes sims; they must never run over an + unverified canonical claim. +- Idempotent across rounds: the `Pending` set empties after the first + successful pass. + +### 3.6 Snapshot-generation & verdict semantics (WS-8/WS-9 coherence) + +- `seed_account_code` and `etch_account_code` **bump** the snapshot + generation: they change executable state. (The no-bump exemption stays + storage-prefetch-only: `inject_storage_batch` materializes values the + pinned chain already had; a seed is a *claim*, an etch is a *mutation*.) +- `verify_code_seeds` on match does **not** bump: confirming a claim and + patching balance to pinned-block truth is materialization. On mismatch the + purge bumps (existing `apply_update` path) — exactly when readers must + notice. +- Freshness verdicts are unchanged by design: the validator's verify set is + storage slots; code is never in it. Etched code re-executes identically in + validator overlays (same cache), so `Confirmed*` verdicts over etched + contracts remain honest — they attest storage freshness, not code + canonicality, and the etched set is queryable for consumers who care. +- Root-gate hygiene (documented, not enforced): adapters should not list + etched accounts in `probe_roots` if they also locally mutate that + account's storage — the storageHash gate would report expected noise. + +## 4. Live registration — the new-pool path (primary consumer flow) + +Cache already running; factory event announces pool `P` with known +`(token0, token1, fee, tickSpacing)`: + +1. Adapter patches its runtime template → `seed_account_code(P, code)` + (no RPC; generation bumps). +2. Fire **two independent `eth_call`s concurrently** (both pinned to the + same block): + a. `verify_code_seeds()` — settles the claim, injects real balance. + b. the V3 full-sync `StorageProgram` — injects the pool's entire tick + range/liquidity/observations. + They must be separate calls: the sync program *overrides P's code* inside + its call, so `EXTCODEHASH(P)` in a merged multicall would hash the + override, not the deployment. Concurrent issue keeps it ~1 RTT. +3. Storage injections are valid regardless of the verify outcome (chain + storage is code-independent). On mismatch: `P` was purged — re-seed with + a corrected template or let `ensure_account` fetch real code, then + re-inject the (still valid) storage snapshot. On `not_deployed`: re-pin + forward, retry. + +Net: zero `eth_getCode`, zero `basic_ref` triple, one round trip in the +happy path. Cold start is the same story ×N pools: seed all, one fields +call verifies the fleet, `ensure_account` triples only run for failures. + +## 5. Consumer contract (evm-amm-state, informative) + +- **V2-style pairs / EIP-1167 clones:** one blob (or one designator) per + factory per chain covers every instance — immutable-free runtime code is + byte-identical, so one stored template seeds unbounded pools. +- **V3-style pools:** `factory/token0/token1/fee/tickSpacing/ + maxLiquidityPerTick` are Solidity `immutable`s **baked into runtime + code** — every pool's code hash differs. The adapter stores one template + per (factory, chain, compiler build) plus the immutable byte offsets + (discoverable by diffing two known pools' deployed code), patches per + pool, and lets code-hash verification catch any wrong offset/compiler + variant — mismatch degrades to one `eth_getCode`, never to wrong sims. +- Seed-state file and marks live entirely in evm-fork-cache; the adapter's + only obligations are (a) seed before cold start / on detection, (b) gate + serving on `CodeVerifyReport` if it requires verified code, (c) keep + etched addresses out of `probe_roots` it also mutates. + +## 6. Companion workstream: proof batching & root-gate cadence + +Same release window, separate commits. The reactive root gate today issues +one **serial** `eth_getProof` per tracked account per canonical block — the +default fetcher's own comment encodes the assumption being retired here +("Account targets are few, so a straightforward per-request loop … is +sufficient", `mod.rs:1715`). Two changes fix the shape: + +### 6.1 Batch the proof fan-out (call shape + default fetcher) + +- **Call sites**: `run_root_gate` (`reactive/mod.rs:1993`) and the + account-resync executor (`reactive/mod.rs:2938`) each invoke the seam once + per address. Both switch to **one seam invocation carrying the full target + list**, exactly as the cold-start `probe_roots` phase already does + (`driver.rs:176`). Failure handling is untouched: the contract already + returns per-address `Result`s, and both callers already treat a + failed/omitted entry as "no signal this round". +- **Default fetcher** (`mod.rs:1717`): replace the serial await loop with a + bounded, order-preserving fan-out + (`futures::stream::iter(..).buffered(cap)`). `eth_getProof` is + single-address at the RPC level, so concurrency is the only lever: + wall-clock per firing drops from `N × RTT` to `~ceil(N / cap) × RTT`. +- **Knob**: `EvmCacheBuilder::max_concurrent_proofs(usize)`, default **8** + (name-symmetric with `BulkCallConfig::max_concurrent_calls`). The seam + signature is unchanged; custom fetchers keep working as-is. + +### 6.2 `RootGateCadence` — per-block proofing is never the default + +```rust +/// How often the reactive root gate probes tracked accounts. +pub enum RootGateCadence { + /// Probe at most once every `n` canonical blocks. `EveryNBlocks(1)` is + /// the old per-block behavior; the default is 16. + EveryNBlocks(NonZeroU64), + /// Root gate off: coverage gaps surface only via decoders + freshness. + Disabled, +} +``` + +- Field + setter on `ReactiveRuntime`, default `EveryNBlocks(16)`. Applies + to `WholeAccount` and `Scalars` alike (`Slots` was never gated). +- **Why skipping blocks is safe**: the gate diffs `root_now` against the + *persisted baseline*, never block-over-block (phase-8 §6: a currency gate + across time). A move in any skipped block is still visible at the next + probe — cadence trades detection lag (≤ `n−1` blocks) for cost, never + eventual detection. Missed-block ranges stay caught for the same reason. +- **Correctness obligation — accumulate `touched`**: the gap rule is "root + moved ∧ addr ∉ touched". Under cadence, `touched` must be the **union of + decoder-touched addresses since the last firing** + (`touched_since_gate: HashSet
` on the runtime, drained when the + gate fires). Without accumulation, a decoder-covered write in a skipped + block would false-positive as a `CoverageGap` at the next probe. +- Firing rule: the first canonical block ever seen fires the gate (baseline + adoption must not wait `n` blocks); thereafter at most once per `n`. + Accounts registered mid-flight adopt at the next scheduled firing (≤ `n` + blocks; pre-adoption writes are not gap-checked — same as today, just up + to `n` blocks later). +- Cost shape, 100 `WholeAccount` pools on mainnet: per-block serial ≈ 600k + CU/h of `eth_getProof`; every-16-blocks ≈ 37k CU/h, with wall-clock per + firing cut ~8× by §6.1. Fast-block L2s should raise `n`, not lower it. + +## 7. Tests — the acceptance contract (offline, stub fetchers) + +1. Seed → `Pending`; stubbed match ⇒ `Verified`, balance patched, fields + fetcher **called exactly once ever** (call-count proves no + re-verification, including across a save/load). +2. Stubbed mismatch ⇒ purged from both layers, mark cleared, reported; + subsequent `ensure_account` refetches via stub backend. +3. `not_deployed` (zero hash) and `codeless` (`keccak("")`) classified into + their buckets, purged. +4. Transport failure ⇒ still `Pending`, reported `unverifiable`, nothing + purged. +5. Seed over RPC-origin equal-hash ⇒ instant `Verified`, no fetcher call; + over RPC-origin different-hash ⇒ `CodeSeedConflict`, cached code intact. +6. Etch ⇒ `Etched`, excluded from `verify_code_seeds`, sims read etched + code, `etched_accounts()` reports it; `override_account_code*` and + `deploy_contract` targets also appear. +7. Persistence round-trip: `Pending`/`Verified`/`Etched` survive save/load; + unmarked (RPC-origin) accounts untouched; missing `code_seeds.bin` loads + as empty (legacy caches); mark-without-code pruned on load. +8. Generation: seed bumps, etch bumps, verify-match does not, mismatch-purge + does (extends `snapshot_generation_bumps_on_writes_and_repins_not_prefetch`). +9. Driver: `verify_code` runs before `accounts` (mismatch in round N is + refetched by round N's accounts phase); guard fires as + `NoAccountFieldsFetcher` only for pending-bearing rounds; report survives + an accounts-phase hard error. +10. Seeded account never triggers `basic_ref` (call-count on stub backend). +11. Root gate fires on the first canonical block, then only on cadence + boundaries; `Disabled` never invokes the seam; `EveryNBlocks(1)` + reproduces the old per-block behavior. +12. Touched accumulation: a decoder-covered write in a skipped block does + **not** report a `CoverageGap` at the next firing (and the accumulator + drains); an uncovered root move still does. +13. One seam invocation per gate firing and per account-resync batch, + carrying every target (stub fetcher records call count + batch size). +14. A root move occurring in a skipped block is detected at the next firing + — cadence delays detection, never loses it. +15. (RPC-gated, `#[ignore]`) fan-out sanity: a ~50-address proof batch + through the default fetcher completes in ≪ 50 × single-proof latency. + +## 8. Decisions (locked) + +1. Two classes, three marks; absence of mark = RPC-origin. +2. Chain-fetched code beats templates: equal-hash seeds verify free, + conflicting seeds error, never overwrite. +3. Verification is one bulk fields call; proof-based (`AccountProof. + code_hash`) verification is a later zero-RPC optimization, not v1. +4. Mismatch ⇒ purge + report; refetch rides existing paths. Transport + failure ⇒ keep `Pending`. +5. `Verified` is durable (post-6780 immutability); escape hatch is + `purge_account`. +6. Persist all three marks; save marks before code (fail-closed ordering). +7. Driver phase is implicit over `pending_code_seeds()`, runs first, + guarded like the other fetcher-dependent phases. +8. Seed/etch bump the snapshot generation; verify-match does not. +9. `override_account_code*` / `deploy_contract` record `Etched`. +10. Root-gate and account-resync proof reads are single batched seam + invocations; the default fetcher fans out with bounded, order-preserving + concurrency (`max_concurrent_proofs`, default 8). +11. Root gate defaults to `EveryNBlocks(16)`; per-block is opt-in, never a + default. `touched` accumulates across skipped blocks, drained per firing. +12. Cold-start `probe_roots` is untouched (already batched; once per boot). + +## 9. Open decisions (confirm before building) + +1. **Nonce refinement via proofs** — when an account-proof fetcher is + installed, `verify_code_seeds` *could* also patch exact nonce/balance + from `eth_getProof`. Recommend: defer (fields sample suffices for AMMs; + revisit with a consumer that CREATEs). +2. **`etch_account_code` naming** — `etch` (foundry vocabulary, proposed) + vs. `set_local_account_code`. Recommend `etch`. +3. **Auto-verify outside the driver** — should the reactive runtime also + sweep `Pending` seeds on a timer/trigger, or is adapter-driven + `verify_code_seeds()` + the driver phase enough for v1? Recommend: + adapter-driven for v1 (the live-registration flow calls it explicitly). +4. **Time-based cadence** (`MinInterval(Duration)`) for chains with + irregular block times. Recommend: defer — blocks are the ingest loop's + native unit and need no clock injection in tests. +5. **Per-account cadence overrides** (gate a flagship pool tighter than the + fleet). Recommend: defer until a consumer asks. + +## 10. Build order (commit per step, green each time) + +- **C1** — marks + `seed_account_code`/`etch_account_code` + conflict rules + + `Etched` unification + generation semantics + `code_seeds.bin` + persistence (tests 1-partial, 5, 6, 7, 8, 10). +- **C2** — `AccountFieldsFetchFn` seam + default wiring + + `verify_code_seeds` + `CodeVerifyReport` (tests 1–4). +- **C3** — driver `verify_code` phase + `NoAccountFieldsFetcher` + + `ColdStartResults.code_verifications` (test 9); docs: README feature + bullet + production-checklist line, INTERNALS section, CHANGELOG 0.2.0 + Added entry, KNOWN_ISSUES note for the pre-6780 caveat. +- **C4** — batched proof fan-out: single seam invocation in `run_root_gate` + and the account-resync executor; `buffered(cap)` default fetcher + + `EvmCacheBuilder::max_concurrent_proofs` (tests 13, 15). +- **C5** — `RootGateCadence` + `touched_since_gate` accumulation + firing + rules (tests 11, 12, 14); CHANGELOG: the root gate's default moves from + per-block to every-16-blocks inside the existing 0.2.0 entry (branch is + unpublished, so it is a pre-release default choice, not a breaking + change). + +Estimated at the established quality bar: ~1.5 focused days (C1–C3 ≈ one +day, C4–C5 ≈ a half; no published-surface semantics change anywhere). + +## 11. Release integration + +Recommendation: **fold into 0.2.0.** The crate is unpublished (no semver +cost), `evm-amm-state`'s initial release wants the behavior, and the two +risks that argued for deferral — unverified-seed persistence and verdict +interaction — are resolved by §3.4 and §3.6 rather than left to +implementation judgment. The docs review then covers the final surface once +instead of twice. + +The companion workstream strengthens the case: C5 changes the root gate's +*default* behavior, and defaults are exactly what you want settled before +the first `cargo publish` — the published 0.2.0 should never have shipped a +per-block `eth_getProof` loop as its out-of-the-box posture. diff --git a/examples/bulk_storage_bench.rs b/examples/bulk_storage_bench.rs new file mode 100644 index 0000000..6fa354f --- /dev/null +++ b/examples/bulk_storage_bench.rs @@ -0,0 +1,986 @@ +//! Live benchmark: bulk storage extraction vs batched `eth_getStorageAt`. +//! +//! Measures the call-override bulk loader (`bulk_storage` module) against the +//! crate's default point-read batch fetcher on a real endpoint, and captures +//! the numbers recorded in `docs/bulk-storage-extraction.md`: +//! +//! 1. **Correctness spot-check** — bulk values must equal `eth_getStorageAt` +//! ground truth at the same pinned block. +//! 2. **Single-target scaling** — N slots of one contract (WETH), N up to 15k. +//! 3. **Multi-contract multicall** — 20 mainnet tokens × 25 slots in one call. +//! 4. **Uniswap V3 pool tick-range load** — the `evm-amm-state` cold-start +//! shape: statics + full tickBitmap, then every initialized tick + +//! observations, in 2-3 `eth_call`s total. +//! 5. **Gzip on/off** — end-to-end latency and raw wire bytes for the largest +//! nonzero-heavy response. +//! 6. **`eth_callMany` vs `eth_call`** — same payloads through both dispatch +//! modes (20 CU/request vs 26 CU/call on Alchemy). +//! 7. **Contract fleet** — 100 distinct contracts × 30 slots in one dispatch. +//! 8. **Custom storage program** — the one-shot V3 observation-ring loader +//! (data-dependent loads derived in-EVM, zero calldata). +//! 9. **Companion extractors** — account fields (balance + codehash) and +//! block context in one call each. +//! 10. **Chunk-ceiling probe** — raise slots-per-call until the provider +//! rejects it, to find the endpoint's real `eth_call` budget. +//! 11. **Verified code seeding** — cold-start materialization of N known +//! contracts: locally seeded templates + one bulk `verify_code_seeds` +//! call vs per-account `ensure_account` (balance + nonce + code reads). +//! +//! Gated on `RPC_URL` (skips when unset, like the other live benches): +//! +//! ```sh +//! RPC_URL=https://eth-mainnet.g.alchemy.com/v2/ \ +//! cargo run --release --example bulk_storage_bench +//! ``` +//! +//! Optional env knobs: `BULK_BENCH_SAMPLES` (default 3), +//! `BULK_BENCH_BASELINE_MAX` (default 1000 — caps how many slots the +//! *point-read* baseline fetches, since that path costs 20 CU per slot), +//! `BULK_BENCH_PROBE=0` (skip the ceiling probe), +//! `BULK_BENCH_SCENARIOS=4,11` (comma-separated scenario numbers to run; +//! default all — handy for refreshing one table without paying for the rest). +//! +//! The full default run costs roughly 130k CU on Alchemy, dominated by the +//! point-read baselines the bulk path is being compared against. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use alloy_eips::{BlockId, BlockNumberOrTag}; +use alloy_primitives::{Address, Bytes, I256, U256, address, hex, keccak256}; +use alloy_provider::network::AnyNetwork; +use alloy_provider::{Provider, RootProvider}; +use alloy_rpc_client::RpcClient; +use alloy_transport_http::Http; +use anyhow::{Context, Result, bail}; +use evm_fork_cache::bulk_storage::{ + BulkCallConfig, CallDispatch, STORAGE_EXTRACTOR_CODE, StorageProgram, + bulk_call_storage_fetcher, fetch_account_fields_bulk, fetch_block_context, pack_slots_calldata, + planned_call_count, run_storage_program, +}; +use evm_fork_cache::cache::{EvmCache, StorageBatchFetchFn}; + +/// Alchemy CU prices (https://www.alchemy.com/docs/reference/compute-unit-costs). +const CU_GET_STORAGE_AT: u64 = 20; +const CU_ETH_CALL: u64 = 26; + +const WETH: Address = address!("C02aaa39b223FE8D0A0e5C4F27eAD9083C756Cc2"); +/// Uniswap V3 USDC/WETH 0.05% pool (fee 500, tick spacing 10). +const USDC_WETH_V3_POOL: Address = address!("88e6A0c2dDD26FEEb64F039a2c41296FcB3f5640"); +const POOL_TICK_SPACING: i32 = 10; +/// Uniswap V3 pool storage layout: `ticks` mapping and `tickBitmap` mapping. +const POOL_TICKS_SLOT: u64 = 5; +const POOL_TICK_BITMAP_SLOT: u64 = 6; +const POOL_OBSERVATIONS_SLOT: u64 = 8; + +/// Well-known mainnet ERC-20s for the multi-contract scenario. Only used as +/// storage sources — the measured cost is identical whatever the slots hold. +const TOKENS: [Address; 20] = [ + WETH, + address!("A0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"), // USDC + address!("dAC17F958D2ee523a2206206994597C13D831ec7"), // USDT + address!("6B175474E89094C44Da98b954EedeAC495271d0F"), // DAI + address!("2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599"), // WBTC + address!("1f9840a85d5aF5bf1D1762F925BDADdC4201F984"), // UNI + address!("514910771AF9Ca656af840dff83E8264EcF986CA"), // LINK + address!("7Fc66500c84A76Ad7e9c93437bFc5Ac33E2DDaE9"), // AAVE + address!("9f8F72aA9304c8B593d555F12eF6589cC3A579A2"), // MKR + address!("5A98FcBEA516Cf06857215779Fd812CA3beF1B32"), // LDO + address!("ae7ab96520DE3A18E5e111B5EaAb095312D7fE84"), // stETH + address!("D533a949740bb3306d119CC777fa900bA034cd52"), // CRV + address!("c00e94Cb662C3520282E6f5717214004A7f26888"), // COMP + address!("C011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F"), // SNX + address!("0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e"), // YFI + address!("6B3595068778DD592e39A122f4f5a5cF09C90fE2"), // SUSHI + address!("111111111117dC0aa78b770fA6A738034120C302"), // 1INCH + address!("c944E90C64B2c07662A292be6244BDf05Cda44a7"), // GRT + address!("7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0"), // MATIC + address!("95aD61b0a150d79219dCF64E1E6Cc01f0B64C4cE"), // SHIB +]; + +fn env_usize(name: &str, default: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +fn make_provider(rpc_url: &str, gzip: bool) -> Result>> { + let mut builder = reqwest::Client::builder(); + builder = if gzip { + builder.gzip(true) + } else { + builder.no_gzip() + }; + let client = builder.build().context("build reqwest client")?; + let http = Http::with_client(client, rpc_url.parse().context("parse RPC_URL")?); + Ok(Arc::new(RootProvider::::new(RpcClient::new( + http, false, + )))) +} + +/// Deterministic pseudo-random slot keys (uniform 256-bit, like the Dedaub +/// harness) so runs are reproducible at a pinned block. +fn synthetic_slots(n: usize) -> Vec { + (0..n) + .map(|i| { + let mut seed = [0u8; 40]; + seed[..8].copy_from_slice(b"efc-bulk"); + seed[8..16].copy_from_slice(&(i as u64).to_be_bytes()); + U256::from_be_bytes(keccak256(seed).0) + }) + .collect() +} + +/// Storage key of `mapping(intN => ..)` entry: `keccak256(int256(key) . slot)`. +fn signed_mapping_key(key: i32, mapping_slot: u64) -> U256 { + let mut buf = [0u8; 64]; + buf[..32].copy_from_slice( + &I256::try_from(key) + .expect("fits i32") + .into_raw() + .to_be_bytes::<32>(), + ); + buf[32..].copy_from_slice(&U256::from(mapping_slot).to_be_bytes::<32>()); + U256::from_be_bytes(keccak256(buf).0) +} + +struct SampleStats { + median: Duration, + min: Duration, + max: Duration, + ok: usize, + err: usize, + first_error: Option, +} + +/// Run `fetcher` `samples` times over the same request set and report medians. +/// Fetchers are the crate's synchronous seam; on this multi-thread runtime the +/// internal `block_in_place` bridge is valid. +fn run_samples( + fetcher: &StorageBatchFetchFn, + requests: &[(Address, U256)], + block: BlockId, + samples: usize, +) -> SampleStats { + let mut durations = Vec::with_capacity(samples); + let (mut ok, mut err, mut first_error) = (0usize, 0usize, None); + for _ in 0..samples { + let started = Instant::now(); + let results = fetcher(requests.to_vec(), block); + durations.push(started.elapsed()); + ok = results.iter().filter(|(_, _, r)| r.is_ok()).count(); + err = results.len() - ok; + if first_error.is_none() { + first_error = results + .iter() + .find_map(|(_, _, r)| r.as_ref().err().map(|e| e.to_string())); + } + std::thread::sleep(Duration::from_millis(250)); + } + durations.sort(); + SampleStats { + median: durations[durations.len() / 2], + min: durations[0], + max: durations[durations.len() - 1], + ok, + err, + first_error, + } +} + +fn fetch_map( + fetcher: &StorageBatchFetchFn, + requests: &[(Address, U256)], + block: BlockId, +) -> Result> { + let mut map = std::collections::HashMap::with_capacity(requests.len()); + for (addr, slot, result) in fetcher(requests.to_vec(), block) { + match result { + Ok(value) => { + map.insert((addr, slot), value); + } + Err(e) => bail!("fetch failed for {addr} slot {slot:#x}: {e}"), + } + } + Ok(map) +} + +fn ms(d: Duration) -> String { + format!("{:.0} ms", d.as_secs_f64() * 1000.0) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let rpc_url = match std::env::var("RPC_URL") { + Ok(url) if !url.trim().is_empty() => url, + _ => { + eprintln!( + "RPC_URL not set — skipping bulk_storage_bench. \ + Set RPC_URL= to run it." + ); + return Ok(()); + } + }; + let samples = env_usize("BULK_BENCH_SAMPLES", 3).max(1); + let baseline_max = env_usize("BULK_BENCH_BASELINE_MAX", 1000); + let run_probe = std::env::var("BULK_BENCH_PROBE").as_deref() != Ok("0"); + let scenarios: Option> = + std::env::var("BULK_BENCH_SCENARIOS").ok().map(|raw| { + raw.split(',') + .filter_map(|s| s.trim().parse().ok()) + .collect() + }); + let enabled = |n: usize| scenarios.as_ref().is_none_or(|set| set.contains(&n)); + + let gzip_provider = make_provider(&rpc_url, true)?; + let identity_provider = make_provider(&rpc_url, false)?; + + let chain_id = gzip_provider.get_chain_id().await?; + let latest = gzip_provider.get_block_number().await?; + // Pin a few blocks back so every sample reads settled, identical state. + let pinned = latest.saturating_sub(8); + let block = BlockId::Number(BlockNumberOrTag::Number(pinned)); + println!("# Bulk storage extraction — live benchmark\n"); + println!("- chain id: {chain_id}"); + println!("- pinned block: {pinned}"); + println!("- samples per measurement: {samples}"); + println!("- CU prices: eth_getStorageAt = {CU_GET_STORAGE_AT}, eth_call = {CU_ETH_CALL}\n"); + + // The production baseline: the cache's own default point-read batch + // fetcher (JSON-RPC batches of eth_getStorageAt, Slow preset by default). + let cache = EvmCache::builder(gzip_provider.clone()) + .block(block) + .build() + .await; + let point_fetcher = cache + .storage_batch_fetcher() + .cloned() + .context("provider-backed cache exposes the default fetcher")?; + let config = BulkCallConfig::default(); + let bulk = bulk_call_storage_fetcher(gzip_provider.clone(), config); + let bulk_identity = bulk_call_storage_fetcher(identity_provider.clone(), config); + + if enabled(1) { + scenario_correctness(&bulk, &point_fetcher, block)?; + } + if enabled(2) { + scenario_single_target(&bulk, &point_fetcher, block, samples, baseline_max, config)?; + } + if enabled(3) { + scenario_multi_target(&bulk, &point_fetcher, block, samples, baseline_max, config)?; + } + let tick_slots = if enabled(4) { + scenario_univ3_pool(&bulk, &point_fetcher, block, samples, config)? + } else { + Vec::new() + }; + if enabled(5) { + scenario_gzip( + &bulk, + &bulk_identity, + &rpc_url, + block, + pinned, + samples, + &tick_slots, + ) + .await?; + } + if enabled(6) { + scenario_call_many(&gzip_provider, &bulk, block, samples, &tick_slots)?; + } + if enabled(7) { + scenario_fleet(&bulk, block, samples, config)?; + } + if enabled(8) { + scenario_custom_program(&gzip_provider, &bulk, block).await?; + } + if enabled(9) { + scenario_companion_extractors(&gzip_provider, block, pinned).await?; + } + if run_probe && enabled(10) { + scenario_ceiling(&gzip_provider, block).await; + } + if enabled(11) { + scenario_code_seeding(&gzip_provider, block, samples).await?; + } + + println!("\nDone. Copy the tables above into docs/bulk-storage-extraction.md."); + Ok(()) +} + +/// 1. Bulk values must be byte-identical to eth_getStorageAt ground truth. +fn scenario_correctness( + bulk: &StorageBatchFetchFn, + point: &StorageBatchFetchFn, + block: BlockId, +) -> Result<()> { + println!("## 1. Correctness spot-check\n"); + let mut requests: Vec<(Address, U256)> = Vec::new(); + for slot in 0..5u64 { + requests.push((WETH, U256::from(slot))); + } + for slot in 0..9u64 { + requests.push((USDC_WETH_V3_POOL, U256::from(slot))); + } + // Two real WETH balance slots: keccak256(holder . 3). + for holder in [ + address!("28C6c06298d514Db089934071355E5743bf21d60"), + address!("F04a5cC80B1E94C69B48f5ee68a08CD2F09A7c3E"), + ] { + let mut buf = [0u8; 64]; + buf[12..32].copy_from_slice(holder.as_slice()); + buf[32..].copy_from_slice(&U256::from(3u64).to_be_bytes::<32>()); + requests.push((WETH, U256::from_be_bytes(keccak256(buf).0))); + } + + let bulk_values = fetch_map(bulk, &requests, block)?; + let point_values = fetch_map(point, &requests, block)?; + let mut nonzero = 0usize; + for key in bulk_values.keys() { + let (b, p) = (bulk_values[key], point_values[key]); + if b != p { + bail!( + "MISMATCH at {} slot {:#x}: bulk={b:#x} point={p:#x}", + key.0, + key.1 + ); + } + if !b.is_zero() { + nonzero += 1; + } + } + println!( + "{} slots verified identical to eth_getStorageAt ({} nonzero).\n", + requests.len(), + nonzero + ); + Ok(()) +} + +/// 2. N slots of one contract: bulk vs point-read baseline. +fn scenario_single_target( + bulk: &StorageBatchFetchFn, + point: &StorageBatchFetchFn, + block: BlockId, + samples: usize, + baseline_max: usize, + config: BulkCallConfig, +) -> Result<()> { + println!("## 2. Single-target scaling (WETH, pseudo-random slots)\n"); + println!("| Slots | Bulk calls | Bulk median | Bulk CU | Point median | Point CU | CU ratio |"); + println!("| ---: | ---: | ---: | ---: | ---: | ---: | ---: |"); + for n in [10usize, 100, 1_000, 5_000, 10_000, 15_000] { + let requests: Vec<(Address, U256)> = + synthetic_slots(n).into_iter().map(|s| (WETH, s)).collect(); + let calls = planned_call_count(&requests, &config); + let bulk_stats = run_samples(bulk, &requests, block, samples); + let bulk_cu = calls as u64 * CU_ETH_CALL; + let point_cu = n as u64 * CU_GET_STORAGE_AT; + + let point_cell = if n <= baseline_max { + let stats = run_samples(point, &requests, block, samples); + if stats.err > 0 { + format!( + "{} ({} errs: {})", + ms(stats.median), + stats.err, + stats.first_error.as_deref().unwrap_or("?") + ) + } else { + ms(stats.median) + } + } else { + "— (skipped)".to_string() + }; + let bulk_cell = if bulk_stats.err > 0 { + format!( + "{} ({} errs: {})", + ms(bulk_stats.median), + bulk_stats.err, + bulk_stats.first_error.as_deref().unwrap_or("?") + ) + } else { + ms(bulk_stats.median) + }; + println!( + "| {n} | {calls} | {bulk_cell} | {bulk_cu} | {point_cell} | {point_cu} | {:.0}x |", + point_cu as f64 / bulk_cu as f64 + ); + let _ = bulk_stats.ok; + } + println!(); + Ok(()) +} + +/// 3. Many contracts in one multicall dispatch. +fn scenario_multi_target( + bulk: &StorageBatchFetchFn, + point: &StorageBatchFetchFn, + block: BlockId, + samples: usize, + baseline_max: usize, + config: BulkCallConfig, +) -> Result<()> { + println!("## 3. Multi-contract multicall (20 tokens x 25 slots = 500 slots)\n"); + let mut requests: Vec<(Address, U256)> = Vec::new(); + for token in TOKENS { + for slot in 0..25u64 { + requests.push((token, U256::from(slot))); + } + } + let calls = planned_call_count(&requests, &config); + let bulk_stats = run_samples(bulk, &requests, block, samples); + println!( + "- bulk: {} calls, median {} (min {}, max {}), {} CU, {} errs", + calls, + ms(bulk_stats.median), + ms(bulk_stats.min), + ms(bulk_stats.max), + calls as u64 * CU_ETH_CALL, + bulk_stats.err, + ); + if requests.len() <= baseline_max { + let point_stats = run_samples(point, &requests, block, samples); + println!( + "- point reads: median {} (min {}, max {}), {} CU, {} errs", + ms(point_stats.median), + ms(point_stats.min), + ms(point_stats.max), + requests.len() as u64 * CU_GET_STORAGE_AT, + point_stats.err, + ); + } + println!(); + Ok(()) +} + +/// Scenario 4 — the evm-amm-state shape: statics + full tickBitmap, then +/// every initialized tick + observations. Returns the phase-2 slot list for +/// the gzip scenario. +fn scenario_univ3_pool( + bulk: &StorageBatchFetchFn, + point: &StorageBatchFetchFn, + block: BlockId, + samples: usize, + config: BulkCallConfig, +) -> Result> { + println!("## 4. Uniswap V3 USDC/WETH 0.05% pool — full tick-range load\n"); + + // Phase 1: statics (slot0..8) + every tickBitmap word over the full range. + let compressed_bound = 887_272 / POOL_TICK_SPACING; // MIN/MAX_TICK / spacing + let (min_word, max_word) = ( + (-compressed_bound) >> 8, // arithmetic shift = floor division + compressed_bound >> 8, + ); + let mut phase1: Vec<(Address, U256)> = (0..9u64) + .map(|slot| (USDC_WETH_V3_POOL, U256::from(slot))) + .collect(); + let mut word_keys = Vec::new(); + for word in min_word..=max_word { + let key = signed_mapping_key(word, POOL_TICK_BITMAP_SLOT); + word_keys.push((word, key)); + phase1.push((USDC_WETH_V3_POOL, key)); + } + + let phase1_calls = planned_call_count(&phase1, &config); + let phase1_stats = run_samples(bulk, &phase1, block, samples); + let values = fetch_map(bulk, &phase1, block)?; + + // Decode: observation cardinality from slot0, initialized ticks from the + // bitmap words. + let slot0 = values[&(USDC_WETH_V3_POOL, U256::from(0u64))]; + let cardinality: u64 = ((slot0 >> 200usize) & U256::from(0xffffu64)).to::(); + let mut ticks: Vec = Vec::new(); + for (word, key) in &word_keys { + let bitmap = values[&(USDC_WETH_V3_POOL, *key)]; + if bitmap.is_zero() { + continue; + } + for bit in 0..256usize { + if bitmap.bit(bit) { + ticks.push((word * 256 + bit as i32) * POOL_TICK_SPACING); + } + } + } + + // Phase 2: 4 slots per initialized tick + the observation ring. + let mut phase2: Vec<(Address, U256)> = Vec::new(); + for tick in &ticks { + let base = signed_mapping_key(*tick, POOL_TICKS_SLOT); + for offset in 0..4u64 { + phase2.push((USDC_WETH_V3_POOL, base + U256::from(offset))); + } + } + for i in 0..cardinality { + phase2.push((USDC_WETH_V3_POOL, U256::from(POOL_OBSERVATIONS_SLOT + i))); + } + let phase2_calls = planned_call_count(&phase2, &config); + let phase2_stats = run_samples(bulk, &phase2, block, samples); + let phase2_values = fetch_map(bulk, &phase2, block)?; + + // Sanity: every initialized tick must have nonzero liquidityGross, and a + // spot sample must match point-read ground truth. + let mut zero_gross = 0usize; + for tick in &ticks { + let base = signed_mapping_key(*tick, POOL_TICKS_SLOT); + if phase2_values[&(USDC_WETH_V3_POOL, base)].is_zero() { + zero_gross += 1; + } + } + let spot: Vec<(Address, U256)> = phase2 + .iter() + .step_by(phase2.len().max(1) / 8 + 1) + .copied() + .collect(); + let ground_truth = fetch_map(point, &spot, block)?; + for (key, expected) in &ground_truth { + if phase2_values[key] != *expected { + bail!("tick-slot mismatch at {:#x}", key.1); + } + } + + let total_slots = phase1.len() + phase2.len(); + let total_calls = phase1_calls + phase2_calls; + println!( + "- initialized ticks: {}, observation cardinality: {cardinality}", + ticks.len() + ); + println!( + "- phase 1 (statics + {} bitmap words): {} slots, {} call(s), median {}", + word_keys.len(), + phase1.len(), + phase1_calls, + ms(phase1_stats.median), + ); + println!( + "- phase 2 (ticks + observations): {} slots, {} call(s), median {}", + phase2.len(), + phase2_calls, + ms(phase2_stats.median), + ); + println!( + "- total: {} slots in {} eth_calls = {} CU (vs {} CU as point reads, {:.0}x cheaper)", + total_slots, + total_calls, + total_calls as u64 * CU_ETH_CALL, + total_slots as u64 * CU_GET_STORAGE_AT, + (total_slots as u64 * CU_GET_STORAGE_AT) as f64 / (total_calls as u64 * CU_ETH_CALL) as f64, + ); + println!( + "- spot-verified {} slots against eth_getStorageAt; {} ticks with zero liquidityGross\n", + ground_truth.len(), + zero_gross, + ); + Ok(phase2) +} + +/// 5. Gzip vs identity on the largest nonzero-heavy payload. +async fn scenario_gzip( + bulk_gzip: &StorageBatchFetchFn, + bulk_identity: &StorageBatchFetchFn, + rpc_url: &str, + block: BlockId, + pinned: u64, + samples: usize, + tick_slots: &[(Address, U256)], +) -> Result<()> { + println!("## 5. Gzip vs identity (tick-range payload)\n"); + if tick_slots.is_empty() { + println!("(no tick slots — skipped)\n"); + return Ok(()); + } + let gzip_stats = run_samples(bulk_gzip, tick_slots, block, samples); + let identity_stats = run_samples(bulk_identity, tick_slots, block, samples); + println!( + "- end-to-end ({} slots): gzip median {}, identity median {}", + tick_slots.len(), + ms(gzip_stats.median), + ms(identity_stats.median), + ); + + // Wire-level: one raw eth_call with auto-decompression disabled so the + // compressed byte count is observable. + let slots: Vec = tick_slots.iter().map(|(_, s)| *s).collect(); + let calldata = pack_slots_calldata(&slots); + let mut overrides = serde_json::Map::new(); + overrides.insert( + format!("{USDC_WETH_V3_POOL}"), + serde_json::json!({ "code": format!("0x{}", hex::encode(STORAGE_EXTRACTOR_CODE)) }), + ); + let body = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_call", + "params": [ + { "to": USDC_WETH_V3_POOL, "data": format!("0x{}", hex::encode(&calldata)) }, + format!("{pinned:#x}"), + overrides, + ], + }); + let raw_client = reqwest::Client::builder().no_gzip().build()?; + for encoding in ["identity", "gzip"] { + let started = Instant::now(); + let response = raw_client + .post(rpc_url) + .header("Accept-Encoding", encoding) + .json(&body) + .send() + .await?; + let served = response + .headers() + .get("content-encoding") + .and_then(|v| v.to_str().ok()) + .unwrap_or("identity") + .to_string(); + let bytes = response.bytes().await?; + println!( + "- wire ({encoding:>8}): {:>9} bytes in {}, content-encoding: {served}", + bytes.len(), + ms(started.elapsed()), + ); + } + println!(); + Ok(()) +} + +/// Scenario 6 — eth_callMany (20 CU/request on Alchemy) vs per-call +/// eth_call (26 CU each): same payloads, both dispatch modes. +fn scenario_call_many( + provider: &Arc>, + bulk_per_call: &StorageBatchFetchFn, + block: BlockId, + samples: usize, + tick_slots: &[(Address, U256)], +) -> Result<()> { + println!("## 6. eth_callMany vs eth_call dispatch\n"); + let call_many = bulk_call_storage_fetcher( + provider.clone(), + BulkCallConfig { + dispatch: CallDispatch::CallMany, + ..BulkCallConfig::default() + }, + ); + + if !tick_slots.is_empty() { + let per_call = run_samples(bulk_per_call, tick_slots, block, samples); + let many = run_samples(&call_many, tick_slots, block, samples); + println!( + "- tick payload ({} slots, 1 chunk): eth_call median {} / 26 CU, eth_callMany median {} / 20 CU ({} errs)", + tick_slots.len(), + ms(per_call.median), + ms(many.median), + many.err, + ); + } + + // A 25k-slot job: three 10k chunks per-call vs one callMany request. + let big: Vec<(Address, U256)> = synthetic_slots(25_000) + .into_iter() + .map(|s| (WETH, s)) + .collect(); + let per_call = run_samples(bulk_per_call, &big, block, samples); + let many = run_samples(&call_many, &big, block, samples); + println!( + "- 25,000 slots: eth_call 3 chunks median {} / 78 CU, eth_callMany 1 request median {} / 20 CU ({} errs)\n", + ms(per_call.median), + ms(many.median), + many.err, + ); + Ok(()) +} + +/// Scenario 7 — fleet dispatch: 100 distinct contracts × 30 slots through one +/// multicall. Synthetic addresses (empty accounts) keep this honest about +/// dispatch overhead — gas costs are identical whatever the slots hold. +fn scenario_fleet( + bulk: &StorageBatchFetchFn, + block: BlockId, + samples: usize, + config: BulkCallConfig, +) -> Result<()> { + println!("## 7. Contract fleet (100 contracts x 30 slots = 3,000 slots)\n"); + let mut requests: Vec<(Address, U256)> = Vec::new(); + for c in 0..100u64 { + let mut seed = [0u8; 16]; + seed[..8].copy_from_slice(b"efcfleet"); + seed[8..].copy_from_slice(&c.to_be_bytes()); + let addr = Address::from_slice(&keccak256(seed)[12..]); + for slot in 0..30u64 { + requests.push((addr, U256::from(slot))); + } + } + let calls = planned_call_count(&requests, &config); + let stats = run_samples(bulk, &requests, block, samples); + println!( + "- {} slots across 100 contracts: {} call(s), median {} (min {}, max {}), {} CU vs {} CU as point reads ({} errs)\n", + requests.len(), + calls, + ms(stats.median), + ms(stats.min), + ms(stats.max), + calls as u64 * CU_ETH_CALL, + requests.len() as u64 * CU_GET_STORAGE_AT, + stats.err, + ); + Ok(()) +} + +/// The one-shot Uniswap V3 observation-ring loader: reads the ring +/// cardinality from slot0 *inside the EVM*, then returns the whole ring — +/// zero calldata, one call. The offline revm test for this exact bytecode +/// lives in `tests/bulk_storage.rs`. +const OBSERVATION_RING_PROGRAM: &[u8] = + &hex!("5f5460c81c61ffff165f5b81811460215780600801548160051b52600101600a565b5060051b5ff3"); + +/// Scenario 8 — a custom storage program (data-dependent loads in-EVM). +async fn scenario_custom_program( + provider: &Arc>, + bulk: &StorageBatchFetchFn, + block: BlockId, +) -> Result<()> { + println!("## 8. Custom storage program: one-shot V3 observation ring\n"); + let program = StorageProgram { + target: USDC_WETH_V3_POOL, + code: alloy_primitives::Bytes::from_static(OBSERVATION_RING_PROGRAM), + calldata: alloy_primitives::Bytes::new(), + }; + let started = Instant::now(); + let bytes = run_storage_program(provider.as_ref(), block, &program) + .await + .map_err(|e| anyhow::anyhow!("program failed: {e}"))?; + let elapsed = started.elapsed(); + let cardinality = bytes.len() / 32; + + // Ground truth: the same ring via slot-list extraction. + let ring_slots: Vec<(Address, U256)> = (0..cardinality as u64) + .map(|i| (USDC_WETH_V3_POOL, U256::from(POOL_OBSERVATIONS_SLOT + i))) + .collect(); + let expected = fetch_map(bulk, &ring_slots, block)?; + for (i, chunk) in bytes.chunks_exact(32).enumerate() { + let key = ( + USDC_WETH_V3_POOL, + U256::from(POOL_OBSERVATIONS_SLOT + i as u64), + ); + if U256::from_be_slice(chunk) != expected[&key] { + anyhow::bail!("program output diverged from slot-list extraction at index {i}"); + } + } + println!( + "- {cardinality} observation slots in ONE call with ZERO calldata, {} — the program \ + derived the ring size from slot0 in-EVM; all values match slot-list extraction.\n", + ms(elapsed), + ); + Ok(()) +} + +/// Scenario 9 — companion extractors: account fields + block context. +async fn scenario_companion_extractors( + provider: &Arc>, + block: BlockId, + pinned: u64, +) -> Result<()> { + println!("## 9. Companion extractors\n"); + let started = Instant::now(); + let fields = fetch_account_fields_bulk(provider.as_ref(), &TOKENS, block) + .await + .map_err(|e| anyhow::anyhow!("account fields failed: {e}"))?; + let nonzero_balances = fields.iter().filter(|(_, f)| !f.balance.is_zero()).count(); + println!( + "- account fields: balance + codehash for {} contracts in one 26-CU call, {} \ + (vs {} CU via eth_getBalance + eth_getCode); {} with nonzero native balance", + fields.len(), + ms(started.elapsed()), + fields.len() as u64 * 2 * 20, + nonzero_balances, + ); + + let started = Instant::now(); + let ctx = fetch_block_context(provider.as_ref(), block) + .await + .map_err(|e| anyhow::anyhow!("block context failed: {e}"))?; + anyhow::ensure!(ctx.number == pinned, "context block must match the pin"); + println!( + "- block context: number={} timestamp={} basefee={} gas_limit={} chain_id={} in one call, {}\n", + ctx.number, + ctx.timestamp, + ctx.basefee, + ctx.gas_limit, + ctx.chain_id, + ms(started.elapsed()), + ); + Ok(()) +} + +/// Scenario 10 — raise slots-per-call until the provider says no. +async fn scenario_ceiling(provider: &Arc>, block: BlockId) { + println!("## 10. Chunk-ceiling probe (single eth_call, ~2,664 gas/slot)\n"); + println!("| Slots/call | Est. gas | Result |"); + println!("| ---: | ---: | --- |"); + for n in [15_000usize, 20_000, 25_000, 30_000, 40_000, 50_000] { + let config = BulkCallConfig { + max_slots_per_call: n, + max_concurrent_calls: 1, + ..BulkCallConfig::default() + }; + let fetcher = bulk_call_storage_fetcher(provider.clone(), config); + let requests: Vec<(Address, U256)> = + synthetic_slots(n).into_iter().map(|s| (WETH, s)).collect(); + let started = Instant::now(); + let results = tokio::task::spawn_blocking({ + let requests = requests.clone(); + move || fetcher(requests, block) + }) + .await + .expect("probe task"); + let elapsed = started.elapsed(); + let errs = results.iter().filter(|(_, _, r)| r.is_err()).count(); + let est_gas = n as u64 * 2_664; + if errs == 0 { + println!("| {n} | {est_gas} | ok in {} |", ms(elapsed)); + } else { + let first = results + .iter() + .find_map(|(_, _, r)| r.as_ref().err().map(|e| e.to_string())) + .unwrap_or_default(); + let trimmed: String = first.chars().take(120).collect(); + println!("| {n} | {est_gas} | FAILED: {trimmed} |"); + break; + } + } + println!(); +} + +/// Scenario 11 — verified code seeding: the 0.2.0 cold-start path where the +/// adapter already embeds each contract's deployed bytecode. Seeding writes +/// the templates locally and ONE bulk account-fields `eth_call` verifies every +/// claim (and materializes real balances), versus the classic materialization +/// of the same accounts via `ensure_account` — three point reads each +/// (`eth_getBalance` + `eth_getTransactionCount` + `eth_getCode`), with the +/// full runtime bytecode on the wire. +async fn scenario_code_seeding( + provider: &Arc>, + block: BlockId, + samples: usize, +) -> Result<()> { + println!("## 11. Verified code seeding (cold-start account materialization)\n"); + + // The templates an adapter would embed at build time — fetched once here, + // untimed, purely to have byte-exact runtime code for the pinned block. + let mut templates: Vec<(Address, Bytes)> = Vec::with_capacity(TOKENS.len()); + for token in TOKENS { + let code = provider.get_code_at(token).block_id(block).await?; + anyhow::ensure!(!code.is_empty(), "{token} should have runtime code"); + templates.push((token, code)); + } + let code_bytes: usize = templates.iter().map(|(_, code)| code.len()).sum(); + + // Baseline: fresh cache, `ensure_account` per contract — the pre-seeding + // way to materialize known accounts before simulating against them. + let mut baseline = Vec::with_capacity(samples); + for _ in 0..samples { + let mut cache = EvmCache::builder(provider.clone()) + .block(block) + .build() + .await; + let started = Instant::now(); + for (token, _) in &templates { + cache + .ensure_account(*token) + .await + .map_err(|e| anyhow::anyhow!("ensure_account({token}): {e}"))?; + } + baseline.push(started.elapsed()); + tokio::time::sleep(Duration::from_millis(250)).await; + } + baseline.sort(); + + // Seeded: fresh cache, templates written locally, one bulk verify call. + let mut seeded = Vec::with_capacity(samples); + for _ in 0..samples { + let mut cache = EvmCache::builder(provider.clone()) + .block(block) + .build() + .await; + let started = Instant::now(); + for (token, code) in &templates { + cache + .seed_account_code(*token, code.clone()) + .map_err(|e| anyhow::anyhow!("seed_account_code({token}): {e}"))?; + } + let report = cache + .verify_code_seeds() + .map_err(|e| anyhow::anyhow!("verify_code_seeds: {e}"))?; + seeded.push(started.elapsed()); + anyhow::ensure!( + report.verified.len() == templates.len(), + "every template should verify: {report:?}" + ); + anyhow::ensure!( + cache.pending_code_seeds().is_empty(), + "no claim should stay pending after a clean sweep" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + seeded.sort(); + + let n = templates.len() as u64; + // ensure_account = eth_getBalance + eth_getTransactionCount + eth_getCode, + // each 20 CU on Alchemy. + let baseline_cu = n * 3 * CU_GET_STORAGE_AT; + let (baseline_median, seeded_median) = (baseline[samples / 2], seeded[samples / 2]); + println!( + "- templates: {} contracts, {} bytes of runtime code", + templates.len(), + code_bytes, + ); + println!( + "- baseline ensure_account x {}: median {} (min {}, max {}), {} RPCs, {} CU, ~{} KB code on the wire", + templates.len(), + ms(baseline_median), + ms(baseline[0]), + ms(baseline[samples - 1]), + n * 3, + baseline_cu, + code_bytes * 2 / 1024, // hex-encoded JSON roughly doubles the bytes + ); + println!( + "- seed + verify_code_seeds: median {} (min {}, max {}), ONE eth_call, {} CU, 0 code bytes on the wire ({:.0}x cheaper, {:.1}x faster; real balances materialized from the same call)", + ms(seeded_median), + ms(seeded[0]), + ms(seeded[samples - 1]), + CU_ETH_CALL, + baseline_cu as f64 / CU_ETH_CALL as f64, + baseline_median.as_secs_f64() / seeded_median.as_secs_f64(), + ); + + // Fail-closed spot check: a wrong template and a nonexistent address are + // classified (and purged) by the same single call, cache left clean. + let mut cache = EvmCache::builder(provider.clone()) + .block(block) + .build() + .await; + let wrong_template = templates[0].1.clone(); // WETH runtime... + cache + .seed_account_code(templates[1].0, wrong_template.clone()) + .map_err(|e| anyhow::anyhow!("seed wrong template: {e}"))?; // ...claimed at USDC + let ghost = Address::from_slice(&keccak256(b"efc-seed-ghost")[12..]); + cache + .seed_account_code(ghost, wrong_template) + .map_err(|e| anyhow::anyhow!("seed ghost: {e}"))?; + let report = cache + .verify_code_seeds() + .map_err(|e| anyhow::anyhow!("verify fail-closed pair: {e}"))?; + anyhow::ensure!( + report.mismatched.len() == 1 && report.not_deployed.len() == 1, + "expected one mismatch + one not-deployed: {report:?}" + ); + println!( + "- fail-closed: wrong template -> mismatched (expected {}.. vs actual {}..) and purged; \ + unknown address -> not_deployed; one call classified both\n", + &format!("{:#x}", report.mismatched[0].expected)[..10], + &format!("{:#x}", report.mismatched[0].actual)[..10], + ); + Ok(()) +} diff --git a/examples/bundle_simulation.rs b/examples/bundle_simulation.rs index ba67cb7..bf06aa0 100644 --- a/examples/bundle_simulation.rs +++ b/examples/bundle_simulation.rs @@ -91,7 +91,7 @@ async fn main() -> Result<()> { ), ]; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let result = overlay.simulate_bundle( &bundle, &BundleOptions { @@ -122,7 +122,7 @@ async fn main() -> Result<()> { BundleTx::new(alice, token, transfer(bob, 100)), BundleTx::new(alice, token, Bytes::from(vec![0xde, 0xad, 0xbe, 0xef])), ]; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut atomic = EvmOverlay::new(snapshot.clone(), None); let r = atomic.simulate_bundle( diff --git a/examples/call_tracer.rs b/examples/call_tracer.rs index c41af67..de9cdba 100644 --- a/examples/call_tracer.rs +++ b/examples/call_tracer.rs @@ -101,7 +101,7 @@ async fn main() -> Result<()> { ]; let calldata = Bytes::from(IMulticall3::aggregate3Call { calls }.abi_encode()); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let (_result, tracer) = overlay.call_raw_with_inspector( caller, MULTICALL3_ADDRESS, @@ -125,7 +125,7 @@ async fn main() -> Result<()> { } .abi_encode(), ); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let (_result, stack) = overlay.call_raw_with_inspector( alice, token, diff --git a/examples/cold_start.rs b/examples/cold_start.rs index 2a0f74f..b533821 100644 --- a/examples/cold_start.rs +++ b/examples/cold_start.rs @@ -110,8 +110,8 @@ async fn main() -> Result<()> { // A stub fetcher standing in for the chain: the owner's true on-chain balance // is 1_000_000; everything else reads as a genuine zero. - let fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { + let fetcher: StorageBatchFetchFn = + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { requests .into_iter() .map(|(a, s)| { @@ -123,8 +123,7 @@ async fn main() -> Result<()> { (a, s, Ok(value)) }) .collect() - }, - ); + }); cache.set_storage_batch_fetcher(fetcher); println!( diff --git a/examples/fetch_minimization_counted.rs b/examples/fetch_minimization_counted.rs index e160970..b26dd0a 100644 --- a/examples/fetch_minimization_counted.rs +++ b/examples/fetch_minimization_counted.rs @@ -50,15 +50,13 @@ fn balance_slot(owner: Address) -> U256 { /// An `AtomicUsize`-counting batch fetcher: tallies every requested slot and /// returns a canned non-zero balance, standing in for the RPC backend. fn counting_fetcher(counter: Arc) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { - counter.fetch_add(requests.len(), Ordering::Relaxed); - requests - .into_iter() - .map(|(addr, slot)| (addr, slot, Ok(U256::from(1_000u64)))) - .collect() - }, - ) + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + counter.fetch_add(requests.len(), Ordering::Relaxed); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::from(1_000u64)))) + .collect() + }) } /// Build an offline cache with a MockERC20 installed at `token` (storage left @@ -102,7 +100,7 @@ async fn main() -> Result<()> { let warmup_fetches = counter.load(Ordering::Relaxed); // Freeze the warmed state, then fan N candidates out over cheap overlays. - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); counter.store(0, Ordering::Relaxed); // measure only the fan-out from here for c in 0..N_CANDIDATES { diff --git a/examples/freshness_multi_sim.rs b/examples/freshness_multi_sim.rs index c837b12..2f048ef 100644 --- a/examples/freshness_multi_sim.rs +++ b/examples/freshness_multi_sim.rs @@ -84,8 +84,8 @@ async fn main() -> Result<()> { ((token, balance_slot(senders[1])), U256::from(50)), // changed! ((token, balance_slot(senders[2])), U256::from(1000)), ]); - let fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { + let fetcher: StorageBatchFetchFn = + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { requests .into_iter() .map(|(addr, slot)| { @@ -93,8 +93,7 @@ async fn main() -> Result<()> { (addr, slot, Ok(v)) }) .collect() - }, - ); + }); cache.set_storage_batch_fetcher(fetcher); // ── Classification layer ─────────────────────────────────────────────── @@ -137,13 +136,17 @@ async fn main() -> Result<()> { .collect(); println!("optimistic (against the snapshot): {optimistic_ok:?} (all succeed)"); - match sim.validate().await { - Validation::Corrected { results, changed } => { + match sim.validate().await? { + Validation::Corrected { + results, + changed_slots, + .. + } => { println!( "\nvalidation: Corrected — {} slot(s) changed:", - changed.len() + changed_slots.len() ); - for c in &changed { + for c in &changed_slots { println!(" sender slot {} : {} -> {}", c.slot, c.old, c.new); } let corrected_ok: Vec = results @@ -154,14 +157,20 @@ async fn main() -> Result<()> { // Exactly the second sim flipped success -> revert; the others are // untouched (selective re-run). - assert_eq!(changed.len(), 1, "only one sender's balance changed"); + assert_eq!(changed_slots.len(), 1, "only one sender's balance changed"); assert_eq!(corrected_ok, vec![true, false, true]); println!( "\n→ only sim #2 was re-run (its balance fell below the transfer); \ sims #1 and #3 were left as-is." ); } - Validation::Confirmed => println!("validation: Confirmed (unexpected here)"), + Validation::ConfirmedStorage => println!( + "validation: ConfirmedStorage — storage-only success, account fields \ + NOT verified (unexpected here)" + ), + Validation::ConfirmedFull => { + println!("validation: ConfirmedFull — storage + account verified (unexpected here)") + } Validation::Unverified { reason } => println!("validation: Unverified — {reason}"), } diff --git a/examples/freshness_optimistic.rs b/examples/freshness_optimistic.rs index d4b8c5d..5735c3f 100644 --- a/examples/freshness_optimistic.rs +++ b/examples/freshness_optimistic.rs @@ -61,8 +61,8 @@ async fn main() -> Result<()> { // (An unmapped slot reads as zero, matching how a sim reads an unseen slot.) let fresh: HashMap<(Address, U256), U256> = HashMap::from([((token, owner_slot), U256::from(50))]); - let fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { + let fetcher: StorageBatchFetchFn = + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { requests .into_iter() .map(|(addr, slot)| { @@ -70,8 +70,7 @@ async fn main() -> Result<()> { (addr, slot, Ok(value)) }) .collect() - }, - ); + }); cache.set_storage_batch_fetcher(fetcher); // Classification: by default every slot is volatile (re-verified); we pin @@ -118,13 +117,26 @@ async fn main() -> Result<()> { ); // Now await the deferred validation verdict. - match sim.validate().await { - Validation::Confirmed => { - println!("validation: Confirmed — nothing the sim read had changed"); + match sim.validate().await? { + Validation::ConfirmedStorage => { + println!( + "validation: ConfirmedStorage — no volatile storage slot the sim read \ + had changed (account-level balance/nonce/code was NOT verified)" + ); + } + Validation::ConfirmedFull => { + println!( + "validation: ConfirmedFull — storage + account fields verified, \ + nothing the sim read had changed" + ); } - Validation::Corrected { results, changed } => { + Validation::Corrected { + results, + changed_slots, + .. + } => { println!("validation: Corrected — a slot the sim read had changed:"); - for c in &changed { + for c in &changed_slots { println!(" {} slot {} : {} -> {}", c.address, c.slot, c.old, c.new); } let corrected = &results[0]; diff --git a/examples/parallel_overlays.rs b/examples/parallel_overlays.rs index 5688711..2b23fd9 100644 --- a/examples/parallel_overlays.rs +++ b/examples/parallel_overlays.rs @@ -1,7 +1,7 @@ //! Fan one frozen snapshot out to many parallel, isolated simulations. //! //! This is the crate's headline workflow: freeze the cache into an immutable -//! `Arc` with `create_snapshot()`, then give each task its own +//! `Arc` with `snapshot()`, then give each task its own //! `EvmOverlay` (a cheap `Arc::clone` of the snapshot plus a private dirty //! layer). Overlays are `Send`, so the tasks can run on separate threads, and a //! write committed in one overlay is invisible to its siblings. @@ -49,7 +49,7 @@ async fn main() -> Result<()> { cache.insert_mapping_storage_slot(token, slot, recipient, U256::ZERO)?; // Freeze the current state into an immutable, Send + Sync snapshot. - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); println!("frozen snapshot: sender starts with {start}\n"); // Fan out: each thread gets a cheap Arc::clone of the snapshot and its own diff --git a/examples/reactive_cache.rs b/examples/reactive_cache.rs index 45544c3..d086f60 100644 --- a/examples/reactive_cache.rs +++ b/examples/reactive_cache.rs @@ -86,15 +86,14 @@ async fn main() -> Result<()> { let counter = fetches.clone(); let fresh: HashMap<(Address, U256), U256> = HashMap::from([((token, bob_slot), U256::from(260))]); - let fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { + let fetcher: StorageBatchFetchFn = + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { counter.fetch_add(requests.len(), Ordering::Relaxed); requests .into_iter() .map(|(a, s)| (a, s, Ok(fresh.get(&(a, s)).copied().unwrap_or(U256::ZERO)))) .collect() - }, - ); + }); cache.set_storage_batch_fetcher(fetcher); let block = 100u64; diff --git a/examples/reactive_engine_lifecycle.rs b/examples/reactive_engine_lifecycle.rs new file mode 100644 index 0000000..11aef4a --- /dev/null +++ b/examples/reactive_engine_lifecycle.rs @@ -0,0 +1,318 @@ +//! `ReactiveEngine` adapter lifecycle: register and unregister handlers while the +//! engine is live — the flow an AMM indexer runs when a pool is created or retired +//! mid-stream. +//! +//! [`ReactiveEngine`] binds a [`ReactiveRuntime`] to an [`EventSubscriber`] and +//! drives handler lifecycle as one operation, keyed by a stable [`HandlerId`]. +//! This example shows the whole loop with **no network** — a small scripted +//! subscriber stands in for a live `AlloySubscriber` so the mechanics are +//! deterministic: +//! +//! 1. Register an initial pool handler. On a fresh runtime (nothing ingested yet) +//! registration is **live-only** — there is no processed position to backfill +//! from. +//! 2. Ingest a canonical block. The runtime now has a canonical head. +//! 3. Discover a new pool mid-stream and [`register_handler`](ReactiveEngine::register_handler) +//! it. Because the runtime has a canonical head, the new handler is +//! **backfilled from that block automatically** — the discovery→subscription +//! window closes with no caller bookkeeping. (`register_handler_with_backfill` +//! for deeper history; `register_handler_live_only` to opt out.) +//! 4. Retire a pool with the teardown recipe: +//! [`unregister_handler`](ReactiveEngine::unregister_handler) (routing + +//! transport) plus [`untrack_account`](ReactiveRuntime::untrack_account) (stop +//! root-gate `eth_getProof` probes) and +//! [`cancel_pending_resyncs`](ReactiveRuntime::cancel_pending_resyncs) (drop +//! queued repairs). Cache eviction stays an explicit caller action. +//! +//! In production the scripted subscriber is replaced by `AlloySubscriber` (which +//! implements [`InterestOwnerSubscriber`]); the engine calls are identical. +//! +//! Runs fully offline against a mocked provider — no network, no RPC key. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example reactive_engine_lifecycle +//! ``` + +#[path = "support/mock.rs"] +mod mock; + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; + +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256, keccak256}; +use alloy_rpc_types_eth::{Filter, Log}; +use anyhow::{Result, anyhow}; +use evm_fork_cache::StateUpdate; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + BlockRef, ChainStatus, EventSubscriber, HandlerError, HandlerId, HandlerOutcome, InputSource, + InterestOwnerSubscriber, LogInterest, ReactiveConfig, ReactiveContext, ReactiveEffect, + ReactiveEngine, ReactiveHandler, ReactiveInput, ReactiveInputBatch, ReactiveInputRecord, + ReactiveInterest, ReactiveRuntime, RouteKeySpec, StateEffectQuality, SubscriberBackfill, + SubscriberError, SubscriberNextBatch, TrackingPolicy, +}; + +/// The storage slot each pool handler maintains from its swap logs (a stand-in +/// for a packed price/liquidity word). The post-state value rides in the log +/// `data`, so the handler decodes and writes it — no RPC. +const POOL_SLOT: u64 = 0; + +/// A protocol-neutral pool handler: every matching log carries the new slot value +/// in its `data`, written straight into `POOL_SLOT`. +struct PoolHandler { + id: HandlerId, + pool: Address, +} + +impl ReactiveHandler for PoolHandler { + fn id(&self) -> HandlerId { + self.id.clone() + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.pool), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + let ReactiveInput::Log(log) = input else { + return Ok(HandlerOutcome::empty(StateEffectQuality::NoStateEffect)); + }; + let value = U256::from_be_slice(log.data().data.as_ref()); + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + log.address(), + U256::from(POOL_SLOT), + value, + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +/// A minimal in-example [`EventSubscriber`] that hands back scripted batches and +/// tracks interest owners. It exists so the example is deterministic and +/// offline; production code uses `AlloySubscriber`, which implements the same two +/// traits over a live WebSocket transport. +#[derive(Default)] +struct ScriptedSubscriber { + owners: HashMap>>, + /// Records `(owner, anchor)` for every backfill the engine requested — so the + /// example can show that mid-lifecycle registration auto-anchors. + backfills: Vec<(HandlerId, SubscriberBackfill)>, + batches: VecDeque>, +} + +impl EventSubscriber for ScriptedSubscriber { + fn register_interests( + &mut self, + _interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + // Full-replacement setup path — unused here; the engine drives owners. + self.owners.clear(); + Ok(()) + } + + fn next_batch(&mut self) -> SubscriberNextBatch<'_, Ethereum> { + Box::pin(async move { Ok(self.batches.pop_front()) }) + } +} + +impl InterestOwnerSubscriber for ScriptedSubscriber { + fn add_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + self.owners.insert(owner, interests.to_vec()); + Ok(()) + } + + fn add_interest_owner_with_backfill( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: SubscriberBackfill, + ) -> Result<(), SubscriberError> { + self.add_interest_owner(owner.clone(), interests)?; + self.backfills.push((owner, backfill)); + Ok(()) + } + + fn remove_interest_owner( + &mut self, + owner: &HandlerId, + ) -> Option>> { + self.owners.remove(owner) + } + + fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest]> { + self.owners.get(owner).map(Vec::as_slice) + } +} + +/// Build a canonical-block batch carrying one swap log for `pool` whose data is +/// the new slot value. +fn swap_batch(pool: Address, block_number: u64, value: u64) -> ReactiveInputBatch { + let block = BlockRef { + number: block_number, + hash: B256::repeat_byte(block_number as u8), + parent_hash: Some(B256::repeat_byte(block_number.saturating_sub(1) as u8)), + timestamp: Some(1_700_000_000 + block_number), + }; + let log = Log { + inner: PrimitiveLog::new_unchecked( + pool, + vec![keccak256(b"Swap()")], + Bytes::from(U256::from(value).to_be_bytes::<32>().to_vec()), + ), + block_hash: Some(block.hash), + block_number: Some(block_number), + block_timestamp: block.timestamp, + transaction_hash: Some(B256::repeat_byte(0xcc)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + }; + let ctx = ReactiveContext { + chain_id: Some(1), + source: InputSource::Subscription, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(0), + }; + ReactiveInputBatch::new(vec![ReactiveInputRecord::new(ReactiveInput::Log(log), ctx)]) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + let pool_a = Address::repeat_byte(0xa1); + let pool_b = Address::repeat_byte(0xb2); + + let mut cache = mock::offline_cache().await?; + mock::install_mock_erc20(&mut cache, pool_a); + mock::install_mock_erc20(&mut cache, pool_b); + + let runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + let mut engine = ReactiveEngine::new(runtime, ScriptedSubscriber::default()); + + // 1. Register the first pool on a fresh runtime → live-only (no canonical + // head to backfill from yet). + engine.register_handler(Arc::new(PoolHandler { + id: HandlerId::new("pool-a"), + pool: pool_a, + }))?; + // Track pool-A so the root gate re-verifies its storage root on a cadence. + engine + .runtime_mut() + .track_account(pool_a, TrackingPolicy::WholeAccount); + println!( + "1. registered pool-a on a fresh runtime → backfills requested: {} (live-only)", + engine.subscriber().backfills.len() + ); + + // 2. Ingest a canonical block. Recommended loop: next_ingest_with_resync, + // which executes any coverage-gap resyncs the runtime surfaces. + engine + .subscriber_mut() + .batches + .push_back(swap_batch(pool_a, 100, 111)); + engine.next_ingest_with_resync(&mut cache).await?; + let head = engine + .runtime() + .last_canonical_block() + .ok_or_else(|| anyhow!("expected a canonical head after ingest"))?; + println!( + "2. ingested block {head}; pool-a slot0 = {}", + cache + .cached_storage_value(pool_a, U256::from(POOL_SLOT)) + .unwrap_or_default(), + head = head.number + ); + + // 3. A PoolCreated event surfaces pool-B mid-stream. Registering it now + // auto-anchors its log backfill to the runtime's canonical head — no + // caller bookkeeping, no discovery→subscription gap. + engine.register_handler(Arc::new(PoolHandler { + id: HandlerId::new("pool-b"), + pool: pool_b, + }))?; + let (owner, backfill) = engine + .subscriber() + .backfills + .last() + .cloned() + .ok_or_else(|| anyhow!("expected an auto-anchored backfill for pool-b"))?; + println!( + "3. discovered pool-b → registered with automatic backfill: owner={} from block {}", + owner.as_str(), + backfill.start_block() + ); + + // 4. Both handlers are live. Ingest a block touching each. + engine + .subscriber_mut() + .batches + .push_back(swap_batch(pool_b, 101, 222)); + engine.next_ingest_with_resync(&mut cache).await?; + println!( + "4. ingested block 101; pool-b slot0 = {}", + cache + .cached_storage_value(pool_b, U256::from(POOL_SLOT)) + .unwrap_or_default() + ); + + // 5. Retire pool-A. The full teardown recipe: stop routing/transport, stop + // root-gate probes, and drop any queued repairs. Cache eviction (if you + // want the state gone) stays an explicit `StateUpdate::purge` / cache API + // call — deliberately not implied by unregistration. + let removed = engine.unregister_handler(&HandlerId::new("pool-a")); + let untracked = engine.runtime_mut().untrack_account(pool_a); + let cancelled = engine.runtime_mut().cancel_pending_resyncs(pool_a); + println!( + "5. retired pool-a → handler removed: {}, untracked: {}, resyncs cancelled: {}", + removed.is_some(), + untracked, + cancelled.len() + ); + + // 6. Prove the teardown took effect: a further pool-A log is no longer routed + // (its handler is gone), while pool-B keeps updating. + engine + .subscriber_mut() + .batches + .push_back(swap_batch(pool_a, 102, 999)); + engine + .subscriber_mut() + .batches + .push_back(swap_batch(pool_b, 102, 333)); + while engine.next_ingest_with_resync(&mut cache).await?.is_some() {} + println!( + "6. after teardown, block 102: pool-a slot0 = {} (unchanged — not routed), pool-b slot0 = {}", + cache + .cached_storage_value(pool_a, U256::from(POOL_SLOT)) + .unwrap_or_default(), + cache + .cached_storage_value(pool_b, U256::from(POOL_SLOT)) + .unwrap_or_default() + ); + + println!("\nlifecycle complete — registered, auto-anchored, and torn down with no RPC."); + Ok(()) +} diff --git a/examples/reactive_runtime.rs b/examples/reactive_runtime.rs index 0b3210e..440a189 100644 --- a/examples/reactive_runtime.rs +++ b/examples/reactive_runtime.rs @@ -227,15 +227,14 @@ async fn main() -> Result<()> { // counter would catch it. let fetches = Arc::new(AtomicUsize::new(0)); let counter = fetches.clone(); - let fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { + let fetcher: StorageBatchFetchFn = + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { counter.fetch_add(requests.len(), Ordering::Relaxed); requests .into_iter() .map(|(a, s)| (a, s, Ok(U256::ZERO))) .collect() - }, - ); + }); cache.set_storage_batch_fetcher(fetcher); let hook = Arc::new(AppliedLog::default()); diff --git a/examples/snapshot_and_restore.rs b/examples/snapshot_and_restore.rs index e8e3622..03daf3f 100644 --- a/examples/snapshot_and_restore.rs +++ b/examples/snapshot_and_restore.rs @@ -1,12 +1,12 @@ -//! Snapshot cache state, mutate it, then roll back — the core primitive for +//! Checkpoint cache state, mutate it, then roll back — the core primitive for //! evaluating many candidate transactions from the same starting point. //! -//! `snapshot()` captures a cheap in-memory copy of the cache's state; `restore()` +//! `checkpoint()` captures a cheap in-memory copy of the cache's state; `restore()` //! resets to it. Here we transfer tokens (committing the change), observe the new //! balances, then restore and confirm the transfer was undone. //! //! This is the in-place rollback API on a single `EvmCache`. It is distinct from -//! `create_snapshot()`, which returns an `Arc` for sharing one frozen +//! `snapshot()`, which returns an `Arc` for sharing one frozen //! state across many parallel `EvmOverlay` simulations — see the //! `parallel_overlays` example for that workflow. //! @@ -51,7 +51,7 @@ async fn main() -> Result<()> { ); // Capture a restore point. - let snapshot = cache.snapshot(); + let checkpoint = cache.checkpoint(); // Commit a transfer of 250 from alice to bob. let transfer = mock::MockERC20::transferCall { @@ -65,8 +65,8 @@ async fn main() -> Result<()> { mock::balance_of(&mut cache, token, bob)? ); - // Roll back to the snapshot — the transfer is undone. - cache.restore(snapshot); + // Roll back to the checkpoint — the transfer is undone. + cache.restore(checkpoint); println!( "after restore: alice={}, bob={}", mock::balance_of(&mut cache, token, alice)?, diff --git a/fixtures/README.md b/fixtures/README.md index 60703de..8dca8c2 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -53,6 +53,13 @@ jq -r '.bytecode.object' out/MockERC20.sol/MockERC20.json \ suite exercise the real `aggregate3` build/execute/decode path (input-order results and `allowFailure` semantics) with no network. + It is also compiled into the library by `src/bulk_storage.rs` + (`multicall3_runtime_code()`), which injects it as an `eth_call` state + override so multi-contract bulk storage extraction works on chains — and at + historical blocks — where Multicall3 is not deployed. Last verified + byte-identical to the on-chain mainnet code (`eth_getCode`) across two + independent providers on 2026-07-02. + ### Regenerating ```sh diff --git a/src/access_list.rs b/src/access_list.rs index 39d495b..babcdad 100644 --- a/src/access_list.rs +++ b/src/access_list.rs @@ -29,11 +29,11 @@ use alloy_provider::Provider; use alloy_rlp::Encodable; use alloy_rpc_types_eth::{TransactionInput, TransactionRequest}; use alloy_sol_types::{SolCall, sol}; -use anyhow::{Context as _, Result}; use revm::context::result::ExecutionResult; use tracing::{debug, info}; use crate::cache::EvmCache; +use crate::errors::{AccessListError, AccessListResult as Result}; /// Arbitrum ArbGasInfo precompile. const ARB_GAS_INFO: Address = address!("000000000000000000000000000000000000006C"); @@ -200,12 +200,12 @@ impl SmartAccessList { let prices = prices_call .call() .await - .context("failed to query ArbGasInfo prices for access-list profitability")?; + .map_err(|e| AccessListError::query("ArbGasInfo prices", e))?; let l1_fee_call = arb.getL1BaseFeeEstimate(); let l1_base_fee = l1_fee_call .call() .await - .context("failed to query ArbGasInfo L1 base fee for access-list profitability")?; + .map_err(|e| AccessListError::query("ArbGasInfo L1 base fee", e))?; let l2_gas_price = prices.perArbGas; @@ -261,12 +261,12 @@ pub async fn access_list_if_profitable( .getPricesInWei() .call() .await - .context("failed to query ArbGasInfo prices for access-list profitability")?; + .map_err(|e| AccessListError::query("ArbGasInfo prices", e))?; let l1_base_fee = arb .getL1BaseFeeEstimate() .call() .await - .context("failed to query ArbGasInfo L1 base fee for access-list profitability")?; + .map_err(|e| AccessListError::query("ArbGasInfo L1 base fee", e))?; let l2_gas_price = prices.perArbGas; @@ -329,17 +329,25 @@ async fn access_list_if_profitable_op_stack( tx_without_access_list: Bytes, tx_with_access_list: Bytes, ) -> Result> { - let l2_gas_price = - U256::from(provider.get_gas_price().await.context( - "failed to query OP Stack provider gas price for access-list profitability", - )?); + let l2_gas_price = U256::from( + provider + .get_gas_price() + .await + .map_err(|e| AccessListError::query("OP Stack provider gas price", e))?, + ); let l1_fee_without = query_op_l1_fee(provider, tx_without_access_list) .await - .context("failed to query OP Stack GasPriceOracle L1 fee without access list")?; + .map_err(|e| AccessListError::Query { + operation: "OP Stack GasPriceOracle L1 fee without access list", + details: e.to_string(), + })?; let l1_fee_with = query_op_l1_fee(provider, tx_with_access_list) .await - .context("failed to query OP Stack GasPriceOracle L1 fee with access list")?; + .map_err(|e| AccessListError::Query { + operation: "OP Stack GasPriceOracle L1 fee with access list", + details: e.to_string(), + })?; let incremental_l1_fee = l1_fee_with.saturating_sub(l1_fee_without); let total_entries = access_list_entry_count(&access_list); @@ -375,7 +383,7 @@ async fn query_op_l1_fee(provider: &P, tx_data: Bytes) -> Result +//! - reference implementation: +//! +//! # Mechanism +//! +//! `eth_call` accepts a *state-override set* that can replace the **code** at +//! any address while leaving its **storage** intact. We override the target +//! contract with a 23-byte handwritten extractor ([`STORAGE_EXTRACTOR_CODE`], +//! Dedaub's bytecode, credited above) that treats calldata as a raw array of +//! 32-byte slot keys, `SLOAD`s each one, and returns the packed values — +//! no function selector, no ABI: +//! +//! ```text +//! [00] PUSH0 counter = 0 +//! [01] JUMPDEST loop: +//! [02] DUP1 CALLDATASIZE EQ +//! [05] PUSH1 0x13 JUMPI -> exit when counter == calldatasize +//! [08] DUP1 CALLDATALOAD slot key at calldata[counter] +//! [0a] SLOAD +//! [0b] DUP2 MSTORE mem[counter] = value (counter doubles as mem offset) +//! [0d] PUSH1 0x20 ADD counter += 32 +//! [10] PUSH1 0x01 JUMP +//! [13] JUMPDEST CALLDATASIZE PUSH0 RETURN +//! ``` +//! +//! Marginal cost is ~2,664 gas per slot (cold `SLOAD` 2,100 + calldata ~510 + +//! loop ~30 + memory), so a default 50M-gas `eth_call` fits ~18,500 slots. +//! [`BulkCallConfig::max_slots_per_call`] defaults to a conservative 10,000 +//! (~27M gas), splitting larger requests across concurrent calls. +//! +//! # Multi-contract batches +//! +//! `SLOAD` reads the storage of the *executing* contract, so each target must +//! run the extractor at its own address. To read many contracts in one round +//! trip we additionally override [`MULTICALL3_ADDRESS`] with the canonical +//! Multicall3 runtime ([`multicall3_runtime_code`]) and dispatch one +//! `aggregate3` call whose subcalls hit each overridden target. Overriding the +//! dispatcher code unconditionally makes the scheme work on chains — and at +//! historical blocks — where Multicall3 is not deployed. +//! +//! # Semantics & caveats +//! +//! - Results are identical to `eth_getStorageAt` at the same block: absent +//! slots (and slots of code-less accounts) read as zero. +//! - The provider must support the state-override parameter of `eth_call` +//! (Geth-lineage nodes, Reth, Erigon, and the major hosted providers all +//! do). Providers that reject it surface a per-slot error; install a +//! fallback via [`bulk_call_storage_fetcher_with_fallback`] to repair those +//! with classic point reads. +//! - True precompile addresses (`0x01..=0x11` on mainnet) execute the +//! precompile regardless of a code override; slots requested there fail the +//! response-length check and surface as errors (repaired by the fallback +//! when configured) rather than silently returning garbage. +//! - [`STORAGE_EXTRACTOR_CODE`] uses `PUSH0` (Shanghai). For pre-Shanghai +//! chains set [`BulkCallConfig::pre_shanghai_extractor`] to use the +//! equivalent [`STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI`]. +//! +//! # Wiring it into a cache +//! +//! **Since 0.2.0 this is every provider-backed cache's default storage +//! fetcher** — no wiring needed. Tune it with +//! [`EvmCacheBuilder::bulk_call_config`](crate::cache::EvmCacheBuilder::bulk_call_config), +//! opt out with +//! [`StorageFetchStrategy::PointRead`](crate::cache::StorageFetchStrategy::PointRead), +//! or compose it manually as below (e.g. over a custom fallback): +//! +//! ```no_run +//! # use std::sync::Arc; +//! # use alloy_provider::{ProviderBuilder, network::AnyNetwork}; +//! # use evm_fork_cache::cache::EvmCache; +//! # use evm_fork_cache::bulk_storage::{BulkCallConfig, bulk_call_storage_fetcher_with_fallback}; +//! # async fn example() -> Result<(), Box> { +//! let provider = Arc::new( +//! ProviderBuilder::new() +//! .network::() +//! .connect_http("https://example-rpc.invalid".parse()?), +//! ); +//! let mut cache = EvmCache::builder(provider.clone()).build().await; +//! +//! // Keep the default point-read fetcher as a repair path, then route all +//! // batch storage fetches through call-override bulk extraction. +//! let fallback = cache +//! .storage_batch_fetcher() +//! .cloned() +//! .expect("provider-backed cache has a default fetcher"); +//! cache.set_storage_batch_fetcher(bulk_call_storage_fetcher_with_fallback( +//! provider, +//! BulkCallConfig::default(), +//! fallback, +//! )); +//! # Ok(()) +//! # } +//! ``` + +use std::collections::HashMap; +use std::sync::{Arc, OnceLock}; + +use alloy_eips::BlockId; +use alloy_primitives::{Address, B256, Bytes, U256, hex}; +use alloy_provider::Provider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_types_eth::TransactionRequest; +use alloy_rpc_types_eth::state::{AccountOverride, StateOverride}; +use alloy_sol_types::SolCall; +use futures::stream::{self, StreamExt}; +use tracing::{debug, warn}; + +use crate::cache::{StorageBatchFetchFn, block_in_place_handle}; +use crate::errors::{StorageFetchError, StorageFetchResult}; +use crate::multicall::{IMulticall3, MULTICALL3_ADDRESS}; + +/// Dedaub's 23-byte storage extractor (see the module docs for the annotated +/// disassembly). Calldata is a contiguous array of 32-byte slot keys; the +/// return data is the corresponding array of 32-byte values. Requires +/// `PUSH0` (Shanghai). +/// +/// Source: (`extractor.hex`). +pub const STORAGE_EXTRACTOR_CODE: &[u8] = &hex!("5f5b80361460135780355481526020016001565b365ff3"); + +/// [`STORAGE_EXTRACTOR_CODE`] with both `PUSH0`s replaced by `PUSH1 0x00` +/// (jump targets re-pointed), for chains that have not activated Shanghai. +pub const STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI: &[u8] = + &hex!("60005b80361460145780355481526020016002565b366000f3"); + +/// Runtime bytecode of Multicall3 (`0xcA11bde05977b3631167028862bE2a173976CA11`), +/// as deployed on Ethereum mainnet. Injected as a code override at +/// [`MULTICALL3_ADDRESS`] for multi-contract extraction so the dispatcher +/// exists on every chain and at every historical block. +/// +/// The fixture was fetched via `eth_getCode` and verified byte-identical +/// across independent providers; `fixtures/README.md` records provenance. +pub fn multicall3_runtime_code() -> &'static Bytes { + static CODE: OnceLock = OnceLock::new(); + CODE.get_or_init(|| { + let raw = include_str!("../fixtures/multicall3_runtime.hex"); + Bytes::from(hex::decode(raw.trim()).expect("valid multicall3 runtime hex fixture")) + }) +} + +/// How planned extraction chunks are shipped to the provider. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CallDispatch { + /// One `eth_call` per planned chunk. Universally supported; chunks run + /// concurrently up to [`BulkCallConfig::max_concurrent_calls`]. The + /// default. + #[default] + PerCall, + /// Ship many chunks as the transactions of a single `eth_callMany` + /// bundle (Erigon-lineage providers, including Alchemy — where it costs + /// 20 CU per *request* vs 26 per `eth_call`). Requests are bounded by + /// [`BulkCallConfig::max_slots_per_request`]; a request-level failure + /// (e.g. the method is unsupported) transparently re-dispatches that + /// request's chunks per-call. Hash-pinned blocks always dispatch + /// per-call (`eth_callMany` takes a number/tag block context). + CallMany, +} + +/// Tuning knobs for the call-override bulk storage fetcher. +/// +/// The defaults target Geth-default RPC limits (50M gas per `eth_call`): +/// 10,000 slots ≈ 27M gas, comfortably under the cap while leaving headroom +/// for multicall dispatch overhead. Providers with higher caps can raise +/// `max_slots_per_call` substantially (measure before relying on it — +/// Alchemy accepted 30k slots/call in testing, bounded by request body size +/// rather than gas; see `docs/bulk-storage-extraction.md`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BulkCallConfig { + /// Maximum storage slots packed into one `eth_call` (across all targets + /// in that call). ~2,664 gas per slot; keep the product under the + /// provider's `eth_call` gas cap. + pub max_slots_per_call: usize, + /// Maximum distinct target contracts dispatched through one multicall + /// (~8k gas of call/ABI overhead per target). + pub max_targets_per_call: usize, + /// Maximum `eth_call`s in flight at once when a request spans multiple + /// calls. + pub max_concurrent_calls: usize, + /// Requests with fewer than this many slots are routed to the fallback + /// fetcher when one is installed (an `eth_call` costs slightly more than + /// a single `eth_getStorageAt` on CU-metered providers). Ignored when no + /// fallback is available. + pub point_read_threshold: usize, + /// Use the `PUSH0`-free extractor for chains without Shanghai. + pub pre_shanghai_extractor: bool, + /// How chunks are shipped: one `eth_call` each, or batched through + /// `eth_callMany`. + pub dispatch: CallDispatch, + /// [`CallDispatch::CallMany`] only: maximum total slots per + /// `eth_callMany` request. Slot keys are incompressible calldata + /// (~64 bytes each in the JSON body), and providers cap request bodies — + /// Alchemy rejects ~2.5 MB with HTTP 413. The default (25,000 ≈ 1.6 MB) + /// stays inside that. + pub max_slots_per_request: usize, +} + +impl Default for BulkCallConfig { + fn default() -> Self { + Self { + max_slots_per_call: 10_000, + max_targets_per_call: 250, + max_concurrent_calls: 4, + point_read_threshold: 2, + pre_shanghai_extractor: false, + dispatch: CallDispatch::PerCall, + max_slots_per_request: 25_000, + } + } +} + +impl BulkCallConfig { + fn normalized(self) -> Self { + Self { + max_slots_per_call: self.max_slots_per_call.max(1), + max_targets_per_call: self.max_targets_per_call.max(1), + max_concurrent_calls: self.max_concurrent_calls.max(1), + max_slots_per_request: self.max_slots_per_request.max(1), + ..self + } + } + + fn extractor(&self) -> Bytes { + if self.pre_shanghai_extractor { + Bytes::from_static(STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI) + } else { + Bytes::from_static(STORAGE_EXTRACTOR_CODE) + } + } +} + +/// Pack slot keys into extractor calldata: the raw concatenation of each +/// key's 32-byte big-endian representation (no selector, no ABI). +pub fn pack_slots_calldata(slots: &[U256]) -> Bytes { + let mut out = Vec::with_capacity(slots.len() * 32); + for slot in slots { + out.extend_from_slice(&slot.to_be_bytes::<32>()); + } + out.into() +} + +/// Decode extractor return data (packed 32-byte words) into values. +/// +/// Returns `None` when the payload is not exactly `expected` words — the +/// signature of a call that did not actually execute the extractor (e.g. a +/// provider that ignored the code override, or a precompile target). +pub fn decode_packed_values(data: &[u8], expected: usize) -> Option> { + if data.len() != expected * 32 { + return None; + } + Some(data.chunks_exact(32).map(U256::from_be_slice).collect()) +} + +/// ABI-encode one `aggregate3` dispatch whose subcalls run the extractor at +/// each `(target, slots)` pair. Subcalls use `allowFailure = true` so one +/// failing target degrades to per-target errors instead of reverting the +/// whole batch. +pub fn encode_multi_target_calldata(targets: &[(Address, Vec)]) -> Bytes { + let calls: Vec = targets + .iter() + .map(|(target, slots)| IMulticall3::Call3 { + target: *target, + allowFailure: true, + callData: pack_slots_calldata(slots), + }) + .collect(); + IMulticall3::aggregate3Call { calls }.abi_encode().into() +} + +/// Decode an `aggregate3` response produced by [`encode_multi_target_calldata`] +/// back into one result tuple per requested `(target, slot)` pair. +pub fn decode_multi_target_response( + targets: &[(Address, Vec)], + response: &[u8], +) -> Vec<(Address, U256, StorageFetchResult)> { + let decoded = match IMulticall3::aggregate3Call::abi_decode_returns(response) { + Ok(results) if results.len() == targets.len() => results, + Ok(results) => { + return per_target_errors(targets, || { + StorageFetchError::custom(format!( + "aggregate3 returned {} results for {} extraction targets", + results.len(), + targets.len() + )) + }); + } + Err(e) => { + return per_target_errors(targets, || { + StorageFetchError::custom(format!("failed to decode aggregate3 response: {e}")) + }); + } + }; + + let mut out = Vec::with_capacity(targets.iter().map(|(_, s)| s.len()).sum()); + for ((target, slots), result) in targets.iter().zip(decoded) { + if !result.success { + out.extend(slots.iter().map(|slot| { + ( + *target, + *slot, + Err(StorageFetchError::custom( + "extractor subcall failed (allowFailure=true); the target may be a precompile", + )), + ) + })); + continue; + } + match decode_packed_values(&result.returnData, slots.len()) { + Some(values) => out.extend( + slots + .iter() + .zip(values) + .map(|(slot, value)| (*target, *slot, Ok(value))), + ), + None => out.extend(slots.iter().map(|slot| { + ( + *target, + *slot, + Err(StorageFetchError::custom(format!( + "extractor at {target} returned {} bytes, expected {}", + result.returnData.len(), + slots.len() * 32 + ))), + ) + })), + } + } + out +} + +fn per_target_errors( + targets: &[(Address, Vec)], + make: impl Fn() -> StorageFetchError, +) -> Vec<(Address, U256, StorageFetchResult)> { + targets + .iter() + .flat_map(|(target, slots)| slots.iter().map(|slot| (*target, *slot, Err(make())))) + .collect() +} + +/// One planned `eth_call`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum CallPlan { + /// Direct extractor call: `to = target`, calldata = packed slots. + Single { target: Address, slots: Vec }, + /// Multicall3 dispatch across several targets, each running the extractor. + Multi { targets: Vec<(Address, Vec)> }, +} + +impl CallPlan { + fn request_slot_count(&self) -> usize { + match self { + Self::Single { slots, .. } => slots.len(), + Self::Multi { targets } => targets.iter().map(|(_, s)| s.len()).sum(), + } + } +} + +/// Split requests into `eth_call`-sized plans. +/// +/// Groups by address in first-seen order; targets with more than +/// `max_slots_per_call` slots are split into dedicated single-target calls, +/// and the remaining small groups are greedily packed into multicall +/// dispatches bounded by both the slot and target budgets. A target equal to +/// [`MULTICALL3_ADDRESS`] always gets a dedicated call so its code override +/// cannot collide with the dispatcher's. +fn plan_calls(requests: &[(Address, U256)], config: &BulkCallConfig) -> Vec { + let mut order: Vec
= Vec::new(); + let mut groups: HashMap> = HashMap::new(); + for (address, slot) in requests { + groups + .entry(*address) + .or_insert_with(|| { + order.push(*address); + Vec::new() + }) + .push(*slot); + } + + let mut plans = Vec::new(); + let mut packable: Vec<(Address, Vec)> = Vec::new(); + for address in order { + let slots = groups.remove(&address).expect("grouped above"); + for chunk in slots.chunks(config.max_slots_per_call) { + let full = chunk.len() == config.max_slots_per_call; + // The dispatcher address must never share a multicall with other + // targets: its extractor override would clobber the dispatcher + // code override at the same key. Only the final chunk of a target + // can be partial, so at most one packable remainder per target. + if full || address == MULTICALL3_ADDRESS { + plans.push(CallPlan::Single { + target: address, + slots: chunk.to_vec(), + }); + } else { + packable.push((address, chunk.to_vec())); + } + } + } + + // Greedily pack the small per-target remainders into multicall dispatches. + let mut current: Vec<(Address, Vec)> = Vec::new(); + let mut current_slots = 0usize; + let flush = + |current: &mut Vec<(Address, Vec)>, plans: &mut Vec| match current.len() { + 0 => {} + 1 => { + let (target, slots) = current.pop().expect("len checked"); + plans.push(CallPlan::Single { target, slots }); + } + _ => plans.push(CallPlan::Multi { + targets: std::mem::take(current), + }), + }; + for (address, slots) in packable { + let would_overflow = current_slots + slots.len() > config.max_slots_per_call + || current.len() >= config.max_targets_per_call; + if !current.is_empty() && would_overflow { + flush(&mut current, &mut plans); + current_slots = 0; + } + current_slots += slots.len(); + current.push((address, slots)); + } + flush(&mut current, &mut plans); + + plans +} + +/// Build the state-override set for one plan: the extractor at every target, +/// plus the Multicall3 runtime at the dispatcher for multi-target plans. +fn overrides_for_plan(plan: &CallPlan, extractor: &Bytes) -> StateOverride { + let mut overrides = StateOverride::default(); + match plan { + CallPlan::Single { target, .. } => { + overrides.insert( + *target, + AccountOverride::default().with_code(extractor.clone()), + ); + } + CallPlan::Multi { targets } => { + overrides.insert( + MULTICALL3_ADDRESS, + AccountOverride::default().with_code(multicall3_runtime_code().clone()), + ); + for (target, _) in targets { + overrides.insert( + *target, + AccountOverride::default().with_code(extractor.clone()), + ); + } + } + } + overrides +} + +/// The `to` address and calldata for one planned call. +fn plan_call_parts(plan: &CallPlan) -> (Address, Bytes) { + match plan { + CallPlan::Single { target, slots } => (*target, pack_slots_calldata(slots)), + CallPlan::Multi { targets } => (MULTICALL3_ADDRESS, encode_multi_target_calldata(targets)), + } +} + +/// Decode one plan's successful call output into per-slot results. +fn decode_plan_response( + plan: &CallPlan, + bytes: &[u8], +) -> Vec<(Address, U256, StorageFetchResult)> { + match plan { + CallPlan::Single { target, slots } => match decode_packed_values(bytes, slots.len()) { + Some(values) => slots + .iter() + .zip(values) + .map(|(slot, value)| (*target, *slot, Ok(value))) + .collect(), + None => slots + .iter() + .map(|slot| { + ( + *target, + *slot, + Err(StorageFetchError::custom(format!( + "extractor at {target} returned {} bytes, expected {} — the \ + provider may not support eth_call state overrides, or the \ + target is a precompile", + bytes.len(), + slots.len() * 32 + ))), + ) + }) + .collect(), + }, + CallPlan::Multi { targets } => decode_multi_target_response(targets, bytes), + } +} + +/// Report `err` for every slot the plan covers. +fn plan_error_results( + plan: &CallPlan, + err: StorageFetchError, +) -> Vec<(Address, U256, StorageFetchResult)> { + match plan { + CallPlan::Single { target, slots } => slots + .iter() + .map(|slot| (*target, *slot, Err(err.clone()))) + .collect(), + CallPlan::Multi { targets } => targets + .iter() + .flat_map(|(target, slots)| { + slots.iter().map({ + let err = err.clone(); + move |slot| (*target, *slot, Err(err.clone())) + }) + }) + .collect(), + } +} + +async fn execute_plan>( + provider: &P, + block: BlockId, + plan: CallPlan, + extractor: &Bytes, +) -> Vec<(Address, U256, StorageFetchResult)> { + let overrides = overrides_for_plan(&plan, extractor); + let (to, data) = plan_call_parts(&plan); + let tx = TransactionRequest::default().to(to).input(data.into()); + + let response: Result = provider + .client() + .request("eth_call", (tx, block, overrides)) + .await; + + match response { + Ok(bytes) => decode_plan_response(&plan, &bytes), + Err(e) => plan_error_results(&plan, StorageFetchError::provider("eth_call", &e)), + } +} + +/// One entry of an `eth_callMany` response: `{"value": "0x.."}` on success, +/// `{"error": ..}` on per-transaction failure (Erigon-style). +#[derive(Debug, serde::Deserialize)] +struct CallManyEntry { + value: Option, + error: Option, +} + +/// Execute several plans as the transactions of one `eth_callMany` bundle. +/// +/// Returns `Err` only for request-level failures (method unsupported, +/// transport error, malformed response) so the caller can re-dispatch the +/// same plans per-call; per-transaction failures are mapped to per-slot +/// errors in the `Ok` payload. +async fn execute_plans_call_many>( + provider: &P, + number: alloy_eips::BlockNumberOrTag, + plans: &[CallPlan], + extractor: &Bytes, +) -> Result)>, StorageFetchError> { + // One shared override map across every plan in the bundle. Merging is + // safe: every target maps to the extractor and the dispatcher maps to + // Multicall3; plans targeting the dispatcher itself are routed per-call + // by the caller. + let mut overrides = StateOverride::default(); + let mut transactions = Vec::with_capacity(plans.len()); + for plan in plans { + for (address, account) in overrides_for_plan(plan, extractor) { + overrides.insert(address, account); + } + let (to, data) = plan_call_parts(plan); + transactions.push(serde_json::json!({ "to": to, "data": data })); + } + + let bundles = serde_json::json!([{ "transactions": transactions }]); + let context = serde_json::json!({ "blockNumber": number, "transactionIndex": -1 }); + let response: Vec> = provider + .client() + .request("eth_callMany", (bundles, context, overrides)) + .await + .map_err(|e| StorageFetchError::provider("eth_callMany", &e))?; + + let entries: Vec = response.into_iter().flatten().collect(); + if entries.len() != plans.len() { + return Err(StorageFetchError::custom(format!( + "eth_callMany returned {} results for {} bundled calls", + entries.len(), + plans.len() + ))); + } + + let mut out = Vec::new(); + for (plan, entry) in plans.iter().zip(entries) { + match entry.value { + Some(bytes) => out.extend(decode_plan_response(plan, &bytes)), + None => { + let detail = entry + .error + .map(|e| e.to_string()) + .unwrap_or_else(|| "no value returned".to_string()); + out.extend(plan_error_results( + plan, + StorageFetchError::custom(format!("eth_callMany transaction failed: {detail}")), + )); + } + } + } + Ok(out) +} + +/// Group plans into `eth_callMany` requests bounded by the per-request slot +/// budget (the request *body* is the binding provider limit, not gas). +fn group_plans_for_call_many( + plans: Vec, + max_slots_per_request: usize, +) -> Vec> { + let mut requests: Vec> = Vec::new(); + let mut current: Vec = Vec::new(); + let mut current_slots = 0usize; + for plan in plans { + let slots = plan.request_slot_count(); + if !current.is_empty() && current_slots + slots > max_slots_per_request { + requests.push(std::mem::take(&mut current)); + current_slots = 0; + } + current_slots += slots; + current.push(plan); + } + if !current.is_empty() { + requests.push(current); + } + requests +} + +/// Fetch storage slots in bulk via `eth_call` code overrides (async core). +/// +/// Returns exactly one result tuple per requested `(address, slot)` pair +/// (order not preserved, duplicates included), matching the +/// [`StorageBatchFetchFn`] contract. Chunk-level failures (transport errors, +/// providers without state-override support) surface as per-slot errors. +/// +/// This is the direct entry point for async callers — e.g. loading an entire +/// AMM pool's tick range during cold start — while +/// [`bulk_call_storage_fetcher`] adapts it to the cache's synchronous fetcher +/// seam. +pub async fn fetch_slots_bulk>( + provider: &P, + requests: Vec<(Address, U256)>, + block: BlockId, + config: BulkCallConfig, +) -> Vec<(Address, U256, StorageFetchResult)> { + let config = config.normalized(); + if requests.is_empty() { + return Vec::new(); + } + let extractor = config.extractor(); + let plans = plan_calls(&requests, &config); + debug!( + slots = requests.len(), + calls = plans.len(), + dispatch = ?config.dispatch, + "bulk storage extraction dispatch" + ); + + let extractor = &extractor; + // eth_callMany takes a number/tag block context; hash pins dispatch + // per-call instead. + let call_many_number = match (config.dispatch, block) { + (CallDispatch::CallMany, BlockId::Number(number)) => Some(number), + _ => None, + }; + + let Some(number) = call_many_number else { + let results: Vec> = stream::iter( + plans + .into_iter() + .map(|plan| execute_plan(provider, block, plan, extractor)), + ) + .buffer_unordered(config.max_concurrent_calls) + .collect() + .await; + return results.into_iter().flatten().collect(); + }; + + // Plans targeting the dispatcher address itself cannot share a bundle's + // override map (their extractor override would clobber the dispatcher + // override); ship those per-call. + let (conflicting, bundleable): (Vec<_>, Vec<_>) = plans.into_iter().partition( + |plan| matches!(plan, CallPlan::Single { target, .. } if *target == MULTICALL3_ADDRESS), + ); + let groups = group_plans_for_call_many(bundleable, config.max_slots_per_request); + let group_futs = groups.into_iter().map(|group| async move { + match execute_plans_call_many(provider, number, &group, extractor).await { + Ok(results) => results, + Err(e) => { + // Request-level failure (method unsupported, transport, + // malformed response): re-dispatch this request's chunks as + // plain eth_calls so the fetch still succeeds. + warn!( + error = %e, + chunks = group.len(), + "eth_callMany dispatch failed; re-dispatching per-call" + ); + let mut results = Vec::new(); + for plan in group { + results.extend(execute_plan(provider, block, plan, extractor).await); + } + results + } + } + }); + let mut results: Vec<_> = stream::iter(group_futs) + .buffer_unordered(config.max_concurrent_calls) + .collect::>>() + .await + .into_iter() + .flatten() + .collect(); + for plan in conflicting { + results.extend(execute_plan(provider, block, plan, extractor).await); + } + results +} + +/// Number of `eth_call`s a request set will be split into under `config`. +/// +/// Useful for CU budgeting on metered providers: the bulk path costs +/// `planned_call_count(..) × cost(eth_call)` (26 CU each on Alchemy) versus +/// `requests.len() × cost(eth_getStorageAt)` (20 CU each) for point reads. +pub fn planned_call_count(requests: &[(Address, U256)], config: &BulkCallConfig) -> usize { + plan_calls(requests, &config.normalized()).len() +} + +/// Build a [`StorageBatchFetchFn`] backed by call-override bulk extraction. +/// +/// Install it with [`EvmCache::set_storage_batch_fetcher`](crate::cache::EvmCache::set_storage_batch_fetcher); +/// every batch consumer (freshness verification, cold-start verify/probe, +/// reactive point-read resyncs, prefetch) then loads storage through bulk +/// `eth_call`s. Requires a multi-thread tokio runtime, like the default +/// fetcher. +/// +/// Failed slots are reported as errors; use +/// [`bulk_call_storage_fetcher_with_fallback`] to repair them with classic +/// point reads instead. +pub fn bulk_call_storage_fetcher + 'static>( + provider: Arc

, + config: BulkCallConfig, +) -> StorageBatchFetchFn { + make_fetcher(provider, config, None) +} + +/// [`bulk_call_storage_fetcher`] with a repair path. +/// +/// `fallback` (typically the cache's default point-read fetcher, obtained via +/// [`EvmCache::storage_batch_fetcher`](crate::cache::EvmCache::storage_batch_fetcher) +/// before replacing it) is invoked for: +/// - requests smaller than [`BulkCallConfig::point_read_threshold`], where a +/// point read is cheaper than an `eth_call` on CU-metered providers; and +/// - any pairs the bulk path reported as errors (provider without +/// state-override support, precompile targets, transport failures). +pub fn bulk_call_storage_fetcher_with_fallback + 'static>( + provider: Arc

, + config: BulkCallConfig, + fallback: StorageBatchFetchFn, +) -> StorageBatchFetchFn { + make_fetcher(provider, config, Some(fallback)) +} + +fn make_fetcher + 'static>( + provider: Arc

, + config: BulkCallConfig, + fallback: Option, +) -> StorageBatchFetchFn { + let config = config.normalized(); + // After this many *consecutive* batches where every slot failed with a + // provider-level error (the signature of an endpoint without + // state-override support), stop attempting bulk extraction and route + // straight to the fallback. Sticky for the fetcher's lifetime — install a + // fresh fetcher to retry bulk extraction. Only meaningful when a fallback + // exists; without one the bulk attempt is the only option anyway. + const OVERRIDE_FAILURE_LATCH: usize = 2; + let consecutive_failures = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + Arc::new(move |requests: Vec<(Address, U256)>, block: BlockId| { + use std::sync::atomic::Ordering; + if requests.is_empty() { + return Vec::new(); + } + if let Some(fallback) = &fallback + && (requests.len() < config.point_read_threshold + || consecutive_failures.load(Ordering::Relaxed) >= OVERRIDE_FAILURE_LATCH) + { + return fallback(requests, block); + } + + // Guard against panicking inside `block_in_place` on a current-thread + // runtime (or when no runtime is present): report an `Err` result for + // every requested slot instead, mirroring the default fetcher. The + // guard errors still flow through the fallback repair below — a + // synchronous fallback can serve them even where the bulk path can't + // run. + let bulk_results = match block_in_place_handle() { + Ok(handle) => tokio::task::block_in_place(|| { + handle.block_on(fetch_slots_bulk(provider.as_ref(), requests, block, config)) + }), + Err(e) => requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Err(StorageFetchError::Runtime(e.clone())))) + .collect(), + }; + + let Some(fallback) = &fallback else { + return bulk_results; + }; + + // Latch bookkeeping: any success resets the streak; a batch where + // *everything* failed at the provider level counts toward latching. + if bulk_results.iter().any(|(_, _, r)| r.is_ok()) { + consecutive_failures.store(0, Ordering::Relaxed); + } else if bulk_results + .iter() + .any(|(_, _, r)| matches!(r, Err(StorageFetchError::Provider { .. }))) + { + let streak = consecutive_failures.fetch_add(1, Ordering::Relaxed) + 1; + if streak == OVERRIDE_FAILURE_LATCH { + warn!( + streak, + "bulk storage extraction failed consecutive batches with provider errors; \ + latching this fetcher to the point-read fallback (install a fresh fetcher \ + to retry bulk extraction)" + ); + } + } + + // Repair failed pairs (with multiplicity) through the fallback, + // preserving the one-result-per-request contract. + let mut repaired = Vec::with_capacity(bulk_results.len()); + let mut failed: Vec<(Address, U256)> = Vec::new(); + for (addr, slot, result) in bulk_results { + match result { + Ok(value) => repaired.push((addr, slot, Ok(value))), + Err(_) => failed.push((addr, slot)), + } + } + if !failed.is_empty() { + warn!( + failed = failed.len(), + "bulk storage extraction failed for some slots; repairing via fallback fetcher" + ); + repaired.extend(fallback(failed, block)); + } + repaired + }) +} + +// --------------------------------------------------------------------------- +// Custom storage programs & companion extractors +// --------------------------------------------------------------------------- + +/// A caller-supplied extraction program: arbitrary bytecode injected at +/// `target` through a code override and executed by one `eth_call`. +/// +/// The slot-list extractor requires the client to know every slot key up +/// front. A custom program removes that constraint — it can *derive* what to +/// read inside the EVM. Example: a Uniswap V3 loader that walks the +/// `tickBitmap` words on-chain and returns every initialized tick's data in a +/// single round trip, with no calldata at all (the two-phase +/// bitmap-then-ticks pattern collapsed into one call). The program runs at +/// the target's address, so `SLOAD` reads the target's real storage; the +/// output format is whatever the program returns — decoding is the caller's +/// contract with its own bytecode. +/// +/// See `examples/bulk_storage_bench.rs` for a worked program (a one-shot +/// Uniswap V3 observation-ring loader that reads the cardinality from +/// `slot0` and returns the whole ring) and the offline revm tests in +/// `tests/bulk_storage.rs` that execute it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageProgram { + /// Address whose storage the program reads (its code is replaced by + /// `code` for the duration of the call). + pub target: Address, + /// Runtime bytecode to inject at `target`. + pub code: Bytes, + /// Calldata passed to the program (may be empty). + pub calldata: Bytes, +} + +/// Execute one [`StorageProgram`] via `eth_call` and return its raw output. +pub async fn run_storage_program>( + provider: &P, + block: BlockId, + program: &StorageProgram, +) -> StorageFetchResult { + let mut overrides = StateOverride::default(); + overrides.insert( + program.target, + AccountOverride::default().with_code(program.code.clone()), + ); + let tx = TransactionRequest::default() + .to(program.target) + .input(program.calldata.clone().into()); + provider + .client() + .request("eth_call", (tx, block, overrides)) + .await + .map_err(|e| StorageFetchError::provider("eth_call", &e)) +} + +/// Execute several [`StorageProgram`]s, batching programs with distinct +/// targets into a single Multicall3-dispatched `eth_call`. +/// +/// Programs that share a target address (each needs its own code override at +/// that key) or that target the dispatcher address run as individual calls. +/// Results are returned in input order, one per program. +pub async fn run_storage_programs>( + provider: &P, + block: BlockId, + programs: &[StorageProgram], +) -> Vec> { + let mut seen = std::collections::HashSet::new(); + let mut bundle: Vec = Vec::new(); + let mut individual: Vec = Vec::new(); + for (index, program) in programs.iter().enumerate() { + if program.target != MULTICALL3_ADDRESS && seen.insert(program.target) { + bundle.push(index); + } else { + individual.push(index); + } + } + // A bundle of one is just an ordinary call with dispatch overhead. + if bundle.len() == 1 { + individual.append(&mut bundle); + } + + let mut out: Vec>> = vec![None; programs.len()]; + + if !bundle.is_empty() { + let mut overrides = StateOverride::default(); + overrides.insert( + MULTICALL3_ADDRESS, + AccountOverride::default().with_code(multicall3_runtime_code().clone()), + ); + let calls: Vec = bundle + .iter() + .map(|&index| { + let program = &programs[index]; + overrides.insert( + program.target, + AccountOverride::default().with_code(program.code.clone()), + ); + IMulticall3::Call3 { + target: program.target, + allowFailure: true, + callData: program.calldata.clone(), + } + }) + .collect(); + let data: Bytes = IMulticall3::aggregate3Call { calls }.abi_encode().into(); + let tx = TransactionRequest::default() + .to(MULTICALL3_ADDRESS) + .input(data.into()); + let response: Result = provider + .client() + .request("eth_call", (tx, block, overrides)) + .await; + match response + .map_err(|e| StorageFetchError::provider("eth_call", &e)) + .and_then(|bytes| { + IMulticall3::aggregate3Call::abi_decode_returns(&bytes).map_err(|e| { + StorageFetchError::custom(format!("failed to decode aggregate3 response: {e}")) + }) + }) { + Ok(results) if results.len() == bundle.len() => { + for (&index, result) in bundle.iter().zip(results) { + out[index] = Some(if result.success { + Ok(result.returnData) + } else { + Err(StorageFetchError::custom( + "storage program subcall failed (allowFailure=true)", + )) + }); + } + } + Ok(results) => { + let err = StorageFetchError::custom(format!( + "aggregate3 returned {} results for {} programs", + results.len(), + bundle.len() + )); + for &index in &bundle { + out[index] = Some(Err(err.clone())); + } + } + Err(err) => { + for &index in &bundle { + out[index] = Some(Err(err.clone())); + } + } + } + } + + for index in individual { + out[index] = Some(run_storage_program(provider, block, &programs[index]).await); + } + + out.into_iter() + .map(|entry| entry.expect("every program resolved")) + .collect() +} + +/// Account-fields extractor: calldata is a contiguous array of 32-byte +/// left-padded addresses; the return data is `[balance, extcodehash]` (two +/// words) per address, via the `BALANCE` and `EXTCODEHASH` opcodes. +/// +/// ```text +/// [00] PUSH0 counter = 0 +/// [01] JUMPDEST loop: exit when counter == calldatasize +/// [08] DUP1 CALLDATALOAD addr +/// [0a] DUP1 BALANCE mem[2*counter] = balance(addr) +/// [11] EXTCODEHASH mem[2*counter + 32] = extcodehash(addr) +/// [1a] counter += 32; loop +/// [20] JUMPDEST RETURN(0, 2*calldatasize) +/// ``` +/// +/// Requires `PUSH0` (Shanghai). Nonces and storage roots are **not** +/// EVM-visible — use `eth_getProof` (the [`AccountProofFetchFn`] path) when +/// those are needed. +/// +/// [`AccountProofFetchFn`]: crate::cache::AccountProofFetchFn +pub const ACCOUNT_FIELDS_EXTRACTOR_CODE: &[u8] = + &hex!("5f5b803614602057803580318260011b523f8160011b602001526020016001565b3660011b5ff3"); + +/// Balance + code hash of one account, as sampled in-EVM by +/// [`ACCOUNT_FIELDS_EXTRACTOR_CODE`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AccountFieldsSample { + /// Native balance (`BALANCE`). + pub balance: U256, + /// `EXTCODEHASH` semantics (EIP-1052): zero for a non-existent account, + /// `keccak256("")` for an existing code-less account (EOA). + pub code_hash: B256, +} + +/// Fetch balance + code hash for many accounts in **one** `eth_call`. +/// +/// `BALANCE`/`EXTCODEHASH` read *other* accounts, so only one code override +/// (the extractor host, at [`MULTICALL3_ADDRESS`]) is injected and the +/// queried accounts are untouched. Costs ~5.3k gas per address (two cold +/// account accesses), so thousands of accounts fit in one call. Querying the +/// host address itself reports the extractor's own code hash — give it a +/// dedicated `eth_getProof` instead. +pub async fn fetch_account_fields_bulk>( + provider: &P, + addresses: &[Address], + block: BlockId, +) -> StorageFetchResult> { + if addresses.is_empty() { + return Ok(Vec::new()); + } + let mut calldata = Vec::with_capacity(addresses.len() * 32); + for address in addresses { + calldata.extend_from_slice(&[0u8; 12]); + calldata.extend_from_slice(address.as_slice()); + } + let program = StorageProgram { + target: MULTICALL3_ADDRESS, + code: Bytes::from_static(ACCOUNT_FIELDS_EXTRACTOR_CODE), + calldata: calldata.into(), + }; + let bytes = run_storage_program(provider, block, &program).await?; + if bytes.len() != addresses.len() * 64 { + return Err(StorageFetchError::custom(format!( + "account-fields extractor returned {} bytes, expected {}", + bytes.len(), + addresses.len() * 64 + ))); + } + Ok(addresses + .iter() + .enumerate() + .map(|(i, address)| { + ( + *address, + AccountFieldsSample { + balance: U256::from_be_slice(&bytes[i * 64..i * 64 + 32]), + code_hash: B256::from_slice(&bytes[i * 64 + 32..i * 64 + 64]), + }, + ) + }) + .collect()) +} + +/// Block-context extractor: no calldata; returns seven words — +/// `NUMBER`, `TIMESTAMP`, `BASEFEE`, `COINBASE`, `PREVRANDAO`, `GASLIMIT`, +/// `CHAINID` — straight from the EVM environment of the queried block. +/// Piggybacks block-header context onto the same transport as slot loads +/// without an `eth_getBlockByNumber`. Requires `PUSH0` (Shanghai). +pub const BLOCK_CONTEXT_EXTRACTOR_CODE: &[u8] = + &hex!("435f52426020524860405241606052446080524560a0524660c05260e05ff3"); + +/// One block's EVM-visible context, as sampled by +/// [`BLOCK_CONTEXT_EXTRACTOR_CODE`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockContextSample { + /// `NUMBER`. + pub number: u64, + /// `TIMESTAMP`. + pub timestamp: u64, + /// `BASEFEE` (wei). + pub basefee: U256, + /// `COINBASE`. + pub coinbase: Address, + /// `PREVRANDAO` (post-merge mix hash). + pub prevrandao: B256, + /// `GASLIMIT`. + pub gas_limit: u64, + /// `CHAINID`. + pub chain_id: u64, +} + +/// Sample a block's EVM context in one `eth_call` (see +/// [`BLOCK_CONTEXT_EXTRACTOR_CODE`]). +pub async fn fetch_block_context>( + provider: &P, + block: BlockId, +) -> StorageFetchResult { + let program = StorageProgram { + target: MULTICALL3_ADDRESS, + code: Bytes::from_static(BLOCK_CONTEXT_EXTRACTOR_CODE), + calldata: Bytes::new(), + }; + let bytes = run_storage_program(provider, block, &program).await?; + if bytes.len() != 7 * 32 { + return Err(StorageFetchError::custom(format!( + "block-context extractor returned {} bytes, expected 224", + bytes.len() + ))); + } + let word = |i: usize| U256::from_be_slice(&bytes[i * 32..(i + 1) * 32]); + let to_u64 = |v: U256| u64::try_from(v).unwrap_or(u64::MAX); + Ok(BlockContextSample { + number: to_u64(word(0)), + timestamp: to_u64(word(1)), + basefee: word(2), + coinbase: Address::from_slice(&bytes[3 * 32 + 12..4 * 32]), + prevrandao: B256::from_slice(&bytes[4 * 32..5 * 32]), + gas_limit: to_u64(word(5)), + chain_id: to_u64(word(6)), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn addr(byte: u8) -> Address { + Address::repeat_byte(byte) + } + + fn cfg(max_slots: usize, max_targets: usize) -> BulkCallConfig { + BulkCallConfig { + max_slots_per_call: max_slots, + max_targets_per_call: max_targets, + ..BulkCallConfig::default() + } + } + + #[test] + fn pack_and_decode_roundtrip() { + let slots = vec![U256::ZERO, U256::from(1u64), U256::MAX]; + let packed = pack_slots_calldata(&slots); + assert_eq!(packed.len(), 96); + assert_eq!(&packed[32..64], &U256::from(1u64).to_be_bytes::<32>()); + let decoded = decode_packed_values(&packed, 3).expect("exact length"); + assert_eq!(decoded, slots); + assert!(decode_packed_values(&packed, 2).is_none()); + assert!(decode_packed_values(&packed[..95], 3).is_none()); + } + + #[test] + fn extractor_constants_are_wellformed() { + // Anchor the exact published bytecode; the EVM-level behavior of both + // variants is exercised end-to-end in tests/bulk_storage.rs. + assert_eq!(STORAGE_EXTRACTOR_CODE.len(), 23); + assert_eq!(STORAGE_EXTRACTOR_CODE[0], 0x5f, "PUSH0 entry"); + assert_eq!(STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI.len(), 25); + assert!( + !STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI.contains(&0x5f), + "pre-Shanghai variant must not use PUSH0" + ); + assert!(multicall3_runtime_code().len() > 1_000); + } + + #[test] + fn planning_single_small_group() { + let requests = vec![ + (addr(0xaa), U256::from(1u64)), + (addr(0xaa), U256::from(2u64)), + ]; + let plans = plan_calls(&requests, &cfg(100, 10)); + assert_eq!( + plans, + vec![CallPlan::Single { + target: addr(0xaa), + slots: vec![U256::from(1u64), U256::from(2u64)], + }] + ); + } + + #[test] + fn planning_splits_oversized_target_and_packs_remainder() { + // 7 slots with a 3-slot budget: two full single-target chunks + the + // 1-slot remainder packed with the other small target. + let mut requests: Vec<_> = (0..7u64).map(|i| (addr(0x01), U256::from(i))).collect(); + requests.push((addr(0x02), U256::from(99u64))); + let plans = plan_calls(&requests, &cfg(3, 10)); + assert_eq!(plans.len(), 3); + assert_eq!( + plans[0], + CallPlan::Single { + target: addr(0x01), + slots: (0..3u64).map(U256::from).collect(), + } + ); + assert_eq!( + plans[1], + CallPlan::Single { + target: addr(0x01), + slots: (3..6u64).map(U256::from).collect(), + } + ); + assert_eq!( + plans[2], + CallPlan::Multi { + targets: vec![ + (addr(0x01), vec![U256::from(6u64)]), + (addr(0x02), vec![U256::from(99u64)]), + ], + } + ); + let planned: usize = plans.iter().map(CallPlan::request_slot_count).sum(); + assert_eq!(planned, requests.len()); + } + + #[test] + fn planning_respects_target_budget() { + let requests: Vec<_> = (0..5u8) + .map(|i| (addr(i + 1), U256::from(i as u64))) + .collect(); + let plans = plan_calls(&requests, &cfg(100, 2)); + // 5 single-slot targets with a 2-target budget: 2 + 2 + 1. + assert_eq!(plans.len(), 3); + assert!(matches!(&plans[0], CallPlan::Multi { targets } if targets.len() == 2)); + assert!(matches!(&plans[1], CallPlan::Multi { targets } if targets.len() == 2)); + assert!(matches!(&plans[2], CallPlan::Single { .. })); + } + + #[test] + fn planning_lone_remainder_degrades_to_single_call() { + let requests: Vec<_> = (0..4u64).map(|i| (addr(0x01), U256::from(i))).collect(); + let plans = plan_calls(&requests, &cfg(3, 10)); + assert_eq!(plans.len(), 2); + assert!(matches!(&plans[1], CallPlan::Single { slots, .. } if slots.len() == 1)); + } + + #[test] + fn planning_isolates_dispatcher_address_collision() { + let requests = vec![ + (MULTICALL3_ADDRESS, U256::from(1u64)), + (addr(0x02), U256::from(2u64)), + (addr(0x03), U256::from(3u64)), + ]; + let plans = plan_calls(&requests, &cfg(100, 10)); + assert_eq!( + plans[0], + CallPlan::Single { + target: MULTICALL3_ADDRESS, + slots: vec![U256::from(1u64)], + } + ); + assert!(matches!(&plans[1], CallPlan::Multi { targets } if targets.len() == 2)); + } + + #[test] + fn multi_target_overrides_include_dispatcher_and_extractors() { + let plan = CallPlan::Multi { + targets: vec![ + (addr(0x02), vec![U256::from(1u64)]), + (addr(0x03), vec![U256::from(2u64)]), + ], + }; + let extractor = Bytes::from_static(STORAGE_EXTRACTOR_CODE); + let overrides = overrides_for_plan(&plan, &extractor); + assert_eq!(overrides.len(), 3); + assert_eq!( + overrides[&MULTICALL3_ADDRESS].code.as_ref(), + Some(multicall3_runtime_code()) + ); + assert_eq!(overrides[&addr(0x02)].code.as_ref(), Some(&extractor)); + assert_eq!(overrides[&addr(0x03)].code.as_ref(), Some(&extractor)); + } + + #[test] + fn call_many_grouping_respects_request_budget() { + let plans = vec![ + CallPlan::Single { + target: addr(0x01), + slots: (0..6u64).map(U256::from).collect(), + }, + CallPlan::Single { + target: addr(0x02), + slots: (0..6u64).map(U256::from).collect(), + }, + CallPlan::Single { + target: addr(0x03), + slots: (0..2u64).map(U256::from).collect(), + }, + ]; + let groups = group_plans_for_call_many(plans, 10); + // 6 + 6 > 10 → split; 6 + 2 ≤ 10 → packed together. + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].len(), 1); + assert_eq!(groups[1].len(), 2); + let total: usize = groups + .iter() + .flatten() + .map(CallPlan::request_slot_count) + .sum(); + assert_eq!(total, 14); + } + + #[test] + fn multi_target_response_decodes_per_target_failures() { + let targets = vec![ + (addr(0x02), vec![U256::from(1u64), U256::from(2u64)]), + (addr(0x03), vec![U256::from(3u64)]), + ]; + let response = IMulticall3::aggregate3Call::abi_encode_returns(&vec![ + IMulticall3::Result { + success: true, + returnData: pack_slots_calldata(&[U256::from(11u64), U256::from(22u64)]), + }, + IMulticall3::Result { + success: false, + returnData: Bytes::new(), + }, + ]); + let results = decode_multi_target_response(&targets, &response); + assert_eq!(results.len(), 3); + assert!(matches!(results[0], (_, _, Ok(v)) if v == U256::from(11u64))); + assert!(matches!(results[1], (_, _, Ok(v)) if v == U256::from(22u64))); + assert!(results[2].2.is_err()); + } +} diff --git a/src/bundle.rs b/src/bundle.rs index fe0652b..3fb2c9a 100644 --- a/src/bundle.rs +++ b/src/bundle.rs @@ -133,9 +133,27 @@ pub struct BundleResult { /// transactions (priority fee + direct coinbase tips; the base fee is already /// excluded by revm — see the [module docs](self#coinbase-accounting)). /// Saturating; `0` for an [`Atomic`](RevertPolicy::Atomic) bundle that aborted. + /// + /// Under [`AllowReverts`](RevertPolicy::AllowReverts) a whitelisted tx that + /// reverts contributes **nothing** to this figure (its beneficiary credit is + /// rolled back with the rest of its effects), even though it still burned gas. + /// A searcher's net cost therefore also includes that wasted gas: + /// `coinbase_payment + reverted_tx_gas` approximates net searcher cost under + /// `AllowReverts`. pub coinbase_payment: U256, - /// Total gas used across the executed transactions. + /// Total gas used across **all** executed transactions — successful and + /// reverted alike. Equal to `successful_tx_gas + reverted_tx_gas`. Saturating. pub gas_used: u64, + /// Sum of [`gas_used`](TxOutcome::gas_used) over the executed transactions that + /// did **not** revert (`!reverted`). This is the gas backing + /// [`coinbase_payment`](Self::coinbase_payment). Saturating. + pub successful_tx_gas: u64, + /// Sum of [`gas_used`](TxOutcome::gas_used) over the executed transactions that + /// **reverted** (`reverted`). Under [`AllowReverts`](RevertPolicy::AllowReverts) + /// this is the gas a whitelisted revert wasted without contributing to + /// [`coinbase_payment`](Self::coinbase_payment); a searcher recovers it here + /// instead of iterating [`per_tx`](Self::per_tx). Saturating. + pub reverted_tx_gas: u64, /// `false` iff an [`Atomic`](RevertPolicy::Atomic) bundle aborted on a /// revert/halt (or an `AllowReverts` bundle hit a revert at a non-whitelisted /// index). diff --git a/src/cache/binary_state.rs b/src/cache/binary_state.rs index 369d433..c81a0d6 100644 --- a/src/cache/binary_state.rs +++ b/src/cache/binary_state.rs @@ -13,13 +13,13 @@ use std::time::Instant; use alloy_primitives::map::HashMap; use alloy_primitives::{Address, B256, U256}; -use anyhow::{Context as _, Result}; use foundry_fork_db::BlockchainDb; use revm::state::AccountInfo; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; use super::versioned; +use crate::errors::PersistenceError; const BINARY_STATE_MAGIC: &[u8; 8] = b"EFCSTAT\0"; const BINARY_STATE_VERSION: u32 = 1; @@ -54,7 +54,10 @@ struct BinaryAccountInfo { /// The on-disk format carries magic bytes and a version number before the /// bincode payload. Unknown versions are treated as a cache miss rather than /// being migrated. -pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> Result<()> { +pub fn save_binary_state( + blockchain_db: &BlockchainDb, + path: &Path, +) -> Result<(), PersistenceError> { let start = Instant::now(); let accounts: Vec<(Address, BinaryAccountInfo)> = blockchain_db @@ -89,11 +92,9 @@ pub fn save_binary_state(blockchain_db: &BlockchainDb, path: &Path) -> Result<() "binary EVM state", )?; if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("failed to create binary EVM state directory {parent:?}"))?; + std::fs::create_dir_all(parent).map_err(|err| PersistenceError::create_dir(parent, err))?; } - std::fs::write(path, &data) - .with_context(|| format!("failed to write binary EVM state to {path:?}"))?; + std::fs::write(path, &data).map_err(|err| PersistenceError::write(path, err))?; let ms = start.elapsed().as_millis(); debug!( diff --git a/src/cache/bytecode.rs b/src/cache/bytecode.rs index 1ef00bc..2946dc2 100644 --- a/src/cache/bytecode.rs +++ b/src/cache/bytecode.rs @@ -14,11 +14,11 @@ use std::collections::HashMap; use std::path::Path; use alloy_primitives::Address; -use anyhow::Result; use foundry_fork_db::BlockchainDb; use serde::{Deserialize, Serialize}; use super::versioned; +use crate::errors::PersistenceError; const BYTECODE_CACHE_MAGIC: &[u8; 8] = b"EFCBYTE\0"; const BYTECODE_CACHE_VERSION: u32 = 1; @@ -62,9 +62,10 @@ impl BytecodeCache { /// /// Returns an error if the parent directory cannot be created, if bincode /// serialization fails, or if writing the file fails. - pub(crate) fn save(&self, path: &Path) -> Result<()> { + pub(crate) fn save(&self, path: &Path) -> Result<(), PersistenceError> { if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; } let data = versioned::encode( BYTECODE_CACHE_MAGIC, @@ -72,7 +73,7 @@ impl BytecodeCache { self, "bytecode cache", )?; - std::fs::write(path, data)?; + std::fs::write(path, data).map_err(|err| PersistenceError::write(path, err))?; Ok(()) } @@ -107,7 +108,12 @@ mod tests { use revm::state::{AccountInfo, Bytecode}; fn temp_path(tag: &str) -> std::path::PathBuf { - let dir = std::env::temp_dir().join(format!("evm_fork_cache_bytecode_{tag}")); + // Keyed by pid so concurrent `cargo test` processes never share (and + // never `remove_dir_all`) each other's directory. + let dir = std::env::temp_dir().join(format!( + "evm_fork_cache_bytecode_{tag}_{}", + std::process::id() + )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).expect("create temp dir"); dir.join("bytecodes.bin") diff --git a/src/cache/code_seeds.rs b/src/cache/code_seeds.rs new file mode 100644 index 0000000..8c7f999 --- /dev/null +++ b/src/cache/code_seeds.rs @@ -0,0 +1,260 @@ +//! Code-seed marks: provenance + trust state for bytecode that did **not** +//! arrive via the lazy RPC backend. +//! +//! Adapters can push runtime code into the cache instead of paying an +//! `eth_getCode` per address (see `EvmCache::seed_account_code` / +//! `EvmCache::etch_account_code`). Every such write records a +//! [`CodeSeedState`] mark; the *absence* of a mark means the code is +//! RPC-origin (fetched from the provider and trusted as chain state). +//! +//! Marks persist across restarts in `code_seeds.bin` so a `Pending` claim can +//! never masquerade as chain-fetched after a reload. The file is written as a +//! crate-specific versioned envelope followed by bincode payload, so +//! incompatible versions are detected as cache misses. Unlike `bytecodes.bin` +//! (load-merge-save, correct for immutable code), this file is saved as a +//! **full replace** of the in-memory map: marks are mutable trust state, and +//! a merge would resurrect marks that were purged this session. + +use std::collections::HashMap; +use std::path::Path; + +use alloy_primitives::{Address, B256}; +use serde::{Deserialize, Serialize}; + +use super::versioned; +use crate::errors::PersistenceError; + +const CODE_SEED_CACHE_MAGIC: &[u8; 8] = b"EFCSEED\0"; +const CODE_SEED_CACHE_VERSION: u32 = 1; + +/// Provenance + trust state of an address's cached bytecode, for code that +/// did **not** arrive via the lazy RPC backend. +/// +/// Absence of a mark means RPC-origin (fetched from the provider, trusted as +/// chain state). See the two write primitives: +/// [`seed_account_code`](crate::cache::EvmCache::seed_account_code) (canonical +/// claim, verified once) and +/// [`etch_account_code`](crate::cache::EvmCache::etch_account_code) +/// (deliberate local divergence). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CodeSeedState { + /// Canonical claim awaiting on-chain code-hash verification + /// (`EvmCache::verify_code_seeds`). + Pending { + /// keccak256 of the seeded runtime code. + code_hash: B256, + }, + /// Canonical claim confirmed against the chain. Never re-verified: post + /// EIP-6780, deployed code is immutable, so one confirmation is durable. + /// On chains without 6780 the escape hatch is + /// [`purge_account`](crate::cache::EvmCache::purge_account), which clears + /// the mark. + Verified { + /// keccak256 of the verified runtime code. + code_hash: B256, + /// Pinned block number at which the on-chain code hash matched. + verified_at_block: u64, + }, + /// Deliberate local divergence (an unreleased contract, a test harness). + /// Never verified, excluded from all canonical machinery, and reported on + /// the health surface via + /// [`etched_accounts`](crate::cache::EvmCache::etched_accounts). + Etched { + /// keccak256 of the etched runtime code. + code_hash: B256, + }, +} + +impl CodeSeedState { + /// The keccak256 code hash this mark refers to. + pub fn code_hash(&self) -> B256 { + match self { + Self::Pending { code_hash } + | Self::Verified { code_hash, .. } + | Self::Etched { code_hash } => *code_hash, + } + } +} + +/// Serializable code-seed mark store (`code_seeds.bin`). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub(crate) struct CodeSeedCache { + /// Map of address to its code-seed mark. + pub(crate) entries: HashMap, +} + +impl CodeSeedCache { + /// Load the mark store from disk (binary format). + /// + /// Returns `None` if `path` cannot be read, fails the magic/version check, + /// or fails to decode as bincode for this type — legacy/missing files are + /// cache misses, never errors. + pub(crate) fn load(path: &Path) -> Option { + let data = std::fs::read(path).ok()?; + versioned::decode( + &data, + CODE_SEED_CACHE_MAGIC, + CODE_SEED_CACHE_VERSION, + "code seed cache", + ) + } + + /// Save the mark store to disk (binary format), replacing any previous + /// file wholesale (see the module docs for why this is not a merge). + /// + /// # Errors + /// + /// Returns an error if the parent directory cannot be created, if bincode + /// serialization fails, or if writing the file fails. + pub(crate) fn save(&self, path: &Path) -> Result<(), PersistenceError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; + } + let data = versioned::encode( + CODE_SEED_CACHE_MAGIC, + CODE_SEED_CACHE_VERSION, + self, + "code seed cache", + )?; + std::fs::write(path, data).map_err(|err| PersistenceError::write(path, err))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_path(tag: &str) -> std::path::PathBuf { + // Keyed by pid so concurrent `cargo test` processes never share (and + // never `remove_dir_all`) each other's directory. + let dir = std::env::temp_dir().join(format!( + "evm_fork_cache_code_seeds_{tag}_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir.join("code_seeds.bin") + } + + #[test] + fn save_load_round_trip_preserves_all_three_marks() { + let path = temp_path("roundtrip"); + let pending = Address::repeat_byte(0x01); + let verified = Address::repeat_byte(0x02); + let etched = Address::repeat_byte(0x03); + + let mut cache = CodeSeedCache::default(); + cache.entries.insert( + pending, + CodeSeedState::Pending { + code_hash: B256::repeat_byte(0xaa), + }, + ); + cache.entries.insert( + verified, + CodeSeedState::Verified { + code_hash: B256::repeat_byte(0xbb), + verified_at_block: 123, + }, + ); + cache.entries.insert( + etched, + CodeSeedState::Etched { + code_hash: B256::repeat_byte(0xcc), + }, + ); + cache.save(&path).expect("save code seed cache"); + + let bytes = std::fs::read(&path).expect("read saved code seed cache"); + assert!( + bytes.starts_with(b"EFCSEED\0"), + "code seed cache must carry a magic header" + ); + assert_eq!( + &bytes[8..12], + &1u32.to_le_bytes(), + "code seed cache must carry an explicit version" + ); + + let loaded = CodeSeedCache::load(&path).expect("load code seed cache"); + assert_eq!(loaded.entries.len(), 3); + assert_eq!( + loaded.entries.get(&pending), + Some(&CodeSeedState::Pending { + code_hash: B256::repeat_byte(0xaa) + }), + "Pending survives a reload as Pending — it must never masquerade as RPC-origin" + ); + assert_eq!( + loaded.entries.get(&verified), + Some(&CodeSeedState::Verified { + code_hash: B256::repeat_byte(0xbb), + verified_at_block: 123 + }) + ); + assert_eq!( + loaded.entries.get(&etched), + Some(&CodeSeedState::Etched { + code_hash: B256::repeat_byte(0xcc) + }) + ); + + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn load_unversioned_or_missing_is_none() { + let path = temp_path("legacy"); + let mut cache = CodeSeedCache::default(); + cache.entries.insert( + Address::repeat_byte(0x42), + CodeSeedState::Etched { + code_hash: B256::repeat_byte(0x42), + }, + ); + std::fs::write(&path, bincode::serialize(&cache).unwrap()).expect("write legacy cache"); + assert!( + CodeSeedCache::load(&path).is_none(), + "unversioned legacy bincode must be treated as a cache miss" + ); + assert!(CodeSeedCache::load(std::path::Path::new("/nonexistent/code_seeds.bin")).is_none()); + + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn save_is_full_replace_not_merge() { + let path = temp_path("replace"); + let stale = Address::repeat_byte(0x01); + let kept = Address::repeat_byte(0x02); + + let mut first = CodeSeedCache::default(); + first.entries.insert( + stale, + CodeSeedState::Pending { + code_hash: B256::repeat_byte(0xaa), + }, + ); + first.save(&path).expect("save first"); + + // A second save without the stale entry must not resurrect it: a + // purged mark staying purged is the whole point of replace semantics. + let mut second = CodeSeedCache::default(); + second.entries.insert( + kept, + CodeSeedState::Etched { + code_hash: B256::repeat_byte(0xbb), + }, + ); + second.save(&path).expect("save second"); + + let loaded = CodeSeedCache::load(&path).expect("load code seed cache"); + assert_eq!(loaded.entries.len(), 1, "stale entry must not survive"); + assert!(loaded.entries.contains_key(&kept)); + assert!(!loaded.entries.contains_key(&stale)); + + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } +} diff --git a/src/cache/metadata.rs b/src/cache/metadata.rs index 4a7708e..0fec753 100644 --- a/src/cache/metadata.rs +++ b/src/cache/metadata.rs @@ -10,12 +10,12 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use alloy_primitives::{Address, U256}; -use anyhow::Result; use serde::{Deserialize, Serialize}; use std::collections::HashSet; use super::versioned; +use crate::errors::PersistenceError; const IMMUTABLE_CACHE_MAGIC: &[u8; 8] = b"EFCMETA\0"; const IMMUTABLE_CACHE_VERSION: u32 = 2; @@ -79,6 +79,14 @@ impl CacheConfig { self.chain_dir().join("immutable_data.bin") } + /// Get the path for the code-seed mark cache file (binary format). + /// + /// Saved by `flush()` strictly before `bytecodes.bin` so persisted code + /// can never outrun the trust marks describing it. + pub(crate) fn code_seeds_cache_path(&self) -> PathBuf { + self.chain_dir().join("code_seeds.bin") + } + /// Get the path for the EVM state cache file (bincode format). /// /// This cache stores the complete EVM state (accounts + storage) in @@ -126,9 +134,10 @@ impl ImmutableDataCache { /// /// Returns an error if the parent directory cannot be created, if bincode /// serialization fails, or if writing the file fails. - pub fn save(&self, path: &Path) -> Result<()> { + pub fn save(&self, path: &Path) -> Result<(), PersistenceError> { if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; } let data = versioned::encode( IMMUTABLE_CACHE_MAGIC, @@ -136,7 +145,7 @@ impl ImmutableDataCache { self, "immutable data cache", )?; - std::fs::write(path, data)?; + std::fs::write(path, data).map_err(|err| PersistenceError::write(path, err))?; Ok(()) } diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 28209b7..4131a5c 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1,5 +1,21 @@ +//! The forked-EVM state cache: lazy RPC loading, a layered write funnel, and +//! cheap copy-on-write snapshots. +//! +//! [`EvmCache`] is the core handle. It fronts a [`foundry_fork_db`]-backed fork +//! database with a hot [`revm`] cache layer, lazily fetching account and storage +//! state from a provider on the first miss and serving it locally thereafter. +//! Targeted writes and purges ([`StateUpdate`], balance/code overrides, verified +//! code seeds) flow through a single write +//! funnel — never the RPC path — so event-driven state maintenance never round-trips. +//! [`EvmCache::snapshot`] produces an immutable, `Arc`-shared, cross-thread +//! [`EvmSnapshot`] (see [`snapshot`]) for parallel fan-out; [`overlay`] +//! layers per-simulation state on top. See the crate-root docs for the full +//! state stack and [`docs/INTERNALS.md`](https://github.com/KaiCode2/evm-fork-cache/blob/main/docs/INTERNALS.md) +//! for the snapshot cost model. + mod binary_state; mod bytecode; +mod code_seeds; mod journal_access_list; mod metadata; pub mod overlay; @@ -18,10 +34,7 @@ use std::{ collections::{HashMap, HashSet}, fs, rc::Rc, - sync::{ - Arc, Mutex, - atomic::{AtomicU8, Ordering}, - }, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; @@ -33,7 +46,6 @@ use alloy_primitives::{Address, B256, Bytes, I256, Log, TxKind, U256, keccak256} use alloy_provider::{Provider, network::AnyNetwork}; use alloy_rpc_types_eth::TransactionRequest; use alloy_sol_types::{SolCall, SolValue, sol}; -use anyhow::{Context as _, Result, anyhow}; use foundry_fork_db::{BlockchainDb, SharedBackend, cache::BlockchainDbMeta}; use revm::{ Context, ExecuteCommitEvm, ExecuteEvm, InspectEvm, MainBuilder, MainContext, @@ -46,7 +58,11 @@ use revm::{ use tracing::{debug, instrument, trace, warn}; use crate::access_set::StorageAccessList; -use crate::errors::{SimError, SimulationError, SimulationResult}; +use crate::bulk_storage::AccountFieldsSample; +use crate::errors::{ + BlockContextError, CacheError, CacheResult as Result, RpcError, RuntimeError, SimError, + SimHostError, SimulationError, SimulationResult, StorageFetchError, StorageFetchResult, +}; use crate::freshness::{SlotChange, SlotFetch, SlotOutcome}; use crate::inspector::TransferInspector; use crate::state_update::{ @@ -55,6 +71,8 @@ use crate::state_update::{ }; use bytecode::BytecodeCache; +use code_seeds::CodeSeedCache; +pub use code_seeds::CodeSeedState; use journal_access_list::{extract_access_list, merge_access_lists}; /// Re-export AnyNetwork for callers that need to construct providers. @@ -67,7 +85,7 @@ pub type ForkCacheDB = CacheDB; /// Callback for making direct RPC `eth_call` requests, bypassing revm simulation. /// Used when batch-querying many contracts where revm's lazy storage fetching would /// be prohibitively slow (e.g. querying 500+ gauge contracts). -pub type RpcCallFn = Arc Result + Send + Sync>; +pub type RpcCallFn = Arc Result + Send + Sync>; /// Callback for batch-fetching storage slots directly from RPC, bypassing SharedBackend. /// @@ -75,13 +93,14 @@ pub type RpcCallFn = Arc Result + Send + Sync>; /// round-trips through SharedBackend. Fires concurrent `eth_getStorageAt` calls /// directly via the provider and returns results for bulk injection into /// BlockchainDb. +/// Users may replace the provider-backed implementation with their own fetcher via +/// [`EvmCache::set_storage_batch_fetcher`]. /// -/// The second argument pins the fetch to a specific block: `Some(block)` fetches -/// at exactly that block, while `None` uses the fetcher's configured block (the -/// cache's currently-pinned block). The freshness validator passes the block its -/// snapshot was built from, so a concurrent [`EvmCache::set_block`] cannot make -/// the deferred fetch read a *different* block than the snapshot it is compared -/// against. +/// The second argument pins the fetch to a specific block. Callers pass the +/// cache's pinned block at the point they schedule the fetch; deferred callers +/// such as the freshness validator pass the block their snapshot was built from, +/// so a concurrent [`EvmCache::set_block`] cannot make the deferred fetch read a +/// *different* block than the snapshot it is compared against. /// /// **Contract:** an implementation must return **exactly one** result tuple per /// requested `(address, slot)` (order does not matter). Callers — `verify_slots`, @@ -90,11 +109,109 @@ pub type RpcCallFn = Arc Result + Send + Sync>; /// reorders-and-truncates, or duplicates entries breaks the "one outcome per /// requested slot" guarantee those APIs document. pub type StorageBatchFetchFn = Arc< - dyn Fn(Vec<(Address, U256)>, Option) -> Vec<(Address, U256, Result)> + dyn Fn(Vec<(Address, U256)>, BlockId) -> Vec<(Address, U256, StorageFetchResult)> + Send + Sync, >; +/// Account header + optional storage-proof slots from `eth_getProof`. +/// `slots` is populated only for requested storage keys; an empty key list is a +/// root-only probe (account fields + `storage_hash`, no slot payload). +#[derive(Clone, Debug)] +pub struct AccountProof { + /// Merkle root of the account's storage trie (`storageHash`). + pub storage_hash: B256, + /// Account balance. + pub balance: U256, + /// Account nonce. + pub nonce: u64, + /// Hash of the account's runtime code (`codeHash`). + pub code_hash: B256, + /// Proven `(slot, value)` pairs for the requested storage keys. Empty for a + /// root-only probe. + pub slots: Vec<(U256, U256)>, +} + +/// Callback for fetching account headers (and optional storage-proof slots) +/// directly from RPC via `eth_getProof`, mirroring [`StorageBatchFetchFn`]. +/// +/// Used by callers that need authoritative account fields (balance/nonce/code +/// hash) plus the account's `storageHash`, e.g. account-target resyncs and +/// account-level freshness. Each request is a `(address, keys)` pair; an empty +/// `keys` list is a root-only probe. +/// +/// The second argument pins the fetch to a specific block, matching +/// [`StorageBatchFetchFn`]'s block semantics. +/// +/// **Contract:** an implementation returns at most one result per requested +/// address. An address present with `Ok(..)` succeeded; present with `Err(..)` +/// failed; omitted entirely means the fetcher produced no result for it. Callers +/// derive their per-address outcome from whether the address appears and, if so, +/// whether it is `Ok`/`Err`. +pub type AccountProofFetchFn = Arc< + dyn Fn(Vec<(Address, Vec)>, BlockId) -> Vec<(Address, StorageFetchResult)> + + Send + + Sync, +>; + +/// Callback fetching `(balance, EXTCODEHASH)` samples for many addresses at a +/// pinned block — one bulk `eth_call` by default (the +/// [`ACCOUNT_FIELDS_EXTRACTOR_CODE`](crate::bulk_storage::ACCOUNT_FIELDS_EXTRACTOR_CODE) +/// program). Sync and type-erased with the same bridging rules as +/// [`StorageBatchFetchFn`] (multi-thread tokio runtime required for the +/// default provider-backed implementation). +/// +/// **Contract:** the call is all-or-nothing — `Ok` carries one sample per +/// requested address (an omitted address is treated by callers as +/// unverifiable), `Err` means the whole fetch failed and nothing can be +/// concluded about any address. Used by +/// [`EvmCache::verify_code_seeds`](EvmCache::verify_code_seeds) and the +/// cold-start `verify_code` phase. +pub type AccountFieldsFetchFn = Arc< + dyn Fn(Vec

, BlockId) -> StorageFetchResult> + + Send + + Sync, +>; + +/// Final state changes observed from a block-level state-diff trace. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BlockStateDiff { + /// Accounts changed by the traced block. + pub accounts: Vec, +} + +/// Final account/storage values observed for one account in a block trace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BlockStateAccountDiff { + /// Changed account address. + pub address: Address, + /// Final balance when the trace reports a balance change. + pub balance: Option, + /// Final nonce when the trace reports a nonce change. + pub nonce: Option, + /// Final runtime bytecode when the trace reports a code change. + pub code: Option, + /// Final storage-slot values reported for this account. + pub storage: Vec, +} + +/// Final value for one storage slot observed from a block trace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BlockStateStorageDiff { + /// Storage slot key. + pub slot: U256, + /// Final slot value after the block. Cleared slots are represented as zero. + pub value: U256, +} + +/// Callback for fetching one block's state diff through debug/trace RPC. +/// +/// The callback returns final post-block values for accounts/storage slots that +/// changed in the block. Callers may resolve matching resync targets from this +/// diff before falling back to point reads. +pub type BlockStateDiffFetchFn = + Arc StorageFetchResult + Send + Sync>; + /// Return a tokio runtime [`Handle`] suitable for `block_in_place` + `block_on`, /// or an error describing why one is unavailable. /// @@ -105,23 +222,244 @@ pub type StorageBatchFetchFn = Arc< /// degrade to a typed error instead. /// /// Requires a **multi-thread** tokio runtime. -pub(crate) fn block_in_place_handle() -> Result { +pub(crate) fn block_in_place_handle() -> Result { match tokio::runtime::Handle::try_current() { Ok(handle) => match handle.runtime_flavor() { - tokio::runtime::RuntimeFlavor::CurrentThread => Err(anyhow!( - "evm-fork-cache RPC operations require a multi-thread tokio runtime; \ - found a current-thread runtime (block_in_place is not supported there). \ - Build the runtime with `tokio::runtime::Builder::new_multi_thread()` \ - or annotate with `#[tokio::main(flavor = \"multi_thread\")]`" - )), + tokio::runtime::RuntimeFlavor::CurrentThread => Err(RuntimeError::CurrentThreadRuntime), _ => Ok(handle), }, - Err(e) => Err(anyhow!( - "evm-fork-cache RPC operations require a running multi-thread tokio runtime: {e}" - )), + Err(e) => Err(RuntimeError::MissingRuntime { + details: e.to_string(), + }), + } +} + +fn trace_rpc_method_and_params(block: BlockId) -> (&'static str, serde_json::Value) { + let tracer = serde_json::json!({ + "tracer": "prestateTracer", + "tracerConfig": { + "diffMode": true, + }, + }); + match block { + BlockId::Hash(hash) => ( + "debug_traceBlockByHash", + serde_json::json!([hash.block_hash, tracer]), + ), + BlockId::Number(number) => ( + "debug_traceBlockByNumber", + serde_json::json!([block_number_or_tag_param(number), tracer]), + ), + } +} + +fn block_number_or_tag_param(number: BlockNumberOrTag) -> serde_json::Value { + match number { + BlockNumberOrTag::Number(number) => serde_json::json!(format!("{number:#x}")), + BlockNumberOrTag::Latest => serde_json::json!("latest"), + BlockNumberOrTag::Finalized => serde_json::json!("finalized"), + BlockNumberOrTag::Safe => serde_json::json!("safe"), + BlockNumberOrTag::Earliest => serde_json::json!("earliest"), + BlockNumberOrTag::Pending => serde_json::json!("pending"), + } +} + +fn parse_block_state_diff_trace(value: &serde_json::Value) -> Result { + let mut accounts: HashMap = HashMap::new(); + match value { + serde_json::Value::Array(traces) => { + for trace in traces { + merge_trace_diff(trace, &mut accounts)?; + } + } + trace => merge_trace_diff(trace, &mut accounts)?, + } + + let mut accounts: Vec<_> = accounts.into_values().collect(); + accounts.sort_by_key(|account| account.address); + for account in &mut accounts { + account.storage.sort_by_key(|slot| slot.slot); + } + Ok(BlockStateDiff { accounts }) +} + +fn merge_trace_diff( + trace: &serde_json::Value, + accounts: &mut HashMap, +) -> Result<()> { + let diff = trace.get("result").unwrap_or(trace); + let Some(pre) = diff.get("pre").and_then(serde_json::Value::as_object) else { + return Ok(()); + }; + let Some(post) = diff.get("post").and_then(serde_json::Value::as_object) else { + return Ok(()); + }; + + for (address, post_account) in post { + let address = parse_trace_address(address)?; + let entry = accounts + .entry(address) + .or_insert_with(|| empty_block_state_account_diff(address)); + + if let Some(balance) = post_account.get("balance") { + entry.balance = Some(parse_trace_u256(balance)?); + } + if let Some(nonce) = post_account.get("nonce") { + entry.nonce = Some(parse_trace_u64(nonce)?); + } + if let Some(code) = post_account.get("code") { + entry.code = Some(parse_trace_bytes(code)?); + } + if let Some(storage) = post_account + .get("storage") + .and_then(serde_json::Value::as_object) + { + for (slot, value) in storage { + upsert_block_state_storage_diff( + entry, + parse_trace_u256_str(slot)?, + parse_trace_u256(value)?, + ); + } + } + } + + // In diff mode, state that ends the block *absent* appears in `pre` but is + // omitted from `post`. Convert those omissions into explicit final values: + for (address_key, pre_account) in pre { + let address = parse_trace_address(address_key)?; + let post_account = post.get(address_key); + + // An account entirely absent from `post` was deleted by the block + // (SELFDESTRUCT; post-Cancun, a same-tx create+destruct). Synthesize + // the explicit post-deletion fields so account-target resyncs resolve + // authoritatively from the trace instead of falling back to point + // reads. A later transaction in the same block re-creating the account + // overwrites these in its own merge pass (entries merge in tx order). + if post_account.is_none() { + let entry = accounts + .entry(address) + .or_insert_with(|| empty_block_state_account_diff(address)); + entry.balance = Some(U256::ZERO); + entry.nonce = Some(0); + entry.code = Some(Bytes::new()); + } + + // Storage cleared to zero: present in `pre`, omitted from `post`. + let Some(pre_storage) = pre_account + .get("storage") + .and_then(serde_json::Value::as_object) + else { + continue; + }; + let post_storage = post_account + .and_then(|account| account.get("storage")) + .and_then(serde_json::Value::as_object); + for slot in pre_storage.keys() { + let cleared = post_storage.is_none_or(|storage| !storage.contains_key(slot)); + if cleared { + let entry = accounts + .entry(address) + .or_insert_with(|| empty_block_state_account_diff(address)); + upsert_block_state_storage_diff(entry, parse_trace_u256_str(slot)?, U256::ZERO); + } + } + } + + Ok(()) +} + +fn empty_block_state_account_diff(address: Address) -> BlockStateAccountDiff { + BlockStateAccountDiff { + address, + balance: None, + nonce: None, + code: None, + storage: Vec::new(), + } +} + +fn upsert_block_state_storage_diff(account: &mut BlockStateAccountDiff, slot: U256, value: U256) { + if let Some(existing) = account.storage.iter_mut().find(|entry| entry.slot == slot) { + existing.value = value; + } else { + account.storage.push(BlockStateStorageDiff { slot, value }); + } +} + +fn parse_trace_address(value: &str) -> Result
{ + value.parse().map_err(|err| CacheError::TraceParse { + details: format!("invalid address `{value}`: {err}"), + }) +} + +fn parse_trace_u256(value: &serde_json::Value) -> Result { + match value { + serde_json::Value::String(value) => parse_trace_u256_str(value), + serde_json::Value::Number(value) => parse_trace_u256_str(&value.to_string()), + other => Err(CacheError::TraceParse { + details: format!("expected U256 string/number, got {other:?}"), + }), + } +} + +fn parse_trace_u256_str(value: &str) -> Result { + if let Some(value) = value.strip_prefix("0x") { + if value.is_empty() { + return Ok(U256::ZERO); + } + return U256::from_str_radix(value, 16).map_err(|err| CacheError::TraceParse { + details: format!("invalid U256 `0x{value}`: {err}"), + }); + } + if value.is_empty() { + return Ok(U256::ZERO); + } + U256::from_str_radix(value, 10).map_err(|err| CacheError::TraceParse { + details: format!("invalid U256 `{value}`: {err}"), + }) +} + +fn parse_trace_u64(value: &serde_json::Value) -> Result { + match value { + serde_json::Value::Number(value) => value.as_u64().ok_or_else(|| CacheError::TraceParse { + details: format!("invalid u64 number `{value}`"), + }), + serde_json::Value::String(value) => { + if let Some(value) = value.strip_prefix("0x") { + if value.is_empty() { + return Ok(0); + } + return u64::from_str_radix(value, 16).map_err(|err| CacheError::TraceParse { + details: format!("invalid u64 `0x{value}`: {err}"), + }); + } + if value.is_empty() { + return Ok(0); + } + value.parse().map_err(|err| CacheError::TraceParse { + details: format!("invalid u64 `{value}`: {err}"), + }) + } + other => Err(CacheError::TraceParse { + details: format!("expected u64 string/number, got {other:?}"), + }), } } +fn parse_trace_bytes(value: &serde_json::Value) -> Result { + let Some(value) = value.as_str() else { + return Err(CacheError::TraceParse { + details: format!("expected bytecode string, got {value:?}"), + }); + }; + let value = value.strip_prefix("0x").unwrap_or(value); + let bytes = alloy_primitives::hex::decode(value).map_err(|err| CacheError::TraceParse { + details: format!("invalid bytecode hex: {err}"), + })?; + Ok(Bytes::from(bytes)) +} + pub(crate) fn unix_timestamp_secs_saturating(time: SystemTime) -> u64 { time.duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs()) @@ -183,16 +521,14 @@ fn account_patch_is_empty(patch: &AccountPatch) -> bool { patch.balance.is_none() && patch.nonce.is_none() && patch.code.is_none() } -static CACHE_SPEED_MODE: AtomicU8 = AtomicU8::new(CacheSpeedMode::Slow as u8); - -/// Runtime tuning profile for cache-side batch storage fetches. +/// Preset runtime tuning profile for cache-side batch storage fetches. /// -/// Selects the per-batch size and concurrency used by [`StorageBatchFetchFn`]: -/// faster modes send larger batches with more in-flight HTTP requests, slower -/// modes throttle to avoid RPC rate-limiting (e.g. HTTP 429 on Base). The -/// selected mode is **process-global** state, set via [`set_cache_speed_mode`] -/// and read via [`cache_speed_mode`]; it affects every cache in the process. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Converts into [`StorageBatchConfig`]: faster modes send larger batches with +/// more in-flight HTTP requests, slower modes throttle to avoid RPC rate-limiting +/// (e.g. HTTP 429 on Base). Configure a preset per cache with +/// [`EvmCacheBuilder::speed_mode`], or supply exact values with +/// [`EvmCacheBuilder::storage_batch_config`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] #[repr(u8)] pub enum CacheSpeedMode { /// Largest batches, highest concurrency — fastest, most likely to trip rate limits. @@ -200,39 +536,285 @@ pub enum CacheSpeedMode { /// Moderate batch size and concurrency. Normal = 1, /// Conservative batch size and concurrency. The default. + #[default] Slow = 2, /// Smallest batches, single in-flight request — slowest, gentlest on the RPC provider. XSlow = 3, } -impl CacheSpeedMode { - fn from_u8(value: u8) -> Self { - match value { - 0 => Self::Fast, - 1 => Self::Normal, - 2 => Self::Slow, - 3 => Self::XSlow, - _ => Self::Slow, +/// Concrete tuning knobs for the provider-backed [`StorageBatchFetchFn`]. +/// +/// `slots_per_batch` controls how many `eth_getStorageAt` calls are packed into +/// each JSON-RPC batch request. `max_concurrent_batches` controls how many of +/// those HTTP batch requests may be in flight at once. Larger values can improve +/// cold-start / verification throughput on tolerant RPC endpoints; smaller +/// values are gentler on rate-limited providers. +/// +/// This config only affects the fetcher created by the cache constructors. If a +/// caller installs a custom fetcher with +/// [`set_storage_batch_fetcher`](EvmCache::set_storage_batch_fetcher), that +/// fetcher owns its own batching and throttling. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StorageBatchConfig { + /// Number of storage slots to include in one JSON-RPC batch request. + pub slots_per_batch: usize, + /// Maximum number of JSON-RPC batch requests in flight at once. + pub max_concurrent_batches: usize, +} + +impl StorageBatchConfig { + /// Construct a config, normalizing zero values to one. + pub fn new(slots_per_batch: usize, max_concurrent_batches: usize) -> Self { + Self { + slots_per_batch, + max_concurrent_batches, + } + .normalized() + } + + fn normalized(self) -> Self { + Self { + slots_per_batch: self.slots_per_batch.max(1), + max_concurrent_batches: self.max_concurrent_batches.max(1), } } } -/// Set the process-global cache batch-fetch speed profile. +impl Default for StorageBatchConfig { + fn default() -> Self { + CacheSpeedMode::default().into() + } +} + +impl From for StorageBatchConfig { + fn from(mode: CacheSpeedMode) -> Self { + match mode { + CacheSpeedMode::Fast => Self::new(150, 8), + CacheSpeedMode::Normal => Self::new(100, 6), + CacheSpeedMode::Slow => Self::new(75, 4), + CacheSpeedMode::XSlow => Self::new(25, 1), + } + } +} + +/// How a cache's batch storage fetcher loads slots. /// -/// This mutates a single static shared by every cache in the process, so it -/// affects all in-flight and future batch fetches, not just one [`EvmCache`]. -/// Read the current value with [`cache_speed_mode`]. -pub fn set_cache_speed_mode(mode: CacheSpeedMode) { - CACHE_SPEED_MODE.store(mode as u8, Ordering::Relaxed); +/// The default is [`BulkCall`](Self::BulkCall): bulk `eth_call` state-override +/// extraction (one call covers thousands of slots) with the classic +/// point-read fetcher as its fallback and repair path. Requests below the +/// bulk config's `point_read_threshold`, providers without state-override +/// support, and precompile targets all degrade gracefully to point reads. +/// See the [`bulk_storage`](crate::bulk_storage) module and +/// `docs/bulk-storage-extraction.md` for mechanism and measured economics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageFetchStrategy { + /// Bulk `eth_call` extraction with point-read fallback (the default). + BulkCall(crate::bulk_storage::BulkCallConfig), + /// Classic JSON-RPC-batched `eth_getStorageAt` point reads only + /// (pre-0.2.0 behavior), tuned by [`StorageBatchConfig`]. + PointRead, +} + +impl Default for StorageFetchStrategy { + fn default() -> Self { + Self::BulkCall(crate::bulk_storage::BulkCallConfig::default()) + } +} + +/// Build the classic point-read [`StorageBatchFetchFn`]: JSON-RPC batches of +/// `eth_getStorageAt`, sized and throttled by [`StorageBatchConfig`]. +/// +/// This is the default fetcher's *fallback* path (see +/// [`StorageFetchStrategy`]) and the whole fetcher under +/// [`StorageFetchStrategy::PointRead`]. It is public so callers composing +/// their own fetchers — e.g. +/// [`bulk_call_storage_fetcher_with_fallback`](crate::bulk_storage::bulk_call_storage_fetcher_with_fallback) +/// over a differently-tuned repair path — can reuse it. +pub fn point_read_storage_fetcher

( + provider: Arc

, + config: StorageBatchConfig, +) -> StorageBatchFetchFn +where + P: Provider + 'static, +{ + let config = config.normalized(); + Arc::new( + move |requests: Vec<(Address, U256)>, current_block: BlockId| { + use futures::stream::{self, StreamExt}; + // Max items per JSON-RPC batch. RPC providers typically limit batch + // size to ~1000 items. Kept conservative to avoid 429s on Base. + let batch_size = config.slots_per_batch; + // Max concurrent HTTP batch requests. Each batch contains batch_size + // individual eth_getStorageAt calls. Limiting concurrency prevents + // thundering herd when prefetching thousands of storage slots. + let max_concurrent = config.max_concurrent_batches; + + // Guard against panicking inside `block_in_place` on a + // current-thread runtime (or when no runtime is present): return + // an `Err` result for every requested slot instead. + let handle = match block_in_place_handle() { + Ok(handle) => handle, + Err(e) => { + return requests + .into_iter() + .map(|(addr, slot)| { + ( + addr, + slot, + Err(StorageFetchError::Runtime(RuntimeError::MissingRuntime { + details: e.to_string(), + })), + ) + }) + .collect(); + } + }; + // The caller supplies the exact block this fetch must observe. + // Capturing it at the call site is what lets the deferred + // freshness validator fetch at the snapshot's block despite a + // later `set_block`. + tokio::task::block_in_place(|| { + handle.block_on(async { + let mut results = Vec::with_capacity(requests.len()); + + // Build and send JSON-RPC batches (each batch = one HTTP request) + let batch_futs: Vec<_> = requests + .chunks(batch_size) + .map(|chunk| { + let client = provider.client(); + let mut batch = alloy_rpc_client::BatchRequest::new(client); + let mut waiters = Vec::with_capacity(chunk.len()); + + for &(addr, slot) in chunk { + let params = (addr, slot, current_block); + match batch.add_call::<_, U256>("eth_getStorageAt", ¶ms) { + Ok(waiter) => waiters.push((addr, slot, Ok(waiter))), + Err(e) => { + // Serialization error — rare, treat as failure + tracing::warn!( + ?addr, + ?slot, + "batch request serialization failed: {}", + e + ); + waiters.push(( + addr, + slot, + Err(StorageFetchError::serialization(e)), + )); + } + } + } + + async move { + // Send the batch as a single HTTP request + let send_result = batch.send().await; + let mut chunk_results = Vec::with_capacity(waiters.len()); + + let batch_error = + send_result.as_ref().err().map(|err| err.to_string()); + for (addr, slot, waiter) in waiters { + match waiter { + Ok(waiter) => { + if let Some(source) = &batch_error { + chunk_results.push(( + addr, + slot, + Err(StorageFetchError::batch_send(source)), + )); + continue; + } + match waiter.await { + Ok(value) => { + chunk_results.push((addr, slot, Ok(value))); + } + Err(e) => { + chunk_results.push(( + addr, + slot, + Err(StorageFetchError::provider( + "eth_getStorageAt", + e, + )), + )); + } + } + } + Err(err) => { + chunk_results.push((addr, slot, Err(err))); + } + } + } + chunk_results + } + }) + .collect(); + + // Fire batches with bounded concurrency (`max_concurrent`) to avoid + // a thundering herd; per-batch size is the configured `batch_size` + // chosen above, so throughput scales without overwhelming RPC providers. + let all_batch_results: Vec> = stream::iter(batch_futs) + .buffer_unordered(max_concurrent) + .collect() + .await; + for batch_results in all_batch_results { + results.extend(batch_results); + } + results + }) + }) + }, + ) } -/// Return the current process-global cache batch-fetch speed profile. +/// Outcome of [`EvmCache::prewarm_slots`]. +#[derive(Debug, Default)] +pub struct PrewarmReport { + /// Slots fetched and injected into the cache. + pub loaded: usize, + /// Pairs the fetcher failed to load, with the per-slot error. + pub failed: Vec<(Address, U256, StorageFetchError)>, +} + +/// Outcome of [`EvmCache::verify_code_seeds`]: how each `Pending` canonical +/// code claim resolved against the chain at the pinned block. /// -/// Defaults to [`CacheSpeedMode::Slow`] until changed via -/// [`set_cache_speed_mode`]. The value is shared across all caches in the -/// process. -pub fn cache_speed_mode() -> CacheSpeedMode { - CacheSpeedMode::from_u8(CACHE_SPEED_MODE.load(Ordering::Relaxed)) +/// Fail-closed on trust, fail-safe on transport: `mismatched` / +/// `not_deployed` / `codeless` entries were **purged** (both cache layers and +/// the mark — the next touch refetches authoritative chain state), while +/// `unverifiable` entries are **still `Pending`** (a failed read proves +/// nothing, so the seed is neither promoted nor destroyed). +#[derive(Clone, Debug, Default)] +pub struct CodeVerifyReport { + /// Claims confirmed: marked [`CodeSeedState::Verified`], real balance + /// injected from the same response. + pub verified: Vec

, + /// Claims contradicted by on-chain code — purged. Usually a wrong + /// template or immutable-patch offset; the mismatching hashes are + /// included for debugging. + pub mismatched: Vec, + /// `EXTCODEHASH == 0`: no account at the pinned block — purged. This is + /// the live-registration race (the deployment is newer than the pin); + /// re-pin forward and re-seed rather than debugging the template. + pub not_deployed: Vec
, + /// `EXTCODEHASH == keccak256("")`: the address exists but holds no code + /// (an EOA) — purged. + pub codeless: Vec
, + /// The fetch failed (transport error, omitted address, or the + /// [`MULTICALL3_ADDRESS`](crate::multicall::MULTICALL3_ADDRESS) host + /// caveat) — each still `Pending`, with the reason. + pub unverifiable: Vec<(Address, String)>, +} + +/// One contradicted code claim from [`EvmCache::verify_code_seeds`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CodeMismatch { + /// The seeded address. + pub address: Address, + /// The hash the seed claimed (keccak256 of the seeded bytes). + pub expected: B256, + /// The on-chain `EXTCODEHASH` observed at the pinned block. + pub actual: B256, } /// Behavior when overriding code at a target account that is not known to the cache/backend. @@ -272,6 +854,97 @@ pub struct TxConfig { pub access_list: Option, } +/// Which block-context header fields a cache requires to be present. +/// +/// Block-env fields (`NUMBER` / `BASEFEE` / `COINBASE` / `PREVRANDAO` / +/// `GASLIMIT`) are populated from a fetched block header. When a field is +/// absent — because a fetch failed or the chain does not carry it — the EVM +/// silently defaults it, which can steer contracts that branch on block context +/// down a different code path and produce quietly-wrong simulations. +/// +/// These per-field requirements let a caller opt into failing loudly instead. +/// [`strict()`](Self::strict) requires every field; [`lenient()`](Self::lenient) +/// (the [`Default`]) requires none and reproduces the historical +/// silently-default behavior. A chain without EIP-1559, for example, can start +/// from [`strict()`](Self::strict) and clear [`require_basefee`](Self::require_basefee). +/// +/// Requirements are checked by [`validate_header`](Self::validate_header), which +/// [`EvmCache::advance_block`] and [`EvmCacheBuilder::try_build`] call. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BlockContextRequirements { + /// Require the header to carry a block number (`NUMBER`). + pub require_number: bool, + /// Require the header to carry an EIP-1559 base fee (`BASEFEE`). + pub require_basefee: bool, + /// Require the header to carry a beneficiary (`COINBASE`). + pub require_coinbase: bool, + /// Require the header to carry a `prevrandao` / mix hash (`PREVRANDAO`). + pub require_prevrandao: bool, + /// Require the header to carry a gas limit (`GASLIMIT`). + pub require_gas_limit: bool, +} + +impl Default for BlockContextRequirements { + fn default() -> Self { + Self::lenient() + } +} + +impl BlockContextRequirements { + /// Require every block-context field to be present. + /// + /// Under this policy a header missing any required field is rejected rather + /// than silently defaulted. + pub const fn strict() -> Self { + Self { + require_number: true, + require_basefee: true, + require_coinbase: true, + require_prevrandao: true, + require_gas_limit: true, + } + } + + /// Require no block-context field (the [`Default`]). + /// + /// Reproduces the historical behavior: a missing field is silently + /// defaulted by the EVM. + pub const fn lenient() -> Self { + Self { + require_number: false, + require_basefee: false, + require_coinbase: false, + require_prevrandao: false, + require_gas_limit: false, + } + } + + /// Validate that a header carries every required block-context field. + /// + /// Only the two `Option`-typed header fields can actually be absent: + /// [`require_basefee`](Self::require_basefee) checks + /// [`base_fee_per_gas`](alloy_consensus::BlockHeader::base_fee_per_gas) and + /// [`require_prevrandao`](Self::require_prevrandao) checks + /// [`mix_hash`](alloy_consensus::BlockHeader::mix_hash). Number, beneficiary + /// and gas limit are non-`Option` on the [`BlockHeader`] trait, so those + /// requirement flags are satisfied whenever a header is present. Returns + /// `Ok(())` when all required fields are satisfied. + pub fn validate_header(&self, header: &H) -> Result<(), BlockContextError> { + // `number`, `coinbase` (beneficiary) and `gas_limit` are non-`Option` on + // the `BlockHeader` trait: they are always present when a header exists, + // so their requirement flags are trivially satisfied here. + if self.require_basefee && header.base_fee_per_gas().is_none() { + return Err(BlockContextError::MissingField { field: "basefee" }); + } + if self.require_prevrandao && header.mix_hash().is_none() { + return Err(BlockContextError::MissingField { + field: "prevrandao", + }); + } + Ok(()) + } +} + /// Fluent builder for [`EvmCache`]. /// /// A readable alternative to the positional [`EvmCache::with_cache`] @@ -282,7 +955,7 @@ pub struct TxConfig { /// # use alloy_provider::{ProviderBuilder, network::AnyNetwork}; /// # use revm::primitives::hardfork::SpecId; /// # use evm_fork_cache::cache::EvmCache; -/// # async fn example() -> anyhow::Result<()> { +/// # async fn example() -> Result<(), Box> { /// let provider = ProviderBuilder::new() /// .network::() /// .connect_http("https://example-rpc.invalid".parse()?); @@ -301,7 +974,11 @@ pub struct EvmCacheBuilder

{ cache_config: Option, spec_id: SpecId, shared_memory_capacity: SharedMemoryCapacity, + storage_batch_config: StorageBatchConfig, + storage_fetch_strategy: StorageFetchStrategy, chain_id: Option, + block_context_requirements: BlockContextRequirements, + max_concurrent_proofs: usize, } impl

EvmCacheBuilder

@@ -316,10 +993,30 @@ where cache_config: None, spec_id: SpecId::CANCUN, shared_memory_capacity: SharedMemoryCapacity::default(), + storage_batch_config: StorageBatchConfig::default(), + storage_fetch_strategy: StorageFetchStrategy::default(), chain_id: None, + block_context_requirements: BlockContextRequirements::lenient(), + max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS, } } + /// Cap the default account-proof fetcher's concurrent `eth_getProof` + /// fan-out (default 8, name-symmetric with + /// [`BulkCallConfig::max_concurrent_calls`](crate::bulk_storage::BulkCallConfig)). + /// + /// `eth_getProof` is single-address at the RPC level, so when the root + /// gate or an account resync probes N tracked accounts in one seam call, + /// concurrency is the only wall-clock lever: `N × RTT` serial becomes + /// `~ceil(N / cap) × RTT`. Values are clamped to at least 1. Custom + /// fetchers installed via + /// [`set_account_proof_fetcher`](EvmCache::set_account_proof_fetcher) + /// ignore this knob. + pub fn max_concurrent_proofs(mut self, cap: usize) -> Self { + self.max_concurrent_proofs = cap.max(1); + self + } + /// Pin simulations and RPC fetches to a specific block. /// /// Use this to fork at a fixed height for reproducible simulation. Without @@ -384,19 +1081,102 @@ where self } + /// Set the concrete storage batch-fetch configuration for this cache instance. + /// + /// The config controls the batch size and concurrency used by the + /// provider-backed [`StorageBatchFetchFn`]. Defaults to + /// [`StorageBatchConfig::default`] (the [`CacheSpeedMode::Slow`] preset). + /// Different cache instances can use different values in the same process. + /// Zero values are normalized to one. + pub fn storage_batch_config(mut self, config: impl Into) -> Self { + self.storage_batch_config = config.into().normalized(); + self + } + + /// Set the storage batch-fetch profile from a preset. + /// + /// Shorthand for [`storage_batch_config`](Self::storage_batch_config) with + /// `mode.into()`. + pub fn speed_mode(self, mode: CacheSpeedMode) -> Self { + self.storage_batch_config(mode) + } + + /// Choose how the cache's batch storage fetcher loads slots. + /// + /// Defaults to [`StorageFetchStrategy::BulkCall`] with + /// [`BulkCallConfig::default`](crate::bulk_storage::BulkCallConfig::default): + /// bulk `eth_call` state-override extraction, repaired by (and degrading + /// to) the point-read fetcher that [`storage_batch_config`](Self::storage_batch_config) + /// tunes. Use [`StorageFetchStrategy::PointRead`] to restore the classic + /// per-slot behavior. + pub fn storage_fetch_strategy(mut self, strategy: StorageFetchStrategy) -> Self { + self.storage_fetch_strategy = strategy; + self + } + + /// Tune the bulk `eth_call` extraction path. + /// + /// Shorthand for [`storage_fetch_strategy`](Self::storage_fetch_strategy) + /// with [`StorageFetchStrategy::BulkCall`]`(config)` — e.g. raising + /// `max_slots_per_call` on a provider with a generous gas cap, or + /// selecting [`CallDispatch::CallMany`](crate::bulk_storage::CallDispatch::CallMany) + /// on Erigon-lineage endpoints. + pub fn bulk_call_config(self, config: crate::bulk_storage::BulkCallConfig) -> Self { + self.storage_fetch_strategy(StorageFetchStrategy::BulkCall(config)) + } + + /// Set which block-context header fields the cache requires. + /// + /// See [`BlockContextRequirements`]. Defaults to + /// [`lenient`](BlockContextRequirements::lenient). Only [`try_build`](Self::try_build) + /// enforces non-lenient requirements at construction; the infallible + /// [`build`](Self::build) always stays lenient. + pub fn block_context_requirements(mut self, reqs: BlockContextRequirements) -> Self { + self.block_context_requirements = reqs; + self + } + + /// Convenience toggle: require every block-context field (`true`) or none + /// (`false`). + /// + /// Equivalent to + /// [`block_context_requirements`](Self::block_context_requirements) with + /// [`strict`](BlockContextRequirements::strict) / + /// [`lenient`](BlockContextRequirements::lenient). Enforced only by + /// [`try_build`](Self::try_build). + pub fn strict_block_context(mut self, strict: bool) -> Self { + self.block_context_requirements = if strict { + BlockContextRequirements::strict() + } else { + BlockContextRequirements::lenient() + }; + self + } + /// Build the [`EvmCache`], fetching the pinned block's header for context. /// /// If a chain ID was not set via [`chain_id`](Self::chain_id), it is inferred /// from the provider (`eth_chainId`); see [`chain_id`](Self::chain_id) for the /// full resolution order. + /// + /// This constructor is infallible and always uses + /// [`lenient`](BlockContextRequirements::lenient) enforcement (a missing + /// block-context field is silently defaulted). To enforce + /// [`BlockContextRequirements`] at construction, use + /// [`try_build`](Self::try_build) instead. pub async fn build(self) -> EvmCache { let explicit_chain_id = self.chain_id; - let mut cache = EvmCache::with_cache_capacity( + let provider = self.provider.clone(); + let strategy = self.storage_fetch_strategy; + let storage_batch_config = self.storage_batch_config; + let mut cache = EvmCache::with_cache_capacity_and_storage_batch_config( self.provider, self.block, self.cache_config, self.spec_id, self.shared_memory_capacity, + self.storage_batch_config, + self.max_concurrent_proofs, ) .await; // An explicit builder value is authoritative for the `CHAINID` opcode and @@ -404,8 +1184,95 @@ where if let Some(chain_id) = explicit_chain_id { cache.set_chain_id(chain_id); } + apply_storage_fetch_strategy(&mut cache, provider, strategy, storage_batch_config); cache } + + /// Build the [`EvmCache`], enforcing the configured + /// [`BlockContextRequirements`] against the fetched block header. + /// + /// Builds the cache the same way [`build`](Self::build) does, then, if the + /// requirements are non-lenient, validates the pinned block's header: + /// - if the header could not be fetched (the provider errored or returned no + /// block), returns [`BlockContextError::FetchFailed`]; + /// - otherwise validates the fetched header via + /// [`BlockContextRequirements::validate_header`] and propagates any + /// [`BlockContextError::MissingField`]. + /// + /// A [`lenient`](BlockContextRequirements::lenient) build never errors (it + /// does not fetch a header solely to validate). On success the requirements + /// are stored on the returned cache so a later + /// [`advance_block`](EvmCache::advance_block) enforces them too. + pub async fn try_build(self) -> Result { + let explicit_chain_id = self.chain_id; + let reqs = self.block_context_requirements; + let block = self.block; + let provider = self.provider.clone(); + let strategy = self.storage_fetch_strategy; + let storage_batch_config = self.storage_batch_config; + + let mut cache = EvmCache::with_cache_capacity_and_storage_batch_config( + self.provider, + self.block, + self.cache_config, + self.spec_id, + self.shared_memory_capacity, + self.storage_batch_config, + self.max_concurrent_proofs, + ) + .await; + if let Some(chain_id) = explicit_chain_id { + cache.set_chain_id(chain_id); + } + cache.set_block_context_requirements(reqs); + apply_storage_fetch_strategy(&mut cache, provider.clone(), strategy, storage_batch_config); + + // Only a non-lenient policy fetches a header to validate: a lenient + // build must never error and must not incur an extra RPC round-trip. + if reqs != BlockContextRequirements::lenient() { + match provider.get_block(block).await { + Ok(Some(blk)) => reqs.validate_header(blk.header())?, + Ok(None) => { + return Err(BlockContextError::FetchFailed(format!( + "no block header returned for {block:?}" + ))); + } + Err(e) => return Err(BlockContextError::FetchFailed(e.to_string())), + } + } + + Ok(cache) + } +} + +/// Install the fetcher a [`StorageFetchStrategy`] describes on a built cache. +/// +/// The constructor already installs the default strategy (bulk extraction +/// wrapping the point-read fetcher), so the default case is a no-op rather +/// than a redundant re-wrap. +fn apply_storage_fetch_strategy

( + cache: &mut EvmCache, + provider: Arc

, + strategy: StorageFetchStrategy, + batch_config: StorageBatchConfig, +) where + P: Provider + 'static, +{ + match strategy { + StorageFetchStrategy::BulkCall(config) + if config == crate::bulk_storage::BulkCallConfig::default() => {} + StorageFetchStrategy::BulkCall(config) => { + let fallback = point_read_storage_fetcher(provider.clone(), batch_config); + cache.set_storage_batch_fetcher( + crate::bulk_storage::bulk_call_storage_fetcher_with_fallback( + provider, config, fallback, + ), + ); + } + StorageFetchStrategy::PointRead => { + cache.set_storage_batch_fetcher(point_read_storage_fetcher(provider, batch_config)); + } + } } type CacheEvm<'a> = revm::MainnetEvm< @@ -422,6 +1289,10 @@ type InspectorCacheEvm<'a, INSP> = revm::MainnetEvm< /// [`SharedMemoryCapacity`]. const DEFAULT_SHARED_MEMORY_CAPACITY: usize = 64 * 1024; +/// Default cap on the default account-proof fetcher's concurrent +/// `eth_getProof` fan-out (see [`EvmCacheBuilder::max_concurrent_proofs`]). +const DEFAULT_MAX_CONCURRENT_PROOFS: usize = 8; + /// How much EVM shared memory (per-context working memory) to pre-allocate for /// simulations. /// @@ -514,6 +1385,13 @@ pub struct EvmCache { prevrandao: Option, /// Block gas limit for EVM simulations (GASLIMIT opcode). block_gas_limit: Option, + /// Which block-context header fields this cache requires to be present. + /// [`lenient`](BlockContextRequirements::lenient) by default; the strict + /// builder path sets it before returning. Enforced by + /// [`advance_block`](Self::advance_block). + block_context_requirements: BlockContextRequirements, + /// Cache-side batch-fetch configuration for this instance. + storage_batch_config: StorageBatchConfig, /// Shared memory buffer reused across EVM simulations. /// This avoids repeated allocations and allows measuring peak memory usage. shared_memory_buffer: Rc>>, @@ -523,10 +1401,28 @@ pub struct EvmCache { rpc_caller: Option, /// Optional batch storage fetcher that bypasses SharedBackend. /// Captures a provider clone and fires concurrent `eth_getStorageAt` calls directly. + /// Monotonic snapshot-consistency generation (see + /// [`snapshot_generation`](Self::snapshot_generation)). Bumped by targeted + /// state writes (`apply_update` / `apply_updates` / `modify_slot`) and + /// block re-pins (`set_block` / `advance_block`); cold prefetch + /// (`inject_storage_batch`) does not bump it. + snapshot_generation: u64, storage_batch_fetcher: Option, - /// Shared block ID for the batch storage fetcher closure. - /// Updated by `set_block()` so batch fetches always use the current block. - batch_block_id: Arc>, + /// Optional account/root fetcher that bypasses SharedBackend. + /// Captures a provider clone and fires `eth_getProof` calls directly to fetch + /// authoritative account fields (balance/nonce/code hash) and `storageHash`. + account_proof_fetcher: Option, + /// Optional block state-diff fetcher backed by debug/trace RPC. + block_state_diff_fetcher: Option, + /// Optional bulk account-fields fetcher (balance + `EXTCODEHASH` in one + /// `eth_call`), the read side of code-seed verification. + account_fields_fetcher: Option, + /// Provenance + trust marks for bytecode that did not arrive via the lazy + /// RPC backend (see [`CodeSeedState`]). Absence of a mark = RPC-origin. + /// Persisted to `code_seeds.bin` (saved before `bytecodes.bin`, full + /// replace) so a `Pending` claim never masquerades as chain-fetched + /// across restarts. + code_seeds: HashMap, /// Best-known ERC20 `balanceOf` mapping slot per token contract. /// /// Used by `set_erc20_balance_with_slot_scan` to avoid re-scanning slots @@ -537,7 +1433,7 @@ pub struct EvmCache { /// in `chains.toml`. spec_id: SpecId, /// Memoized, `Arc`-shared flatten of the cold layer-2 index, reused across - /// successive [`create_snapshot`](Self::create_snapshot) calls (Pillar A). + /// successive [`snapshot`](Self::snapshot) calls (Pillar A). /// `None` until the first snapshot. Rebuilt copy-on-write by /// [`refresh_base`](Self::refresh_base); never mutated in place once shared. /// Not part of any public API and not serialized. @@ -590,6 +1486,9 @@ pub enum SimStatus { }, } +/// Outcome of a simulated call: status, return data, gas used, and the touched +/// access list. `#[non_exhaustive]` — construct via the simulation APIs and match +/// with a wildcard arm. #[derive(Clone, Debug)] #[non_exhaustive] pub struct CallSimulationResult { @@ -731,10 +1630,37 @@ impl EvmCache { spec_id: SpecId, shared_memory_capacity: SharedMemoryCapacity, ) -> Self + where + P: Provider + 'static, + { + Self::with_cache_capacity_and_storage_batch_config( + provider, + block, + cache_config, + spec_id, + shared_memory_capacity, + StorageBatchConfig::default(), + DEFAULT_MAX_CONCURRENT_PROOFS, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn with_cache_capacity_and_storage_batch_config

( + provider: Arc

, + block: BlockId, + cache_config: Option, + spec_id: SpecId, + shared_memory_capacity: SharedMemoryCapacity, + storage_batch_config: StorageBatchConfig, + max_concurrent_proofs: usize, + ) -> Self where P: Provider + 'static, { let block_id = block; + let storage_batch_config = storage_batch_config.normalized(); + let max_concurrent_proofs = max_concurrent_proofs.max(1); // Fetch the pinned block header for accurate block context (NUMBER, // BASEFEE, COINBASE, PREVRANDAO, GASLIMIT opcodes). Without this, revm @@ -850,6 +1776,36 @@ impl EvmCache { } } + // Restore code-seed marks. Pruning rule: a mark is kept only while the + // account it describes still holds code with the marked hash — a mark + // whose code did not survive (evicted, never persisted, or clobbered) + // is meaningless and must not outlive it. The reverse orphan + // (code-without-mark) is prevented by `flush()` writing + // `code_seeds.bin` BEFORE `bytecodes.bin`. + let code_seeds: HashMap = cache_config + .as_ref() + .and_then(|cfg| CodeSeedCache::load(&cfg.code_seeds_cache_path())) + .map(|cache| { + let accounts = blockchain_db.accounts().read(); + let before = cache.entries.len(); + let mut entries = cache.entries; + entries.retain(|addr, state| { + accounts.get(addr).is_some_and(|info| { + info.code.as_ref().is_some_and(|code| !code.is_empty()) + && info.code_hash == state.code_hash() + }) + }); + if entries.len() < before { + debug!( + pruned = before - entries.len(), + kept = entries.len(), + "Pruned code-seed marks whose code did not survive the reload" + ); + } + entries + }) + .unwrap_or_default(); + // Load immutable data cache (token decimals). // This is still needed for validation and metadata lookups let immutable_cache = cache_config @@ -884,141 +1840,147 @@ impl EvmCache { provider_for_rpc .call(tx.into()) .await - .map_err(|e| anyhow!("{}", e)) + .map_err(|e| RpcError::provider("eth_call", e)) }) }) }); - // Create a batch storage fetcher that bypasses SharedBackend for bulk prefetch. - // Uses JSON-RPC batch requests to send multiple eth_getStorageAt calls in a - // single HTTP request, dramatically reducing round-trip overhead. - let provider_for_batch = provider.clone(); - let batch_block_id = Arc::new(Mutex::new(block_id)); - let batch_block_ref = batch_block_id.clone(); - let storage_batch_fetcher: StorageBatchFetchFn = Arc::new( - move |requests: Vec<(Address, U256)>, block: Option| { - use futures::stream::{self, StreamExt}; - // Max items per JSON-RPC batch. RPC providers typically limit batch - // size to ~1000 items. Reduced from 200 to avoid 429s on Base. - let batch_size: usize = match cache_speed_mode() { - CacheSpeedMode::Fast => 150, - CacheSpeedMode::Normal => 100, - CacheSpeedMode::Slow => 75, - CacheSpeedMode::XSlow => 25, - }; - // Max concurrent HTTP batch requests. Each batch contains batch_size - // individual eth_getStorageAt calls. Limiting concurrency prevents - // thundering herd when prefetching thousands of storage slots. - let max_concurrent: usize = match cache_speed_mode() { - CacheSpeedMode::Fast => 8, - CacheSpeedMode::Normal => 6, - CacheSpeedMode::Slow => 4, - CacheSpeedMode::XSlow => 1, - }; + // Batch storage fetcher: bulk `eth_call` state-override extraction by + // default, with the classic point-read fetcher as its fallback and + // repair path (see the `bulk_storage` module and + // docs/bulk-storage-extraction.md). `StorageBatchConfig` tunes the + // point-read path; `EvmCacheBuilder::storage_fetch_strategy` swaps or + // tunes the bulk path. + let storage_batch_fetcher: StorageBatchFetchFn = + crate::bulk_storage::bulk_call_storage_fetcher_with_fallback( + provider.clone(), + crate::bulk_storage::BulkCallConfig::default(), + point_read_storage_fetcher(provider.clone(), storage_batch_config), + ); + // Create an account/root fetcher that bypasses SharedBackend, firing + // `eth_getProof` calls directly for authoritative account fields plus the + // account's `storageHash`. `eth_getProof` is single-address at the RPC + // level, so a multi-account seam call (the reactive root gate, account + // resyncs, cold-start probe_roots) fans out with bounded, order- + // preserving concurrency (`buffered`): wall clock drops from N × RTT to + // ~ceil(N / max_concurrent_proofs) × RTT. + let provider_for_proof = provider.clone(); + let account_proof_fetcher: AccountProofFetchFn = Arc::new( + move |requests: Vec<(Address, Vec)>, current_block: BlockId| { // Guard against panicking inside `block_in_place` on a // current-thread runtime (or when no runtime is present): return - // an `Err` result for every requested slot instead. + // an `Err` result for every requested address instead. let handle = match block_in_place_handle() { Ok(handle) => handle, Err(e) => { - let msg = e.to_string(); - return requests - .into_iter() - .map(|(addr, slot)| (addr, slot, Err(anyhow!("{}", msg)))) - .collect(); - } - }; - // Pin to the explicitly-requested block when given, else the - // cache's currently-pinned block. Capturing the block at the call - // site is what lets the deferred freshness validator fetch at the - // snapshot's block despite a later `set_block`. - let current_block = block.unwrap_or_else(|| *batch_block_ref.lock().unwrap()); - tokio::task::block_in_place(|| { - handle.block_on(async { - let mut results = Vec::with_capacity(requests.len()); - - // Build and send JSON-RPC batches (each batch = one HTTP request) - let batch_futs: Vec<_> = requests - .chunks(batch_size) - .map(|chunk| { - let client = provider_for_batch.client(); - let mut batch = alloy_rpc_client::BatchRequest::new(client); - let mut waiters = Vec::with_capacity(chunk.len()); - - for &(addr, slot) in chunk { - let params = (addr, slot, current_block); - match batch.add_call::<_, U256>("eth_getStorageAt", ¶ms) { - Ok(waiter) => waiters.push((addr, slot, Some(waiter))), - Err(e) => { - // Serialization error — rare, treat as failure - tracing::warn!( - ?addr, - ?slot, - "batch request serialization failed: {}", - e - ); - waiters.push((addr, slot, None)); - } - } - } - - async move { - // Send the batch as a single HTTP request - let send_result = batch.send().await; - let mut chunk_results = Vec::with_capacity(waiters.len()); - - for (addr, slot, waiter) in waiters { - if let Some(waiter) = waiter { - if send_result.is_ok() { - match waiter.await { - Ok(value) => { - chunk_results.push((addr, slot, Ok(value))); - } - Err(e) => { - chunk_results.push(( - addr, - slot, - Err(anyhow!("{}", e)), - )); - } - } - } else { - chunk_results.push(( - addr, - slot, - Err(anyhow!("batch send failed")), - )); - } - } else { - chunk_results.push(( - addr, - slot, - Err(anyhow!("serialization failed")), - )); - } - } - chunk_results - } + return requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Err(StorageFetchError::Runtime(RuntimeError::MissingRuntime { + details: e.to_string(), + })), + ) }) .collect(); - - // Fire batches with bounded concurrency (`max_concurrent`) to avoid - // a thundering herd; per-batch size is the speed-mode `batch_size` - // chosen above, so throughput scales without overwhelming RPC providers. - let all_batch_results: Vec> = stream::iter(batch_futs) - .buffer_unordered(max_concurrent) - .collect() - .await; - for batch_results in all_batch_results { - results.extend(batch_results); - } - results + } + }; + let provider = provider_for_proof.clone(); + // The caller supplies the exact block this proof fetch must observe. + tokio::task::block_in_place(|| { + handle.block_on(async { + use futures::StreamExt; + futures::stream::iter(requests.into_iter().map(|(addr, keys)| { + let provider = provider.clone(); + async move { + // `eth_getProof` takes slot keys as 32-byte B256. + let proof_keys: Vec = + keys.iter().map(|slot| B256::from(*slot)).collect(); + let outcome = provider + .get_proof(addr, proof_keys) + .block_id(current_block) + .await; + match outcome { + Ok(response) => { + // Map proven storage-proof entries back to + // the requested `(slot, value)` pairs. + let slots = response + .storage_proof + .iter() + .map(|proof| { + ( + U256::from_be_bytes(proof.key.as_b256().0), + proof.value, + ) + }) + .collect(); + ( + addr, + Ok(AccountProof { + storage_hash: response.storage_hash, + balance: response.balance, + nonce: response.nonce, + code_hash: response.code_hash, + slots, + }), + ) + } + Err(e) => { + (addr, Err(StorageFetchError::provider("eth_getProof", e))) + } + } + } + })) + .buffered(max_concurrent_proofs) + .collect::>() + .await }) }) }, ); + // Create a bulk account-fields fetcher: balance + EXTCODEHASH for many + // addresses in ONE eth_call via the account-fields extractor program. + // This is the read side of code-seed verification; the call is + // all-or-nothing per the `AccountFieldsFetchFn` contract. + let provider_for_fields = provider.clone(); + let account_fields_fetcher: AccountFieldsFetchFn = + Arc::new(move |addresses: Vec

, block: BlockId| { + // Guard against panicking inside `block_in_place` on a + // current-thread runtime (or with no runtime present): degrade + // to a typed error, matching the sibling fetchers. + let handle = block_in_place_handle()?; + tokio::task::block_in_place(|| { + handle.block_on(crate::bulk_storage::fetch_account_fields_bulk( + provider_for_fields.as_ref(), + &addresses, + block, + )) + }) + }); + + // Create a block-level state-diff fetcher over debug trace RPC. The + // reactive runtime uses this as a trace-first accelerator before falling + // back to point reads for unresolved cold targets. + let provider_for_trace = provider.clone(); + let block_state_diff_fetcher: BlockStateDiffFetchFn = Arc::new(move |block: BlockId| { + let handle = block_in_place_handle()?; + tokio::task::block_in_place(|| { + handle.block_on(async { + let (method, params) = trace_rpc_method_and_params(block); + let response = provider_for_trace + .client() + .request::<_, serde_json::Value>(method, params) + .await + .map_err(|e| StorageFetchError::provider(method, e))?; + parse_block_state_diff_trace(&response) + .map_err(|err| StorageFetchError::custom(err.to_string())) + }) + }) + }); + // Resolve the chain ID reported to simulations (the `CHAINID` opcode). A // disk `CacheConfig` is authoritative (its `chain_id` also namespaces the // on-disk cache directory); otherwise infer it from the provider via @@ -1074,10 +2036,16 @@ impl EvmCache { coinbase, prevrandao, block_gas_limit, + block_context_requirements: BlockContextRequirements::lenient(), + storage_batch_config, shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity(shared_memory_capacity))), + snapshot_generation: 0, rpc_caller: Some(rpc_caller), storage_batch_fetcher: Some(storage_batch_fetcher), - batch_block_id, + account_proof_fetcher: Some(account_proof_fetcher), + block_state_diff_fetcher: Some(block_state_diff_fetcher), + account_fields_fetcher: Some(account_fields_fetcher), + code_seeds, erc20_balance_slots: HashMap::new(), spec_id, base: None, @@ -1156,12 +2124,18 @@ impl EvmCache { coinbase: None, prevrandao: None, block_gas_limit: None, + block_context_requirements: BlockContextRequirements::lenient(), + storage_batch_config: StorageBatchConfig::default(), + snapshot_generation: 0, shared_memory_buffer: Rc::new(RefCell::new(Vec::with_capacity( DEFAULT_SHARED_MEMORY_CAPACITY, ))), rpc_caller: None, storage_batch_fetcher: None, - batch_block_id: Arc::new(Mutex::new(block)), + account_proof_fetcher: None, + block_state_diff_fetcher: None, + account_fields_fetcher: None, + code_seeds: HashMap::new(), erc20_balance_slots: HashMap::new(), spec_id, base: None, @@ -1186,16 +2160,29 @@ impl EvmCache { if let Some(cfg) = &self.cache_config { // Save EVM state to binary cache (bincode format) let binary_path = cfg.binary_state_cache_path(); - binary_state::save_binary_state(&self.blockchain_db, &binary_path) - .with_context(|| format!("failed to save binary state cache to {binary_path:?}"))?; + binary_state::save_binary_state(&self.blockchain_db, &binary_path)?; + + // Save code-seed marks BEFORE bytecodes (fail-closed ordering: a + // mark without code is pruned on load and harmless; code without + // its mark would let a Pending seed masquerade as RPC-origin). + // Full replace, not merge: marks are mutable trust state, and a + // merge would resurrect marks purged this session. + let code_seeds_path = cfg.code_seeds_cache_path(); + CodeSeedCache { + entries: self.code_seeds.clone(), + } + .save(&code_seeds_path)?; + debug!( + count = self.code_seeds.len(), + path = ?code_seeds_path, + "Updated code-seed mark cache (binary format)" + ); // Save bytecode cache let bytecode_path = cfg.bytecode_cache_path(); let mut bytecode_cache = BytecodeCache::load(&bytecode_path).unwrap_or_default(); bytecode_cache.merge_from_db(&self.blockchain_db); - bytecode_cache - .save(&bytecode_path) - .with_context(|| format!("failed to save bytecode cache to {bytecode_path:?}"))?; + bytecode_cache.save(&bytecode_path)?; debug!( count = bytecode_cache.contracts.len(), path = ?bytecode_path, @@ -1204,11 +2191,7 @@ impl EvmCache { // Save the immutable data cache let immutable_path = cfg.immutable_cache_path(); - self.immutable_cache - .save(&immutable_path) - .with_context(|| { - format!("failed to save immutable data cache to {immutable_path:?}") - })?; + self.immutable_cache.save(&immutable_path)?; debug!( token_decimals = self.immutable_cache.token_decimals.len(), path = ?immutable_path, @@ -1255,7 +2238,7 @@ impl EvmCache { /// # Snapshot base /// Writing layer 2 directly through this unchecked handle also bypasses the /// memoized copy-on-write snapshot base (Pillar A). The next - /// [`create_snapshot`](Self::create_snapshot) only performs a count/absence + /// [`snapshot`](Self::snapshot) only performs a count/absence /// growth scan over layer 2, which catches lazy RPC-populated accounts/slots /// because that path only appends at a fixed block. It does **not** catch /// direct in-place changes where cardinality is unchanged: overwriting an @@ -1289,7 +2272,7 @@ impl EvmCache { /// handler by reading back the updated account/slot through `SharedBackend` /// (for example via `basic_ref` / `storage_ref`), then call /// [`invalidate_snapshot_base`](Self::invalidate_snapshot_base) before the next - /// [`create_snapshot`](Self::create_snapshot). Calling + /// [`snapshot`](Self::snapshot). Calling /// `invalidate_snapshot_base` immediately after `insert_or_update_*` is not, by /// itself, a guarantee that the queued update has been applied before the next /// snapshot. @@ -1321,7 +2304,7 @@ impl EvmCache { /// runtime), the callback degrades to an `Err` rather than panicking, but /// `block_in_place` itself will panic if invoked from a non-worker thread of /// a multi-thread runtime. - pub fn rpc_call(&self, to: Address, calldata: Bytes) -> Option> { + pub fn rpc_call(&self, to: Address, calldata: Bytes) -> Option> { self.rpc_caller .as_ref() .map(|caller| (caller)(to, calldata)) @@ -1343,6 +2326,32 @@ impl EvmCache { self.storage_batch_fetcher.as_ref() } + /// Get the account/root proof fetcher, if available. + /// + /// Returns `None` when constructed via `from_backend` (no provider + /// available) unless a fetcher was injected via + /// [`set_account_proof_fetcher`](Self::set_account_proof_fetcher). + /// + /// # Panics + /// The returned [`AccountProofFetchFn`] must be invoked from within a + /// **multi-thread** tokio runtime: it drives `eth_getProof` calls to + /// completion via `tokio::task::block_in_place`. On a current-thread runtime + /// (or with no runtime) it degrades to an `Err` result for every requested + /// address rather than panicking, but `block_in_place` itself will panic if + /// invoked from a non-worker thread of a multi-thread runtime. + pub fn account_proof_fetcher(&self) -> Option<&AccountProofFetchFn> { + self.account_proof_fetcher.as_ref() + } + + /// Get the block state-diff fetcher, if available. + /// + /// Returns `None` when constructed via `from_backend` (no provider + /// available) unless a fetcher was injected via + /// [`set_block_state_diff_fetcher`](Self::set_block_state_diff_fetcher). + pub fn block_state_diff_fetcher(&self) -> Option<&BlockStateDiffFetchFn> { + self.block_state_diff_fetcher.as_ref() + } + /// Inject batch-fetched storage values directly into BlockchainDb (layer 2). /// /// This bypasses SharedBackend and makes values available for subsequent @@ -1373,7 +2382,7 @@ impl EvmCache { /// address that *already* has a CacheDB overlay entry (layer 1), it writes /// the slot into that overlay too. /// - /// This matters because both [`create_snapshot`](Self::create_snapshot) and + /// This matters because both [`snapshot`](Self::snapshot) and /// the synchronous EVM SLOAD path let the overlay win over the backend. A /// correction written only to layer 2 would be shadowed by a stale layer-1 /// slot, so the cache could never converge — the freshness validator would @@ -1395,6 +2404,53 @@ impl EvmCache { let _ = self.apply_updates(&updates); } + /// Bulk-load the given slots into the cache at its pinned block. + /// + /// Fetches through the installed [`StorageBatchFetchFn`] — bulk `eth_call` + /// extraction by default, so thousands of slots (across many contracts) + /// arrive in a handful of calls — and injects every successfully fetched + /// value into layer 2 via + /// [`inject_storage_batch`](Self::inject_storage_batch), the cold-prefetch + /// write. Use it to prewarm a declared working set (an AMM pool's tick + /// range, a protocol's config slots) before entering a simulation or + /// reactive loop, complementing the *recorded* working sets that + /// [`prefetch_registry`](crate::prefetch_registry) replays. + /// + /// Duplicate pairs are fetched once each and injected idempotently. + /// Returns how many slots loaded and which pairs failed; failures leave + /// the cache unchanged (those slots lazily point-read later as usual). + pub fn prewarm_slots(&mut self, requests: &[(Address, U256)]) -> PrewarmReport { + let Some(fetcher) = self.storage_batch_fetcher.clone() else { + return PrewarmReport { + loaded: 0, + failed: requests + .iter() + .map(|&(addr, slot)| { + ( + addr, + slot, + StorageFetchError::custom("no storage batch fetcher installed"), + ) + }) + .collect(), + }; + }; + let results = fetcher(requests.to_vec(), self.block); + let mut to_inject = Vec::with_capacity(results.len()); + let mut failed = Vec::new(); + for (addr, slot, result) in results { + match result { + Ok(value) => to_inject.push((addr, slot, value)), + Err(e) => failed.push((addr, slot, e)), + } + } + self.inject_storage_batch(&to_inject); + PrewarmReport { + loaded: to_inject.len(), + failed, + } + } + /// Apply a single targeted [`StateUpdate`], returning a [`StateDiff`] of what /// actually changed. /// @@ -1450,6 +2506,7 @@ impl EvmCache { /// # } /// ``` pub fn apply_update(&mut self, update: &StateUpdate) -> StateDiff { + self.bump_snapshot_generation(); let mut diff = StateDiff::default(); match update { StateUpdate::Slot { @@ -1578,6 +2635,9 @@ impl EvmCache { /// [`StateDiff::len`]. After a batch with relative updates, check /// [`StateDiff::has_skipped`]. pub fn apply_updates(&mut self, updates: &[StateUpdate]) -> StateDiff { + if !updates.is_empty() { + self.bump_snapshot_generation(); + } let mut diff = StateDiff::default(); let mut i = 0; while i < updates.len() { @@ -1765,6 +2825,7 @@ impl EvmCache { let current = self.cached_storage_value(address, slot); let new = f(current)?; + self.bump_snapshot_generation(); self.write_slot_through(address, slot, new); let old = current.unwrap_or(U256::ZERO); @@ -2018,10 +3079,54 @@ impl EvmCache { /// This is the seam the freshness controller and tests use to drive /// re-verification without a live provider: a stubbed /// [`StorageBatchFetchFn`] can be injected over a mocked-provider cache. + /// Production callers can also inject their own transport, retry, batching, + /// or rate-limiting strategy here. Once replaced, the cache's + /// [`StorageBatchConfig`] no longer controls batching; the custom fetcher is + /// responsible for honoring the [`StorageBatchFetchFn`] contract. pub fn set_storage_batch_fetcher(&mut self, f: StorageBatchFetchFn) { self.storage_batch_fetcher = Some(f); } + /// Set (or replace) the account/root proof fetcher. + /// + /// This is the seam account-target resyncs and account-level freshness use to + /// drive `eth_getProof` fetches without a live provider: a stubbed + /// [`AccountProofFetchFn`] can be injected over a mocked-provider cache, + /// mirroring [`set_storage_batch_fetcher`](Self::set_storage_batch_fetcher). + pub fn set_account_proof_fetcher(&mut self, f: AccountProofFetchFn) { + self.account_proof_fetcher = Some(f); + } + + /// Set (or replace) the block state-diff fetcher. + /// + /// This is the seam trace-backed reactive resync uses to resolve matching + /// targets from one block-level debug trace before falling back to storage or + /// account proof point reads. + pub fn set_block_state_diff_fetcher(&mut self, f: BlockStateDiffFetchFn) { + self.block_state_diff_fetcher = Some(f); + } + + /// Set (or replace) the bulk account-fields fetcher. + /// + /// This is the seam [`verify_code_seeds`](Self::verify_code_seeds) (and + /// the cold-start `verify_code` phase) reads through: a stubbed + /// [`AccountFieldsFetchFn`] can be injected over a mocked-provider cache, + /// mirroring [`set_storage_batch_fetcher`](Self::set_storage_batch_fetcher). + pub fn set_account_fields_fetcher(&mut self, f: AccountFieldsFetchFn) { + self.account_fields_fetcher = Some(f); + } + + /// The installed bulk account-fields fetcher, if any. + /// + /// `Some` on provider-backed caches (default-wired to + /// [`fetch_account_fields_bulk`](crate::bulk_storage::fetch_account_fields_bulk)); + /// `None` on [`from_backend`](Self::from_backend) caches until one is + /// installed via + /// [`set_account_fields_fetcher`](Self::set_account_fields_fetcher). + pub fn account_fields_fetcher(&self) -> Option<&AccountFieldsFetchFn> { + self.account_fields_fetcher.as_ref() + } + /// Return the currently-cached value for a storage slot, if any. /// /// Mirrors what the EVM would `SLOAD` from the cached layers (it never touches @@ -2101,7 +3206,7 @@ impl EvmCache { /// Shared with the cold-start probe phase /// ([`execute_cold_start_round`](Self::execute_cold_start_round)) so the /// single classification is reused rather than duplicated. - pub(crate) fn classify(fetched: Result) -> SlotFetch { + pub(crate) fn classify(fetched: StorageFetchResult) -> SlotFetch { match fetched { Ok(v) if v != U256::ZERO => SlotFetch::Value(v), Ok(_) => SlotFetch::Zero, @@ -2140,7 +3245,7 @@ impl EvmCache { let fetcher = self .storage_batch_fetcher .as_ref() - .ok_or_else(|| anyhow!("verify_slots requires a storage batch fetcher"))? + .ok_or(CacheError::MissingStorageBatchFetcher)? .clone(); // Snapshot the cached values before fetching so we compare against a @@ -2150,7 +3255,7 @@ impl EvmCache { .map(|&(addr, slot)| ((addr, slot), self.cached_storage_value(addr, slot))) .collect(); - let results = (fetcher)(slots.to_vec(), Some(self.block)); + let results = (fetcher)(slots.to_vec(), self.block); let mut changed = Vec::new(); let mut outcomes = Vec::with_capacity(results.len()); @@ -2159,7 +3264,7 @@ impl EvmCache { // Read 1: classify the fetch outcome for every slot, failed or not. let fetch = Self::classify(match &fetched { Ok(v) => Ok(*v), - Err(e) => Err(anyhow!("{e}")), + Err(e) => Err(StorageFetchError::custom(e.to_string())), }); outcomes.push(SlotOutcome { address: addr, @@ -2230,11 +3335,9 @@ impl EvmCache { pub fn reconcile_slots(&mut self, slots: &[(Address, U256)]) -> Result> { let (changed, fetched_ok) = self.verify_slots_inner(slots)?; if !slots.is_empty() && fetched_ok == 0 { - return Err(anyhow!( - "reconcile could not fetch any of the {} requested slot(s) \ - (no usable storage fetcher / provider unreachable)", - slots.len() - )); + return Err(CacheError::ReconcileFetchFailed { + requested: slots.len(), + }); } Ok(changed) } @@ -2259,6 +3362,13 @@ impl EvmCache { /// `(backend_slots_removed, account_removed)` where `account_removed` is true /// if an account entry was removed from either account layer. fn purge_account_inner(&mut self, addr: Address) -> (usize, bool) { + // An account-scope purge also discards any code-seed mark: whatever + // trust state the code carried, the code itself is gone, and the next + // touch refetches authoritative chain state (which is unmarked + // RPC-origin by definition). This is also the documented escape hatch + // for re-seeding after a believed redeploy. + self.code_seeds.remove(&addr); + // Layer 1: CacheDB overlay (accounts + their storage live together). let overlay_removed = self.db.cache.accounts.remove(&addr).is_some(); @@ -2299,13 +3409,13 @@ impl EvmCache { /// Prefer setting this at construction through /// [`EvmCacheBuilder::chain_id`]. This setter exists for cases where the /// chain ID must change after construction. It takes effect on the next - /// [`create_snapshot`](Self::create_snapshot) / `build_evm`; existing + /// [`snapshot`](Self::snapshot) / `build_evm`; existing /// snapshots and overlays keep the chain ID captured when they were created. pub fn set_chain_id(&mut self, chain_id: u64) { self.chain_id = chain_id; } - /// Take a low-level, same-thread snapshot of the CacheDB overlay for + /// Take a low-level, same-thread checkpoint of the CacheDB overlay for /// in-place restore. /// /// Clones the inner [`revm::database::Cache`] (the layer-1 overlay's @@ -2314,23 +3424,23 @@ impl EvmCache { /// overlay back on the same `EvmCache` after speculative mutations (this is /// how the balance-slot scan probes and rewinds). /// - /// For cross-thread fan-out use [`create_snapshot`](Self::create_snapshot) + /// For cross-thread fan-out use [`snapshot`](Self::snapshot) /// instead: it merges both layers into an `Arc<`[`EvmSnapshot`]`>` that is /// `Send + Sync` and can be shared with parallel simulators via /// [`EvmOverlay`]. - pub fn snapshot(&self) -> revm::database::Cache { + pub fn checkpoint(&self) -> revm::database::Cache { self.db.cache.clone() } - /// Restore the CacheDB overlay from a snapshot taken with - /// [`snapshot`](Self::snapshot). + /// Restore the CacheDB overlay from a checkpoint taken with + /// [`checkpoint`](Self::checkpoint). /// - /// Overwrites the layer-1 overlay wholesale with `snapshot`, discarding any + /// Overwrites the layer-1 overlay wholesale with `checkpoint`, discarding any /// overlay mutations made since it was taken. The BlockchainDb backend is /// untouched. This is the in-place counterpart to the cross-thread - /// [`create_snapshot`](Self::create_snapshot) / [`EvmOverlay`] path. - pub fn restore(&mut self, snapshot: revm::database::Cache) { - self.db.cache = snapshot; + /// [`snapshot`](Self::snapshot) / [`EvmOverlay`] path. + pub fn restore(&mut self, checkpoint: revm::database::Cache) { + self.db.cache = checkpoint; } /// Create a new session for executing multiple operations. @@ -2354,13 +3464,13 @@ impl EvmCache { /// delta over it. Layer-1 values shadow the base on reads, reproducing the /// live cache's layered semantics; the resulting [`EvmSnapshot`] is shared /// across threads via `Arc`. Its cost tracks *changed* state, not *total* - /// state. (The retained [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone) + /// state. (The retained [`snapshot_deep_clone`](Self::snapshot_deep_clone) /// is the read-equivalent O(total) reference, kept for benchmarking/testing.) /// /// Takes `&mut self` because it refreshes and memoizes the base. For cheap /// same-thread save/restore of just the overlay, prefer - /// [`snapshot`](Self::snapshot) / [`restore`](Self::restore) instead. - pub fn create_snapshot(&mut self) -> Arc { + /// [`checkpoint`](Self::checkpoint) / [`restore`](Self::restore) instead. + pub fn snapshot(&mut self) -> Arc { // 1. Refresh / memoize the cold layer-2 base, then take a cheap Arc handle // (O(1) when layer 2 is unchanged since the last snapshot). self.refresh_base(); @@ -2432,7 +3542,7 @@ impl EvmCache { }) } - /// Force the next [`create_snapshot`](Self::create_snapshot) to rebuild the + /// Force the next [`snapshot`](Self::snapshot) to rebuild the /// memoized copy-on-write base from scratch (Pillar A). /// /// The crate's own mutators keep the base honest automatically. This is the @@ -2461,7 +3571,7 @@ impl EvmCache { /// Refresh the memoized cold layer-2 [`BaseState`](snapshot::BaseState), /// reusing the previous `Arc` wherever layer 2 is unchanged (Pillar A). /// - /// Called at the top of [`create_snapshot`](Self::create_snapshot). It never + /// Called at the top of [`snapshot`](Self::snapshot). It never /// mutates an `Arc` that may already be shared with a live /// snapshot: on any change it builds a *new* `BaseState` that shares the `Arc` /// handles of unchanged accounts and rebuilds only the changed ones @@ -2476,7 +3586,7 @@ impl EvmCache { /// any address whose slot count differs from the recorded length, or any /// account absent from the base, as dirty. /// 3. **Nothing dirty** → reuse the existing `Arc` unchanged (the - /// common hot-loop case; the base side of `create_snapshot` is then O(1)). + /// common hot-loop case; the base side of `snapshot` is then O(1)). /// 4. **Some addresses dirty** → build a new `BaseState` sharing the `Arc`s of /// unchanged accounts and rebuilding only the dirty ones. fn refresh_base(&mut self) { @@ -2564,7 +3674,7 @@ impl EvmCache { // Rebuild the code index from the refreshed accounts (NOT cloned from the // previous base): a purged or recoded dirty account must not leave a stale - // `code_by_hash` entry, which would diverge from `create_snapshot_deep_clone` + // `code_by_hash` entry, which would diverge from `snapshot_deep_clone` // on a direct `code_by_hash(old_hash)` lookup. Rebuilding from scratch also // handles shared code hashes correctly (a hash survives iff some present // account still carries it). @@ -2582,7 +3692,7 @@ impl EvmCache { /// the deep-clone reference: a hash is present iff some account carries that /// code inline. Rebuilt from scratch on every base (re)build so a purged or /// recoded account never leaves a stale entry — preserving read-equivalence - /// with [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone). + /// with [`snapshot_deep_clone`](Self::snapshot_deep_clone). fn code_index(accounts: &HashMap) -> HashMap { accounts .values() @@ -2596,7 +3706,7 @@ impl EvmCache { /// Build a fresh [`BaseState`](snapshot::BaseState) by flattening all of layer /// 2, recording `base_storage_lens`. Shared by `refresh_base`'s full-rebuild - /// path and [`create_snapshot_deep_clone`](Self::create_snapshot_deep_clone). + /// path and [`snapshot_deep_clone`](Self::snapshot_deep_clone). fn build_base_full(&mut self) -> snapshot::BaseState { let mut accounts = HashMap::new(); { @@ -2627,16 +3737,16 @@ impl EvmCache { /// A/B benchmarking and as the read-equivalence reference (Decision D3). /// /// Produces the same two-tier [`EvmSnapshot`](snapshot::EvmSnapshot) shape as - /// [`create_snapshot`](Self::create_snapshot), but with `base` set to the + /// [`snapshot`](Self::snapshot), but with `base` set to the /// fully-merged flatten of **both** layers and **empty** overlay maps (the /// cleared / not-existing sets still in place). It is read-indistinguishable - /// from `create_snapshot` by construction (the `tests/cow_snapshot.rs` + /// from `snapshot` by construction (the `tests/cow_snapshot.rs` /// differential gate pins this), at the cost of an O(total state) deep copy - /// every call — exactly the cost `create_snapshot` now amortizes away. + /// every call — exactly the cost `snapshot` now amortizes away. /// /// Stays `&self`: it does not touch the memoized base. #[doc(hidden)] - pub fn create_snapshot_deep_clone(&self) -> Arc { + pub fn snapshot_deep_clone(&self) -> Arc { let mut accounts = HashMap::new(); let mut storage: HashMap> = HashMap::new(); let mut code_by_hash = HashMap::new(); @@ -2661,7 +3771,7 @@ impl EvmCache { // 2. Overlay from CacheDB (Layer 1, takes precedence). Merge into the same // flat maps, dropping shadowed entries, exactly as the original - // `create_snapshot` did. A cleared account's storage is routed into + // `snapshot` did. A cleared account's storage is routed into // `overlay_storage` (not the base), because `EvmSnapshot::storage_value` // only applies the cleared-as-ZERO rule for an address with an // `overlay_storage` entry — so the cleared semantics must be expressed @@ -2780,11 +3890,11 @@ impl EvmCache { }; if changed { self.block = block; + self.bump_snapshot_generation(); // Re-pinning replaces layer 2 wholesale (state at a new block): the // memoized base must be rebuilt from scratch on the next snapshot. self.invalidate_base(); let _ = self.backend.set_pinned_block(block); - *self.batch_block_id.lock().unwrap() = block; } if changed || concrete_number.is_none() { self.basefee = None; @@ -2801,6 +3911,49 @@ impl EvmCache { self.block } + /// Monotonic generation counter for snapshot consistency (G6). + /// + /// Increments on every targeted state write ([`apply_update`](Self::apply_update), + /// [`apply_updates`](Self::apply_updates), [`modify_slot`](Self::modify_slot) + /// — and therefore everything built on them: reactive ingestion, freshness + /// corrections, fresh injections) and on block re-pins + /// ([`set_block`](Self::set_block), [`advance_block`](Self::advance_block)). + /// Cold prefetch ([`inject_storage_batch`](Self::inject_storage_batch)) and + /// lazy backend fetches do **not** increment it: they materialize the pinned + /// block's existing state rather than changing it. + /// + /// The magnitude is opaque — how much one call increments it is + /// unspecified — so compare values for **equality only**. + /// + /// The fan-out pattern: read the generation, take the + /// [`snapshot`](Self::snapshot), read the generation again. If the two + /// reads differ, state mutated in between (e.g. your event loop applied + /// part of a block between the reads) — discard and re-snapshot to avoid + /// fanning out simulations over a mid-block state. + /// + /// ```no_run + /// # fn demo(cache: &mut evm_fork_cache::cache::EvmCache) { + /// let snapshot = loop { + /// let generation = cache.snapshot_generation(); + /// let snapshot = cache.snapshot(); + /// if cache.snapshot_generation() == generation { + /// break snapshot; + /// } + /// // A mutation interleaved: try again at the next stable point. + /// }; + /// # let _ = snapshot; + /// # } + /// ``` + pub fn snapshot_generation(&self) -> u64 { + self.snapshot_generation + } + + /// Advance the snapshot-consistency generation (see + /// [`snapshot_generation`](Self::snapshot_generation)). + fn bump_snapshot_generation(&mut self) { + self.snapshot_generation = self.snapshot_generation.wrapping_add(1); + } + /// Set a custom timestamp for EVM simulations. /// /// When set, all EVM executions will use this timestamp instead of the current @@ -2855,7 +4008,7 @@ impl EvmCache { } /// Set the block base fee (the `BASEFEE` opcode) for subsequent simulations, - /// propagated into the next [`create_snapshot`](Self::create_snapshot). + /// propagated into the next [`snapshot`](Self::snapshot). /// /// Offline caches built over a mocked provider have no fetched block header, /// so the base fee is unset (and the `BASEFEE` opcode reads `0`). Use this to @@ -2896,6 +4049,100 @@ impl EvmCache { self.block_gas_limit = gas_limit; } + /// Get the block beneficiary used for EVM simulations (the `COINBASE` + /// opcode). + /// + /// Fetched from the pinned block's header at construction, refreshed by + /// [`advance_block`](Self::advance_block), or overridden via + /// [`set_coinbase`](Self::set_coinbase). `None` means revm uses its default + /// beneficiary. + pub fn coinbase(&self) -> Option
{ + self.coinbase + } + + /// Get `prevrandao` used for EVM simulations (the `PREVRANDAO` opcode, the + /// post-merge header mix hash). + /// + /// Fetched from the pinned block's header at construction, refreshed by + /// [`advance_block`](Self::advance_block), or overridden via + /// [`set_prevrandao`](Self::set_prevrandao). `None` leaves revm's default in + /// place. + pub fn prevrandao(&self) -> Option { + self.prevrandao + } + + /// Get the block gas limit used for EVM simulations (the `GASLIMIT` opcode). + /// + /// Fetched from the pinned block's header at construction, refreshed by + /// [`advance_block`](Self::advance_block), or overridden via + /// [`set_block_gas_limit`](Self::set_block_gas_limit). `None` lets revm use + /// its default. + pub fn block_gas_limit(&self) -> Option { + self.block_gas_limit + } + + /// Set which block-context header fields subsequent + /// [`advance_block`](Self::advance_block) calls require to be present. + /// + /// See [`BlockContextRequirements`]. Under + /// [`strict`](BlockContextRequirements::strict) enforcement, + /// [`advance_block`](Self::advance_block) rejects a header missing a required + /// field rather than silently defaulting it. + pub fn set_block_context_requirements(&mut self, reqs: BlockContextRequirements) { + self.block_context_requirements = reqs; + } + + /// Engine-driven per-block env refresh from a canonical block header. + /// + /// First validates the header against the configured + /// [`BlockContextRequirements`] (set via + /// [`set_block_context_requirements`](Self::set_block_context_requirements) + /// or the strict builder path). Under strict/partial requirements a header + /// missing a required field is rejected with [`BlockContextError`] instead of + /// being silently defaulted; under the [`lenient`](BlockContextRequirements::lenient) + /// default this never errors. + /// + /// On success it refreshes the full EVM block env from the header — block + /// number (`NUMBER`), base fee (`BASEFEE`), beneficiary (`COINBASE`), + /// `prevrandao` (`PREVRANDAO`), gas limit (`GASLIMIT`) and timestamp — and + /// re-pins **every** RPC fetch path (the SharedBackend lazy fallback, the + /// batch storage fetcher, and the account-proof fetcher) to the header's + /// block number, so a lazy miss after the advance reads state at the + /// advanced block, in lockstep with the env. Intended to be driven once per + /// canonical block (e.g. by the reactive runtime as new headers arrive). + /// + /// Unlike [`set_block`](Self::set_block), this does **not** invalidate the + /// memoized COW snapshot base: an advance is a forward roll of the same live + /// view (canonical mutations flow through the write funnel, which already + /// marks the base dirty; lazy fetches stay insert-only-on-miss, which the + /// base's growth scan catches), whereas `set_block` is a wholesale re-fork + /// that must rebuild layer 2. Re-pinning to an *older* block is a re-fork, + /// not an advance — use `set_block` for that. + pub fn advance_block(&mut self, header: &H) -> Result<(), BlockContextError> { + self.block_context_requirements.validate_header(header)?; + + self.block_number = Some(header.number()); + self.basefee = header.base_fee_per_gas(); + self.coinbase = Some(header.beneficiary()); + self.prevrandao = header.mix_hash(); + self.block_gas_limit = Some(header.gas_limit()); + self.timestamp_override = Some(header.timestamp()); + + // Advance every fetch path to the new height in lockstep with the env: + // the SharedBackend lazy fallback (a miss must not serve state from the + // previously pinned block) and the pin accessor. Mirrors `set_block`'s + // pin updates, minus the base invalidation (see the method docs for why + // an advance keeps the memoized base). + let block = BlockId::number(header.number()); + self.block = block; + let _ = self.backend.set_pinned_block(block); + // A snapshot spanning the env refresh would pair the new block context + // with pre-advance state — bump so consumers can detect it (G6). + self.bump_snapshot_generation(); + + Ok(()) + } + /// Re-pin the cache to a specific block number. /// /// Updates the SharedBackend pinned block, the batch fetcher block, and the @@ -2938,7 +4185,10 @@ impl EvmCache { let info = self .backend .basic_ref(address) - .map_err(|e| anyhow!("Failed to fetch account: {:?}", e))?; + .map_err(|e| CacheError::AccountFetch { + address, + details: format!("{e:?}"), + })?; if let Some(info) = info { self.db.insert_account_info(address, info); @@ -2956,7 +4206,11 @@ impl EvmCache { use revm::database_interface::DatabaseRef; self.backend .storage_ref(address, slot) - .map_err(|e| anyhow!("storage read failed for {address} slot {slot}: {e}")) + .map_err(|e| CacheError::StorageRead { + address, + slot, + details: e.to_string(), + }) } /// Write a raw storage slot value directly into the CacheDB layer. @@ -2965,7 +4219,13 @@ impl EvmCache { /// without any RPC call. Used for hot-state injection where we already know the /// current on-chain value from WebSocket events. pub fn insert_storage_slot(&mut self, address: Address, slot: U256, value: U256) -> Result<()> { - self.db.insert_account_storage(address, slot, value)?; + self.db + .insert_account_storage(address, slot, value) + .map_err(|e| CacheError::StorageInsert { + address, + slot, + details: e.to_string(), + })?; Ok(()) } @@ -3004,7 +4264,12 @@ impl EvmCache { ) -> Result<()> { let hashed_balance_slot = keccak256((slot_address, slot).abi_encode()); self.db - .insert_account_storage(contract, hashed_balance_slot.into(), value)?; + .insert_account_storage(contract, hashed_balance_slot.into(), value) + .map_err(|e| CacheError::StorageInsert { + address: contract, + slot: hashed_balance_slot.into(), + details: e.to_string(), + })?; Ok(()) } @@ -3054,7 +4319,7 @@ impl EvmCache { return Ok(Some(slot)); } - let baseline_snapshot = self.snapshot(); + let baseline_snapshot = self.checkpoint(); let baseline_balance = self.erc20_balance_of(token, owner)?; // Choose a probe value distinct from baseline to avoid false positives. @@ -3120,7 +4385,7 @@ impl EvmCache { /// # use alloy_primitives::Address; /// # use alloy_sol_types::sol; /// # use evm_fork_cache::cache::EvmCache; - /// # fn example(cache: &mut EvmCache, token: Address, owner: Address) -> anyhow::Result<()> { + /// # fn example(cache: &mut EvmCache, token: Address, owner: Address) -> Result<(), Box> { /// sol! { /// function balanceOf(address account) external view returns (uint256); /// } @@ -3200,15 +4465,13 @@ impl EvmCache { let mut evm = self.build_evm(); if commit { - return evm - .transact_commit(tx_env) - .map_err(|e| anyhow!("Failed to transact: {:?}", e)); + return evm.transact_commit(tx_env).map_err(CacheError::transact); } let checkpoint = evm.journaled_state.checkpoint(); let result = evm.transact_one(tx_env); evm.journaled_state.checkpoint_revert(checkpoint); - result.map_err(|e| anyhow!("Failed to transact: {:?}", e)) + result.map_err(CacheError::transact) } /// Execute a non-committing call and extract the access list of touched @@ -3245,7 +4508,7 @@ impl EvmCache { // Revert the checkpoint even on a host/transact error so the EVM // journal is not left dirty (mirrors `call_raw`). evm.journaled_state.checkpoint_revert(checkpoint); - Err(anyhow!("Failed to transact: {:?}", e)) + Err(CacheError::transact(e)) } } } @@ -3270,7 +4533,9 @@ impl EvmCache { if let ExecutionResult::Success { logs, gas_used, .. } = result { Ok((logs, gas_used)) } else { - Err(anyhow!("Failed to call: {:?}", result)) + Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }) } } @@ -3289,11 +4554,17 @@ impl EvmCache { match result { ExecutionResult::Success { output, .. } => { let out = output.into_data(); - let balance = IERC20::balanceOfCall::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode balanceOf: {:?}", e))?; + let balance = IERC20::balanceOfCall::abi_decode_returns(&out).map_err(|e| { + CacheError::Decode { + what: "ERC20 balanceOf return data", + details: format!("{e:?}"), + } + })?; Ok(balance) } - _ => Err(anyhow!("balanceOf call failed: {:?}", result)), + _ => Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }), } } @@ -3317,11 +4588,17 @@ impl EvmCache { match result { ExecutionResult::Success { output, .. } => { let out = output.into_data(); - let allowance = IERC20::allowanceCall::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode allowance: {:?}", e))?; + let allowance = IERC20::allowanceCall::abi_decode_returns(&out).map_err(|e| { + CacheError::Decode { + what: "ERC20 allowance return data", + details: format!("{e:?}"), + } + })?; Ok(allowance) } - _ => Err(anyhow!("allowance call failed: {:?}", result)), + _ => Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }), } } @@ -3351,14 +4628,20 @@ impl EvmCache { match result { ExecutionResult::Success { output, .. } => { let out = output.into_data(); - let decimals = IERC20::decimalsCall::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode decimals: {:?}", e))?; + let decimals = IERC20::decimalsCall::abi_decode_returns(&out).map_err(|e| { + CacheError::Decode { + what: "ERC20 decimals return data", + details: format!("{e:?}"), + } + })?; self.token_decimals.insert(token, decimals); // Also update immutable cache for persistence self.immutable_cache.set_token_decimals(token, decimals); Ok(decimals) } - _ => Err(anyhow!("decimals call failed: {:?}", result)), + _ => Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }), } } @@ -3475,12 +4758,17 @@ impl EvmCache { /// [`EvmCacheBuilder`] resolved to a concrete size (with /// [`SharedMemoryCapacity::Auto`] resolved against the state loaded at /// construction), raised by any later [`reserve_shared_memory`](Self::reserve_shared_memory). - /// Each [`create_snapshot`](Self::create_snapshot) copies it onto the snapshot + /// Each [`snapshot`](Self::snapshot) copies it onto the snapshot /// so snapshot-backed [`EvmOverlay`]s pre-allocate the same amount. pub fn shared_memory_capacity(&self) -> usize { self.shared_memory_capacity } + /// The cache-side storage batch-fetch configuration for this instance. + pub fn storage_batch_config(&self) -> StorageBatchConfig { + self.storage_batch_config + } + /// Purge all storage slots for a specific contract from both cache layers. /// /// This clears: @@ -3792,9 +5080,7 @@ impl EvmCache { let mut evm = self.build_evm(); let synthetic_beneficiary = Self::seed_synthetic_beneficiary(&mut evm); let target_checkpoint = evm.journaled_state.checkpoint(); - let result = evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; + let result = evm.transact_one(tx).map_err(CacheError::transact)?; let (logs, gas_used, output) = match result { ExecutionResult::Success { logs, @@ -3805,7 +5091,9 @@ impl EvmCache { _ => { evm.journaled_state.checkpoint_revert(target_checkpoint); Self::remove_synthetic_beneficiary(&mut evm, synthetic_beneficiary); - return Err(anyhow!("Failed to call: {:?}", result)); + return Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }); } }; access_lists.push(extract_access_list(&evm.journaled_state.state)); @@ -3864,14 +5152,14 @@ impl EvmCache { tokens: Option>, commit: bool, ) -> SimulationResult { - let tx = Self::build_tx_env(from, to, calldata).map_err(SimError::Other)?; + let tx = Self::build_tx_env(from, to, calldata).map_err(SimError::from)?; let inspector = TransferInspector::new(); let mut evm = self.build_evm_with_inspector(inspector); let checkpoint = evm.journaled_state.checkpoint(); let result = evm .inspect_one_tx(tx) - .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); + .map_err(|e| SimError::Other(SimHostError::transact(e))); match result { Ok(ExecutionResult::Success { @@ -3975,7 +5263,7 @@ impl EvmCache { txs: &[crate::bundle::BundleTx], opts: &crate::bundle::BundleOptions, ) -> SimulationResult { - let snapshot = self.create_snapshot(); + let snapshot = self.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); overlay.simulate_bundle(txs, opts) } @@ -3999,28 +5287,41 @@ impl EvmCache { .data(creation_code) .value(U256::ZERO) .build() - .map_err(|e| anyhow!("Failed to build CREATE tx: {:?}", e))?; + .map_err(CacheError::tx_env)?; // Use a relaxed contract size limit for deployment. Arbitrum supports // larger contracts than the EIP-170 24576-byte limit via ArbOS. let mut evm = self.build_evm(); evm.cfg.limit_contract_code_size = Some(usize::MAX); - let result = evm - .transact_commit(tx) - .map_err(|e| anyhow!("Contract deployment failed: {:?}", e))?; + let result = evm.transact_commit(tx).map_err(CacheError::transact)?; match result { - ExecutionResult::Success { output, .. } => output - .address() - .copied() - .ok_or_else(|| anyhow!("Contract deployment succeeded but no address returned")), - ExecutionResult::Revert { output, .. } => Err(anyhow!( - "Contract deployment reverted: 0x{}", - alloy_primitives::hex::encode(&output) - )), - ExecutionResult::Halt { reason, .. } => { - Err(anyhow!("Contract deployment halted: {:?}", reason)) + ExecutionResult::Success { output, .. } => { + let address = output + .address() + .copied() + .ok_or(CacheError::DeploymentMissingAddress)?; + // A locally-deployed contract is divergence by construction: + // record it so `etched_accounts` reports every non-chain code + // site. The committed create left the runtime code in the + // overlay; hash from there. + let code_hash = self + .db + .cache + .accounts + .get(&address) + .map(|account| account.info.code_hash) + .unwrap_or(revm::primitives::KECCAK_EMPTY); + self.code_seeds + .insert(address, CodeSeedState::Etched { code_hash }); + Ok(address) } + ExecutionResult::Revert { output, .. } => Err(CacheError::DeploymentReverted { + output_hex: alloy_primitives::hex::encode(&output), + }), + ExecutionResult::Halt { reason, .. } => Err(CacheError::DeploymentHalted { + reason: format!("{reason:?}"), + }), } } @@ -4083,7 +5384,9 @@ impl EvmCache { .accounts .get(&source) .and_then(|a| a.info.code.clone()) - .ok_or_else(|| anyhow!("No bytecode found at source address {}", source))?; + .ok_or(CacheError::MissingSourceBytecode { + source_address: source, + })?; Self::ensure_runtime_code(source, Some(&source_code), "source")?; let code_hash = source_code.hash_slow(); @@ -4118,9 +5421,351 @@ impl EvmCache { // site (D2), so base correctness never relies on that shadowing invariant. self.mark_base_dirty(target); + // Every locally-divergent code write is visible in one place: the + // override target joins the etched set (see `etched_accounts`). + self.code_seeds + .insert(target, CodeSeedState::Etched { code_hash }); + Ok(()) } + /// Verify every [`CodeSeedState::Pending`] canonical code claim against + /// the chain at the pinned block — one bulk `eth_call` for the whole set. + /// + /// Per-address outcomes (see [`CodeVerifyReport`]): + /// - **match** ⇒ marked [`CodeSeedState::Verified`] (never re-checked; + /// post-EIP-6780 code is immutable) and the account's real balance is + /// patched in from the same response — pure materialization of + /// pinned-block truth, so it does **not** bump the + /// [snapshot generation](Self::snapshot_generation); + /// - **mismatch / not-deployed / code-less** ⇒ + /// [`purge_account`](Self::purge_account) (both layers **and** the + /// mark; the purge path bumps the generation) — the next touch + /// refetches authoritative chain state; + /// - **transport failure** (the whole call, an omitted address, or the + /// `MULTICALL3_ADDRESS` extractor-host caveat) ⇒ still `Pending`, + /// reported `unverifiable` — a failed read proves nothing, so it never + /// promotes and never destroys a seed. + /// + /// With no pending seeds this is a no-op that needs no fetcher. Verified + /// seeds are skipped forever, so calling this repeatedly (or from every + /// cold-start round) costs nothing once the set is settled. + /// + /// # Errors + /// [`CacheError::MissingAccountFieldsFetcher`] when pending seeds exist + /// but no [`AccountFieldsFetchFn`] is installed (a + /// [`from_backend`](Self::from_backend) cache without + /// [`set_account_fields_fetcher`](Self::set_account_fields_fetcher)). + pub fn verify_code_seeds(&mut self) -> Result { + let pending = self.pending_code_seeds(); + if pending.is_empty() { + return Ok(CodeVerifyReport::default()); + } + let fetcher = self + .account_fields_fetcher + .clone() + .ok_or(CacheError::MissingAccountFieldsFetcher)?; + + let mut report = CodeVerifyReport::default(); + + // The extractor is hosted at MULTICALL3_ADDRESS under the eth_call + // override, so querying that address would report the extractor's own + // hash — a seed there is unverifiable by this path (use eth_getProof). + let (host, query): (Vec
, Vec
) = pending + .into_iter() + .partition(|address| *address == crate::multicall::MULTICALL3_ADDRESS); + for address in host { + report.unverifiable.push(( + address, + "the account-fields extractor is hosted at this address under the eth_call \ + override; verify it via the eth_getProof path instead" + .to_string(), + )); + } + if query.is_empty() { + return Ok(report); + } + + let samples = match (fetcher)(query.clone(), self.block) { + Ok(samples) => samples, + Err(error) => { + // Fail-safe on transport: every seed stays Pending. + let reason = error.to_string(); + report + .unverifiable + .extend(query.into_iter().map(|address| (address, reason.clone()))); + return Ok(report); + } + }; + let by_address: HashMap = samples.into_iter().collect(); + + let verified_at_block = self.block_number.unwrap_or_default(); + for address in query { + let Some(CodeSeedState::Pending { + code_hash: expected, + }) = self.code_seeds.get(&address).cloned() + else { + // Unreachable in practice (the set was snapshotted above); + // skip rather than misclassify. + continue; + }; + let Some(sample) = by_address.get(&address) else { + report.unverifiable.push(( + address, + "account-fields fetcher returned no sample for this address".to_string(), + )); + continue; + }; + + if sample.code_hash == expected { + self.code_seeds.insert( + address, + CodeSeedState::Verified { + code_hash: expected, + verified_at_block, + }, + ); + self.materialize_verified_balance(address, sample.balance); + report.verified.push(address); + } else if sample.code_hash == B256::ZERO { + self.purge_account(address); + report.not_deployed.push(address); + } else if sample.code_hash == revm::primitives::KECCAK_EMPTY { + self.purge_account(address); + report.codeless.push(address); + } else { + self.purge_account(address); + report.mismatched.push(CodeMismatch { + address, + expected, + actual: sample.code_hash, + }); + } + } + Ok(report) + } + + /// Patch a just-verified seed's balance to the on-chain value from the + /// verification sample — in both layers, **without** a + /// snapshot-generation bump: confirming a claim and materializing + /// pinned-block truth is the prefetch class of write, not a mutation. + /// The overlay is only patched when the account already has an entry + /// there (it always does for a seeded account), mirroring the layer + /// policy of [`inject_storage_batch_fresh`](Self::inject_storage_batch_fresh). + fn materialize_verified_balance(&mut self, address: Address, balance: U256) { + if let Some(account) = self.db.cache.accounts.get_mut(&address) { + account.info.balance = balance; + } + { + let mut accounts = self.blockchain_db.accounts().write(); + if let Some(info) = accounts.get_mut(&address) { + info.balance = balance; + } + } + self.mark_base_dirty(address); + } + + /// Local (already-materialized) account info for `address` — CacheDB + /// overlay first, then the BlockchainDb backend. Never fetches: code-seed + /// decisions are made strictly against what the cache already holds. + fn local_account_info(&self, address: Address) -> Option { + if let Some(account) = self.db.cache.accounts.get(&address) { + return Some(account.info.clone()); + } + self.blockchain_db.accounts().read().get(&address).cloned() + } + + /// Dual-layer account write shared by [`seed_account_code_with`](Self::seed_account_code_with) + /// and [`etch_account_code`](Self::etch_account_code): CacheDB overlay + /// (the primary EVM read path) plus the BlockchainDb backend (shared with + /// parallel tasks), base invalidation, and a snapshot-generation bump — + /// a code write changes executable state (see + /// [`snapshot_generation`](Self::snapshot_generation)). + fn write_marked_code(&mut self, address: Address, info: AccountInfo) { + self.db.insert_account_info(address, info.clone()); + { + let mut accounts = self.blockchain_db.accounts().write(); + accounts.insert(address, info); + } + self.mark_base_dirty(address); + self.bump_snapshot_generation(); + } + + /// Seed canonical runtime code for `address` without fetching it. + /// + /// The claim is marked [`CodeSeedState::Pending`] until + /// [`verify_code_seeds`](Self::verify_code_seeds) confirms it against the + /// on-chain `EXTCODEHASH` (or the cold-start driver's `verify_code` phase + /// does). Because the account is materialized in both cache layers, the + /// lazy backend never fires its balance/nonce/code RPC triple for it. + /// + /// Defaults: nonce 1 (the EIP-161 contract minimum — exact for any + /// contract that never `CREATE`s) and balance `ZERO` until verification + /// patches the real value from the same response. Use + /// [`seed_account_code_with`](Self::seed_account_code_with) to supply + /// both explicitly. + /// + /// Conflict rules (chain-fetched state is authoritative over templates): + /// seeding an **unmarked** address that already holds RPC-origin code + /// with the same hash marks it `Verified` immediately (zero RPC — the + /// warm-cache fast path); a differing hash (including a code-less EOA) is + /// [`CacheError::CodeSeedConflict`] and leaves the cached code untouched. + /// Re-seeding a marked address overwrites and restarts the claim as + /// `Pending`. + /// + /// Returns the keccak256 hash recorded for the claim. + /// + /// # Errors + /// [`CacheError::CodeSeedEmpty`] for empty `code`; + /// [`CacheError::CodeSeedConflict`] as above. + pub fn seed_account_code(&mut self, address: Address, code: Bytes) -> Result { + self.seed_account_code_with(address, code, 1, U256::ZERO) + } + + /// [`seed_account_code`](Self::seed_account_code) with explicit `nonce` + /// and provisional `balance` for the materialized account. Verification + /// still overwrites the balance with the on-chain value on a match; the + /// nonce keeps the supplied value (an exact nonce needs the + /// `eth_getProof` path and only matters for contracts that `CREATE`). + pub fn seed_account_code_with( + &mut self, + address: Address, + code: Bytes, + nonce: u64, + balance: U256, + ) -> Result { + if code.is_empty() { + return Err(CacheError::CodeSeedEmpty { address }); + } + let bytecode = Bytecode::new_raw(code); + let code_hash = bytecode.hash_slow(); + + // Unmarked + locally present ⇒ RPC-origin, which is authoritative. + if !self.code_seeds.contains_key(&address) + && let Some(existing) = self.local_account_info(address) + { + if existing.code_hash == code_hash { + // Hash equality proves byte equality: the claim is already + // confirmed by chain-fetched state, zero RPC. If the restored + // account is missing its code *bytes* (binary state without a + // bytecodes.bin entry), the seed supplies exactly the bytes + // the recorded hash committed to — a free repair. + if existing + .code + .as_ref() + .is_none_or(|existing_code| existing_code.is_empty()) + { + let mut info = existing; + info.code = Some(bytecode); + info.code_hash = code_hash; + self.write_marked_code(address, info); + } + self.code_seeds.insert( + address, + CodeSeedState::Verified { + code_hash, + verified_at_block: self.block_number.unwrap_or_default(), + }, + ); + return Ok(code_hash); + } + return Err(CacheError::CodeSeedConflict { + address, + cached: existing.code_hash, + seeded: code_hash, + }); + } + + // Absent, or an existing mark being re-seeded: write the claim. + // A marked account keeps its current balance/nonce; a fresh one gets + // the caller's provisional values. + let mut info = self.local_account_info(address).unwrap_or(AccountInfo { + balance, + nonce, + code_hash: revm::primitives::KECCAK_EMPTY, + code: None, + account_id: None, + }); + info.code = Some(bytecode); + info.code_hash = code_hash; + self.write_marked_code(address, info); + self.code_seeds + .insert(address, CodeSeedState::Pending { code_hash }); + Ok(code_hash) + } + + /// Etch deliberately-local runtime code at `address` — the raw-bytes + /// sibling of [`override_or_create_account_code`](Self::override_or_create_account_code), + /// with no source account needed. + /// + /// Marks [`CodeSeedState::Etched`]: never verified, excluded from all + /// canonical machinery, and reported via + /// [`etched_accounts`](Self::etched_accounts) so local divergence stays + /// visible. Preserves the existing balance/nonce/storage when the account + /// is already present; creates a default account otherwise. Overwrites + /// any prior code or mark — divergence is the caller's explicit intent. + /// + /// Returns the keccak256 hash of the etched code. + /// + /// # Errors + /// [`CacheError::CodeSeedEmpty`] for empty `code`. + pub fn etch_account_code(&mut self, address: Address, code: Bytes) -> Result { + if code.is_empty() { + return Err(CacheError::CodeSeedEmpty { address }); + } + let bytecode = Bytecode::new_raw(code); + let code_hash = bytecode.hash_slow(); + let mut info = self.local_account_info(address).unwrap_or_default(); + info.code = Some(bytecode); + info.code_hash = code_hash; + self.write_marked_code(address, info); + self.code_seeds + .insert(address, CodeSeedState::Etched { code_hash }); + Ok(code_hash) + } + + /// The code-seed mark for `address`, if any. `None` means RPC-origin: + /// the code (if present) was fetched from the provider and is trusted as + /// chain state. + pub fn code_seed_state(&self, address: &Address) -> Option<&CodeSeedState> { + self.code_seeds.get(address) + } + + /// Addresses whose canonical code claims still await verification + /// ([`CodeSeedState::Pending`]), sorted for deterministic iteration. + /// This is the implicit work set of + /// [`verify_code_seeds`](Self::verify_code_seeds) and the cold-start + /// `verify_code` phase. + pub fn pending_code_seeds(&self) -> Vec
{ + let mut pending: Vec
= self + .code_seeds + .iter() + .filter_map(|(addr, state)| { + matches!(state, CodeSeedState::Pending { .. }).then_some(*addr) + }) + .collect(); + pending.sort(); + pending + } + + /// Addresses whose code deliberately diverges from the chain + /// ([`CodeSeedState::Etched`]), sorted for deterministic iteration. This + /// is the health surface for local divergence: everything written through + /// [`etch_account_code`](Self::etch_account_code), + /// [`override_account_code`](Self::override_account_code) and friends, or + /// [`deploy_contract`](Self::deploy_contract) appears here. + pub fn etched_accounts(&self) -> Vec
{ + let mut etched: Vec
= self + .code_seeds + .iter() + .filter_map(|(addr, state)| { + matches!(state, CodeSeedState::Etched { .. }).then_some(*addr) + }) + .collect(); + etched.sort(); + etched + } + pub(crate) fn require_contract_target(&self, target: Address) -> Result<()> { let target_info = self.target_account_info(target, MissingTargetBehavior::Error)?; Self::ensure_runtime_code(target, target_info.code.as_ref(), "target") @@ -4146,13 +5791,11 @@ impl EvmCache { use revm::database_interface::DatabaseRef; self.backend .basic_ref(target) - .map_err(|e| anyhow!("Failed to fetch target account {}: {:?}", target, e))? - .ok_or_else(|| { - anyhow!( - "Target account {} not found; use override_or_create_account_code for synthetic targets", - target - ) - }) + .map_err(|e| CacheError::TargetAccountFetch { + target, + details: format!("{e:?}"), + })? + .ok_or(CacheError::MissingTargetAccount { target }) } } } @@ -4162,11 +5805,14 @@ impl EvmCache { return Ok(()); } - Err(anyhow!( - "{} account {} has no runtime bytecode", - role, - address - )) + Err(CacheError::MissingRuntimeCode { + role: match role { + "source" => "source", + "target" => "target", + _ => "account", + }, + address, + }) } } @@ -4304,9 +5950,7 @@ impl EvmCache { if let Some(access_list) = &tx.access_list { builder = builder.access_list(access_list.clone()); } - builder - .build() - .map_err(|e| anyhow!("Failed to build tx env: {:?}", e)) + builder.build().map_err(CacheError::tx_env) } fn call_sol_with_commit( @@ -4321,14 +5965,7 @@ impl EvmCache { C: SolCall, { let calldata = Bytes::from(call.abi_encode()); - let result = self - .call_raw_with(from, to, calldata, commit, tx) - .with_context(|| { - format!( - "failed to execute Solidity call {} from {from:?} to {to:?}", - C::SIGNATURE - ) - })?; + let result = self.call_raw_with(from, to, calldata, commit, tx)?; Self::decode_sol_call_result::(from, to, result) } @@ -4343,20 +5980,20 @@ impl EvmCache { match result { ExecutionResult::Success { output, .. } => { let output = output.into_data(); - C::abi_decode_returns(&output).map_err(|error| { - anyhow!( - "failed to decode Solidity call {} return data from {from:?} to {to:?}: output_len={}, error: {:?}", - C::SIGNATURE, - output.len(), - error - ) + C::abi_decode_returns(&output).map_err(|error| CacheError::SolCallDecode { + signature: C::SIGNATURE, + from, + to, + output_len: output.len(), + details: format!("{error:?}"), }) } - other => Err(anyhow!( - "Solidity call {} from {from:?} to {to:?} did not succeed: {:?}", - C::SIGNATURE, - other - )), + other => Err(CacheError::SolCallFailed { + signature: C::SIGNATURE, + from, + to, + result: format!("{other:?}"), + }), } } @@ -4368,18 +6005,22 @@ impl EvmCache { ) -> Result { let call = IERC20::balanceOfCall { target: owner }; let tx = Self::build_tx_env(caller, token, Bytes::from(call.abi_encode()))?; - let result = evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e))?; + let result = evm.transact_one(tx).map_err(CacheError::transact)?; match result { ExecutionResult::Success { output, .. } => { let out = output.into_data(); - let balance = IERC20::balanceOfCall::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode balanceOf: {:?}", e))?; + let balance = IERC20::balanceOfCall::abi_decode_returns(&out).map_err(|e| { + CacheError::Decode { + what: "ERC20 balanceOf return data", + details: format!("{e:?}"), + } + })?; Ok(balance) } - _ => Err(anyhow!("balanceOf call failed: {:?}", result)), + _ => Err(CacheError::CallNotSuccessful { + result: format!("{result:?}"), + }), } } @@ -4422,8 +6063,9 @@ impl EvmCache { /// persist changes to the underlying database, or simply drop the session to discard /// all changes. /// -/// Note: For snapshot/restore functionality across multiple transactions, use `EvmCache::snapshot()` -/// and `EvmCache::restore()` instead, as the EVM journal is cleared after each transaction. +/// Note: For checkpoint/restore functionality across multiple transactions, use +/// `EvmCache::checkpoint()` and `EvmCache::restore()` instead, as the EVM journal +/// is cleared after each transaction. pub struct EvmSession<'a> { evm: CacheEvm<'a>, } @@ -4446,14 +6088,12 @@ impl<'a> EvmSession<'a> { let tx = EvmCache::build_tx_env(from, to, calldata)?; if commit { - self.evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e)) + self.evm.transact_one(tx).map_err(CacheError::transact) } else { let checkpoint = self.evm.journaled_state.checkpoint(); let result = self.evm.transact_one(tx); self.evm.journaled_state.checkpoint_revert(checkpoint); - result.map_err(|e| anyhow!("Failed to transact: {:?}", e)) + result.map_err(CacheError::transact) } } @@ -4522,6 +6162,210 @@ mod shared_memory_capacity_tests { mod core_tests { use super::*; + #[test] + fn parses_prestate_diff_trace_values_and_cleared_slots() { + let trace = serde_json::json!([ + { + "result": { + "pre": { + "0x4242424242424242424242424242424242424242": { + "storage": { + "0x01": "0x05", + "0x02": "0x06" + } + } + }, + "post": { + "0x4242424242424242424242424242424242424242": { + "balance": 10, + "nonce": "0x0a", + "code": "0x6001", + "storage": { + "0x01": "0x0b" + } + } + } + } + } + ]); + + let diff = parse_block_state_diff_trace(&trace).unwrap(); + + assert_eq!(diff.accounts.len(), 1); + let account = &diff.accounts[0]; + assert_eq!(account.address, Address::repeat_byte(0x42)); + assert_eq!(account.balance, Some(U256::from(10))); + assert_eq!(account.nonce, Some(10)); + assert_eq!(account.code, Some(Bytes::from(vec![0x60, 0x01]))); + assert_eq!( + account.storage, + vec![ + BlockStateStorageDiff { + slot: U256::from(1), + value: U256::from(11), + }, + BlockStateStorageDiff { + slot: U256::from(2), + value: U256::ZERO, + }, + ] + ); + } + + #[test] + fn parses_prestate_diff_trace_account_deletion() { + // A SELFDESTRUCTed account appears in `pre` but is entirely absent + // from `post`. The merged diff must carry its explicit post-deletion + // fields (zero balance/nonce, empty code) — and, when the account had + // storage, zeroed slots — so account-target resyncs resolve from the + // trace instead of falling back to point reads. + let trace = serde_json::json!([ + { + "result": { + "pre": { + // Deleted WITH storage history in the trace. + "0x4242424242424242424242424242424242424242": { + "balance": "0x64", + "nonce": "0x01", + "code": "0x6001", + "storage": { "0x01": "0x05" } + }, + // Deleted WITHOUT any storage entry (the previously + // missed case). + "0x1111111111111111111111111111111111111111": { + "balance": "0x0a" + } + }, + "post": {} + } + } + ]); + + let diff = parse_block_state_diff_trace(&trace).unwrap(); + assert_eq!(diff.accounts.len(), 2); + + let bare = &diff.accounts[0]; // 0x11.. sorts first + assert_eq!(bare.address, Address::repeat_byte(0x11)); + assert_eq!(bare.balance, Some(U256::ZERO)); + assert_eq!(bare.nonce, Some(0)); + assert_eq!(bare.code, Some(Bytes::new())); + assert!(bare.storage.is_empty()); + + let stored = &diff.accounts[1]; + assert_eq!(stored.address, Address::repeat_byte(0x42)); + assert_eq!(stored.balance, Some(U256::ZERO)); + assert_eq!(stored.nonce, Some(0)); + assert_eq!(stored.code, Some(Bytes::new())); + assert_eq!( + stored.storage, + vec![BlockStateStorageDiff { + slot: U256::from(1), + value: U256::ZERO, + }] + ); + } + + #[test] + fn parses_prestate_diff_trace_deletion_then_recreation_keeps_final_state() { + // tx1 deletes the account; tx2 re-creates it. Entries merge in tx + // order, so the final post-block values must win over the synthesized + // deletion zeros. + let trace = serde_json::json!([ + { + "result": { + "pre": { + "0x4242424242424242424242424242424242424242": { "balance": "0x64" } + }, + "post": {} + } + }, + { + "result": { + "pre": {}, + "post": { + "0x4242424242424242424242424242424242424242": { + "balance": "0x07", + "nonce": "0x01", + "code": "0x6002" + } + } + } + } + ]); + + let diff = parse_block_state_diff_trace(&trace).unwrap(); + assert_eq!(diff.accounts.len(), 1); + let account = &diff.accounts[0]; + assert_eq!(account.balance, Some(U256::from(7))); + assert_eq!(account.nonce, Some(1)); + assert_eq!(account.code, Some(Bytes::from(vec![0x60, 0x02]))); + } + + #[test] + fn snapshot_generation_bumps_on_writes_and_repins_not_prefetch() { + use alloy_provider::RootProvider; + use alloy_rpc_client::RpcClient; + use alloy_transport::mock::Asserter; + + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut cache = rt.block_on(EvmCache::new(Arc::new(provider))); + + let addr = Address::repeat_byte(0x77); + let g0 = cache.snapshot_generation(); + + // Targeted writes bump (magnitude is opaque; assert monotonic change). + cache.apply_updates(&[StateUpdate::slot(addr, U256::from(1), U256::from(10))]); + let g1 = cache.snapshot_generation(); + assert!(g1 > g0, "apply_updates must bump the generation"); + + cache.apply_update(&StateUpdate::slot(addr, U256::from(2), U256::from(20))); + let g2 = cache.snapshot_generation(); + assert!(g2 > g1, "apply_update must bump the generation"); + + // An empty batch is a no-op, not a mutation. + cache.apply_updates(&[]); + assert_eq!(cache.snapshot_generation(), g2); + + // modify_slot on a warm slot bumps. + let change = cache.modify_slot(addr, U256::from(1), |v| { + Some(v.unwrap_or_default() + U256::from(1)) + }); + assert!(change.is_some()); + let g3 = cache.snapshot_generation(); + assert!(g3 > g2, "modify_slot must bump the generation"); + + // Cold prefetch materializes existing chain state — no bump. + cache.inject_storage_batch(&[(addr, U256::from(9), U256::from(90))]); + assert_eq!( + cache.snapshot_generation(), + g3, + "inject_storage_batch is prefetch, not mutation" + ); + + // Block re-pins bump; a same-block set_block is a no-op. + cache.set_block(BlockId::Number(BlockNumberOrTag::Number(5))); + let g4 = cache.snapshot_generation(); + assert!(g4 > g3, "set_block to a new pin must bump the generation"); + cache.set_block(BlockId::Number(BlockNumberOrTag::Number(5))); + assert_eq!( + cache.snapshot_generation(), + g4, + "re-pinning to the same block is not a mutation" + ); + + // advance_block refreshes the env — a spanning snapshot would be + // inconsistent, so it bumps too. + let header = alloy_consensus::Header::default(); + cache.advance_block(&header).expect("lenient advance"); + assert!(cache.snapshot_generation() > g4); + } + #[test] fn test_address_to_u256_conversion() { // Test that address conversion preserves the address bytes correctly diff --git a/src/cache/overlay.rs b/src/cache/overlay.rs index c5d11e1..f41e9a6 100644 --- a/src/cache/overlay.rs +++ b/src/cache/overlay.rs @@ -1,3 +1,13 @@ +//! Per-simulation state overlays layered over a snapshot or the live cache. +//! +//! An [`EvmOverlay`] wraps a read-only base (an +//! [`EvmSnapshot`] or the cache itself) with +//! a scratch write layer, so a simulation can mutate balances, storage, and code +//! and run calls without disturbing the base or other overlays. Overlays are the +//! `Send` unit of parallel fan-out: snapshot once, clone cheaply, overlay per +//! candidate. Reads fall through the write layer to the base; writes and reverts +//! stay local to the overlay. + use std::cell::RefCell; use std::collections::HashMap; use std::rc::Rc; @@ -5,7 +15,6 @@ use std::sync::Arc; use alloy_eips::eip2930::{AccessList, AccessListItem}; use alloy_primitives::{Address, B256, Bytes, TxKind, U256}; -use anyhow::{Result, anyhow}; use foundry_fork_db::{DatabaseError, SharedBackend}; use revm::{ Context, ExecuteCommitEvm, ExecuteEvm, InspectEvm, MainBuilder, MainContext, @@ -18,7 +27,10 @@ use super::snapshot::EvmSnapshot; use super::{CallSimulationResult, SimStatus, TxConfig, unix_timestamp_secs_saturating}; use crate::access_set::StorageAccessList; use crate::bundle::{BundleOptions, BundleResult, BundleTx, RevertPolicy, TxOutcome}; -use crate::errors::{SimError, SimulationError, SimulationResult}; +use crate::errors::{ + OverlayError, OverlayResult as Result, SimError, SimHostError, SimulationError, + SimulationResult, +}; use crate::inspector::TransferInspector; type OverlayEvm<'a> = revm::MainnetEvm< @@ -65,6 +77,12 @@ pub struct EvmOverlay { /// [`SharedMemoryCapacity`](super::SharedMemoryCapacity) so overlays honor the /// capacity set on the originating [`EvmCache`]. buffer_capacity: usize, + /// Set when a `BLOCKHASH` read fell through to the ZERO fallback (no + /// snapshot-provided hash and no `ext_db`). The freshness validator reads + /// this via [`Self::blockhash_zero_fallback`] to fail closed instead of + /// confirming a sim whose control flow may rest on a hash its overlays + /// cannot resolve. Cleared by [`Self::reset`]. + blockhash_zero_fallback: bool, } impl EvmOverlay { @@ -82,6 +100,7 @@ impl EvmOverlay { ext_db, reusable_buffer: Vec::with_capacity(buffer_capacity), buffer_capacity, + blockhash_zero_fallback: false, } } @@ -98,10 +117,28 @@ impl EvmOverlay { pub fn reset(&mut self) { self.dirty_accounts.clear(); self.dirty_storage.clear(); + self.blockhash_zero_fallback = false; // Keep: snapshot Arc, ext_db, and the reusable buffer. The buffer is // already cleared after each call, so nothing to do for it here. } + /// `true` if any `BLOCKHASH` read on this overlay fell through to the ZERO + /// fallback (no snapshot-provided hash for that number and no `ext_db`) + /// since construction or the last [`reset`](Self::reset). + /// + /// The freshness validator uses this to **fail closed**: a sim that read a + /// hash its ext-db-less overlays cannot resolve is reported + /// [`Unverified`](crate::freshness::Validation::Unverified) rather than + /// silently confirmed against a ZERO stand-in. + /// + /// Only reads revm actually routes to the database can set this: requests + /// outside the EVM's valid lookback window (`[current − 256, current)`) + /// return spec-mandated ZERO without a database call — that value is + /// correct on-chain too, so such reads are deliberately not flagged. + pub fn blockhash_zero_fallback(&self) -> bool { + self.blockhash_zero_fallback + } + /// Chain ID of the block context captured by the underlying snapshot. /// /// This is the value installed into `cfg.chain_id` by [`Self::build_evm`]. @@ -243,7 +280,7 @@ impl EvmOverlay { /// # use std::sync::Arc; /// # use alloy_primitives::{Address, Bytes}; /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; - /// # fn run(snapshot: Arc) -> anyhow::Result<()> { + /// # fn run(snapshot: Arc) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// let result = overlay.call_raw(Address::ZERO, Address::ZERO, Bytes::new())?; /// // State is reverted; a second call sees the same base state. @@ -264,7 +301,7 @@ impl EvmOverlay { .data(calldata) .value(U256::ZERO) .build() - .map_err(|e| anyhow!("Failed to build tx env: {:?}", e))?; + .map_err(OverlayError::tx_env)?; // Recycle the reusable buffer (Pillar A.2): take it out as a plain Vec // (keeping the overlay Send), lend it to a method-local Rc for @@ -279,9 +316,7 @@ impl EvmOverlay { let mut evm = self.build_evm_with_local(local); use revm::context_interface::JournalTr; let checkpoint = evm.journaled_state.checkpoint(); - let result = evm - .transact_one(tx) - .map_err(|e| anyhow!("Failed to transact: {:?}", e)); + let result = evm.transact_one(tx).map_err(OverlayError::transact); evm.journaled_state.checkpoint_revert(checkpoint); result }; @@ -392,7 +427,7 @@ impl EvmOverlay { /// # use std::sync::Arc; /// # use alloy_primitives::{Address, Bytes}; /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; - /// # fn run(snapshot: Arc, token: Address, owner: Address) -> anyhow::Result<()> { + /// # fn run(snapshot: Arc, token: Address, owner: Address) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// let sim = overlay.simulate_with_transfer_tracking( /// owner, @@ -421,7 +456,7 @@ impl EvmOverlay { .data(calldata) .value(U256::ZERO) .build() - .map_err(|e| SimError::Other(anyhow!("Failed to build tx env: {:?}", e)))?; + .map_err(|e| SimError::Other(SimHostError::tx_env(e)))?; let inspector = TransferInspector::new(); @@ -440,7 +475,7 @@ impl EvmOverlay { let result = evm .inspect_one_tx(tx) - .map_err(|e| SimError::Other(anyhow!("Failed to transact: {:?}", e))); + .map_err(|e| SimError::Other(SimHostError::transact(e))); match result { Ok(ExecutionResult::Success { @@ -535,7 +570,7 @@ impl EvmOverlay { /// # use alloy_primitives::{Address, Bytes}; /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot, TxConfig}; /// # use evm_fork_cache::CallTracer; - /// # fn run(snapshot: Arc, to: Address) -> anyhow::Result<()> { + /// # fn run(snapshot: Arc, to: Address) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// let (result, tracer) = overlay.call_raw_with_inspector( /// Address::ZERO, @@ -590,7 +625,7 @@ impl EvmOverlay { } let tx_env = builder .build() - .map_err(|e| SimError::Other(anyhow!("Failed to build tx env: {:?}", e)))?; + .map_err(|e| SimError::Other(SimHostError::tx_env(e)))?; // Recycle the reusable buffer (Pillar A.2); reclaimed after the EVM drops. let buffer = Rc::new(RefCell::new(std::mem::take(&mut self.reusable_buffer))); @@ -617,7 +652,7 @@ impl EvmOverlay { } Err(e) => { evm.journaled_state.checkpoint_revert(checkpoint); - Err(SimError::Other(anyhow!("Failed to transact: {:?}", e))) + Err(SimError::Other(SimHostError::transact(e))) } } }; @@ -697,9 +732,9 @@ impl EvmOverlay { } builder .build() - .map_err(|e| SimError::Other(anyhow!("Failed to build tx env: {:?}", e))) + .map_err(|e| SimError::Other(SimHostError::tx_env(e))) }) - .collect::>()?; + .collect::>()?; // Resolve the beneficiary and read its pre-bundle balance before the // mutable borrow of `self` by the EVM (the post-bundle delta is the miner @@ -710,7 +745,7 @@ impl EvmOverlay { .unwrap_or_else(|| revm::context::BlockEnv::default().beneficiary); let pre_beneficiary_balance = self .basic(beneficiary) - .map_err(|e| SimError::Other(anyhow!("Failed to load beneficiary: {:?}", e)))? + .map_err(|e| SimError::Other(SimHostError::database(e)))? .map(|info| info.balance) .unwrap_or(U256::ZERO); @@ -744,7 +779,7 @@ impl EvmOverlay { evm.journaled_state.checkpoint_revert(outer); drop(evm); self.reclaim_buffer(buffer); - return Err(SimError::Other(anyhow!("Failed to transact: {:?}", e))); + return Err(SimError::Other(SimHostError::transact(e))); } }; @@ -779,6 +814,19 @@ impl EvmOverlay { // Successful tx: its effects stay journaled for the next tx. } + // Partition total gas into successful/reverted buckets in a single + // pass. Saturating (consistent with `total_gas`); the invariant + // `successful_tx_gas + reverted_tx_gas == total_gas` holds by + // construction since every executed tx lands in exactly one bucket. + let (successful_tx_gas, reverted_tx_gas) = + per_tx.iter().fold((0u64, 0u64), |(succ, rev), tx| { + if tx.reverted { + (succ, rev.saturating_add(tx.gas_used)) + } else { + (succ.saturating_add(tx.gas_used), rev) + } + }); + if aborted { // State is reverted to the pre-bundle outer checkpoint regardless // of `commit`; no payment. @@ -786,6 +834,8 @@ impl EvmOverlay { per_tx, coinbase_payment: U256::ZERO, gas_used: total_gas, + successful_tx_gas, + reverted_tx_gas, succeeded: false, } } else { @@ -814,6 +864,8 @@ impl EvmOverlay { per_tx, coinbase_payment, gas_used: total_gas, + successful_tx_gas, + reverted_tx_gas, succeeded: true, } } @@ -847,7 +899,7 @@ impl EvmOverlay { /// # use std::sync::Arc; /// # use alloy_primitives::{Address, Bytes}; /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; - /// # fn run(snapshot: Arc) -> anyhow::Result<()> { + /// # fn run(snapshot: Arc) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// let (result, access_list) = /// overlay.call_raw_with_access_list(Address::ZERO, Address::ZERO, Bytes::new())?; @@ -899,9 +951,7 @@ impl EvmOverlay { if let Some(access_list) = &tx.access_list { builder = builder.access_list(access_list.clone()); } - let tx_env = builder - .build() - .map_err(|e| anyhow!("Failed to build tx env: {:?}", e))?; + let tx_env = builder.build().map_err(OverlayError::tx_env)?; // Recycle the reusable buffer (Pillar A.2); reclaimed after the EVM drops. let buffer = Rc::new(RefCell::new(std::mem::take(&mut self.reusable_buffer))); @@ -932,7 +982,7 @@ impl EvmOverlay { // Revert the checkpoint even on a host/transact error so the EVM // journal is not left dirty (mirrors `call_raw`). evm.journaled_state.checkpoint_revert(checkpoint); - Err(anyhow!("Failed to transact: {:?}", e)) + Err(OverlayError::transact(e)) } } }; @@ -963,7 +1013,7 @@ impl EvmOverlay { /// # use std::sync::Arc; /// # use alloy_primitives::{Address, Bytes, U256}; /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot}; - /// # fn run(snapshot: Arc, token: Address, slot: U256) -> anyhow::Result<()> { + /// # fn run(snapshot: Arc, token: Address, slot: U256) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// // Inject the fresh value, then re-run to observe the corrected result. /// overlay.override_slot(token, slot, U256::from(42u64)); @@ -1077,8 +1127,10 @@ impl Database for EvmOverlay { // Snapshots never populate `block_hashes` (the live cache does not track // block hashes), so without an `ext_db` the `BLOCKHASH` opcode resolves to // ZERO. Overlays built internally (e.g. the freshness validator) pass - // `ext_db = None`; a contract that reads `BLOCKHASH` through such an - // overlay sees ZERO. Documented in docs/KNOWN_ISSUES.md. + // `ext_db = None`; the fallback is recorded so the validator can fail + // closed (`Unverified`) instead of confirming a sim whose control flow + // may depend on the real hash. See `blockhash_zero_fallback()`. + self.blockhash_zero_fallback = true; Ok(B256::ZERO) } } @@ -1108,7 +1160,7 @@ mod tests { /// Build a two-tier `EvmSnapshot` whose cold base holds the given accounts, /// storage, and code, with an empty hot overlay — the shape - /// `create_snapshot_deep_clone` produces. The `Arc`-per-account storage of the + /// `snapshot_deep_clone` produces. The `Arc`-per-account storage of the /// base is built from the plain per-account maps. fn snap( accounts: HashMap, @@ -1150,6 +1202,31 @@ mod tests { assert_send::(); } + #[test] + fn blockhash_zero_fallback_flags_only_unresolved_reads() { + let known = B256::repeat_byte(0xAB); + let snapshot = snap( + HashMap::new(), + HashMap::new(), + HashMap::new(), + HashMap::from([(5u64, known)]), + ); + let mut overlay = EvmOverlay::new(snapshot, None); + + // A snapshot-provided hash resolves for real: no flag. + assert_eq!(overlay.block_hash(5).unwrap(), known); + assert!(!overlay.blockhash_zero_fallback()); + + // An untracked number falls back to ZERO and is flagged so the + // freshness validator can fail closed. + assert_eq!(overlay.block_hash(6).unwrap(), B256::ZERO); + assert!(overlay.blockhash_zero_fallback()); + + // The flag is per-simulation state: reset clears it. + overlay.reset(); + assert!(!overlay.blockhash_zero_fallback()); + } + #[test] fn test_overlay_basic_from_snapshot() { let mut accounts = HashMap::new(); diff --git a/src/cache/slot_observations.rs b/src/cache/slot_observations.rs index d5a42a2..33d3183 100644 --- a/src/cache/slot_observations.rs +++ b/src/cache/slot_observations.rs @@ -23,6 +23,7 @@ use tracing::{debug, warn}; use crate::freshness::FreshnessParams; use super::versioned; +use crate::errors::PersistenceError; const SLOT_OBSERVATIONS_MAGIC: &[u8; 8] = b"EFC-SOBS"; const SLOT_OBSERVATIONS_VERSION: u32 = 1; @@ -63,6 +64,7 @@ pub struct SlotObservationTracker { } impl SlotObservationTracker { + /// Create an empty tracker with no recorded observations. pub fn new() -> Self { Self { observations: HashMap::new(), @@ -108,12 +110,13 @@ impl SlotObservationTracker { /// Persist observations to disk using the versioned binary format. /// Called at end of cycle or on shutdown. - pub fn save(&mut self, path: &Path) -> anyhow::Result<()> { + pub fn save(&mut self, path: &Path) -> Result<(), PersistenceError> { if !self.dirty { return Ok(()); } if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; } let data = versioned::encode( SLOT_OBSERVATIONS_MAGIC, @@ -121,7 +124,7 @@ impl SlotObservationTracker { &self.observations, "slot observations", )?; - std::fs::write(path, data)?; + std::fs::write(path, data).map_err(|err| PersistenceError::write(path, err))?; self.dirty = false; debug!(entries = self.observations.len(), "Saved slot observations"); Ok(()) diff --git a/src/cache/snapshot.rs b/src/cache/snapshot.rs index 0b6c5ec..b53b08b 100644 --- a/src/cache/snapshot.rs +++ b/src/cache/snapshot.rs @@ -13,10 +13,10 @@ //! (committed sim changes, write-throughs, freshness corrections), which always //! shadows the base on a read. //! -//! [`super::EvmCache::create_snapshot`] memoizes the base (via the internal +//! [`super::EvmCache::snapshot`] memoizes the base (via the internal //! `refresh_base`) and folds only layer 1 fresh, so its cost tracks *changed* //! state, not *total* state. The retained -//! [`super::EvmCache::create_snapshot_deep_clone`] produces the same two-tier +//! [`super::EvmCache::snapshot_deep_clone`] produces the same two-tier //! shape with everything flattened into the base and empty overlay maps; it is the //! A/B benchmark baseline and the read-equivalence reference. //! @@ -75,7 +75,7 @@ pub(crate) struct BaseState { /// public [`storage_value`](Self::storage_value)) are O(1) and lock-free, and /// reproduce the live cache's layered semantics bit-for-bit. /// -/// Created via [`super::EvmCache::create_snapshot()`]. Each parallel simulation +/// Created via [`super::EvmCache::snapshot()`]. Each parallel simulation /// task gets its own [`super::EvmOverlay`] backed by a cheap `Arc::clone` of /// the snapshot. pub struct EvmSnapshot { diff --git a/src/cache/versioned.rs b/src/cache/versioned.rs index 0ed9681..d6eac82 100644 --- a/src/cache/versioned.rs +++ b/src/cache/versioned.rs @@ -1,7 +1,7 @@ use serde::{Serialize, de::DeserializeOwned}; use tracing::warn; -use anyhow::{Context as _, Result}; +use crate::errors::PersistenceError; const VERSION_BYTES: usize = 4; @@ -10,9 +10,9 @@ pub(crate) fn encode( version: u32, value: &T, label: &'static str, -) -> Result> { +) -> Result, PersistenceError> { let payload = - bincode::serialize(value).with_context(|| format!("failed to serialize {label}"))?; + bincode::serialize(value).map_err(|err| PersistenceError::serialize(label, err))?; let mut data = Vec::with_capacity(magic.len() + VERSION_BYTES + payload.len()); data.extend_from_slice(magic); data.extend_from_slice(&version.to_le_bytes()); diff --git a/src/cold_start/config.rs b/src/cold_start/config.rs index 5617d08..4bf27f4 100644 --- a/src/cold_start/config.rs +++ b/src/cold_start/config.rs @@ -94,6 +94,11 @@ pub struct ColdStartRoundSummary { pub probe_requested: usize, /// Probe slots whose fetch failed. pub probe_failed: usize, + /// Probe-roots addresses requested this round. + pub probe_roots_requested: usize, + /// Probe-roots addresses whose root could not be observed + /// (`RootProbeOutcome::root` is `None`). + pub probe_roots_failed: usize, /// Discover calls issued this round. pub discover_calls: usize, /// Slots touched by this round's discover calls. @@ -105,7 +110,11 @@ impl ColdStartRunReport { /// /// Plain field accumulation, no IO. `failed_slots` counts /// [`FetchFailed`](crate::freshness::SlotFetch::FetchFailed) outcomes across - /// both the verify (`fetched`) and probe (`probed`) phases. + /// both the verify (`fetched`) and probe (`probed`) phases. Root probes are + /// not slots, so probe-roots counts are recorded per round + /// (`probe_roots_requested` / `probe_roots_failed`) and never fold into + /// `failed_slots` — mirroring how `probe_requested` / `probe_failed` carry + /// no dedicated run-level totals. pub(crate) fn absorb_round(&mut self, plan: &ColdStartPlan, results: &ColdStartResults) { self.rounds += 1; let verify_failed = results @@ -118,6 +127,11 @@ impl ColdStartRunReport { .iter() .filter(|o| matches!(o.fetch, SlotFetch::FetchFailed { .. })) .count(); + let probe_roots_failed = results + .probed_roots + .iter() + .filter(|o| o.root.is_none()) + .count(); let discovered_slots: usize = results .discovered .iter() @@ -141,6 +155,8 @@ impl ColdStartRunReport { verify_failed, probe_requested: plan.probe.len(), probe_failed, + probe_roots_requested: plan.probe_roots.len(), + probe_roots_failed, discover_calls: plan.discover.len(), discovered_slots, }); diff --git a/src/cold_start/driver.rs b/src/cold_start/driver.rs index 32bedd7..783f018 100644 --- a/src/cold_start/driver.rs +++ b/src/cold_start/driver.rs @@ -4,11 +4,10 @@ //! The driver performs every fetch and call; planners are pure and IO-free. The //! whole module is gated behind the `reactive` feature. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use alloy_eips::BlockId; -use alloy_primitives::Address; -use anyhow::Result; +use alloy_primitives::{Address, B256}; use crate::cache::{EvmCache, block_in_place_handle}; use crate::cold_start::config::{ColdStartConfig, ColdStartPin, ColdStartRunReport}; @@ -16,6 +15,8 @@ use crate::cold_start::error::ColdStartError; use crate::cold_start::plan::ColdStartPlan; use crate::cold_start::planner::{ColdStartPlanner, ColdStartStep}; use crate::cold_start::results::{ColdStartCallResult, ColdStartResults, RoundOutcome}; +use crate::cold_start::roots::RootProbeOutcome; +use crate::errors::CacheResult; use crate::events::StateView; use crate::freshness::{SlotFetch, SlotOutcome}; @@ -40,25 +41,45 @@ impl EvmCache { /// cold-start runtime precondition); on a current-thread runtime or with no /// runtime present it returns a typed error via /// [`block_in_place_handle`](crate::cache::block_in_place_handle). - pub(crate) fn ensure_account_blocking(&mut self, address: Address) -> Result<()> { + pub(crate) fn ensure_account_blocking(&mut self, address: Address) -> CacheResult<()> { let handle = block_in_place_handle()?; tokio::task::block_in_place(|| handle.block_on(self.ensure_account(address))) } /// Execute a single cold-start round and return its (possibly partial) outcome. /// - /// Fixed phase order: **accounts → verify → probe → discover**. + /// Fixed phase order: **verify_code → accounts → verify → probe → + /// probe_roots → discover**. /// - /// Per-round fetcher guard: if the plan declares any verify or probe slots and - /// the cache has no storage batch fetcher, the round short-circuits with - /// [`ColdStartError::NoBatchFetcher`] before issuing any read. A round - /// declaring only accounts/discover runs without a fetcher. + /// Per-round fetcher guards: if the plan declares any verify or probe slots + /// and the cache has no storage batch fetcher, the round short-circuits with + /// [`ColdStartError::NoBatchFetcher`] before issuing any read; likewise a + /// probe_roots-bearing round with no account proof fetcher short-circuits + /// with [`ColdStartError::NoAccountProofFetcher`], and a round with pending + /// code seeds but no account-fields fetcher short-circuits with + /// [`ColdStartError::NoAccountFieldsFetcher`]. A round declaring only + /// accounts/discover (and holding no pending seeds) runs without any + /// fetcher. /// - /// - **accounts (first):** each `plan.accounts` address is pre-seeded via - /// `ensure_account_blocking`. A failure here is a hard error in the first - /// phase, so nothing after it ran: every declared verify/probe slot is marked - /// [`SlotFetch::NotAttempted`] and the round returns with `error: Some(..)`. - /// This is the only producer of `NotAttempted`. + /// - **verify_code (first):** every [`CodeSeedState`](crate::cache::CodeSeedState) + /// `Pending` claim is settled via + /// [`verify_code_seeds`](EvmCache::verify_code_seeds); the work set is + /// the cache's own pending marks, not a plan field. Matches are marked + /// `Verified`; contradicted claims are purged, so an address also listed + /// in `plan.accounts` is refetched by the very next phase. A transport + /// failure is **not** a round hard error — it surfaces in the report's + /// `unverifiable` bucket (matching probe_roots' per-address stance). + /// Running first means no discover sim ever executes over an unverified + /// claim, and `results.code_verifications` survives any later phase's + /// hard error. With no pending seeds the phase is a no-op + /// (`code_verifications: None`). + /// - **accounts:** each `plan.accounts` address is pre-seeded via + /// `ensure_account_blocking`. A failure here is a hard error before the + /// slot phases ran: every declared verify/probe slot is marked + /// [`SlotFetch::NotAttempted`], every declared probe_roots address is + /// synthesized as `root: None`, and the round returns with `error: Some(..)` + /// (the already-computed `code_verifications` is preserved). This is the + /// only producer of `NotAttempted`. /// - **verify:** each verify slot is re-fetched, classified into /// `results.fetched`, and (when changed) injected and recorded in /// `results.verified`. @@ -67,6 +88,11 @@ impl EvmCache { /// classification verify uses. Unlike verify, a probe injects nothing and /// records no [`SlotChange`](crate::freshness::SlotChange): it is the /// archive-miss classification for slots a consumer does not want to warm. + /// - **probe_roots:** each `plan.probe_roots` address is root-only probed + /// (`(addr, vec![])`) through the account proof fetcher at the pinned + /// block and recorded into `results.probed_roots` as a + /// [`RootProbeOutcome`]. Nothing is injected; a per-address failure (or an + /// address the fetcher omitted) is `root: None`, never a round hard error. /// - **discover (last):** each [`ColdStartCall`](crate::cold_start::ColdStartCall) /// is executed via /// [`call_raw_with_access_list`](EvmCache::call_raw_with_access_list), its @@ -88,13 +114,53 @@ impl EvmCache { }; } - // Accounts phase (first): pre-seed each declared account. A failure here + // Per-round proof-fetcher guard: only fires for probe_roots-bearing rounds. + if !plan.probe_roots.is_empty() && self.account_proof_fetcher().is_none() { + return RoundOutcome { + results, + error: Some(ColdStartError::NoAccountProofFetcher), + }; + } + + // Per-round fields-fetcher guard: only fires when the cache actually + // holds pending code seeds (the work set is cache state, not a plan + // declaration). + let pending_seeds = self.pending_code_seeds(); + if !pending_seeds.is_empty() && self.account_fields_fetcher().is_none() { + return RoundOutcome { + results, + error: Some(ColdStartError::NoAccountFieldsFetcher), + }; + } + + // verify_code phase (first): settle every Pending canonical code claim + // before anything simulates over it. Purged mismatches are refetched + // by the accounts phase right below when listed there. The guard above + // front-ran the only error surface of verify_code_seeds; a residual + // error is synthesized exactly like an accounts-phase hard error. + if !pending_seeds.is_empty() { + match self.verify_code_seeds() { + Ok(report) => results.code_verifications = Some(report), + Err(e) => { + results.fetched = not_attempted_outcomes(&plan.verify); + results.probed = not_attempted_outcomes(&plan.probe); + results.probed_roots = not_attempted_root_outcomes(&plan.probe_roots); + return RoundOutcome { + results, + error: Some(ColdStartError::Fetch(e)), + }; + } + } + } + + // Accounts phase: pre-seed each declared account. A failure here // short-circuits the round before verify/probe/discover run, so every // declared verify/probe slot is synthesized as NotAttempted. for &address in &plan.accounts { if let Err(e) = self.ensure_account_blocking(address) { results.fetched = not_attempted_outcomes(&plan.verify); results.probed = not_attempted_outcomes(&plan.probe); + results.probed_roots = not_attempted_root_outcomes(&plan.probe_roots); return RoundOutcome { results, error: Some(ColdStartError::Fetch(e)), @@ -133,7 +199,7 @@ impl EvmCache { .storage_batch_fetcher() .cloned() .expect("probe-bearing round guarded a fetcher above"); - let probed = (fetcher)(plan.probe.clone(), Some(self.block())); + let probed = (fetcher)(plan.probe.clone(), self.block()); results.probed = probed .into_iter() .map(|(address, slot, fetched)| SlotOutcome { @@ -144,6 +210,38 @@ impl EvmCache { .collect(); } + // Probe-roots phase: root-only probe each declared account through the + // account proof fetcher at the pinned block, WITHOUT injecting anything. + // A per-address failure (or an address the fetcher omitted) records + // `root: None` — never a round hard error. + if !plan.probe_roots.is_empty() { + // The per-round guard already ensured a proof fetcher is present + // for a probe_roots-bearing round. + let fetcher = self + .account_proof_fetcher() + .cloned() + .expect("probe_roots-bearing round guarded a proof fetcher above"); + let responses = (fetcher)( + plan.probe_roots.iter().map(|&a| (a, vec![])).collect(), + self.block(), + ); + // Per the AccountProofFetchFn contract, at most one result comes + // back per requested address; Ok carries the root, Err / omitted + // means the root could not be observed. + let observed: HashMap> = responses + .into_iter() + .map(|(address, result)| (address, result.ok().map(|proof| proof.storage_hash))) + .collect(); + results.probed_roots = plan + .probe_roots + .iter() + .map(|&address| RootProbeOutcome { + address, + root: observed.get(&address).copied().flatten(), + }) + .collect(); + } + // Discover phase (last): run each view-call, filter by restrict_to. A // failure drops this and every subsequent call but preserves the // verify/probe outcomes already computed above. @@ -262,3 +360,20 @@ fn not_attempted_outcomes(slots: &[(Address, alloy_primitives::U256)]) -> Vec Vec { + addresses + .iter() + .map(|&address| RootProbeOutcome { + address, + root: None, + }) + .collect() +} diff --git a/src/cold_start/error.rs b/src/cold_start/error.rs index ecf9f85..34c4f47 100644 --- a/src/cold_start/error.rs +++ b/src/cold_start/error.rs @@ -1,8 +1,10 @@ //! The typed error surface for cold-start runs. +use crate::errors::CacheError; + /// A hard error that aborts a cold-start round or run. /// -/// Deliberately typed (no `#[from] anyhow::Error` blanket arm): composed-primitive +/// Deliberately typed (no blanket dynamic-error arm): composed-primitive /// errors are converted explicitly at call sites via [`ColdStartError::Fetch`], so /// a partial round's outcomes are never silently collapsed. #[derive(Debug, thiserror::Error)] @@ -11,6 +13,14 @@ pub enum ColdStartError { /// fetcher configured. #[error("cold-start requires a storage batch fetcher")] NoBatchFetcher, + /// A round declared probe-roots accounts but the cache has no account proof + /// fetcher configured. + #[error("cold-start requires an account proof fetcher")] + NoAccountProofFetcher, + /// The cache holds pending code seeds but has no account-fields fetcher to + /// verify them with (fires only for pending-bearing rounds). + #[error("cold-start code-seed verification requires an account fields fetcher")] + NoAccountFieldsFetcher, /// The planner kept returning `Continue` past `max_rounds` executed rounds. #[error("cold-start round budget exceeded ({max_rounds})")] RoundBudgetExceeded { @@ -19,5 +29,5 @@ pub enum ColdStartError { }, /// A composed fetch/call error, carrying the underlying cause explicitly. #[error("cold-start fetch error: {0}")] - Fetch(anyhow::Error), + Fetch(#[source] CacheError), } diff --git a/src/cold_start/mod.rs b/src/cold_start/mod.rs index d94f3e9..d6cbf5c 100644 --- a/src/cold_start/mod.rs +++ b/src/cold_start/mod.rs @@ -1,8 +1,8 @@ //! Protocol-neutral cold-start sync for [`EvmCache`](crate::cache::EvmCache). //! //! Cold start declares rounds of authoritative slot work — verify, probe, -//! accounts, discover — and drives a bounded multi-round loop via a pure -//! [`ColdStartPlanner`]. Every verify and probe slot yields a per-slot +//! probe_roots, accounts, discover — and drives a bounded multi-round loop via +//! a pure [`ColdStartPlanner`]. Every verify and probe slot yields a per-slot //! [`SlotOutcome`] distinguishing a genuine on-chain zero ([`SlotFetch::Zero`]) //! from a fetch failure ([`SlotFetch::FetchFailed`]) — closing the "archive-miss" //! gap where a transient fetch failure was indistinguishable from absence. @@ -72,12 +72,14 @@ mod error; mod plan; mod planner; mod results; +pub mod roots; pub use config::{ColdStartConfig, ColdStartPin, ColdStartRoundSummary, ColdStartRunReport}; pub use error::ColdStartError; pub use plan::{ColdStartCall, ColdStartPlan}; pub use planner::{ColdStartPlanner, ColdStartStep}; pub use results::{ColdStartCallResult, ColdStartResults, RoundOutcome}; +pub use roots::{RootBaseline, RootBaselinePlanner, RootProbeOutcome}; // The per-slot fetch surface is ungated in `freshness`; re-export so // `evm_fork_cache::cold_start::SlotFetch` resolves for consumers. diff --git a/src/cold_start/plan.rs b/src/cold_start/plan.rs index 8aef18b..f37505b 100644 --- a/src/cold_start/plan.rs +++ b/src/cold_start/plan.rs @@ -1,20 +1,23 @@ //! What a cold-start round declares it wants done. //! //! A [`ColdStartPlan`] is a pure, IO-free description handed to the driver. All -//! four phases are optional and an empty plan is a valid no-op round. +//! phases are optional and an empty plan is a valid no-op round. use alloy_primitives::{Address, Bytes, U256}; /// A single round of cold-start work, declared by a /// [`ColdStartPlanner`](crate::cold_start::ColdStartPlanner). /// -/// All four phases are optional; an empty plan is a valid no-op round. The driver -/// executes the phases in a fixed order: **accounts → verify → probe → discover**. +/// All phases are optional; an empty plan is a valid no-op round. The driver +/// executes the phases in a fixed order: +/// **accounts → verify → probe → probe_roots → discover**. /// /// - `verify` slots are authoritatively re-fetched, classified, and (when changed) /// injected into the cache via the dual-layer /// [`inject_storage_batch_fresh`](crate::cache::EvmCache::inject_storage_batch_fresh). /// - `probe` slots are classified at the pinned block **without** injecting. +/// - `probe_roots` accounts have their storage root observed via the +/// account-proof fetcher at the pinned block, without injecting anything. /// - `accounts` are pre-seeded into the cache before discovery runs. /// - `discover` view-calls capture the `(address, slot)` pairs and accounts they /// touch. @@ -43,6 +46,9 @@ pub struct ColdStartPlan { pub verify: Vec<(Address, U256)>, /// Slots to classify at the pinned block without injecting. pub probe: Vec<(Address, U256)>, + /// Accounts whose storage root is probed via the account-proof fetcher at + /// the pinned block, without injecting anything. + pub probe_roots: Vec
, /// Accounts to pre-seed into the cache before discovery. pub accounts: Vec
, /// View-calls whose touched slots and accounts are captured. diff --git a/src/cold_start/results.rs b/src/cold_start/results.rs index 8c43e1b..d6eb609 100644 --- a/src/cold_start/results.rs +++ b/src/cold_start/results.rs @@ -5,7 +5,9 @@ //! failure — plus the injected [`SlotChange`]s and any discovered access lists. use crate::access_set::StorageAccessList; +use crate::cache::CodeVerifyReport; use crate::cold_start::error::ColdStartError; +use crate::cold_start::roots::RootProbeOutcome; use crate::freshness::{SlotChange, SlotOutcome}; use revm::context::result::ExecutionResult; @@ -30,8 +32,16 @@ pub struct ColdStartResults { pub fetched: Vec, /// One outcome per declared probe slot (classified, not injected). pub probed: Vec, + /// One outcome per declared probe-roots address (`root: None` when the + /// probe failed, the fetcher omitted the address, or the phase never ran). + pub probed_roots: Vec, /// One result per discover call. pub discovered: Vec, + /// The `verify_code` phase's report, when the round found pending code + /// seeds (`None` means the phase was a no-op). The phase runs **first**, + /// so this survives a later phase's hard error. Adapters that require + /// verified code before serving gate on its `unverifiable` bucket. + pub code_verifications: Option, } /// The result of one discover view-call: the raw EVM execution result and the diff --git a/src/cold_start/roots.rs b/src/cold_start/roots.rs new file mode 100644 index 0000000..68e9732 --- /dev/null +++ b/src/cold_start/roots.rs @@ -0,0 +1,284 @@ +//! Cold-start root baseline (`roots.bin`): persist observed storage roots so a +//! restarting process can cheaply detect which tracked accounts changed while +//! it was down. +//! +//! An account's storage-trie root (`storageHash`) is a collision-resistant +//! commitment over *all* of that account's storage, so `root_unchanged ⟹ +//! nothing under the account changed`. The baseline compares the on-chain root +//! **across time** — never a locally-reconstructed root against the chain: on +//! restart, probe each tracked account's root now and, where it equals the +//! persisted baseline, the cached tracked slots are provably current and are +//! **not** re-read. Where it diverges (or no baseline exists, or the probe +//! fails), the tracked slots are re-read and the new root adopted. +//! +//! This is a **currency** gate, not a **completeness** gate (spec §6): an +//! unchanged root proves the tracked subset did not change, but it cannot tell +//! you that a slot you *should* have been tracking was missing all along. + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; + +use alloy_primitives::{Address, B256, U256}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, warn}; + +use crate::cache::versioned; +use crate::cold_start::plan::ColdStartPlan; +use crate::cold_start::planner::{ColdStartPlanner, ColdStartStep}; +use crate::cold_start::results::ColdStartResults; +use crate::errors::PersistenceError; +use crate::events::StateView; + +const ROOT_BASELINE_MAGIC: &[u8; 8] = b"EFCROOT\0"; +const ROOT_BASELINE_VERSION: u32 = 1; + +/// Serialized payload of a [`RootBaseline`]: sorted `(address, root)` pairs. +/// +/// A `Vec` of pairs sorted by address (the [`BTreeMap`] iteration order), so the +/// on-disk bytes are deterministic for a given set of entries. +#[derive(Serialize, Deserialize)] +struct RootBaselineFile { + roots: Vec<(Address, B256)>, +} + +/// A persisted map of `Address -> B256` storage roots — each tracked account's +/// last **observed** on-chain storage root (`storageHash` from `eth_getProof`). +/// +/// Persisted via [`save`](Self::save) / [`load`](Self::load) using the same +/// versioned envelope as the binary state cache (magic bytes + version + bincode +/// payload); an unknown magic, unknown version, or corrupt payload is a cache +/// miss (`None`), never an error. The persistence location is the caller's +/// choice — conventionally `roots.bin` alongside the binary state file +/// (`evm_state.bin`). +/// +/// # Currency, not completeness +/// +/// The baseline is compared **across time** (the root observed now vs. the root +/// observed at the last run), never local-vs-chain. `root_unchanged` proves the +/// tracked subset did not change since the baseline block — it cannot detect +/// that a slot you should have tracked was missing (spec §6). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RootBaseline { + roots: BTreeMap, +} + +impl RootBaseline { + /// Record `root` as the observed storage root of `address`, returning the + /// previously recorded root (if any). + pub fn insert(&mut self, address: Address, root: B256) -> Option { + self.roots.insert(address, root) + } + + /// The recorded storage root of `address`, if one was observed. + pub fn get(&self, address: &Address) -> Option { + self.roots.get(address).copied() + } + + /// Number of recorded `(address, root)` entries. + pub fn len(&self) -> usize { + self.roots.len() + } + + /// `true` when no roots are recorded. + pub fn is_empty(&self) -> bool { + self.roots.is_empty() + } + + /// Persist the baseline to `path` (conventionally `roots.bin` next to + /// `evm_state.bin`). + /// + /// The on-disk format carries magic bytes and a version number before the + /// bincode payload, matching the binary state cache envelope. Entries are + /// written in address order, so the bytes are deterministic. Returns an + /// error if serialization, parent-directory creation, or writing fails. + pub fn save(&self, path: &Path) -> Result<(), PersistenceError> { + let file = RootBaselineFile { + roots: self.roots.iter().map(|(a, r)| (*a, *r)).collect(), + }; + let data = versioned::encode( + ROOT_BASELINE_MAGIC, + ROOT_BASELINE_VERSION, + &file, + "root baseline", + )?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; + } + std::fs::write(path, &data).map_err(|err| PersistenceError::write(path, err))?; + debug!( + entries = self.roots.len(), + bytes = data.len(), + "Saved root baseline" + ); + Ok(()) + } + + /// Load a baseline from `path`. + /// + /// Returns `None` (rather than erroring) for a missing file, an unreadable + /// file, an unknown magic header, an unknown version, or a corrupt payload — + /// all are cache misses, matching the binary state cache's handling. A + /// missing file (the normal first-run case) is logged at `debug`; a read + /// error and any magic/version/decode failure are logged at `warn`. + pub fn load(path: &Path) -> Option { + let data = match std::fs::read(path) { + Ok(d) => d, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debug!("No root baseline file found, starting fresh"); + return None; + } + Err(e) => { + warn!(error = %e, "Failed to read root baseline, starting fresh"); + return None; + } + }; + + let file = versioned::decode::( + &data, + ROOT_BASELINE_MAGIC, + ROOT_BASELINE_VERSION, + "root baseline", + )?; + + let baseline = Self { + roots: file.roots.into_iter().collect(), + }; + debug!( + entries = baseline.roots.len(), + bytes = data.len(), + "Loaded root baseline" + ); + Some(baseline) + } +} + +/// The outcome of one declared probe-roots address: the storage root the +/// account-proof fetcher observed at the pinned block, or `None` when it +/// could not be observed. +#[derive(Clone, Debug)] +pub struct RootProbeOutcome { + /// The probed account. + pub address: Address, + /// The observed storage root; `None` when the probe failed or the fetcher + /// omitted the address (treat as unknown -> conservative re-read). + pub root: Option, +} + +/// Which round the [`RootBaselinePlanner`] is in. +#[derive(Clone, Copy, Debug)] +enum RootBaselinePhase { + /// Round 1: probe every tracked account's root. + Probe, + /// Round 2 (if needed): re-read the tracked slots of diverged/unknown accounts. + Verify, +} + +/// A restart planner that root-gates the tracked working set (Phase-8 §5.5). +/// +/// Round 1 probes every tracked account's storage root via the account-proof +/// fetcher (no reads are injected). For each tracked account: +/// +/// - observed root **equal** to the baseline ⇒ the cached tracked slots are +/// provably current — the root is retained in the updated baseline and the +/// slots are **not** re-read; +/// - observed root **diverged** (or no baseline entry) ⇒ the new root is +/// adopted into the updated baseline and the account's tracked slots are +/// re-read in a second verify round; +/// - probe **failed** (`root: None`) ⇒ conservative: the tracked slots are +/// re-read, and no root is adopted — an unobserved root never clobbers a +/// previously persisted one. +/// +/// When nothing needs re-reading the run finishes after the probe round. +/// [`updated_baseline`](Self::updated_baseline) returns the baseline with this +/// run's adoptions applied — persist it as the next `roots.bin`. +/// +/// Like [`RootBaseline`], this is a **currency** gate, not a completeness gate +/// (spec §6): it detects change in the tracked subset, not slots that were +/// never tracked. +pub struct RootBaselinePlanner { + /// Tracked accounts and the storage slots kept live for each. + tracked: Vec<(Address, Vec)>, + /// The persisted baseline this run compares against (never mutated). + baseline: RootBaseline, + /// Baseline-with-adoptions: starts as a copy of `baseline`, updated with + /// every root observed by this run. + updated: RootBaseline, + /// Which round the planner is in. + phase: RootBaselinePhase, +} + +impl RootBaselinePlanner { + /// Build a planner over `tracked` (`account -> its tracked slots`) and + /// `baseline` loaded from `roots.bin` (empty when nothing was persisted). + pub fn new(tracked: Vec<(Address, Vec)>, baseline: RootBaseline) -> Self { + let updated = baseline.clone(); + Self { + tracked, + baseline, + updated, + phase: RootBaselinePhase::Probe, + } + } + + /// The observed roots adopted by this run, layered over the loaded baseline + /// (persist as the next `roots.bin`). + /// + /// Accounts whose probe failed keep their previous baseline entry (if any); + /// accounts whose root was observed carry the observed root. + pub fn updated_baseline(&self) -> RootBaseline { + self.updated.clone() + } +} + +impl ColdStartPlanner for RootBaselinePlanner { + fn initial_plan(&mut self, _state: &dyn StateView) -> ColdStartPlan { + ColdStartPlan { + probe_roots: self.tracked.iter().map(|&(address, _)| address).collect(), + ..Default::default() + } + } + + fn on_results(&mut self, results: &ColdStartResults, _state: &dyn StateView) -> ColdStartStep { + match self.phase { + RootBaselinePhase::Probe => { + // Index the probe outcomes; an address absent from the results + // entirely is treated the same as a failed probe (unknown). + let observed: HashMap> = results + .probed_roots + .iter() + .map(|o| (o.address, o.root)) + .collect(); + + let mut verify: Vec<(Address, U256)> = Vec::new(); + for (address, slots) in &self.tracked { + match observed.get(address).copied().flatten() { + Some(root) => { + let current = self.baseline.get(address) == Some(root); + // Adopt the observed root either way; skip the + // re-read only when it matches the baseline. + self.updated.insert(*address, root); + if !current { + verify.extend(slots.iter().map(|&slot| (*address, slot))); + } + } + // Probe failed / omitted: conservative re-read, and do + // NOT adopt a root — `updated` keeps the old baseline + // entry (if any) rather than clobbering it. + None => verify.extend(slots.iter().map(|&slot| (*address, slot))), + } + } + + if verify.is_empty() { + return ColdStartStep::Done; + } + self.phase = RootBaselinePhase::Verify; + ColdStartStep::Continue(ColdStartPlan { + verify, + ..Default::default() + }) + } + RootBaselinePhase::Verify => ColdStartStep::Done, + } + } +} diff --git a/src/deploy.rs b/src/deploy.rs index db68b4a..e3eb6fc 100644 --- a/src/deploy.rs +++ b/src/deploy.rs @@ -10,10 +10,10 @@ use std::path::{Path, PathBuf}; use alloy_primitives::{Address, Bytes}; use alloy_sol_types::{SolType, SolValue, abi::TokenSeq}; -use anyhow::{Context, Result, bail}; use tracing::{debug, info}; use crate::cache::{EvmCache, MissingTargetBehavior}; +use crate::errors::{DeployError, DeployResult as Result}; /// A Foundry JSON artifact with decoded creation bytecode. #[derive(Debug, Clone)] @@ -80,7 +80,10 @@ impl FoundryArtifact { let init_code = self.init_code(constructor_args); let deployed = cache .deploy_contract(deployer, init_code) - .with_context(|| format!("deploying Foundry artifact {}", self.path.display()))?; + .map_err(|source| DeployError::ArtifactDeploy { + path: self.path.clone(), + source, + })?; debug!( artifact = %self.path.display(), %deployer, @@ -171,7 +174,7 @@ impl FoundryArtifact { constructor_args: impl AsRef<[u8]>, missing_target: MissingTargetBehavior, ) -> Result { - let snapshot = cache.snapshot(); + let checkpoint = cache.checkpoint(); let result = self.try_etch_with_missing_target_behavior( cache, target, @@ -181,7 +184,7 @@ impl FoundryArtifact { ); if result.is_err() { - cache.restore(snapshot); + cache.restore(checkpoint); } result @@ -198,13 +201,13 @@ impl FoundryArtifact { if matches!(missing_target, MissingTargetBehavior::Error) { cache .require_contract_target(target) - .with_context(|| format!("validating target contract {}", target))?; + .map_err(|source| DeployError::TargetValidation { target, source })?; } let deployed = self.deploy(cache, deployer, constructor_args)?; cache .override_account_code_with_missing_target(deployed, target, missing_target) - .with_context(|| format!("etching runtime bytecode at {}", target))?; + .map_err(|source| DeployError::EtchRuntime { target, source })?; let code_size = cache .db_mut() @@ -294,32 +297,31 @@ where /// placeholders (`__$...$__`), or is otherwise not valid hex. pub fn load_foundry_creation_code(path: impl AsRef) -> Result { let path = path.as_ref(); - let content = std::fs::read_to_string(path) - .with_context(|| format!("failed to read Foundry artifact at {}", path.display()))?; - let json: serde_json::Value = serde_json::from_str(&content).with_context(|| { - format!( - "failed to parse Foundry artifact JSON at {}", - path.display() - ) + let content = std::fs::read_to_string(path).map_err(|source| DeployError::ReadArtifact { + path: path.to_path_buf(), + source, })?; + let json: serde_json::Value = + serde_json::from_str(&content).map_err(|source| DeployError::ParseArtifact { + path: path.to_path_buf(), + source, + })?; let bytecode = json .get("bytecode") - .ok_or_else(|| anyhow::anyhow!("artifact {} has no `bytecode` field", path.display()))?; + .ok_or_else(|| DeployError::MissingBytecodeField { + path: path.to_path_buf(), + })?; let bytecode_hex = bytecode .get("object") .and_then(serde_json::Value::as_str) .or_else(|| bytecode.as_str()) - .ok_or_else(|| { - anyhow::anyhow!( - "artifact {} has no `bytecode.object` string", - path.display() - ) + .ok_or_else(|| DeployError::MissingBytecodeObject { + path: path.to_path_buf(), })?; decode_hex_bytecode(bytecode_hex) - .with_context(|| format!("failed to decode bytecode in {}", path.display())) } /// Build init code from creation bytecode and ABI-encoded constructor args. @@ -429,15 +431,17 @@ pub fn etch_foundry_artifact_or_create( fn decode_hex_bytecode(bytecode_hex: &str) -> Result { let stripped = bytecode_hex.strip_prefix("0x").unwrap_or(bytecode_hex); if stripped.is_empty() { - bail!("empty bytecode"); + return Err(DeployError::EmptyBytecode); } if stripped.contains("__") { - bail!("bytecode contains unresolved library placeholders"); + return Err(DeployError::UnresolvedLibraryPlaceholders); } alloy_primitives::hex::decode(stripped) .map(Bytes::from) - .context("invalid hex bytecode") + .map_err(|source| DeployError::InvalidHex { + details: source.to_string(), + }) } #[cfg(test)] @@ -524,7 +528,7 @@ mod tests { let mut cache = setup_cache(); let deployer = Address::ZERO; let target = Address::repeat_byte(0x33); - let create_address = zero_nonce_create_address()?; + let create_address = zero_nonce_create_address(); let artifact = memory_artifact(non_empty_runtime_creation_code()); cache @@ -555,7 +559,7 @@ mod tests { let mut cache = setup_cache(); let deployer = Address::ZERO; let target = Address::repeat_byte(0x44); - let create_address = zero_nonce_create_address()?; + let create_address = zero_nonce_create_address(); let artifact = memory_artifact(empty_runtime_creation_code()); cache @@ -590,7 +594,8 @@ mod tests { #[test] fn load_foundry_creation_code_reads_bytecode_object() -> Result<()> { let path = temp_artifact_path("foundry-bytecode-object"); - std::fs::write(&path, r#"{"bytecode":{"object":"0x60016002"}}"#)?; + std::fs::write(&path, r#"{"bytecode":{"object":"0x60016002"}}"#) + .expect("write test artifact"); let code = load_foundry_creation_code(&path)?; @@ -602,7 +607,7 @@ mod tests { #[test] fn load_foundry_creation_code_accepts_direct_string_bytecode() -> Result<()> { let path = temp_artifact_path("foundry-bytecode-string"); - std::fs::write(&path, r#"{"bytecode":"0x6001"}"#)?; + std::fs::write(&path, r#"{"bytecode":"0x6001"}"#).expect("write test artifact"); let code = load_foundry_creation_code(&path)?; @@ -676,10 +681,10 @@ mod tests { ]) } - fn zero_nonce_create_address() -> Result
{ + fn zero_nonce_create_address() -> Address { "0xbd770416a3345f91e4b34576cb804a576fa48eb1" .parse() - .context("zero nonce CREATE address should parse") + .expect("zero nonce CREATE address should parse") } fn cached_nonce(cache: &mut EvmCache, address: Address) -> Option { diff --git a/src/errors.rs b/src/errors.rs index 649ada7..32a247a 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -34,12 +34,13 @@ //! } //! ``` -use std::borrow::Cow; use std::collections::HashMap; use std::fmt; +use std::path::PathBuf; use std::sync::{Arc, OnceLock}; +use std::{borrow::Cow, io}; -use alloy_primitives::{Bytes, FixedBytes}; +use alloy_primitives::{Address, B256, Bytes, FixedBytes, U256}; use alloy_sol_types::SolError; use tracing::warn; @@ -583,6 +584,680 @@ impl fmt::Display for SimulationError { impl std::error::Error for SimulationError {} +/// Error raised when validating or refreshing the block-execution context +/// (`NUMBER` / `BASEFEE` / `COINBASE` / `PREVRANDAO` / `GASLIMIT` opcodes). +/// +/// Under strict [`BlockContextRequirements`](crate::cache::BlockContextRequirements) +/// a missing header field surfaces loudly as one of these variants instead of +/// silently defaulting the corresponding EVM block-env field. +#[derive(Debug, thiserror::Error)] +pub enum BlockContextError { + /// The block header could not be fetched (the provider errored or returned + /// no block) while strict requirements demanded one at construction. + #[error("block-context header fetch failed: {0}")] + FetchFailed(String), + /// A header was available but a required block-context field was absent. + /// + /// `field` is the lowercased field token (e.g. `"basefee"`, `"prevrandao"`). + #[error("required block-context field missing: {field}")] + MissingField { + /// The lowercased name of the missing field. + field: &'static str, + }, +} + +/// Error returned when crate-managed synchronous RPC bridges cannot enter the +/// required tokio runtime context. +#[derive(Debug, Clone, thiserror::Error)] +pub enum RuntimeError { + /// A current-thread runtime was active, but the crate needs + /// `tokio::task::block_in_place`, which is available only on multi-thread + /// runtimes. + #[error( + "evm-fork-cache RPC operations require a multi-thread tokio runtime; \ + found a current-thread runtime (block_in_place is not supported there). \ + Build the runtime with `tokio::runtime::Builder::new_multi_thread()` \ + or annotate with `#[tokio::main(flavor = \"multi_thread\")]`" + )] + CurrentThreadRuntime, + /// No usable runtime handle was available. + #[error( + "evm-fork-cache RPC operations require a running multi-thread tokio runtime: {details}" + )] + MissingRuntime { + /// The runtime lookup error rendered by tokio. + details: String, + }, +} + +/// Error returned by direct RPC call callbacks installed on [`EvmCache`]. +/// +/// [`EvmCache`]: crate::cache::EvmCache +#[derive(Debug, thiserror::Error)] +pub enum RpcError { + /// Runtime precondition failure before the RPC call could be made. + #[error(transparent)] + Runtime(#[from] RuntimeError), + /// The provider rejected or failed the RPC request. + #[error("RPC provider call {operation} failed: {details}")] + Provider { + /// JSON-RPC method or high-level operation name. + operation: &'static str, + /// Provider/transport error text. Alloy's provider error type is generic + /// over the transport, so this boundary preserves it as display text. + details: String, + }, + /// Caller-provided callback failure. + #[error("custom RPC callback failed: {message}")] + Custom { + /// Caller-provided error text. + message: String, + }, +} + +impl RpcError { + /// Build a provider-failure error from any displayable provider error. + pub fn provider(operation: &'static str, source: impl fmt::Display) -> Self { + Self::Provider { + operation, + details: source.to_string(), + } + } + + /// Build a custom-callback error. + pub fn custom(message: impl Into) -> Self { + Self::Custom { + message: message.into(), + } + } +} + +/// Error returned by storage/proof batch callbacks. +/// +/// `Clone` lets a batch-level failure (one `eth_call` or one JSON-RPC batch) +/// be reported once per affected slot without re-stringifying the source. +#[derive(Debug, Clone, thiserror::Error)] +pub enum StorageFetchError { + /// Runtime precondition failure before a fetch could be made. + #[error(transparent)] + Runtime(#[from] RuntimeError), + /// The provider rejected or failed an RPC request. + #[error("storage/provider request {operation} failed: {details}")] + Provider { + /// JSON-RPC method or high-level operation name. + operation: &'static str, + /// Provider/transport error text. Alloy's provider error type is generic + /// over the transport, so this boundary preserves it as display text. + details: String, + }, + /// JSON-RPC batch serialization failed before sending. + #[error("failed to serialize storage batch request: {details}")] + Serialization { + /// Serialization error text. + details: String, + }, + /// Sending the JSON-RPC batch failed before per-call waiters could resolve. + #[error("failed to send storage batch request: {details}")] + BatchSend { + /// Send error text. + details: String, + }, + /// Caller-provided callback failure. + #[error("custom storage/proof fetcher failed: {message}")] + Custom { + /// Caller-provided error text. + message: String, + }, +} + +impl StorageFetchError { + /// Build a provider-failure error from any displayable provider error. + pub fn provider(operation: &'static str, source: impl fmt::Display) -> Self { + Self::Provider { + operation, + details: source.to_string(), + } + } + + /// Build a batch-send error. + pub fn batch_send(source: impl fmt::Display) -> Self { + Self::BatchSend { + details: source.to_string(), + } + } + + /// Build a serialization error. + pub fn serialization(source: impl fmt::Display) -> Self { + Self::Serialization { + details: source.to_string(), + } + } + + /// Build a custom-callback error. + pub fn custom(message: impl Into) -> Self { + Self::Custom { + message: message.into(), + } + } +} + +/// Result type returned by storage/proof batch callbacks. +pub type StorageFetchResult = Result; + +/// Persistence failure for crate-owned on-disk cache files. +#[derive(Debug, thiserror::Error)] +pub enum PersistenceError { + /// bincode serialization failed. + #[error("failed to serialize {label}: {source}")] + Serialize { + /// Human-readable cache payload label. + label: &'static str, + /// bincode serialization failure. + #[source] + source: bincode::Error, + }, + /// A parent directory could not be created. + #[error("failed to create directory {path:?}: {source}")] + CreateDir { + /// Directory path. + path: PathBuf, + /// Filesystem error. + #[source] + source: io::Error, + }, + /// A file write failed. + #[error("failed to write {path:?}: {source}")] + Write { + /// File path. + path: PathBuf, + /// Filesystem error. + #[source] + source: io::Error, + }, +} + +impl PersistenceError { + /// Build a bincode serialization error. + pub(crate) fn serialize(label: &'static str, source: bincode::Error) -> Self { + Self::Serialize { label, source } + } + + /// Build a directory creation error. + pub(crate) fn create_dir(path: impl Into, source: io::Error) -> Self { + Self::CreateDir { + path: path.into(), + source, + } + } + + /// Build a file write error. + pub(crate) fn write(path: impl Into, source: io::Error) -> Self { + Self::Write { + path: path.into(), + source, + } + } +} + +/// General cache-operation error for APIs that execute or mutate local fork +/// state but do not classify EVM reverts as [`SimError`]. +#[derive(Debug, thiserror::Error)] +pub enum CacheError { + /// Runtime precondition failure. + #[error(transparent)] + Runtime(#[from] RuntimeError), + /// Direct RPC callback failure. + #[error(transparent)] + Rpc(#[from] RpcError), + /// On-disk persistence failure. + #[error(transparent)] + Persistence(#[from] PersistenceError), + /// A storage verification/reconciliation path had no batch fetcher. + #[error("storage verification requires a storage batch fetcher")] + MissingStorageBatchFetcher, + /// Code-seed verification had pending seeds but no account-fields fetcher. + #[error("code-seed verification requires an account fields fetcher")] + MissingAccountFieldsFetcher, + /// Reconciliation could not fetch any requested slot, so it cannot prove the + /// local state fresh. + #[error( + "reconcile could not fetch any of the {requested} requested slot(s) \ + (no usable storage fetcher / provider unreachable)" + )] + ReconcileFetchFailed { + /// Number of requested slots. + requested: usize, + }, + /// Account fetch failed. + #[error("failed to fetch account {address}: {details}")] + AccountFetch { + /// Account address. + address: Address, + /// Backend/provider error text. + details: String, + }, + /// A canonical code seed contradicts code already fetched from the chain. + /// + /// Chain-fetched state is authoritative over templates: the seed is + /// rejected and the cached code is left untouched. If the caller believes + /// the chain has moved (e.g. a redeploy), purge the account first and + /// re-seed. + #[error( + "code seed for {address} conflicts with chain-fetched code \ + (cached hash {cached}, seeded hash {seeded})" + )] + CodeSeedConflict { + /// Account address. + address: Address, + /// Code hash of the RPC-origin code already in the cache. + cached: B256, + /// Code hash of the rejected seed. + seeded: B256, + }, + /// A code seed/etch was given empty bytes; claiming an address is + /// code-less is not a supported seed (that is what verification's + /// `codeless` classification reports). + #[error("cannot seed/etch empty code at {address}")] + CodeSeedEmpty { + /// Account address. + address: Address, + }, + /// Storage read failed. + #[error("storage read failed for {address} slot {slot}: {details}")] + StorageRead { + /// Contract address. + address: Address, + /// Storage slot. + slot: U256, + /// Backend/provider error text. + details: String, + }, + /// Storage insert failed. + #[error("storage insert failed for {address} slot {slot}: {details}")] + StorageInsert { + /// Contract address. + address: Address, + /// Storage slot. + slot: U256, + /// Backend error text. + details: String, + }, + /// Transaction environment construction failed. + #[error("failed to build transaction environment: {details}")] + TxEnv { + /// Builder error text. + details: String, + }, + /// revm returned a host/database transaction error. + #[error("failed to transact: {details}")] + Transact { + /// revm/database error text. + details: String, + }, + /// A helper that requires a successful EVM call observed a revert or halt. + #[error("EVM call did not succeed: {result}")] + CallNotSuccessful { + /// Debug rendering of the execution result. + result: String, + }, + /// A debug/trace RPC response could not be parsed into the cache's typed + /// block state-diff representation. + #[error("failed to parse block state trace: {details}")] + TraceParse { + /// Parser error text. + details: String, + }, + /// ABI or helper-specific decode failure. + #[error("failed to decode {what}: {details}")] + Decode { + /// Human-readable decode target. + what: &'static str, + /// Decoder error text. + details: String, + }, + /// A typed Solidity call executed but did not succeed. + #[error("Solidity call {signature} from {from:?} to {to:?} did not succeed: {result}")] + SolCallFailed { + /// Solidity function signature. + signature: &'static str, + /// Call sender. + from: Address, + /// Call target. + to: Address, + /// Debug rendering of the execution result. + result: String, + }, + /// A typed Solidity call returned malformed data. + #[error( + "failed to decode Solidity call {signature} return data from {from:?} to {to:?}: \ + output_len={output_len}, error: {details}" + )] + SolCallDecode { + /// Solidity function signature. + signature: &'static str, + /// Call sender. + from: Address, + /// Call target. + to: Address, + /// Return-data length in bytes. + output_len: usize, + /// Decoder error text. + details: String, + }, + /// Deployment succeeded without a created address. + #[error("contract deployment succeeded but no address returned")] + DeploymentMissingAddress, + /// Deployment reverted. + #[error("contract deployment reverted: 0x{output_hex}")] + DeploymentReverted { + /// Hex-encoded revert output. + output_hex: String, + }, + /// Deployment halted. + #[error("contract deployment halted: {reason}")] + DeploymentHalted { + /// Debug rendering of the halt reason. + reason: String, + }, + /// Source account did not contain bytecode for an override. + #[error("no bytecode found at source address {source_address}")] + MissingSourceBytecode { + /// Source address. + source_address: Address, + }, + /// Runtime bytecode was required but absent or empty. + #[error("{role} account {address} has no runtime bytecode")] + MissingRuntimeCode { + /// Account role in the operation. + role: &'static str, + /// Account address. + address: Address, + }, + /// Target account was required but absent. + #[error( + "target account {target} not found; use override_or_create_account_code for synthetic targets" + )] + MissingTargetAccount { + /// Target address. + target: Address, + }, + /// Target account fetch failed while validating an override target. + #[error("failed to fetch target account {target}: {details}")] + TargetAccountFetch { + /// Target address. + target: Address, + /// Backend/provider error text. + details: String, + }, +} + +impl CacheError { + /// Convert a transaction-builder error into [`CacheError::TxEnv`]. + pub fn tx_env(source: impl fmt::Debug) -> Self { + Self::TxEnv { + details: format!("{source:?}"), + } + } + + /// Convert a revm host/database transaction error into + /// [`CacheError::Transact`]. + pub fn transact(source: impl fmt::Debug) -> Self { + Self::Transact { + details: format!("{source:?}"), + } + } +} + +/// Result type returned by cache APIs. +pub type CacheResult = Result; + +/// Error for immutable snapshot-overlay execution helpers that do not classify +/// reverts as [`SimError`]. +#[derive(Debug, thiserror::Error)] +pub enum OverlayError { + /// Transaction environment construction failed. + #[error("failed to build transaction environment: {details}")] + TxEnv { + /// Builder error text. + details: String, + }, + /// revm returned a host/database transaction error. + #[error("failed to transact: {details}")] + Transact { + /// revm/database error text. + details: String, + }, +} + +impl OverlayError { + /// Convert a transaction-builder error into [`OverlayError::TxEnv`]. + pub fn tx_env(source: impl fmt::Debug) -> Self { + Self::TxEnv { + details: format!("{source:?}"), + } + } + + /// Convert a revm host/database transaction error into + /// [`OverlayError::Transact`]. + pub fn transact(source: impl fmt::Debug) -> Self { + Self::Transact { + details: format!("{source:?}"), + } + } +} + +/// Result type returned by overlay APIs that return raw [`ExecutionResult`] +/// values instead of classifying reverts. +/// +/// [`ExecutionResult`]: revm::context::result::ExecutionResult +pub type OverlayResult = Result; + +/// Host-side failure for simulation entry points. +#[derive(Debug, thiserror::Error)] +pub enum SimHostError { + /// Transaction environment construction failed. + #[error("failed to build transaction environment: {details}")] + TxEnv { + /// Builder error text. + details: String, + }, + /// revm returned a host/database transaction error. + #[error("failed to transact: {details}")] + Transact { + /// revm/database error text. + details: String, + }, + /// Database read failed outside a transaction execution. + #[error("database operation failed: {details}")] + Database { + /// Database error text. + details: String, + }, + /// Cache helper failed. + #[error(transparent)] + Cache(#[from] CacheError), + /// Overlay helper failed. + #[error(transparent)] + Overlay(#[from] OverlayError), +} + +impl SimHostError { + /// Convert a transaction-builder error into [`SimHostError::TxEnv`]. + pub fn tx_env(source: impl fmt::Debug) -> Self { + Self::TxEnv { + details: format!("{source:?}"), + } + } + + /// Convert a revm host/database transaction error into + /// [`SimHostError::Transact`]. + pub fn transact(source: impl fmt::Debug) -> Self { + Self::Transact { + details: format!("{source:?}"), + } + } + + /// Convert a database error into [`SimHostError::Database`]. + pub fn database(source: impl fmt::Debug) -> Self { + Self::Database { + details: format!("{source:?}"), + } + } +} + +/// Multicall3 helper failure. +#[derive(Debug, thiserror::Error)] +pub enum MulticallError { + /// Underlying EVM cache call failed. + #[error(transparent)] + Cache(#[from] CacheError), + /// The aggregate3 call reverted or halted. + #[error("Multicall3 aggregate call failed: {result}")] + AggregateFailed { + /// Debug rendering of the execution result. + result: String, + }, + /// A per-call result marked `success = false`. + #[error("multicall result indicates the call failed")] + CallFailed, + /// ABI decode failure. + #[error("failed to decode multicall result: {details}")] + Decode { + /// Decoder error text. + details: String, + }, +} + +/// Result type returned by Multicall3 helpers. +pub type MulticallResult = Result; + +/// Deployment and Foundry-artifact loading failure. +#[derive(Debug, thiserror::Error)] +pub enum DeployError { + /// Artifact file could not be read. + #[error("failed to read Foundry artifact at {path}: {source}")] + ReadArtifact { + /// Artifact path. + path: PathBuf, + /// Filesystem error. + #[source] + source: io::Error, + }, + /// Artifact JSON was invalid. + #[error("failed to parse Foundry artifact JSON at {path}: {source}")] + ParseArtifact { + /// Artifact path. + path: PathBuf, + /// JSON parse error. + #[source] + source: serde_json::Error, + }, + /// Artifact had no bytecode field. + #[error("artifact {path} has no `bytecode` field")] + MissingBytecodeField { + /// Artifact path. + path: PathBuf, + }, + /// Artifact bytecode was not a supported string shape. + #[error("artifact {path} has no `bytecode.object` string")] + MissingBytecodeObject { + /// Artifact path. + path: PathBuf, + }, + /// Bytecode was empty. + #[error("empty bytecode")] + EmptyBytecode, + /// Bytecode still contains unresolved Foundry library placeholders. + #[error("bytecode contains unresolved library placeholders")] + UnresolvedLibraryPlaceholders, + /// Hex bytecode could not be decoded. + #[error("invalid hex bytecode: {details}")] + InvalidHex { + /// Hex decoder error text. + details: String, + }, + /// Underlying cache operation failed. + #[error(transparent)] + Cache(#[from] CacheError), + /// Artifact deployment failed, with path context. + #[error("deploying Foundry artifact {path} failed: {source}")] + ArtifactDeploy { + /// Artifact path. + path: PathBuf, + /// Cache operation failure. + #[source] + source: CacheError, + }, + /// Target validation failed before etching. + #[error("validating target contract {target} failed: {source}")] + TargetValidation { + /// Target address. + target: Address, + /// Cache operation failure. + #[source] + source: CacheError, + }, + /// Runtime bytecode etch failed. + #[error("etching runtime bytecode at {target} failed: {source}")] + EtchRuntime { + /// Target address. + target: Address, + /// Cache operation failure. + #[source] + source: CacheError, + }, +} + +/// Result type returned by deployment helpers. +pub type DeployResult = Result; + +/// Access-list pricing query failure. +#[derive(Debug, thiserror::Error)] +pub enum AccessListError { + /// Provider/oracle query failed. + #[error("failed to query {operation}: {details}")] + Query { + /// Operation being queried. + operation: &'static str, + /// Provider/transport error text. + details: String, + }, +} + +impl AccessListError { + /// Build a query error from any displayable provider error. + pub fn query(operation: &'static str, source: impl fmt::Display) -> Self { + Self::Query { + operation, + details: source.to_string(), + } + } +} + +/// Result type returned by access-list pricing helpers. +pub type AccessListResult = Result; + +/// Error returned by speculative freshness orchestration. +#[derive(Debug, thiserror::Error)] +pub enum FreshnessError { + /// The validation handle was already consumed. + #[error("validation handle already consumed")] + ValidationHandleConsumed, + /// The background validation task failed to join. + #[error("validation task failed: {source}")] + ValidationTaskFailed { + /// Tokio join failure. + #[source] + source: tokio::task::JoinError, + }, + /// An optimistic simulation failed before validation was spawned. + #[error(transparent)] + Overlay(#[from] OverlayError), +} + +/// Result type returned by freshness APIs. +pub type FreshnessResult = Result; + /// Result type returned by simulation entry points: `Ok(T)` on success, or a /// [`SimError`] distinguishing a transaction-level revert, an EVM halt, and a /// host-side failure. @@ -615,7 +1290,7 @@ pub enum SimError { }, /// An unexpected host-side error (RPC, database, ABI encoding). #[error("{0}")] - Other(anyhow::Error), + Other(#[source] SimHostError), } impl SimError { @@ -642,12 +1317,24 @@ impl SimError { } } -impl From for SimError { - fn from(e: anyhow::Error) -> Self { +impl From for SimError { + fn from(e: SimHostError) -> Self { SimError::Other(e) } } +impl From for SimError { + fn from(e: CacheError) -> Self { + SimError::Other(e.into()) + } +} + +impl From for SimError { + fn from(e: OverlayError) -> Self { + SimError::Other(e.into()) + } +} + impl From for SimError { fn from(e: SimulationError) -> Self { SimError::Revert(Box::new(e)) diff --git a/src/events/mod.rs b/src/events/mod.rs index 0bfe210..b47d7f9 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -71,9 +71,9 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use alloy_primitives::{Address, Log, U256}; -use anyhow::Result; use crate::cache::EvmCache; +use crate::errors::CacheResult as Result; use crate::freshness::SlotChange; use crate::state_update::{PurgeScope, StateDiff, StateUpdate}; @@ -223,9 +223,12 @@ pub struct EventPipeline { /// Ring of `(block, touched addresses)` for reorg purge, newest at the back, /// bounded to `reorg.depth`. touched: VecDeque<(u64, Vec
)>, - /// Every event-derived `(address, slot)` seen so far (reconcile sampling - /// source). - derived_slots: HashSet<(Address, U256)>, + /// Ring of `(block, event-derived (address, slot) pairs)`, newest at the + /// back, bounded to the reorg horizon `reorg.depth` in lockstep with + /// `touched`. Only the most-recent `depth` blocks' derived slots are + /// retained (the reconcile sampling source); older entries age out as new + /// blocks are ingested, so steady-state ingestion does not grow unbounded. + derived: VecDeque<(u64, HashSet<(Address, U256)>)>, } impl EventPipeline { @@ -235,7 +238,7 @@ impl EventPipeline { registry, reorg: ReorgConfig::default(), touched: VecDeque::new(), - derived_slots: HashSet::new(), + derived: VecDeque::new(), } } @@ -253,13 +256,15 @@ impl EventPipeline { /// in the same block through the [`StateView`] (e.g. a same-block `Burn` after /// a `Mint`, or two overlapping `Mint`s). The touched addresses are recorded /// in the depth-bounded reorg ring under `block`, and the touched - /// `(address, slot)` pairs into the reconcile-sampling set. + /// `(address, slot)` pairs into the parallel depth-bounded reconcile-sampling + /// ring. pub fn ingest_logs(&mut self, cache: &mut EvmCache, block: u64, logs: &[Log]) -> BlockDigest { let mut digest = BlockDigest { block, ..Default::default() }; let mut touched_addrs: HashSet
= HashSet::new(); + let mut block_derived: HashSet<(Address, U256)> = HashSet::new(); for log in logs { // Decode against the current cache view (immutable borrow), then drop @@ -278,7 +283,12 @@ impl EventPipeline { // freshness + reconcile) from every category of the diff. for change in &diff.slots { touched_addrs.insert(change.address); - self.note_touched_slot(&mut digest, change.address, change.slot); + Self::note_touched_slot( + &mut digest, + &mut block_derived, + change.address, + change.slot, + ); } for change in &diff.accounts { touched_addrs.insert(change.address); @@ -288,14 +298,14 @@ impl EventPipeline { } for skip in &diff.skipped { touched_addrs.insert(skip.address); - self.note_touched_slot(&mut digest, skip.address, skip.slot); + Self::note_touched_slot(&mut digest, &mut block_derived, skip.address, skip.slot); } for skip in &diff.skipped_balances { touched_addrs.insert(skip.address); } for skip in &diff.skipped_masks { touched_addrs.insert(skip.address); - self.note_touched_slot(&mut digest, skip.address, skip.slot); + Self::note_touched_slot(&mut digest, &mut block_derived, skip.address, skip.slot); } // Skipped (cold) `Account` patches carry an address but no slot, so // track the address only — mirroring `skipped_balances`. A third-party @@ -311,6 +321,9 @@ impl EventPipeline { if !touched_addrs.is_empty() { self.touched .push_back((block, touched_addrs.into_iter().collect())); + if !block_derived.is_empty() { + self.derived.push_back((block, block_derived)); + } self.trim_ring(); } @@ -335,8 +348,7 @@ impl EventPipeline { // Drop the rolled-back ring entries and the derived slots they own. self.touched.retain(|(block, _)| *block <= new_head); - self.derived_slots - .retain(|(addr, _)| !to_purge.contains(addr)); + self.derived.retain(|(block, _)| *block <= new_head); let updates: Vec = to_purge .into_iter() @@ -370,26 +382,44 @@ impl EventPipeline { }) } - /// All event-derived slots seen so far (the sampling source for - /// [`reconcile`](Self::reconcile)). + /// All event-derived slots retained within the reorg horizon (the sampling + /// source for [`reconcile`](Self::reconcile)). + /// + /// Flattens the block-horizon ring and dedupes, so each `(address, slot)` + /// pair is yielded exactly once even if it was touched in more than one + /// retained block (set semantics preserved). pub fn derived_slots(&self) -> impl Iterator + '_ { - self.derived_slots.iter().copied() + let mut seen: HashSet<(Address, U256)> = HashSet::new(); + for (_, pairs) in &self.derived { + seen.extend(pairs.iter().copied()); + } + seen.into_iter() } /// Record a touched slot in both the per-block digest (deduped within the - /// block) and the global all-time reconcile-sampling set. - fn note_touched_slot(&mut self, digest: &mut BlockDigest, address: Address, slot: U256) { - self.derived_slots.insert((address, slot)); + /// block) and the current block's derived set (later pushed onto the bounded + /// reconcile-sampling ring). + fn note_touched_slot( + digest: &mut BlockDigest, + block_derived: &mut HashSet<(Address, U256)>, + address: Address, + slot: U256, + ) { + block_derived.insert((address, slot)); if !digest.touched_slots.contains(&(address, slot)) { digest.touched_slots.push((address, slot)); } } - /// Trim the reorg ring to the configured depth, dropping the oldest entries. + /// Trim both reorg-horizon rings (`touched` and `derived`) to the configured + /// depth in lockstep, dropping the oldest front entries. fn trim_ring(&mut self) { while self.touched.len() > self.reorg.depth { self.touched.pop_front(); } + while self.derived.len() > self.reorg.depth { + self.derived.pop_front(); + } } } diff --git a/src/freshness.rs b/src/freshness.rs index 430b6ab..0cd4556 100644 --- a/src/freshness.rs +++ b/src/freshness.rs @@ -25,13 +25,22 @@ //! //! The optimistic loop verifies only **volatile storage slots** in each sim's //! read set. Account-level state — native balance, nonce, and bytecode — is -//! **not** re-fetched or diffed, so [`Validation::Confirmed`] means *"no volatile -//! storage slot the sims read had changed"*, not *"no account state changed"*. A -//! sim whose result depends on a `BALANCE`/`SELFBALANCE` (or nonce/code) that -//! moved on-chain without a co-changing storage slot in its read set can still be -//! reported `Confirmed`. If account-level state matters to a sim, mark the -//! account [`Validity::Pinned`] and keep it fresh via event-driven writes, or -//! reconcile it out of band. See [`Validation`] for the per-verdict note. +//! **not** re-fetched or diffed today, so [`Validation::ConfirmedStorage`] means +//! *"no volatile storage slot the sims read had changed"*, not *"no account state +//! changed"*. A sim whose result depends on a `BALANCE`/`SELFBALANCE` (or +//! nonce/code) that moved on-chain without a co-changing storage slot in its read +//! set can still be reported `ConfirmedStorage`. If account-level state matters to +//! a sim, mark the account [`Validity::Pinned`] and keep it fresh via event-driven +//! writes, or reconcile it out of band. +//! +//! The verdict taxonomy is deliberately split so this over-promise is visible in +//! the type: [`ConfirmedStorage`](Validation::ConfirmedStorage) (storage only, +//! account fields unverified) is distinct from +//! [`ConfirmedFull`](Validation::ConfirmedFull) (storage *and* verified +//! account-level fields both unchanged). `ConfirmedFull` is defined but not yet +//! emitted — a follow-up wave wires validator-side account verification that will +//! populate it and the [`Corrected`](Validation::Corrected) verdict's +//! `changed_accounts`. See [`Validation`] for the per-verdict note. //! //! # Example //! @@ -81,7 +90,8 @@ use crate::cache::{ CallSimulationResult, EvmCache, EvmOverlay, EvmSnapshot, SimStatus, SlotObservationTracker, StorageBatchFetchFn, TxConfig, }; -use crate::state_update::StateUpdate; +use crate::errors::{FreshnessError, FreshnessResult as Result, StorageFetchResult}; +use crate::state_update::{AccountChange, StateUpdate}; /// Default minimum observations before the change-frequency data is trusted. pub const DEFAULT_MIN_OBSERVATIONS: u32 = 10; @@ -521,34 +531,67 @@ pub enum SlotFetch { /// The deferred verdict on a [`SpeculativeSim`]'s optimistic results. /// +/// # Verdict taxonomy +/// +/// The verdict distinguishes *what* was reconciled: +/// +/// - [`ConfirmedStorage`](Validation::ConfirmedStorage): no volatile storage slot +/// the sims read changed. Account-level fields (balance/nonce/code) were **not** +/// verified — this is what today's validator emits on a storage-only success. +/// - [`ConfirmedFull`](Validation::ConfirmedFull): no volatile storage slot **and** +/// no verified account-level field changed. Defined but not yet emitted (a +/// follow-up wave wires validator-side account verification). +/// - [`Corrected`](Validation::Corrected): at least one read slot (and, once +/// account verification lands, account field) changed; carries `changed_slots` +/// and `changed_accounts`. +/// - [`Unverified`](Validation::Unverified): the fetcher failed; results are not +/// trusted. +/// /// # Reconciliation scope /// -/// The verdict reflects only **volatile storage slots** in each sim's read set. -/// Account-level state — native balance, nonce, and bytecode — is **not** +/// Today the verdict reflects only **volatile storage slots** in each sim's read +/// set. Account-level state — native balance, nonce, and bytecode — is **not** /// re-verified, so a sim whose result depends on a `BALANCE`/`SELFBALANCE` (or /// nonce/code) that changed on-chain *without* a co-changing storage slot in its -/// read set can still be reported [`Confirmed`](Validation::Confirmed). Classify -/// such accounts as [`Validity::Pinned`] and keep them fresh via event-driven -/// writes if their account-level state matters to your sims. See the module-level -/// docs for the full freshness contract. +/// read set can still be reported +/// [`ConfirmedStorage`](Validation::ConfirmedStorage). Classify such accounts as +/// [`Validity::Pinned`] and keep them fresh via event-driven writes if their +/// account-level state matters to your sims. A follow-up wave wires validator-side +/// account verification that will populate +/// [`ConfirmedFull`](Validation::ConfirmedFull) and the `changed_accounts` field +/// of [`Corrected`](Validation::Corrected). See the module-level docs for the full +/// freshness contract. pub enum Validation { - /// No **volatile storage slot** in the sims' read sets had changed, so the - /// optimistic results are trusted. Note this does *not* cover account-level - /// balance/nonce/code changes — see the [type-level scope](Validation). - Confirmed, + /// No **volatile storage slot** the sims read changed; account-level + /// balance/nonce/code was **NOT** verified. This is the storage-only success + /// verdict today's validator emits — it does *not* cover account-level state + /// (see the [type-level scope](Validation)). + ConfirmedStorage, + /// No volatile storage slot **AND** no verified account-level field + /// (balance/nonce/code) changed. Not emitted yet: a follow-up wave wires the + /// validator-side account verification that populates it (see the + /// [type-level scope](Validation)). + ConfirmedFull, /// At least one read storage slot changed. `results` is the optimistic set - /// with the affected sims re-run against the fresh values; `changed` lists the - /// slots that differed (also queued for flow-back into the cache). Only - /// storage slots are reconciled — account-level state is not (see the + /// with the affected sims re-run against the fresh values; `changed_slots` + /// lists the slots that differed (also queued for flow-back into the cache) and + /// `changed_accounts` lists any account-level fields that differed. Only + /// storage slots are reconciled today — account-level verification is wired by + /// a follow-up wave, so `changed_accounts` is currently always empty (see the /// [type-level scope](Validation)). Corrected { /// Optimistic results with the affected sims replaced by re-runs. results: Vec, /// Slots whose fresh value differed from the snapshot. - changed: Vec, + changed_slots: Vec, + /// Accounts whose native fields differed from the snapshot. Empty until a + /// follow-up wave wires validator-side account verification. + changed_accounts: Vec, }, - /// The fetcher failed, so the results could not be validated. The optimistic - /// results are *not* trusted. + /// Validation could not complete — the fetcher failed or was missing, a + /// corrected re-run could not execute, the fixed-point round cap was hit, + /// or a sim read `BLOCKHASH` (which validator overlays resolve to ZERO and + /// therefore cannot vouch for). The optimistic results are *not* trusted. Unverified { /// Human-readable description of why validation could not complete. reason: String, @@ -558,10 +601,16 @@ pub enum Validation { impl std::fmt::Debug for Validation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Validation::Confirmed => write!(f, "Confirmed"), - Validation::Corrected { changed, .. } => f + Validation::ConfirmedStorage => write!(f, "ConfirmedStorage"), + Validation::ConfirmedFull => write!(f, "ConfirmedFull"), + Validation::Corrected { + changed_slots, + changed_accounts, + .. + } => f .debug_struct("Corrected") - .field("changed", changed) + .field("changed_slots", changed_slots) + .field("changed_accounts", changed_accounts) .finish_non_exhaustive(), Validation::Unverified { reason } => f .debug_struct("Unverified") @@ -670,14 +719,9 @@ impl SpeculativeSim { /// Consume the handle and return the optimistic results, aborting the /// background validation task. /// - /// # Panics - /// The validation [`JoinHandle`] is single-consumption. Because this takes - /// `self` by value, it and [`validate`](Self::validate) are mutually - /// exclusive: only one of them can ever run for a given `SpeculativeSim`, and - /// each takes the handle. `into_optimistic` takes the handle defensively (it - /// does not panic if the handle is already gone), whereas `validate` panics - /// with `"validation handle taken twice"` if it is invoked once the handle has - /// been consumed. + /// Because this takes `self` by value, it and [`validate`](Self::validate) + /// are mutually exclusive: only one of them can ever run for a given + /// `SpeculativeSim`, and each takes the handle. pub fn into_optimistic(mut self) -> Vec { self.cancelled.store(true, Ordering::Relaxed); if let Some(handle) = self.validation.take() { @@ -688,28 +732,17 @@ impl SpeculativeSim { /// Await the deferred validation verdict. /// - /// If the background task failed to complete (e.g. it panicked), returns - /// [`Validation::Unverified`]. This consumes `self`, so it is mutually + /// If the background task failed to complete (e.g. it panicked), returns an + /// error. This consumes `self`, so it is mutually /// exclusive with the cancel paths ([`into_optimistic`](Self::into_optimistic) /// / drop) — a handle that is awaited here is never cancelled. - /// - /// # Panics - /// The validation [`JoinHandle`] is single-consumption: it is taken by the - /// first of `validate` or [`into_optimistic`](Self::into_optimistic) to run. - /// `validate` panics with `"validation handle taken twice"` if the handle has - /// already been consumed. Both take `self` by value, so under normal ownership - /// this is unreachable. - pub async fn validate(mut self) -> Validation { - let handle = self - .validation - .take() - .expect("validation handle taken twice"); - match handle.await { - Ok(v) => v, - Err(e) => Validation::Unverified { - reason: format!("validation task failed: {e}"), - }, - } + pub async fn validate(mut self) -> Result { + let Some(handle) = self.validation.take() else { + return Err(FreshnessError::ValidationHandleConsumed); + }; + handle + .await + .map_err(|source| FreshnessError::ValidationTaskFailed { source }) } } @@ -867,7 +900,7 @@ impl FreshnessController { &mut self, cache: &mut EvmCache, requests: Vec, - ) -> anyhow::Result { + ) -> Result { let now = self.clock.now(); // 1. Drain pending corrections into the cache before snapshotting. @@ -889,14 +922,18 @@ impl FreshnessController { // 2. Snapshot + fetcher (Arc clones, both Send). Capture the cache's // pinned block now, so the deferred validator fetches at the block the // snapshot was built from even if the cache is re-pinned meanwhile. - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let fetcher = cache.storage_batch_fetcher().cloned(); let validation_block = cache.block(); - // 3. Optimistic sims + per-sim actual volatile read sets. + // 3. Optimistic sims + per-sim actual volatile read sets. Sims whose + // execution read `BLOCKHASH` through the ext-db-less overlay (which + // resolves it to ZERO) are recorded so the validator can fail + // closed instead of confirming a result the replay cannot verify. let mut optimistic = Vec::with_capacity(requests.len()); let mut read_sets: Vec> = Vec::with_capacity(requests.len()); - for req in &requests { + let mut blockhash_readers: Vec = Vec::new(); + for (index, req) in requests.iter().enumerate() { let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); let (result, access) = overlay.call_raw_with_access_list_with( req.from, @@ -904,6 +941,9 @@ impl FreshnessController { req.calldata.clone(), &req.tx, )?; + if overlay.blockhash_zero_fallback() { + blockhash_readers.push(index); + } optimistic.push(result_to_sim(result, &access.to_eip2930())); let volatile: Vec<(Address, U256)> = access @@ -963,6 +1003,7 @@ impl FreshnessController { optimistic: optimistic_for_task, cancelled: cancelled_for_task, validation_block, + blockhash_readers, }) }); @@ -991,6 +1032,11 @@ struct ValidatorInput { /// Block the snapshot was built from; passed to the fetcher so the deferred /// fetch reads the same block the snapshot represents. validation_block: BlockId, + /// Indices of requests whose optimistic run read `BLOCKHASH` through the + /// ZERO fallback. Non-empty ⇒ the validator fails closed (`Unverified`): + /// storage verification cannot vouch for a result whose control flow may + /// depend on a hash these overlays cannot resolve. + blockhash_readers: Vec, } /// Maximum fixed-point iterations the background validator performs while a @@ -1010,8 +1056,8 @@ const MAX_VALIDATION_ROUNDS: u32 = 8; /// false confirmation or correction. fn collect_fetch_results( requested: &[(Address, U256)], - results: Vec<(Address, U256, anyhow::Result)>, -) -> Result, String> { + results: Vec<(Address, U256, StorageFetchResult)>, +) -> std::result::Result, String> { let mut map: HashMap<(Address, U256), U256> = HashMap::new(); for (addr, slot, value) in results { match value { @@ -1048,12 +1094,29 @@ fn run_validator(input: ValidatorInput) -> Validation { optimistic, cancelled, validation_block, + blockhash_readers, } = input; // Checkpoint: cancelled before we even begin (the caller dropped or // `into_optimistic`d the handle while we were parked at the initial yield). if cancelled.load(Ordering::Relaxed) { - return Validation::Confirmed; + return Validation::ConfirmedStorage; + } + + // Fail closed on unverifiable `BLOCKHASH` reads (G5). The overlays these + // sims ran on carry no block hashes, so the opcode resolved to ZERO; + // re-checking storage slots cannot vouch for a result whose control flow + // may depend on the real hash. This must precede the empty-verify-set + // early confirm below — a hash-reading sim that touches no volatile slots + // would otherwise silently confirm. + if let Some(first) = blockhash_readers.first() { + return Validation::Unverified { + reason: format!( + "request {first} read BLOCKHASH, which resolves to ZERO in \ + validator overlays (block hashes are not tracked); the result \ + cannot be verified" + ), + }; } let Some(fetcher) = fetcher else { @@ -1072,19 +1135,19 @@ fn run_validator(input: ValidatorInput) -> Validation { } verify.retain(|(addr, slot)| registry.is_volatile(*addr, *slot, now)); if verify.is_empty() { - return Validation::Confirmed; + return Validation::ConfirmedStorage; } let verify: Vec<(Address, U256)> = verify.into_iter().collect(); // Checkpoint: cancelled before issuing the (costly, side-effecting) fetch. // This is what makes the "dropped before fetching" guarantee hold. if cancelled.load(Ordering::Relaxed) { - return Validation::Confirmed; + return Validation::ConfirmedStorage; } // Fetch fresh values. Any error OR any omitted slot → Unverified (never trust // silently: a missing result must not default to zero). - let results = (fetcher)(verify.clone(), Some(validation_block)); + let results = (fetcher)(verify.clone(), validation_block); let fresh = match collect_fetch_results(&verify, results) { Ok(map) => map, Err(reason) => return Validation::Unverified { reason }, @@ -1094,7 +1157,7 @@ fn run_validator(input: ValidatorInput) -> Validation { // observations or queue a correction. A cancel seen here discards the // verdict's side effects entirely. if cancelled.load(Ordering::Relaxed) { - return Validation::Confirmed; + return Validation::ConfirmedStorage; } // Compare the initial verify set against the snapshot, observe each checked @@ -1123,7 +1186,7 @@ fn run_validator(input: ValidatorInput) -> Validation { } if changed_map.is_empty() { - return Validation::Confirmed; + return Validation::ConfirmedStorage; } // Re-run affected sims to a fixed point. A correction can flip control flow @@ -1177,6 +1240,18 @@ fn run_validator(input: ValidatorInput) -> Validation { }; } }; + // A correction can flip control flow onto a `BLOCKHASH` read the + // optimistic run never made; the re-run saw ZERO for it, so the + // "corrected" result cannot be trusted either (G5, fail closed). + if overlay.blockhash_zero_fallback() { + return Validation::Unverified { + reason: format!( + "corrected re-run for request {i} read BLOCKHASH, which \ + resolves to ZERO in validator overlays; the corrected \ + result cannot be verified" + ), + }; + } results[i] = result_to_sim(result, &access.to_eip2930()); let new_volatile: Vec<(Address, U256)> = access .slots @@ -1218,13 +1293,13 @@ fn run_validator(input: ValidatorInput) -> Validation { // Checkpoint: cancelled mid-loop. Results so far reflect the applied // overrides; do not fetch further or queue corrections. if cancelled.load(Ordering::Relaxed) { - return Validation::Confirmed; + return Validation::ConfirmedStorage; } // Fetch the newly-discovered candidates; any error OR omitted slot → // Unverified (a missing result must not default to zero). let new_vec: Vec<(Address, U256)> = new_candidates.into_iter().collect(); - let fetched = (fetcher)(new_vec.clone(), Some(validation_block)); + let fetched = (fetcher)(new_vec.clone(), validation_block); let new_fresh = match collect_fetch_results(&new_vec, fetched) { Ok(map) => map, Err(reason) => return Validation::Unverified { reason }, @@ -1268,13 +1343,19 @@ fn run_validator(input: ValidatorInput) -> Validation { rerun_count.fetch_add(rerun_indices.len(), Ordering::Relaxed); // Queue every accumulated correction for flow-back into the cache next run. - let changed: Vec = changed_map.into_values().collect(); + let changed_slots: Vec = changed_map.into_values().collect(); { let mut pending = pending.lock().unwrap_or_else(|e| e.into_inner()); - pending.extend(changed.iter().cloned()); + pending.extend(changed_slots.iter().cloned()); } - Validation::Corrected { results, changed } + // Account-level changes are populated by a follow-up wave that wires + // validator-side account verification; empty for now. + Validation::Corrected { + results, + changed_slots, + changed_accounts: Vec::new(), + } } /// Build a [`CallSimulationResult`] from a non-committing execution result and diff --git a/src/lib.rs b/src/lib.rs index 69cf2d0..aadeb5a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,8 +5,9 @@ //! [`alloy`], and [`foundry-fork-db`] to provide a lazy-loading state cache, //! immutable snapshots shareable across threads, per-simulation overlays, a //! freshness control plane, and the state-manipulation helpers a search loop -//! needs (balance overrides, batched multicalls, Foundry-style bytecode etching, -//! CREATE3 address derivation, and an extensible revert decoder). +//! needs (balance overrides, batched multicalls, verified code seeding + +//! Foundry-style bytecode etching, CREATE3 address derivation, and an +//! extensible revert decoder). //! //! [`revm`]: https://github.com/bluealloy/revm //! [`alloy`]: https://github.com/alloy-rs/alloy @@ -21,7 +22,7 @@ //! EvmOverlay × N isolated, Send simulations (cheap Arc clones) //! ▲ clone × N //! EvmSnapshot immutable, point-in-time, Send + Sync -//! ▲ create_snapshot() +//! ▲ snapshot() //! EvmCache lazy RPC fetch + local state cache + targeted writes/purge //! ▲ lazy fetch //! RPC provider @@ -29,7 +30,7 @@ //! //! The entry point is [`cache::EvmCache`]: construct one over an RPC backend //! (see [`cache::EvmCacheBuilder`]), then snapshot it with -//! [`cache::EvmCache::create_snapshot`] to fan out parallel simulations, each +//! [`cache::EvmCache::snapshot`] to fan out parallel simulations, each //! driving its own [`cache::EvmOverlay`]. `EvmCache` is `!Send` (it owns the //! mutable fork and blocks on RPC internally); `EvmSnapshot` is `Send + Sync` //! and `EvmOverlay` is `Send`, so the fan-out parallelizes safely. @@ -63,7 +64,8 @@ //! - `cold_start` — default-enabled (reactive-gated) declarative warming of a //! working set of accounts/slots into the cache in one batched pass //! (`EvmCache::run_cold_start` + `ColdStartPlanner`), returning a structured -//! `ColdStartRunReport`. +//! `ColdStartRunReport`. Each round verifies any pending code seeds first +//! (the `verify_code` phase), so sims never run over unverified claims. //! - [`bundle`] — multi-transaction bundle execution over cumulative block state //! ([`EvmOverlay::simulate_bundle`](cache::EvmOverlay::simulate_bundle)): ordered //! txs, an `Atomic`/`AllowReverts` revert policy, and coinbase/miner-payment @@ -74,9 +76,15 @@ //! ([`CallTracer`] building a [`CallTrace`] tree) plus [`InspectorStack`] for //! composing several inspectors over one pass, driven through //! [`EvmOverlay::call_raw_with_inspector`](cache::EvmOverlay::call_raw_with_inspector). +//! - [`bulk_storage`] — bulk storage extraction over `eth_call` state +//! overrides (the **default** batch storage fetcher): thousands of slots — +//! across many contracts — per call, plus custom storage programs and +//! account-fields/block-context extractors. See +//! `docs/bulk-storage-extraction.md` for measured economics. //! - [`multicall`] — batched read-only calls through Multicall3. //! - [`deploy`] / [`create3`] — contract deployment and CREATE3 address math. -//! - [`prefetch_registry`] — two-stage storage-slot pre-warming. +//! - [`prefetch_registry`] — two-stage storage-slot pre-warming +//! (complemented by the declarative [`cache::EvmCache::prewarm_slots`]). //! //! # Requirements //! @@ -101,9 +109,13 @@ //! //! Simulation entry points that distinguish failure modes return //! [`errors::SimulationResult`] (`Result`), where -//! [`SimError`](errors::SimError) separates a decoded [`Revert`](errors::SimError::Revert), +//! [`SimError`] separates a decoded [`Revert`](errors::SimError::Revert), //! an EVM [`Halt`](errors::SimError::Halt), and an unexpected host-side -//! [`Other`](errors::SimError::Other) error (RPC, database, ABI encoding). The +//! [`Other`](errors::SimError::Other) [`SimHostError`] +//! (RPC, database, ABI encoding). Other fallible modules expose domain errors +//! such as [`CacheError`], [`FreshnessError`], and [`StorageFetchError`] rather +//! than erasing failures +//! into a dynamic catch-all error. The //! freshness loop never silently trusts stale data: a transient RPC failure //! surfaces as [`freshness::Validation::Unverified`] so callers can retry rather //! than act on unverified results. @@ -117,8 +129,11 @@ //! The `examples/` directory has runnable, documented walkthroughs of each //! module — offline ones that need no network, plus a few that fork real chain //! state over RPC. See the crate README for the full list. +#![warn(missing_docs)] + pub mod access_list; pub mod access_set; +pub mod bulk_storage; pub mod bundle; pub mod cache; #[cfg(feature = "reactive")] @@ -137,19 +152,37 @@ pub mod state_update; pub mod tracing; pub use access_set::StorageAccessList; +// Bulk storage extraction over eth_call state overrides — the default batch +// storage fetcher since 0.2.0 (see docs/bulk-storage-extraction.md). +pub use bulk_storage::{ + AccountFieldsSample, BlockContextSample, BulkCallConfig, CallDispatch, StorageProgram, + bulk_call_storage_fetcher, bulk_call_storage_fetcher_with_fallback, fetch_account_fields_bulk, + fetch_block_context, fetch_slots_bulk, planned_call_count, run_storage_program, + run_storage_programs, +}; // Phase 6 Track A+B: bundle simulation + coinbase accounting public vocabulary. pub use bundle::{BundleOptions, BundleResult, BundleTx, RevertPolicy, TxOutcome}; // Primary entry points, hoisted to the crate root for discoverability. The // fully-qualified module paths (`cache::EvmCache`, `reactive::ReactiveRuntime`, // …) remain valid, so this is purely additive. pub use cache::{ - CallSimulationResult, EvmCache, EvmCacheBuilder, EvmOverlay, EvmSnapshot, TxConfig, + AccountFieldsFetchFn, AccountProof, AccountProofFetchFn, BlockContextRequirements, + BlockStateAccountDiff, BlockStateDiff, BlockStateDiffFetchFn, BlockStateStorageDiff, + CacheSpeedMode, CallSimulationResult, CodeMismatch, CodeSeedState, CodeVerifyReport, EvmCache, + EvmCacheBuilder, EvmOverlay, EvmSnapshot, PrewarmReport, StorageBatchConfig, + StorageFetchStrategy, TxConfig, point_read_storage_fetcher, }; #[cfg(feature = "reactive")] pub use cold_start::{ ColdStartCall, ColdStartCallResult, ColdStartConfig, ColdStartError, ColdStartPin, ColdStartPlan, ColdStartPlanner, ColdStartResults, ColdStartRoundSummary, ColdStartRunReport, - ColdStartStep, RoundOutcome, + ColdStartStep, RootBaseline, RootBaselinePlanner, RootProbeOutcome, RoundOutcome, +}; +pub use errors::{ + AccessListError, AccessListResult, BlockContextError, CacheError, CacheResult, DeployError, + DeployResult, FreshnessError, FreshnessResult, MulticallError, MulticallResult, OverlayError, + OverlayResult, PersistenceError, RpcError, RuntimeError, SimError, SimHostError, + SimulationError, SimulationResult, StorageFetchError, StorageFetchResult, }; pub use events::erc20::Erc20TransferDecoder; pub use events::{ @@ -162,7 +195,10 @@ pub use freshness::{ SlotFetch, SlotOutcome, SpeculativeSim, Validation, Validity, WallClock, }; #[cfg(feature = "reactive")] -pub use reactive::{ReactiveConfig, ReactiveHandler, ReactiveRuntime}; +pub use reactive::{ + InterestOwnerSubscriber, ReactiveConfig, ReactiveEngine, ReactiveEngineError, + ReactiveEngineRegisterError, ReactiveHandler, ReactiveRuntime, +}; pub use state_update::{ AccountChange, AccountPatch, PurgeRecord, PurgeScope, SkippedAccountPatch, SkippedBalanceDelta, SkippedDelta, SkippedMask, SlotDelta, StateDiff, StateUpdate, diff --git a/src/multicall.rs b/src/multicall.rs index 2303372..44451fd 100644 --- a/src/multicall.rs +++ b/src/multicall.rs @@ -9,11 +9,11 @@ use alloy_primitives::{Address, Bytes, address}; use alloy_sol_types::{SolCall, sol}; -use anyhow::{Result, anyhow}; use tracing::{debug, instrument}; use crate::access_set::StorageAccessList; use crate::cache::EvmCache; +use crate::errors::{MulticallError, MulticallResult as Result}; /// Multicall3 contract address (same on all EVM chains). pub const MULTICALL3_ADDRESS: Address = address!("cA11bde05977b3631167028862bE2a173976CA11"); @@ -158,14 +158,19 @@ impl MulticallBatch { revm::context::result::ExecutionResult::Success { output, .. } => { let out = output.into_data(); let results: Vec = - IMulticall3::aggregate3Call::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode multicall result: {:?}", e))?; + IMulticall3::aggregate3Call::abi_decode_returns(&out).map_err(|e| { + MulticallError::Decode { + details: format!("{e:?}"), + } + })?; debug!(results = results.len(), "multicall batch executed"); Ok(results) } - other => Err(anyhow!("Multicall failed: {:?}", other)), + other => Err(MulticallError::AggregateFailed { + result: format!("{other:?}"), + }), } } @@ -207,8 +212,11 @@ impl MulticallBatch { revm::context::result::ExecutionResult::Success { output, .. } => { let out = output.into_data(); let results: Vec = - IMulticall3::aggregate3Call::abi_decode_returns(&out) - .map_err(|e| anyhow!("Failed to decode multicall result: {:?}", e))?; + IMulticall3::aggregate3Call::abi_decode_returns(&out).map_err(|e| { + MulticallError::Decode { + details: format!("{e:?}"), + } + })?; debug!( results = results.len(), @@ -217,7 +225,9 @@ impl MulticallBatch { Ok((results, access_list)) } - other => Err(anyhow!("Multicall failed: {:?}", other)), + other => Err(MulticallError::AggregateFailed { + result: format!("{other:?}"), + }), } } } @@ -294,11 +304,12 @@ where /// `result.returnData` cannot be ABI-decoded into `C::Return`. pub fn decode_result(result: &IMulticall3::Result) -> Result { if !result.success { - return Err(anyhow!("Call failed")); + return Err(MulticallError::CallFailed); } - C::abi_decode_returns(&result.returnData) - .map_err(|e| anyhow!("Failed to decode result: {:?}", e)) + C::abi_decode_returns(&result.returnData).map_err(|e| MulticallError::Decode { + details: format!("{e:?}"), + }) } /// Like [`decode_result`], but returns `None` instead of an `Err` when the call diff --git a/src/prefetch_registry.rs b/src/prefetch_registry.rs index 353fd62..e45122d 100644 --- a/src/prefetch_registry.rs +++ b/src/prefetch_registry.rs @@ -16,12 +16,12 @@ use std::collections::{HashMap, HashSet}; use std::path::Path; use alloy_primitives::{Address, U256}; -use anyhow::{Context as _, Result}; use serde::{Deserialize, Serialize}; use tracing::{debug, info, warn}; use crate::StorageAccessList; use crate::cache::{EvmCache, versioned}; +use crate::errors::PersistenceError; const PREFETCH_REGISTRY_MAGIC: &[u8; 8] = b"EFC-PFRG"; const PREFETCH_REGISTRY_VERSION: u32 = 1; @@ -94,11 +94,10 @@ impl PrefetchRegistry { /// /// Returns an error if the parent directory cannot be created, serialization /// fails, or the write fails. - pub fn save(&self, path: &Path) -> Result<()> { + pub fn save(&self, path: &Path) -> Result<(), PersistenceError> { if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!("failed to create prefetch registry directory {parent:?}") - })?; + std::fs::create_dir_all(parent) + .map_err(|err| PersistenceError::create_dir(parent, err))?; } let data = versioned::encode( PREFETCH_REGISTRY_MAGIC, @@ -106,8 +105,7 @@ impl PrefetchRegistry { self, "prefetch registry", )?; - std::fs::write(path, data) - .with_context(|| format!("failed to persist prefetch registry to {path:?}"))?; + std::fs::write(path, data).map_err(|err| PersistenceError::write(path, err))?; let total_slots: usize = self.phases.values().map(|al| al.slots.len()).sum::() + self @@ -265,8 +263,8 @@ fn batch_prefetch( let start = std::time::Instant::now(); let total_requested = requests.len(); - // `None`: fetch at the cache's currently-pinned block (synchronous, no repin race). - let results = fetcher(requests, None); + // Fetch at the cache's currently-pinned block. + let results = fetcher(requests, cache.block()); let mut successes: Vec<(Address, U256, U256)> = Vec::with_capacity(results.len()); let mut errors = 0usize; diff --git a/src/reactive/mod.rs b/src/reactive/mod.rs index 70d662c..243d2ec 100644 --- a/src/reactive/mod.rs +++ b/src/reactive/mod.rs @@ -19,8 +19,12 @@ use std::{ future::Future, hash::Hash, marker::PhantomData, + num::NonZeroU64, pin::Pin, - sync::Arc, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, time::Duration, }; @@ -36,14 +40,15 @@ use alloy_network::{ use alloy_primitives::{Address, B256, Bytes, U256}; use alloy_provider::Provider; use alloy_rpc_types_eth::{Filter, FilterSet, Log}; -use futures::{ - StreamExt, - stream::{self, BoxStream, SelectAll}, -}; +#[cfg(any(feature = "reactive-ws", feature = "reactive-polling", test))] +use futures::{StreamExt, stream}; +use futures::{future::poll_fn, stream::BoxStream}; use crate::{ - cache::EvmCache, + cache::{AccountProof, BlockStateDiff, EvmCache}, + errors::{BlockContextError, StorageFetchResult}, events::{EventDecoder, StateView}, + freshness::FreshnessRegistry, state_update::{AccountPatch, PurgeScope, StateDiff, StateUpdate}, }; @@ -657,6 +662,99 @@ pub trait PendingTxMatcher: Send + Sync { fn matches(&self, tx: &N::TransactionResponse) -> bool; } +/// How a tracked account is kept live by the per-block root gate (Phase-8 step 4). +/// +/// The `storageHash` root gate behaves *oppositely* for two contract shapes, so +/// liveness strategy is per-contract: +/// +/// - A sparse-interest contract (a few balance slots, e.g. WETH) has its root +/// churn on nearly every block, so the root is a noisy gate — [`Slots`] opts +/// out. Its enumerated slots stay fresh via decoders + cadence reconcile. +/// - A whole-economic-state contract (e.g. a Uniswap-V2 pool) has +/// `root_moved ≈ my_state_changed`, so [`WholeAccount`] opts in: probe the root +/// each canonical block; a move a decoder did not cover is a coverage gap. +/// +/// A false-positive resync is never *incorrect* — it costs one batched read — so +/// the policy is a **pure cost knob**, not a correctness lever. +/// +/// [`Slots`]: TrackingPolicy::Slots +/// [`WholeAccount`]: TrackingPolicy::WholeAccount +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum TrackingPolicy { + /// Sparse interest (e.g. WETH: a few balance slots). The root churns on + /// nearly every block, so it is a noisy gate — this policy is **never** + /// root-gated (spec Decision 3). Keep the enumerated slots fresh via decoders + /// and cadence reconcile. + Slots { + /// The enumerated storage slots of interest. + slots: Vec, + }, + /// Whole economic state (e.g. a V2 pool). `root_moved ≈ my_state_changed`, so + /// the root is a tight, cheap gate: probe each canonical block; on a move no + /// decoder covered, emit a [`ReactiveReport::CoverageGap`] and schedule a + /// [`ResyncReason::RootMoved`] repair. + WholeAccount, + /// Balance / nonce / code-hash only — resolved from the same `get_proof` + /// response's account fields; no storage interest. Native balance/nonce + /// changes do **not** move the storage root, so this policy compares the + /// account fields directly across blocks rather than root-gating. + Scalars, +} + +/// How often the reactive root gate probes tracked accounts +/// ([`TrackingPolicy::WholeAccount`] / [`TrackingPolicy::Scalars`]; the +/// `Scalars` account-fields comparison rides the same firing). +/// +/// `eth_getProof` is the slowest read this crate issues, so per-block probing +/// is never the default. Skipping blocks is safe by construction: the gate +/// diffs `root_now` against its **persisted baseline**, never +/// block-over-block, so a move in any skipped block is still visible at the +/// next firing — cadence trades detection lag (at most `n − 1` blocks) for +/// cost, never eventual detection. The decoder-touched set accumulates across +/// skipped blocks and drains per firing, so a covered write in a skipped +/// block never false-positives as a [`ReactiveReport::CoverageGap`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RootGateCadence { + /// Probe at most once every `n` canonical blocks (the first canonical + /// block ever seen always fires, so baseline adoption does not wait a + /// full window). `EveryNBlocks(1)` is per-block probing. + EveryNBlocks(NonZeroU64), + /// Root gate off: coverage gaps surface only via decoders + freshness. + Disabled, +} + +impl RootGateCadence { + /// Probe at most once every `n` canonical blocks, clamping `0` to `1`. + pub fn every_n_blocks(n: u64) -> Self { + Self::EveryNBlocks(NonZeroU64::new(n.max(1)).expect("clamped to at least 1")) + } +} + +impl Default for RootGateCadence { + /// Every 16 canonical blocks — ~3.2 min worst-case detection lag on + /// mainnet for a 16× probe-cost cut. Fast-block chains should *raise* + /// `n`, not lower it. + fn default() -> Self { + Self::every_n_blocks(16) + } +} + +/// Per-account baseline held by the root gate: the last observed on-chain root +/// and account fields, plus the block they were observed at. +/// +/// The gate diffs the on-chain root **across time** (never local-vs-chain, per +/// spec §6): it persists the *observed* root as a baseline and compares +/// `root_now` to it. This is a currency gate, not a completeness gate. +#[derive(Clone, Debug)] +struct TrackedRoot { + last_root: B256, + last_block: u64, + balance: U256, + nonce: u64, + code_hash: B256, +} + /// Request for authoritative state repair. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ResyncRequest { @@ -685,11 +783,29 @@ impl ResyncId { /// Reason for a resync request. #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum ResyncReason { /// Handler requested repair. HandlerRequested, /// State effect could not be applied completely. SkippedStateEffect, + /// A missed block range was detected; caller-scheduled repair. + /// + /// The runtime does not fabricate a targetless [`ResyncRequest`] for a missed + /// range (there are no known targets to resync). This reason is provided so a + /// caller building its own repair in response to a + /// [`ReactiveReport::MissedBlockRange`] can attribute it. + MissedBlockRange, + /// A tracked account's storage root moved with no covering decoder. + /// + /// Emitted by the per-block root gate (Phase-8 step 4). A + /// [`WholeAccount`](TrackingPolicy::WholeAccount)-tracked account's + /// `storageHash` moved between the adopted baseline and the current canonical + /// block, yet no decoder wrote that account during the block — a coverage gap. + /// The gate schedules a resync with this reason to re-read the account + /// authoritatively and self-heal the blind spot. Also used for the + /// [`Scalars`](TrackingPolicy::Scalars) account-field freshness path. + RootMoved, /// Caller-defined reason. Custom(String), } @@ -854,8 +970,97 @@ pub enum HookBackpressure { Error, } +/// Queryable coarse health of the reactive cache. +/// +/// The runtime starts [`Healthy`](CacheHealth::Healthy) and transitions to a +/// degraded or unhealthy state when it detects that its recovery guarantees no +/// longer hold (for example a reorg that runs deeper than the journal, so some +/// dropped effects are neither rolled back nor purged). Later waves report +/// missed-range and coverage-gap conditions into the same state machine. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub enum CacheHealth { + /// All recovery guarantees hold; the cache is fully self-consistent. + #[default] + Healthy, + /// A recoverable inconsistency was detected (for example under-recovered + /// reorg effects); `since_block` records the block that triggered the + /// transition. + Degraded { + /// Block number at which the degradation was first observed. + since_block: u64, + }, + /// A more serious inconsistency was detected; `since_block` records the + /// block that triggered the transition. + Unhealthy { + /// Block number at which the unhealthy condition was first observed. + since_block: u64, + }, +} + +/// Point-in-time copy of the reactive runtime's observability counters. +/// +/// Returned by [`ReactiveRuntime::metrics`]. Each field is a monotonically +/// increasing count over the lifetime of the runtime. Counters wired by later +/// waves (missed-range detection, storage-hash coverage gaps, stale-verdict +/// tracking) remain zero until those waves land. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct CacheMetricsSnapshot { + /// Reorgs that ran deeper than the journal, so aged-out effects could not be + /// rolled back or purged. + pub deep_reorgs: u64, + /// Reorgs for which a [`ReorgReport`] recovery ran (including deep reorgs). + pub reorgs_recovered: u64, + /// Storage resync targets considered by the resync execution pass. + pub resync_requests: u64, + /// Storage resync targets that could not be fetched or applied. + pub resync_failures: u64, + /// Ranges of blocks the runtime detected it did not observe (reserved). + pub missed_ranges: u64, + /// Storage-hash coverage gaps detected (reserved). + pub coverage_gaps: u64, + /// Pending-source inputs that attempted a canonical cache effect. + pub pending_contamination: u64, + /// Verdicts served past their freshness horizon (reserved). + pub stale_verdicts: u64, +} + +/// Internal atomic-backed counters mirrored by [`CacheMetricsSnapshot`]. +/// +/// Fields are [`AtomicU64`] so counters can be incremented behind a shared +/// reference; [`ReactiveRuntime::metrics`] loads each with [`Ordering::Relaxed`] +/// into a plain [`CacheMetricsSnapshot`]. +#[derive(Debug, Default)] +struct CacheMetrics { + deep_reorgs: AtomicU64, + reorgs_recovered: AtomicU64, + resync_requests: AtomicU64, + resync_failures: AtomicU64, + missed_ranges: AtomicU64, + coverage_gaps: AtomicU64, + pending_contamination: AtomicU64, + stale_verdicts: AtomicU64, +} + +impl CacheMetrics { + fn snapshot(&self) -> CacheMetricsSnapshot { + CacheMetricsSnapshot { + deep_reorgs: self.deep_reorgs.load(Ordering::Relaxed), + reorgs_recovered: self.reorgs_recovered.load(Ordering::Relaxed), + resync_requests: self.resync_requests.load(Ordering::Relaxed), + resync_failures: self.resync_failures.load(Ordering::Relaxed), + missed_ranges: self.missed_ranges.load(Ordering::Relaxed), + coverage_gaps: self.coverage_gaps.load(Ordering::Relaxed), + pending_contamination: self.pending_contamination.load(Ordering::Relaxed), + stale_verdicts: self.stale_verdicts.load(Ordering::Relaxed), + } + } +} + /// Runtime report. #[derive(Clone, Debug)] +#[non_exhaustive] pub enum ReactiveReport { /// Input was accepted after deduplication. Input(InputReport), @@ -869,6 +1074,14 @@ pub enum ReactiveReport { BlockCommitted(BlockReport), /// Reorg processing report. Reorg(ReorgReport), + /// A forward gap in the canonical block sequence was detected: blocks between + /// the last-seen head and an arriving block were never observed. + MissedBlockRange(MissedRangeReport), + /// Cache health transitioned between states. + Health(HealthReport), + /// A tracked account's storage root moved with no covering decoder — a + /// coverage gap the per-block root gate detected (Phase-8 step 4). + CoverageGap(CoverageGapReport), /// Runtime or handler error. Error(ReactiveErrorReport), } @@ -954,6 +1167,7 @@ pub struct ResyncFailure { /// Stable classification for a failed resync target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum ResyncFailureKind { /// A storage target could not be fetched because no storage batch fetcher is configured. MissingStorageFetcher, @@ -961,8 +1175,12 @@ pub enum ResyncFailureKind { StorageFetchFailed, /// The storage batch fetcher did not return a result for the requested slot. StorageFetchOmitted, - /// Account-field resync is not supported by the current provider-neutral cache seam. - UnsupportedAccountTarget, + /// An account target could not be fetched because no account proof fetcher is configured. + MissingAccountFetcher, + /// The account proof fetcher returned an error for the requested address. + AccountFetchFailed, + /// The account proof fetcher did not return a result for the requested address. + AccountFetchOmitted, } /// Block processing report. @@ -1021,6 +1239,61 @@ pub enum ReorgReason { ParentMismatch, } +/// Report of a forward gap in the canonical block sequence: an arriving block +/// whose number is more than one past the last-seen head, so the blocks in +/// between were never observed (for example during a subscription disconnect). +/// +/// The arriving block is still accepted and applied — the chain extends — so this +/// report only makes the skipped span observable; it does not drop the block. The +/// span `from..=to` is inclusive of both endpoints. +#[derive(Clone, Debug)] +pub struct MissedRangeReport { + /// First skipped block (`last-seen block number + 1`). + pub from: u64, + /// Last skipped block (`arriving block number - 1`). + pub to: u64, + /// The arriving block's number. + pub block: u64, + /// Network marker. + pub _network: PhantomData, +} + +/// Report of a [`CacheHealth`] transition, emitted into the ingest cycle that +/// caused it and delivered to hooks through the normal dispatch path. +#[derive(Clone, Debug)] +pub struct HealthReport { + /// Health state before the transition. + pub from: CacheHealth, + /// Health state after the transition. + pub to: CacheHealth, + /// Block number associated with the transition, when known. + pub block: Option, + /// Network marker. + pub _network: PhantomData, +} + +/// Report that a tracked account's storage root moved on a canonical block that +/// no decoder covered — a coverage gap surfaced by the per-block root gate +/// (Phase-8 step 4). +/// +/// An account's `storageHash` is a collision-resistant commitment over all of its +/// storage, so a moved root proves *something* under the account changed. When +/// that account is [`WholeAccount`](TrackingPolicy::WholeAccount)-tracked and the +/// batch's touched-address set does not include it, the change arrived through a +/// path no decoder observed. The runtime emits this report (delivered through the +/// normal dispatch path so [`ReactiveHook::on_report`] observers see it), +/// increments [`CacheMetricsSnapshot::coverage_gaps`], and schedules a +/// [`ResyncReason::RootMoved`] repair to re-read the account authoritatively. +#[derive(Clone, Debug)] +pub struct CoverageGapReport { + /// The tracked account whose root moved with no covering decoder. + pub address: Address, + /// The canonical block number at which the gap was observed. + pub block: u64, + /// Network marker. + pub _network: PhantomData, +} + /// Report of a non-fatal error surfaced during an ingest cycle, with the /// associated input (when known) and a human-readable message. #[derive(Clone, Debug)] @@ -1143,6 +1416,30 @@ pub enum RegisterError { DuplicateHandler(HandlerId), } +/// Error returned when [`ReactiveEngine`] cannot register a handler on both the +/// runtime and subscriber sides. +#[derive(Debug, thiserror::Error)] +pub enum ReactiveEngineRegisterError { + /// Runtime registry rejected the handler. + #[error(transparent)] + Register(#[from] RegisterError), + /// Subscriber rejected the handler's interests. + #[error(transparent)] + Subscriber(#[from] SubscriberError), +} + +/// Error returned by [`ReactiveEngine`] helpers that combine subscriber polling +/// and runtime ingestion. +#[derive(Debug, thiserror::Error)] +pub enum ReactiveEngineError { + /// Subscriber polling failed. + #[error(transparent)] + Subscriber(#[from] SubscriberError), + /// Runtime ingestion failed. + #[error(transparent)] + Runtime(#[from] ReactiveError), +} + /// Absolute write target used for conflict reports. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum EffectTarget { @@ -1193,6 +1490,36 @@ pub struct ReactiveRuntime { config: ReactiveConfig, journal: VecDeque>, pending_resyncs: Vec, + health: CacheHealth, + metrics: CacheMetrics, + /// Opt-in freshness registry the runtime stamps for canonical event writes. + /// + /// `None` by default (behavior unchanged); populated by + /// [`enable_freshness_stamping`](Self::enable_freshness_stamping). When + /// present, applying a canonical handler storage-slot effect stamps the + /// touched `(address, slot)` as [`Validity::ValidThrough`](crate::freshness::Validity::ValidThrough)`(N)` + /// so event-maintained slots stop being needlessly re-verified while aging to + /// volatile once the clock passes `N`. + freshness: Option, + /// Per-account tracking registry consulted by the per-block root gate + /// (Phase-8 step 4). Empty by default; populated by + /// [`track_account`](Self::track_account). When empty the gate is a no-op. + tracking: HashMap, + /// Per-account root/field baselines the gate diffs against across blocks. + /// Adopted on first probe and re-adopted on every observed move. + tracked_roots: HashMap, + /// How often the root gate fires (§6.2); see [`RootGateCadence`]. + root_gate_cadence: RootGateCadence, + /// Canonical block of the last root-gate firing. `None` until the first + /// firing (which happens at the first canonical block ever seen, so + /// baseline adoption never waits a full cadence window). + last_gate_block: Option, + /// Union of decoder-touched addresses since the last root-gate firing, + /// drained when it fires. Under cadence the gap rule "root moved ∧ addr ∉ + /// touched" must judge against every covered write in the window, or a + /// decoder-covered write in a skipped block would false-positive as a + /// [`ReactiveReport::CoverageGap`]. + touched_since_gate: HashSet
, } #[derive(Clone, Debug)] @@ -1255,6 +1582,40 @@ impl ReactiveRegistry { Ok(()) } + /// Remove one handler by id, leaving all other handlers and interests intact. + /// + /// Returns the removed handler when the id was registered. Cache eviction is + /// intentionally outside this API: unregistering stops future routing and + /// decode for the handler only. + pub fn unregister_handler(&mut self, id: &HandlerId) -> Option>> { + let index = self + .handlers + .iter() + .position(|registered| ®istered.id == id)?; + Some(self.handlers.remove(index).handler) + } + + /// Return true when `id` is currently registered. + pub fn contains_handler(&self, id: &HandlerId) -> bool { + self.handlers.iter().any(|registered| ®istered.id == id) + } + + /// Ids of all registered handlers, in registration (= routing) order. + pub fn handler_ids(&self) -> Vec { + self.handlers + .iter() + .map(|registered| registered.id.clone()) + .collect() + } + + /// Borrow the interests owned by one handler. + pub fn handler_interests(&self, id: &HandlerId) -> Option<&[ReactiveInterest]> { + self.handlers + .iter() + .find(|registered| ®istered.id == id) + .map(|registered| registered.interests.as_slice()) + } + /// Return all registered interests in handler registration order. pub fn interests(&self) -> Vec> { self.handlers @@ -1315,7 +1676,162 @@ impl ReactiveRuntime { config, journal: VecDeque::new(), pending_resyncs: Vec::new(), + health: CacheHealth::Healthy, + metrics: CacheMetrics::default(), + freshness: None, + tracking: HashMap::new(), + tracked_roots: HashMap::new(), + root_gate_cadence: RootGateCadence::default(), + last_gate_block: None, + touched_since_gate: HashSet::new(), + } + } + + /// Track `address` under `policy` for the per-block root gate (Phase-8 step 4). + /// + /// Tracking is strictly opt-in: a runtime with no tracked accounts runs the + /// gate as a no-op. Registering an account clears any baseline it held (a + /// policy change re-adopts on the next probe rather than diffing against a + /// baseline captured under the old policy). Each [`RootGateCadence`] + /// firing, the gate + /// probes tracked [`WholeAccount`](TrackingPolicy::WholeAccount) and + /// [`Scalars`](TrackingPolicy::Scalars) accounts' roots/fields via the + /// account-proof seam and, on a move no decoder covered, emits a + /// [`ReactiveReport::CoverageGap`] and schedules a + /// [`ResyncReason::RootMoved`] repair. [`Slots`](TrackingPolicy::Slots) + /// accounts are never root-gated (spec Decision 3). + pub fn track_account(&mut self, address: Address, policy: TrackingPolicy) { + self.tracking.insert(address, policy); + self.tracked_roots.remove(&address); + } + + /// Stop tracking `address`, dropping its policy and any adopted baseline. + /// + /// Returns `true` if the account was tracked. + pub fn untrack_account(&mut self, address: Address) -> bool { + self.tracked_roots.remove(&address); + self.tracking.remove(&address).is_some() + } + + /// Set how often the root gate probes tracked accounts (default: + /// [`RootGateCadence::default`] — every 16 canonical blocks; see the + /// [`RootGateCadence`] docs for why skipping blocks loses no detection). + /// + /// Reconfiguring resets the gate's window bookkeeping (the touched-address + /// accumulator and the last-fired block), so a stale window never leaks + /// into the new cadence: the next canonical block fires the gate. + pub fn set_root_gate_cadence(&mut self, cadence: RootGateCadence) { + self.root_gate_cadence = cadence; + self.last_gate_block = None; + self.touched_since_gate.clear(); + } + + /// The configured [`RootGateCadence`]. + pub fn root_gate_cadence(&self) -> RootGateCadence { + self.root_gate_cadence + } + + /// Enable freshness stamping of canonical event-derived writes (opt-in). + /// + /// Installs a [`FreshnessRegistry`] the runtime owns; while it is present, + /// applying a canonical handler storage-slot effect for a block `N` stamps the + /// touched `(address, slot)` as + /// [`Validity::ValidThrough`](crate::freshness::Validity::ValidThrough)`(N)`. + /// The slot is therefore not volatile *at* `N` (event-maintained, no need to + /// re-verify) but ages to volatile once the clock passes `N`. + /// + /// Idempotent: if a registry is already installed it is left untouched, so an + /// existing registry (and any stamps it holds) is never clobbered. + pub fn enable_freshness_stamping(&mut self) { + if self.freshness.is_none() { + self.freshness = Some(FreshnessRegistry::new()); + } + } + + /// Borrow the runtime's freshness registry, if stamping was enabled. + /// + /// Returns `None` unless + /// [`enable_freshness_stamping`](Self::enable_freshness_stamping) was called. + pub fn freshness(&self) -> Option<&FreshnessRegistry> { + self.freshness.as_ref() + } + + /// Mutably borrow the runtime's freshness registry, if stamping was enabled. + /// + /// Returns `None` unless + /// [`enable_freshness_stamping`](Self::enable_freshness_stamping) was called. + pub fn freshness_mut(&mut self) -> Option<&mut FreshnessRegistry> { + self.freshness.as_mut() + } + + /// Return the current queryable [`CacheHealth`] of the runtime. + pub fn health(&self) -> CacheHealth { + self.health + } + + /// Return a point-in-time snapshot of the runtime's observability counters. + pub fn metrics(&self) -> CacheMetricsSnapshot { + self.metrics.snapshot() + } + + /// Complete the caller-driven self-heal by returning health to + /// [`CacheHealth::Healthy`]. + /// + /// A trust-loss event (a reorg deeper than the journal, or a detected missed + /// block range) escalates health toward [`CacheHealth::Unhealthy`] as a + /// "stop until rebuilt" signal that the caller must act on. Once the caller + /// has resynced or rebuilt the affected state, it invokes this to clear the + /// signal. It does not emit a [`ReactiveReport::Health`] report, since it is + /// called outside an ingest cycle. + pub fn reset_health(&mut self) { + self.health = CacheHealth::Healthy; + } + + /// Escalate health one rung up the trust-loss ladder for a trust-loss event + /// observed at `block`, returning a [`ReactiveReport::Health`] report when the + /// state actually changes. + /// + /// The ladder is: + /// - [`Healthy`](CacheHealth::Healthy) -> [`Degraded`](CacheHealth::Degraded) + /// - [`Degraded`](CacheHealth::Degraded) -> [`Unhealthy`](CacheHealth::Unhealthy) + /// - [`Unhealthy`](CacheHealth::Unhealthy) -> no change (`None`) + /// + /// A first event degrades; a second escalates to the terminal + /// [`Unhealthy`](CacheHealth::Unhealthy) stop signal. This is shared by both + /// trust-loss paths (deep reorg beyond the journal and missed-range + /// detection) so mixed event types climb the same ladder. + fn escalate_trust(&mut self, block: u64) -> Option>> { + let to = match self.health { + CacheHealth::Healthy => CacheHealth::Degraded { since_block: block }, + CacheHealth::Degraded { .. } => CacheHealth::Unhealthy { since_block: block }, + CacheHealth::Unhealthy { .. } => return None, + }; + self.transition_health(to, Some(block)) + } + + /// Transition health to `to`, returning a [`ReactiveReport::Health`] report + /// when the state actually changes. + /// + /// The returned report must be threaded into the ingest cycle's dispatched + /// reports so it reaches hooks and appears in + /// [`ReactiveBatchReport::reports`]. Returns `None` when `to` equals the + /// current state (no transition, no report). + fn transition_health( + &mut self, + to: CacheHealth, + block: Option, + ) -> Option>> { + if to == self.health { + return None; } + let from = self.health; + self.health = to; + Some(Arc::new(ReactiveReport::Health(HealthReport { + from, + to, + block, + _network: PhantomData, + }))) } /// Register a handler. @@ -1326,6 +1842,93 @@ impl ReactiveRuntime { self.registry.register_handler(handler) } + /// Remove one handler from the runtime registry without resetting runtime state. + /// + /// This delegates to [`ReactiveRegistry::unregister_handler`] only. It does + /// not clear the reorg journal, health, metrics, hooks, pending resyncs, + /// tracking policy, freshness registry, or root-gate baselines, and it does + /// not purge [`EvmCache`] state. Callers that want cache eviction must issue + /// explicit `StateUpdate::purge` updates or use cache purge APIs separately. + pub fn unregister_handler(&mut self, id: &HandlerId) -> Option>> { + self.registry.unregister_handler(id) + } + + /// Return true when the runtime has a registered handler with `id`. + pub fn contains_handler(&self, id: &HandlerId) -> bool { + self.registry.contains_handler(id) + } + + /// Ids of all registered handlers, in registration (= routing) order. + pub fn handler_ids(&self) -> Vec { + self.registry.handler_ids() + } + + /// Borrow the interests owned by one registered handler. + pub fn handler_interests(&self, id: &HandlerId) -> Option<&[ReactiveInterest]> { + self.registry.handler_interests(id) + } + + /// The most recently journaled canonical block, if any. + /// + /// This is the runtime's current chain position: the canonical block most + /// recently recorded by ingestion. Reorged blocks are dropped from the + /// journal during recovery, so a rolled-back head does not linger here. + /// [`ReactiveEngine::register_handler`] uses it as the default backfill + /// anchor for handlers registered mid-lifecycle. `None` until the first + /// canonical input is journaled, and always `None` when + /// [`ReactiveConfig::journal_depth`] is 0 (journaling disabled). + pub fn last_canonical_block(&self) -> Option { + self.journal.back().map(|entry| entry.block.clone()) + } + + /// Queued resync requests: surfaced by handlers but not yet executed by an + /// [`ingest_batch_with_resync`](Self::ingest_batch_with_resync) pass. + /// + /// Callers driving resync execution themselves (plain + /// [`ingest_batch`](Self::ingest_batch) loops) can read the ledger here; + /// reorg recovery cancels entries whose pinned blocks were dropped, and + /// [`cancel_pending_resyncs`](Self::cancel_pending_resyncs) drops entries + /// for torn-down accounts. + pub fn pending_resyncs(&self) -> &[ResyncRequest] { + &self.pending_resyncs + } + + /// Cancel queued resync work that targets `address`, returning the + /// cancelled portions. + /// + /// Every pending [`ResyncRequest`] target referencing `address` is removed; + /// a request reduced to zero targets is dropped entirely, while + /// mixed-target requests keep their other accounts queued. Each returned + /// request mirrors the original id/reason/block/priority and carries only + /// the targets that were cancelled. + /// + /// This is part of the adapter-teardown recipe (see + /// [`ReactiveEngine::unregister_handler`]): it clears the pending ledger so + /// a dropped pool's queued repairs stop occupying memory and stop + /// surfacing as cancellations in reorg reports. It cannot recall requests + /// already returned to the caller in earlier batch reports. + pub fn cancel_pending_resyncs(&mut self, address: Address) -> Vec { + let mut cancelled = Vec::new(); + self.pending_resyncs.retain_mut(|request| { + let (matching, remaining): (Vec<_>, Vec<_>) = request + .targets + .drain(..) + .partition(|target| resync_target_address(target) == address); + request.targets = remaining; + if !matching.is_empty() { + cancelled.push(ResyncRequest { + id: request.id.clone(), + reason: request.reason.clone(), + block: request.block.clone(), + targets: matching, + priority: request.priority, + }); + } + !request.targets.is_empty() + }); + cancelled + } + /// Register a hook. pub fn register_hook(&mut self, hook: Arc>) -> Result<(), RegisterError> { self.hooks.push(hook); @@ -1368,6 +1971,21 @@ impl ReactiveRuntime { if !batch_report.resyncs.is_empty() { let resync_report = execute_resync_requests(cache, &batch_report.resyncs); + // Count unique logical requests: several handlers may emit the same + // ResyncId in one batch, and duplicates fan out per-origin in the + // report but are one unit of resync work for the metric. + let unique_requests = resync_report + .requested + .iter() + .map(|request| &request.id) + .collect::>() + .len(); + self.metrics + .resync_requests + .fetch_add(unique_requests as u64, Ordering::Relaxed); + self.metrics + .resync_failures + .fetch_add(resync_report.failed.len() as u64, Ordering::Relaxed); self.remove_pending_resyncs(batch_report.resyncs.iter().map(|request| &request.id)); self.record_journal_resync(&resync_report); batch_report @@ -1389,6 +2007,12 @@ impl ReactiveRuntime { let mut batch_report = ReactiveBatchReport::default(); let mut reports_to_dispatch = Vec::new(); + // Phase-8 step 4: accumulate the addresses a decoder actually wrote this + // batch (union of applied `StateDiff` addresses) and the batch's canonical + // block number, so the per-block root gate can run once after the record + // loop with the full touched set. + let mut touched_addrs: HashSet
= HashSet::new(); + let mut canonical_batch_block: Option = None; for record in records { let input_ref = record.input_ref(); @@ -1398,7 +2022,12 @@ impl ReactiveRuntime { _network: PhantomData, }))); - if let Some(reorg_report) = self.recover_for_canonical_input(cache, &record) { + if let Some(reorg_report) = + self.recover_for_canonical_input(cache, &record, &mut reports_to_dispatch) + { + self.metrics + .reorgs_recovered + .fetch_add(1, Ordering::Relaxed); remove_canceled_resyncs_from_batch( &mut batch_report.resyncs, &reorg_report.canceled_resyncs, @@ -1406,7 +2035,12 @@ impl ReactiveRuntime { reports_to_dispatch.push(Arc::new(ReactiveReport::Reorg(reorg_report))); } - if let Some(reorg_report) = self.recover_for_reorged_input(cache, &record) { + if let Some(reorg_report) = + self.recover_for_reorged_input(cache, &record, &mut reports_to_dispatch) + { + self.metrics + .reorgs_recovered + .fetch_add(1, Ordering::Relaxed); remove_canceled_resyncs_from_batch( &mut batch_report.resyncs, &reorg_report.canceled_resyncs, @@ -1416,9 +2050,23 @@ impl ReactiveRuntime { } if let Some(block) = canonical_record_block(&record) { + // Phase-8 step 4: remember the batch's canonical block (the last + // canonical record wins) so the root gate probes at that height. + canonical_batch_block = Some(block.number); self.record_journal_input(block, input_ref); } + // Phase-8 step 2: drive a per-block env refresh from canonical + // headers. Best-effort — a strict validation failure is surfaced as + // a non-fatal error report and does not abort the batch. + if let Some(Err(err)) = advance_block_for_canonical_record(cache, &record) { + reports_to_dispatch.push(Arc::new(ReactiveReport::Error(ReactiveErrorReport { + input_ref: Some(input_ref), + message: err.to_string(), + _network: PhantomData, + }))); + } + let executions = self.execute_handlers(cache, &record, input_ref)?; if executions.is_empty() { continue; @@ -1435,6 +2083,13 @@ impl ReactiveRuntime { detect_conflicts(input_ref, &executions)?; + // Phase-8 step 3: canonical block number for freshness stamping. + // Copied out as a plain `u64` (dropping the borrow of `record`) so it + // can be used while `self.freshness_mut()` mutably borrows `self` + // inside the execution loop. `None` for pending/removed/reorged + // records — those never stamp canonical freshness. + let canonical_block_number = canonical_record_block(&record).map(|block| block.number); + for execution in executions { let diff = if execution.state_updates.is_empty() { StateDiff::default() @@ -1464,6 +2119,31 @@ impl ReactiveRuntime { hook_signals: execution.hook_signals, _network: PhantomData, }; + // Phase-8 step 3 (opt-in): stamp every touched `(address, slot)` + // from this canonical handler write as `ValidThrough(N)`, so an + // event-maintained slot stops being re-verified until the clock + // passes its write block. Read the changed slots straight off + // `applied.diff` (which borrows the local, not `self`) and stamp + // via `self.freshness`, done before `applied` is moved into the + // journal/batch below. Only genuinely-changed slots appear here, + // since a no-op re-write records no `SlotChange`. + if let (Some(number), Some(registry)) = + (canonical_block_number, self.freshness.as_mut()) + { + for change in &applied.diff.slots { + registry.valid_through_slot(change.address, change.slot, number); + } + } + + // Phase-8 step 4: record every address this decoder actually wrote + // (or attempted to write) so the root gate can tell a + // decoder-covered root move from an uncovered coverage gap. Fold in + // the full `StateDiff` address footprint — real changes + // (`slots`/`accounts`/`purged`) and cold-skipped attempts alike, so + // a decoder that tried to write a cold slot still counts as + // covering the account. + collect_diff_addresses(&applied.diff, &mut touched_addrs); + let report = Arc::new(ReactiveReport::Applied(applied.clone())); reports_to_dispatch.push(report); if let Some(block) = canonical_record_block(&record) { @@ -1473,10 +2153,231 @@ impl ReactiveRuntime { } } + // Phase-8 step 4 + §6.2 cadence: accumulate this batch's touched + // addresses (after all handler effects, so the set is complete), then + // fire the root gate only on cadence boundaries. The gate diffs + // against persisted baselines, so skipped blocks lose no detection — + // but the touched set must be the union since the last firing, or a + // decoder-covered write in a skipped block would false-positive as a + // CoverageGap. Fired resyncs surface in `batch_report.resyncs` (so + // callers see them and `ingest_batch_with_resync` executes them) and + // coverage reports go into the dispatched reports. + if self.root_gate_runnable(cache) { + self.touched_since_gate + .extend(touched_addrs.iter().copied()); + if self.root_gate_due(canonical_batch_block) { + let accumulated = std::mem::take(&mut self.touched_since_gate); + self.run_root_gate( + cache, + canonical_batch_block, + &accumulated, + &mut batch_report.resyncs, + &mut reports_to_dispatch, + ); + self.last_gate_block = canonical_batch_block; + } + } else { + // A gate that cannot run (disabled, nothing root-gated, or no + // proof fetcher) must not grow the accumulator unboundedly. + // Dropping it is safe: without a runnable gate no baselines exist + // (a fetcher cannot be uninstalled, and untracking drops the + // baseline), so there is nothing a lost touched set could falsely + // gap against later. + self.touched_since_gate.clear(); + } + batch_report.reports = reports_to_dispatch; Ok(batch_report) } + /// Whether the root gate could produce any signal at all: some tracked + /// account is root-gated (`Slots` never is) and a proof fetcher exists. + /// When this is false the touched accumulator is dropped rather than + /// grown (see the ingest call site for why that is safe). + fn root_gate_runnable(&self, cache: &EvmCache) -> bool { + if matches!(self.root_gate_cadence, RootGateCadence::Disabled) { + return false; + } + let has_gated_targets = self + .tracking + .values() + .any(|policy| !matches!(policy, TrackingPolicy::Slots { .. })); + has_gated_targets && cache.account_proof_fetcher().is_some() + } + + /// Whether the root gate is due at this batch's canonical block (§6.2): + /// the first canonical block ever seen always fires (baseline adoption + /// must not wait a full window), then at most once every `n` blocks. + fn root_gate_due(&self, canonical_block: Option) -> bool { + let Some(block) = canonical_block else { + return false; + }; + match self.root_gate_cadence { + RootGateCadence::Disabled => false, + RootGateCadence::EveryNBlocks(n) => match self.last_gate_block { + None => true, + Some(last) => block >= last.saturating_add(n.get()), + }, + } + } + + /// The `storageHash` root gate (Phase-8 step 4), fired per + /// [`RootGateCadence`] window (§6.2). + /// + /// Runs at the firing batch's canonical block, with `touched` carrying the + /// union of decoder-touched addresses since the previous firing. For each tracked + /// [`WholeAccount`](TrackingPolicy::WholeAccount) / [`Scalars`](TrackingPolicy::Scalars) + /// account, probe the root (and account fields) via the account-proof seam and + /// apply the spec §4 table: + /// + /// - No baseline yet ⇒ **adopt** (no gap, no resync — adoption is not a gap). + /// - [`WholeAccount`](TrackingPolicy::WholeAccount) root unchanged ⇒ nothing. + /// - [`WholeAccount`](TrackingPolicy::WholeAccount) root moved, `addr ∈ touched` + /// ⇒ a decoder covered it; re-adopt, no gap. + /// - [`WholeAccount`](TrackingPolicy::WholeAccount) root moved, `addr ∉ touched` + /// ⇒ emit [`ReactiveReport::CoverageGap`], count it, schedule a + /// [`ResyncReason::RootMoved`] account resync, re-adopt. + /// - [`Scalars`](TrackingPolicy::Scalars) ⇒ compare balance/nonce/code-hash to + /// the baseline (native field changes never move the storage root); on a move + /// with `addr ∉ touched`, schedule a [`ResyncReason::RootMoved`] account + /// resync for the changed fields and re-adopt. + /// + /// No-op when the tracking registry is empty, when the batch has no canonical + /// block, or when the cache has no account-proof fetcher installed. + /// [`Slots`](TrackingPolicy::Slots) accounts are never root-gated (spec + /// Decision 3). + fn run_root_gate( + &mut self, + cache: &EvmCache, + canonical_block: Option, + touched: &HashSet
, + resyncs: &mut Vec, + reports: &mut Vec>>, + ) { + if self.tracking.is_empty() { + return; + } + let Some(block) = canonical_block else { + return; + }; + let Some(fetcher) = cache.account_proof_fetcher().cloned() else { + return; + }; + + // Collect the root-gated targets (Slots opts out) in a stable order so a + // single-block sequence of resyncs/reports is deterministic. + let mut targets: Vec<(Address, bool)> = self + .tracking + .iter() + .filter_map(|(address, policy)| match policy { + TrackingPolicy::Slots { .. } => None, + TrackingPolicy::WholeAccount => Some((*address, true)), + TrackingPolicy::Scalars => Some((*address, false)), + }) + .collect(); + if targets.is_empty() { + return; + } + targets.sort_by_key(|(address, _)| *address); + + let block_id = BlockId::number(block); + // ONE seam invocation carries every root-gated target (root-only + // probes: no storage keys needed). eth_getProof is single-address at + // the RPC level, so batching here lets the fetcher fan the requests + // out concurrently instead of paying N sequential round trips. + let mut probes: HashMap> = (fetcher)( + targets + .iter() + .map(|&(address, _)| (address, vec![])) + .collect(), + block_id, + ) + .into_iter() + .collect(); + for (address, whole_account) in targets { + let Some(Ok(proof)) = probes.remove(&address) else { + // A failed/omitted probe carries no signal; leave the baseline + // untouched and try again next block. + continue; + }; + + let baseline = self.tracked_roots.get(&address).cloned(); + let Some(baseline) = baseline else { + // First observation: adopt the baseline. Not a coverage gap. + self.adopt_root(address, block, &proof); + continue; + }; + + // A stale probe (a batch whose canonical block is not newer than the + // last one we baselined this account against) carries no forward + // signal: skip it rather than diff against — or clobber — a newer + // baseline. + if block <= baseline.last_block { + continue; + } + + if whole_account { + if proof.storage_hash == baseline.last_root { + // Tight steady-state path: unchanged root ⇒ nothing. + continue; + } + // Root moved. + if !touched.contains(&address) { + // Moved with no covering decoder — the coverage gap. + reports.push(Arc::new(ReactiveReport::CoverageGap(CoverageGapReport { + address, + block, + _network: PhantomData, + }))); + self.metrics.coverage_gaps.fetch_add(1, Ordering::Relaxed); + resyncs.push(root_moved_account_resync( + address, + block, + AccountFieldMask { + balance: true, + nonce: true, + code: true, + }, + )); + } + // Adopt the new root whether or not a decoder covered it. + self.adopt_root(address, block, &proof); + } else { + // Scalars: compare the account fields directly (native changes do + // not move the storage root). + let balance_moved = proof.balance != baseline.balance; + let nonce_moved = proof.nonce != baseline.nonce; + let code_moved = proof.code_hash != baseline.code_hash; + if (balance_moved || nonce_moved || code_moved) && !touched.contains(&address) { + resyncs.push(root_moved_account_resync( + address, + block, + AccountFieldMask { + balance: balance_moved, + nonce: nonce_moved, + code: code_moved, + }, + )); + } + self.adopt_root(address, block, &proof); + } + } + } + + /// Adopt (or re-adopt) `proof` as the baseline for `address` at `block`. + fn adopt_root(&mut self, address: Address, block: u64, proof: &AccountProof) { + self.tracked_roots.insert( + address, + TrackedRoot { + last_root: proof.storage_hash, + last_block: block, + balance: proof.balance, + nonce: proof.nonce, + code_hash: proof.code_hash, + }, + ); + } + fn execute_handlers( &self, cache: &EvmCache, @@ -1497,7 +2398,16 @@ impl ReactiveRuntime { source, })?; - validate_effects(input_ref, &record.context, ®istered.id, &outcome.effects)?; + if let Err(error) = + validate_effects(input_ref, &record.context, ®istered.id, &outcome.effects) + { + if matches!(error, ReactiveError::InvalidPendingEffect { .. }) { + self.metrics + .pending_contamination + .fetch_add(1, Ordering::Relaxed); + } + return Err(error); + } executions.push(HandlerExecution::from_outcome( registered.id.clone(), input_ref, @@ -1519,6 +2429,7 @@ impl ReactiveRuntime { &mut self, cache: &mut EvmCache, record: &ReactiveInputRecord, + health_reports: &mut Vec>>, ) -> Option> { let block = canonical_record_block(record)?; let latest = self.journal.back()?.block.clone(); @@ -1537,6 +2448,20 @@ impl ReactiveRuntime { } if block.number > latest.number.saturating_add(1) { + // A forward gap: blocks between the journaled head and the arriving + // block were never observed (e.g. a disconnect). Make it observable + // and escalate health, but still accept the arriving block so it + // journals/applies normally (the chain extends). + self.metrics.missed_ranges.fetch_add(1, Ordering::Relaxed); + health_reports.extend(self.escalate_trust(block.number)); + health_reports.push(Arc::new(ReactiveReport::MissedBlockRange( + MissedRangeReport { + from: latest.number + 1, + to: block.number - 1, + block: block.number, + _network: PhantomData, + }, + ))); return None; } @@ -1548,11 +2473,11 @@ impl ReactiveRuntime { { self.drain_journal_after(parent_index) } else { - self.warn_under_recovery(block.number); + health_reports.extend(self.warn_under_recovery(block.number)); self.drain_journal_from_number(block.number) } } else { - self.warn_under_recovery(block.number); + health_reports.extend(self.warn_under_recovery(block.number)); self.drain_journal_from_number(block.number) }; @@ -1563,6 +2488,7 @@ impl ReactiveRuntime { &mut self, cache: &mut EvmCache, record: &ReactiveInputRecord, + health_reports: &mut Vec>>, ) -> Option> { let (dropped_block, reason) = reorg_signal_block(record)?; let dropped = if let Some(index) = self @@ -1572,7 +2498,7 @@ impl ReactiveRuntime { { self.drain_journal_from(index) } else { - self.warn_under_recovery(dropped_block.number); + health_reports.extend(self.warn_under_recovery(dropped_block.number)); self.drain_journal_from_number(dropped_block.number) }; @@ -1603,7 +2529,14 @@ impl ReactiveRuntime { /// recovery is limited to the blocks still journaled — effects from aged-out /// blocks are neither rolled back nor purged (the freshness/validation loop is /// the backstop). Makes the under-recovery observable instead of silent. - fn warn_under_recovery(&self, reorg_number: u64) { + /// + /// This is a deep reorg: it increments the `deep_reorgs` counter and escalates + /// health along the trust-loss ladder via [`escalate_trust`](Self::escalate_trust) + /// (a first event degrades to [`CacheHealth::Degraded`], a second escalates to + /// [`CacheHealth::Unhealthy`]). Any resulting [`ReactiveReport::Health`] + /// transition is returned so the caller can thread it into the ingest cycle's + /// dispatched reports. + fn warn_under_recovery(&mut self, reorg_number: u64) -> Option>> { let oldest_journaled = self.journal.front().map(|entry| entry.block.number); tracing::warn!( reorg_block = reorg_number, @@ -1615,6 +2548,10 @@ impl ReactiveRuntime { backstop). Increase ReactiveConfig::journal_depth to recover deeper \ reorgs precisely." ); + + self.metrics.deep_reorgs.fetch_add(1, Ordering::Relaxed); + + self.escalate_trust(reorg_number) } fn record_journal_input(&mut self, block: &BlockRef, input_ref: InputRef) { @@ -1759,6 +2696,39 @@ impl ReactiveRuntime { } } +/// Fold every address a [`StateDiff`] references — genuine changes +/// (`slots`/`accounts`/`purged`) and cold-skipped attempts (`skipped*`) alike — +/// into `into`. Used by the per-block root gate to accumulate the batch's +/// decoder-touched address set: an account a decoder wrote (or tried to write) is +/// "covered," so a subsequent root move for it is not a coverage gap. +fn collect_diff_addresses(diff: &StateDiff, into: &mut HashSet
) { + into.extend(diff.slots.iter().map(|change| change.address)); + into.extend(diff.accounts.iter().map(|change| change.address)); + into.extend(diff.purged.iter().map(|purge| purge.address)); + into.extend(diff.skipped.iter().map(|skipped| skipped.address)); + into.extend(diff.skipped_balances.iter().map(|skipped| skipped.address)); + into.extend(diff.skipped_masks.iter().map(|skipped| skipped.address)); + into.extend(diff.skipped_accounts.iter().map(|skipped| skipped.address)); +} + +/// Build the [`ResyncReason::RootMoved`] account resync the root gate schedules +/// for an uncovered move. Re-reads `address`'s `fields` at `block` through the +/// existing account-resync path (Wave 2). The id is derived from the address and +/// block so a repeated move on the same account/block coalesces deterministically. +fn root_moved_account_resync( + address: Address, + block: u64, + fields: AccountFieldMask, +) -> ResyncRequest { + ResyncRequest { + id: ResyncId::new(format!("root-moved:{address:#x}:{block}")), + reason: ResyncReason::RootMoved, + block: ResyncBlock::Number(block), + targets: vec![ResyncTarget::Account { address, fields }], + priority: ResyncPriority::Normal, + } +} + fn canonical_record_block(record: &ReactiveInputRecord) -> Option<&BlockRef> { if matches!(&record.input, ReactiveInput::Log(log) if log.removed) { return None; @@ -1769,6 +2739,28 @@ fn canonical_record_block(record: &ReactiveInputRecord) -> Option None } +/// Best-effort per-block env refresh (Phase-8 step 2). +/// +/// For a canonical record carrying a full header — a +/// [`ReactiveInput::BlockHeader`] or [`ReactiveInput::FullBlock`] — refresh the +/// cache's block env from that header via [`EvmCache::advance_block`]. Returns +/// `Some(result)` when a header was present (so the caller can surface a strict +/// validation error), and `None` for pending/reorged records or non-header +/// inputs, which must never drive a canonical env refresh. +fn advance_block_for_canonical_record( + cache: &mut EvmCache, + record: &ReactiveInputRecord, +) -> Option> { + if !is_canonical_status(&record.context.chain_status) { + return None; + } + match &record.input { + ReactiveInput::BlockHeader(header) => Some(cache.advance_block(header)), + ReactiveInput::FullBlock(block) => Some(cache.advance_block(block.header())), + _ => None, + } +} + fn context_block_ref(ctx: &ReactiveContext) -> Option<&BlockRef> { match &ctx.chain_status { ChainStatus::Included { block, .. } @@ -1833,6 +2825,14 @@ fn remove_canceled_resyncs_from_batch( resyncs.retain(|request| !canceled_ids.contains(&request.id)); } +fn resync_target_address(target: &ResyncTarget) -> Address { + match target { + ResyncTarget::StorageSlot { address, .. } + | ResyncTarget::StorageSlots { address, .. } + | ResyncTarget::Account { address, .. } => *address, + } +} + fn resync_request_targets_dropped_block( request: &ResyncRequest, dropped_blocks: &[BlockRef], @@ -1970,21 +2970,159 @@ struct StorageFetchGroup { seen: HashSet<(Address, U256)>, } -fn execute_resync_requests(cache: &mut EvmCache, requests: &[ResyncRequest]) -> ResyncReport { - let mut failed = Vec::new(); - let mut storage_groups: Vec = Vec::new(); +/// One account-target resync collected during request scanning, resolved through +/// the account proof fetcher after storage groups are processed. +#[derive(Clone, Debug)] +struct AccountResyncTarget { + request_id: ResyncId, + block: ResyncBlock, + address: Address, + fields: AccountFieldMask, +} - for request in requests { - for target in &request.targets { - match target { - ResyncTarget::StorageSlot { address, slot } => { - push_storage_resync_slot( - &mut storage_groups, - &request.id, - &request.block, - *address, - *slot, - ); +fn resolve_trace_resyncs( + cache: &EvmCache, + storage_groups: &mut Vec, + account_targets: &mut Vec, + state_updates: &mut Vec, +) { + let Some(fetcher) = cache.block_state_diff_fetcher().cloned() else { + return; + }; + + let mut blocks = Vec::new(); + let mut seen = HashSet::new(); + for block in storage_groups + .iter() + .map(|group| group.block.clone()) + .chain(account_targets.iter().map(|target| target.block.clone())) + { + if seen.insert(block.clone()) { + blocks.push(block); + } + } + + let mut traces = HashMap::new(); + for block in blocks { + match (fetcher)(resync_block_to_block_id(&block)) { + Ok(diff) => { + traces.insert(block, diff); + } + Err(error) => { + tracing::debug!( + block = ?block, + error = %error, + "block trace resync source failed; falling back to point resync" + ); + } + } + } + + for group in storage_groups.iter_mut() { + let Some(trace) = traces.get(&group.block) else { + continue; + }; + group.slots.retain(|slot| { + if let Some(value) = trace_storage_value(trace, slot.address, slot.slot) { + state_updates.push(StateUpdate::slot(slot.address, slot.slot, value)); + return false; + } + cache + .cached_storage_value(slot.address, slot.slot) + .is_none() + }); + group.seen = group + .slots + .iter() + .map(|slot| (slot.address, slot.slot)) + .collect(); + } + storage_groups.retain(|group| !group.slots.is_empty()); + + let mut unresolved_accounts = Vec::new(); + for mut account in account_targets.drain(..) { + let Some(trace) = traces.get(&account.block) else { + unresolved_accounts.push(account); + continue; + }; + let Some(trace_account) = trace + .accounts + .iter() + .find(|diff| diff.address == account.address) + else { + unresolved_accounts.push(account); + continue; + }; + + let mut patch = AccountPatch::default(); + let mut unresolved = AccountFieldMask::default(); + if account.fields.balance { + if let Some(balance) = trace_account.balance { + patch = patch.balance(balance); + } else { + unresolved.balance = true; + } + } + if account.fields.nonce { + if let Some(nonce) = trace_account.nonce { + patch = patch.nonce(nonce); + } else { + unresolved.nonce = true; + } + } + if account.fields.code { + if let Some(code) = &trace_account.code { + patch = patch.code(code.clone()); + } else { + unresolved.code = true; + } + } + + if patch.balance.is_some() || patch.nonce.is_some() || patch.code.is_some() { + state_updates.push(StateUpdate::account_upsert(account.address, patch)); + } + if !account_field_mask_empty(unresolved) { + account.fields = unresolved; + unresolved_accounts.push(account); + } + } + *account_targets = unresolved_accounts; +} + +fn trace_storage_value(trace: &BlockStateDiff, address: Address, slot: U256) -> Option { + trace + .accounts + .iter() + .find(|account| account.address == address) + .and_then(|account| { + account + .storage + .iter() + .find(|entry| entry.slot == slot) + .map(|entry| entry.value) + }) +} + +fn account_field_mask_empty(mask: AccountFieldMask) -> bool { + !mask.balance && !mask.nonce && !mask.code +} + +fn execute_resync_requests(cache: &mut EvmCache, requests: &[ResyncRequest]) -> ResyncReport { + let mut failed = Vec::new(); + let mut storage_groups: Vec = Vec::new(); + let mut account_targets: Vec = Vec::new(); + + for request in requests { + for target in &request.targets { + match target { + ResyncTarget::StorageSlot { address, slot } => { + push_storage_resync_slot( + &mut storage_groups, + &request.id, + &request.block, + *address, + *slot, + ); } ResyncTarget::StorageSlots { address, slots } => { for slot in slots { @@ -1997,20 +3135,26 @@ fn execute_resync_requests(cache: &mut EvmCache, requests: &[ResyncRequest]) -> ); } } - ResyncTarget::Account { .. } => failed.push(ResyncFailure { - request_id: request.id.clone(), - block: request.block.clone(), - target: target.clone(), - kind: ResyncFailureKind::UnsupportedAccountTarget, - message: - "account resync is unsupported until a provider-neutral account fetcher exists" - .to_string(), - }), + ResyncTarget::Account { address, fields } => { + account_targets.push(AccountResyncTarget { + request_id: request.id.clone(), + block: request.block.clone(), + address: *address, + fields: *fields, + }); + } } } } let mut state_updates = Vec::new(); + resolve_trace_resyncs( + cache, + &mut storage_groups, + &mut account_targets, + &mut state_updates, + ); + if !storage_groups.is_empty() { if let Some(fetcher) = cache.storage_batch_fetcher().cloned() { for group in storage_groups { @@ -2020,7 +3164,7 @@ fn execute_resync_requests(cache: &mut EvmCache, requests: &[ResyncRequest]) -> .iter() .map(|slot| (slot.address, slot.slot)) .collect(); - let results = (fetcher)(fetches, Some(resync_block_to_block_id(&block))); + let results = (fetcher)(fetches, resync_block_to_block_id(&block)); let mut pending: HashMap<(Address, U256), StorageFetchSlot> = group .slots .iter() @@ -2078,6 +3222,103 @@ fn execute_resync_requests(cache: &mut EvmCache, requests: &[ResyncRequest]) -> } } + if !account_targets.is_empty() { + if let Some(fetcher) = cache.account_proof_fetcher().cloned() { + // ONE seam invocation per distinct resync block (targets may pin + // different blocks): eth_getProof is single-address at the RPC + // level, so batching the addresses lets the fetcher fan the + // requests out concurrently instead of paying one round trip per + // account. Root-only probes: account fields need no storage keys. + let mut groups: Vec<(BlockId, Vec<_>)> = Vec::new(); + for account in account_targets { + let block_id = resync_block_to_block_id(&account.block); + match groups + .iter_mut() + .find(|(group_block, _)| *group_block == block_id) + { + Some((_, group)) => group.push(account), + None => groups.push((block_id, vec![account])), + } + } + for (block_id, group) in groups { + let probes: HashMap> = (fetcher)( + group + .iter() + .map(|account| (account.address, vec![])) + .collect(), + block_id, + ) + .into_iter() + .collect(); + for account in group { + // `get` + clone rather than `remove`: two targets for the + // same address in one group must both resolve from the + // single probe. + match probes.get(&account.address).cloned() { + Some(Ok(proof)) => { + // Build an authoritative account update from the requested + // field mask. Use the MATERIALIZING `account_upsert` so a + // resync applies even to a cold account (a partial `Account` + // patch on a cold address is silently skipped). + let mut patch = AccountPatch::default(); + if account.fields.balance { + patch = patch.balance(proof.balance); + } + if account.fields.nonce { + patch = patch.nonce(proof.nonce); + } + // Note: `AccountProof` carries `code_hash`, not code bytes; + // the `eth_getProof` seam cannot supply runtime code, so a + // code-field resync is a no-op here (code freshness is + // handled by a later wave). We still materialize the account + // so requested balance/nonce fields take effect. + state_updates.push(StateUpdate::account_upsert(account.address, patch)); + } + Some(Err(error)) => { + failed.push(ResyncFailure { + request_id: account.request_id, + block: account.block, + target: ResyncTarget::Account { + address: account.address, + fields: account.fields, + }, + kind: ResyncFailureKind::AccountFetchFailed, + message: error.to_string(), + }); + } + None => { + failed.push(ResyncFailure { + request_id: account.request_id, + block: account.block, + target: ResyncTarget::Account { + address: account.address, + fields: account.fields, + }, + kind: ResyncFailureKind::AccountFetchOmitted, + message: + "account proof fetcher did not return a result for address" + .to_string(), + }); + } + } + } + } + } else { + for account in account_targets { + failed.push(ResyncFailure { + request_id: account.request_id, + block: account.block, + target: ResyncTarget::Account { + address: account.address, + fields: account.fields, + }, + kind: ResyncFailureKind::MissingAccountFetcher, + message: "account resync requires an account proof fetcher".to_string(), + }); + } + } + } + let diff = if state_updates.is_empty() { StateDiff::default() } else { @@ -2556,7 +3797,11 @@ impl ReactiveHandler for EventDecoderHandler { /// Provider-agnostic subscriber interface. pub trait EventSubscriber: Send { - /// Register interests with the subscriber. + /// Replace all interests registered with the subscriber. + /// + /// Implementations may use this as a full setup/reset operation. The + /// in-crate [`AlloySubscriber`] clears owner-scoped interest state and + /// delivery/dedupe bookkeeping when this method is called. fn register_interests( &mut self, interests: &[ReactiveInterest], @@ -2644,22 +3889,386 @@ impl Default for SubscriberReconnectConfig { } } +/// Historical log backfill requested when adding subscriber interests. +/// +/// Backfill applies only to [`ReactiveInterest::Logs`] entries. Block and +/// pending-transaction interests are live-only. `AlloySubscriber` emits records +/// fetched through this policy as [`InputSource::Backfill`] before attempting +/// live stream initialization, and a drained backfill seeds the filter's +/// delivery anchor at its resolved upper bound (even when the window held no +/// logs), so the newly added filter gets the same reconnect/catch-up protection +/// an established one has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SubscriberBackfill { + from_block: u64, + to_block: Option, +} + +impl SubscriberBackfill { + /// Backfill an inclusive block range. + pub fn range(from_block: u64, to_block: u64) -> Self { + Self { + from_block, + to_block: Some(to_block), + } + } + + /// Backfill from `from_block` through the provider's latest block. + pub fn from_block(from_block: u64) -> Self { + Self { + from_block, + to_block: None, + } + } + + /// First block included in the backfill. + pub fn start_block(&self) -> u64 { + self.from_block + } + + /// Last block included in the backfill, or `None` for provider latest. + pub fn end_block(&self) -> Option { + self.to_block + } +} + +/// Extension trait for subscribers that can add and remove handler-owned +/// interests incrementally. +/// +/// [`EventSubscriber::register_interests`] remains the full-replacement setup +/// API. Implement this trait when a subscriber can preserve unrelated live +/// sources and delivery state while one handler's interests are added or +/// removed. Implementations should make owner *replacement* continuity-safe: +/// updating an owner's interests must not silently discard delivery progress +/// the previous interests had already established (the in-crate +/// [`AlloySubscriber`] carries the owner's prior delivery anchor over to +/// changed filter shapes and automatically backfills the gap). +pub trait InterestOwnerSubscriber: EventSubscriber { + /// Add or replace the interests owned by `owner`. + fn add_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError>; + + /// Add or replace owner interests and schedule log backfill for that owner. + fn add_interest_owner_with_backfill( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: SubscriberBackfill, + ) -> Result<(), SubscriberError>; + + /// Remove one owner's interests, preserving unrelated interests. + fn remove_interest_owner(&mut self, owner: &HandlerId) -> Option>>; + + /// Borrow the interests currently owned by `owner`. + fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest]>; +} + +/// Binds a [`ReactiveRuntime`] to an [`EventSubscriber`] for the common +/// subscribe-ingest lifecycle. +/// +/// The engine treats the runtime registry as the single source of truth for +/// handler lifecycle: [`register_handler`](Self::register_handler) and +/// [`unregister_handler`](Self::unregister_handler) update runtime routing and +/// subscriber interests as one operation, keyed by the handler's stable +/// [`HandlerId`]. Registration is continuity-safe by default — once the runtime +/// has journaled a canonical block, a newly registered handler is backfilled +/// from that block, so a pool discovered in block *N* (say via a factory +/// `PoolCreated` event) misses none of its own logs from *N* onward even though +/// its live subscription starts later. Overlap between backfill and live +/// delivery is absorbed by subscriber and runtime dedup. +/// +/// Registration methods by intent: +/// +/// | Method | Backfill | +/// |---|---| +/// | [`register_handler`](Self::register_handler) | from the runtime's last canonical block (live-only on a fresh runtime) | +/// | [`register_handler_with_backfill`](Self::register_handler_with_backfill) | explicit range or anchor (deep history) | +/// | [`register_handler_live_only`](Self::register_handler_live_only) | none — future logs only | +/// +/// Unregistering a handler stops future subscription routing and runtime +/// decode for that handler; it deliberately does not evict [`EvmCache`] state +/// or undo runtime side effects. See +/// [`unregister_handler`](Self::unregister_handler) for the complete teardown +/// recipe. +/// +/// The runtime and subscriber stay independently accessible through +/// [`runtime_mut`](Self::runtime_mut) / [`subscriber_mut`](Self::subscriber_mut) +/// for advanced use. One caution: avoid calling +/// [`EventSubscriber::register_interests`] (the full-replacement setup API) on +/// an engine-managed subscriber — implementations may clear owner-scoped +/// bookkeeping, after which per-handler unregistration no longer releases the +/// handler's transport subscriptions. To bootstrap the subscriber from a +/// runtime that already has handlers, use +/// [`sync_handler_interests`](Self::sync_handler_interests), which registers +/// one owner per handler instead of one unowned blob. +pub struct ReactiveEngine { + runtime: ReactiveRuntime, + subscriber: S, +} + +impl ReactiveEngine +where + N: Network, + S: EventSubscriber, +{ + /// Bind a runtime and subscriber. + pub fn new(runtime: ReactiveRuntime, subscriber: S) -> Self { + Self { + runtime, + subscriber, + } + } + + /// Split the engine into its runtime and subscriber parts. + pub fn into_parts(self) -> (ReactiveRuntime, S) { + (self.runtime, self.subscriber) + } + + /// Borrow the runtime. + pub fn runtime(&self) -> &ReactiveRuntime { + &self.runtime + } + + /// Mutably borrow the runtime. + pub fn runtime_mut(&mut self) -> &mut ReactiveRuntime { + &mut self.runtime + } + + /// Borrow the subscriber. + pub fn subscriber(&self) -> &S { + &self.subscriber + } + + /// Mutably borrow the subscriber. + pub fn subscriber_mut(&mut self) -> &mut S { + &mut self.subscriber + } + + /// Poll the subscriber for the next batch. + pub fn next_batch(&mut self) -> SubscriberNextBatch<'_, N> { + self.subscriber.next_batch() + } + + /// Ingest one already-polled batch through the runtime (direct effects + /// only; surfaced resync requests are reported, not executed). + pub fn ingest_batch( + &mut self, + cache: &mut EvmCache, + batch: ReactiveInputBatch, + ) -> Result, ReactiveError> { + self.runtime.ingest_batch(cache, batch) + } + + /// Ingest one already-polled batch and execute the storage/account resyncs + /// it surfaces, exactly like + /// [`ReactiveRuntime::ingest_batch_with_resync`]. + pub fn ingest_batch_with_resync( + &mut self, + cache: &mut EvmCache, + batch: ReactiveInputBatch, + ) -> Result, ReactiveError> { + self.runtime.ingest_batch_with_resync(cache, batch) + } + + /// Poll the subscriber once and ingest the returned batch when present + /// (direct effects only). + pub async fn next_ingest( + &mut self, + cache: &mut EvmCache, + ) -> Result>, ReactiveEngineError> { + let Some(batch) = self.subscriber.next_batch().await? else { + return Ok(None); + }; + Ok(Some(self.runtime.ingest_batch(cache, batch)?)) + } + + /// Poll the subscriber once and ingest the returned batch with resync + /// execution — the loop shape for consumers that rely on coverage-gap + /// repair (root-gate resyncs, handler-requested re-reads). + pub async fn next_ingest_with_resync( + &mut self, + cache: &mut EvmCache, + ) -> Result>, ReactiveEngineError> { + let Some(batch) = self.subscriber.next_batch().await? else { + return Ok(None); + }; + Ok(Some(self.runtime.ingest_batch_with_resync(cache, batch)?)) + } +} + +impl ReactiveEngine +where + N: Network, + S: InterestOwnerSubscriber, +{ + /// Register a handler with both the runtime and subscriber, backfilling its + /// log interests from the runtime's last canonical block. + /// + /// This is the continuity-safe default for mid-lifecycle registration: the + /// runtime already knows how far it has processed the chain, so the new + /// handler's logs are fetched from that block forward and no discovery gap + /// opens between "we decided to track this pool" and "its live subscription + /// started". On a runtime that has not journaled any canonical block yet + /// (fresh start, or `journal_depth` 0) registration is live-only, matching + /// pre-ingestion bootstrap. Use + /// [`register_handler_with_backfill`](Self::register_handler_with_backfill) + /// for deeper history or + /// [`register_handler_live_only`](Self::register_handler_live_only) to opt + /// out of backfill entirely. + /// + /// If subscriber registration fails, the runtime registration is rolled back + /// before the error is returned. + pub fn register_handler( + &mut self, + handler: Arc>, + ) -> Result<(), ReactiveEngineRegisterError> { + let backfill = self + .runtime + .last_canonical_block() + .map(|block| SubscriberBackfill::from_block(block.number)); + self.register_handler_inner(handler, backfill) + } + + /// Register a handler and request an explicit owner-scoped log backfill for + /// its interests (deep history / custom anchors). + /// + /// If subscriber registration fails, the runtime registration is rolled back + /// before the error is returned. + pub fn register_handler_with_backfill( + &mut self, + handler: Arc>, + backfill: SubscriberBackfill, + ) -> Result<(), ReactiveEngineRegisterError> { + self.register_handler_inner(handler, Some(backfill)) + } + + /// Register a handler without any log backfill — only logs delivered after + /// its live subscription starts are routed to it. + /// + /// If subscriber registration fails, the runtime registration is rolled back + /// before the error is returned. + pub fn register_handler_live_only( + &mut self, + handler: Arc>, + ) -> Result<(), ReactiveEngineRegisterError> { + self.register_handler_inner(handler, None) + } + + fn register_handler_inner( + &mut self, + handler: Arc>, + backfill: Option, + ) -> Result<(), ReactiveEngineRegisterError> { + let id = handler.id(); + self.runtime.register_handler(handler)?; + let interests = self + .runtime + .handler_interests(&id) + .expect("handler was just registered") + .to_vec(); + + let subscribed = match backfill { + Some(backfill) => { + self.subscriber + .add_interest_owner_with_backfill(id.clone(), &interests, backfill) + } + None => self.subscriber.add_interest_owner(id.clone(), &interests), + }; + if let Err(error) = subscribed { + self.runtime.unregister_handler(&id); + return Err(error.into()); + } + + Ok(()) + } + + /// Register every handler currently in the runtime registry as a subscriber + /// interest owner. + /// + /// This is the bootstrap path for an engine built around a pre-populated + /// runtime: each handler becomes its own owner (upsert semantics, so + /// rerunning is safe and already-registered owners are refreshed in place). + /// No backfill is requested — bootstrap happens before ingestion starts, so + /// there is no processed position to be continuous with; use + /// [`register_handler_with_backfill`](Self::register_handler_with_backfill) + /// for handlers that need history. Owners are not removed by this call: use + /// [`unregister_handler`](Self::unregister_handler) for lifecycle removal + /// rather than mutating the runtime registry directly. + /// + /// On error, owners already synced stay registered (upserts are + /// independent); the call can simply be retried. + pub fn sync_handler_interests(&mut self) -> Result<(), SubscriberError> { + for id in self.runtime.handler_ids() { + let interests = self + .runtime + .handler_interests(&id) + .map(<[ReactiveInterest]>::to_vec) + .unwrap_or_default(); + self.subscriber.add_interest_owner(id, &interests)?; + } + Ok(()) + } + + /// Unregister a handler from both the subscriber and runtime. + /// + /// Subscriber interests are removed first so no new live records are routed + /// to a handler after it has left the runtime registry. Returns the removed + /// handler when the id was registered. + /// + /// This is the routing/transport half of dropping an adapter. State the + /// handler accumulated is deliberately left in place; the complete teardown + /// for a pool or adapter that will not return is: + /// + /// ```text + /// engine.unregister_handler(&id); + /// for address in handler_addresses { + /// // stop root-gate eth_getProof probes for the account + /// engine.runtime_mut().untrack_account(address); + /// // drop its queued (unexecuted) repair work from the pending ledger + /// engine.runtime_mut().cancel_pending_resyncs(address); + /// } + /// // optional: evict cached state via StateUpdate::purge / cache purge APIs + /// ``` + /// + /// Health, metrics, the reorg journal, hooks, and freshness stamps are + /// runtime-global and are never touched by handler removal. + pub fn unregister_handler(&mut self, id: &HandlerId) -> Option>> { + self.subscriber.remove_interest_owner(id); + self.runtime.unregister_handler(id) + } +} + /// Alloy-backed event subscriber. /// /// The default transport slice drives Alloy pubsub subscriptions for logs, /// block headers, and pending transaction hashes. The HTTP polling `watch_*` /// transport remains available behind the opt-in `reactive-polling` feature. /// Pubsub streams reconnect automatically after termination, and log -/// subscriptions are backfilled from the last seen block. Full pending -/// transaction hydration, full block bodies, and arbitrary historical backfill -/// remain explicit follow-up work. +/// subscriptions are backfilled from the last seen block. Owner-scoped log +/// additions can request backfill from an explicit block anchor. Full pending +/// transaction hydration and full block bodies remain explicit follow-up work. /// With no registered interests, [`EventSubscriber::next_batch`] returns /// `Ok(None)`. pub struct AlloySubscriber { provider: P, mode: SubscriberMode, config: SubscriberConfig, + base_interests: Vec>, + owned_interests: Vec>, interests: Vec>, + /// Stable source id per distinct live log filter. Ids key delivery anchors + /// and live `SubscriberEvent`s; entries are retired (and their anchors + /// pruned) when no base or owner interest references the filter anymore, so + /// long-lived owner churn cannot grow this map unboundedly. + log_source_ids: HashMap, + next_log_source_id: usize, + pending_backfills: VecDeque, + /// Set when interest bookkeeping changed since the last successful stream + /// reconcile, so steady-state polling skips the desired-vs-live diff. + sources_dirty: bool, state: AlloySubscriberState, pending_records: VecDeque>, last_seen_log_blocks: HashMap, @@ -2668,6 +4277,17 @@ pub struct AlloySubscriber { _network: PhantomData, } +struct OwnedSubscriberInterests { + owner: HandlerId, + interests: Vec>, +} + +struct QueuedSubscriberBackfill { + owner: HandlerId, + filter: Filter, + backfill: SubscriberBackfill, +} + /// Best-effort installation of rustls' `ring` crypto provider as the process /// default, so an `wss://` TLS handshake under `reactive-ws` does not panic with /// "no process-level CryptoProvider available". Runs at most once and ignores the @@ -2691,7 +4311,13 @@ impl AlloySubscriber { provider, mode, config, + base_interests: Vec::new(), + owned_interests: Vec::new(), interests: Vec::new(), + log_source_ids: HashMap::new(), + next_log_source_id: 0, + pending_backfills: VecDeque::new(), + sources_dirty: true, state: AlloySubscriberState::Uninitialized, pending_records: VecDeque::new(), last_seen_log_blocks: HashMap::new(), @@ -2716,56 +4342,380 @@ impl AlloySubscriber { &self.config } - /// Registered interests. + /// Registered interests across base and owner-scoped registrations. pub fn registered_interests(&self) -> &[ReactiveInterest] { &self.interests } - fn drain_next_batch(&mut self) -> Option> { - if self.pending_records.is_empty() { - return None; - } - - let len = self.config.max_batch_size.min(self.pending_records.len()); - let records = self.pending_records.drain(..len).collect(); - Some(ReactiveInputBatch::new(records)) + /// Add or replace the interests owned by `owner`. + /// + /// This preserves unrelated owners, queued/pending records, recent dedupe + /// state, and last-seen log anchors. The live transport is reconciled on the + /// next [`EventSubscriber::next_batch`] call so newly added log filters can + /// be subscribed without rebuilding the whole subscriber object. + /// + /// Replacing an existing owner is continuity-safe: filters the owner + /// already had keep their delivery anchors, and any changed or new filter + /// shape is automatically backfilled from the owner's oldest prior anchor — + /// growing a pool set on an established owner does not open a delivery gap + /// for what the old subscription had already covered. A brand-new owner has + /// no anchor to inherit; pass an explicit + /// [`add_interest_owner_with_backfill`](Self::add_interest_owner_with_backfill) + /// anchor (or register through [`ReactiveEngine::register_handler`], which + /// anchors to the runtime's last canonical block). + pub fn add_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + self.set_interest_owner(owner, interests, None) } - fn reset_delivery_state(&mut self) { - self.pending_records.clear(); - self.last_seen_log_blocks.clear(); - self.recent_input_refs.clear(); - self.recent_input_ref_set.clear(); + /// Add or replace owner interests and schedule log backfill for that owner. + /// + /// Backfill is queued only for log interests; block and pending transaction + /// interests are live-only. Queued backfill is drained before live stream + /// initialization, its resolved upper bound seeds the filter's delivery + /// anchor, and the anchored filter is caught up again right after its live + /// stream connects — so the discovery boundary is closed end to end as long + /// as `backfill` starts at (or before) the block the interest was + /// discovered in. Continuity backfill for a replaced owner (see + /// [`add_interest_owner`](Self::add_interest_owner)) is queued in addition, + /// unless this explicit backfill is open-ended and already starts at or + /// below the owner's prior anchor. + pub fn add_interest_owner_with_backfill( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: SubscriberBackfill, + ) -> Result<(), SubscriberError> { + self.set_interest_owner(owner, interests, Some(backfill)) } -} - -enum AlloySubscriberState { - Uninitialized, - Active(SubscriberStreams), - Empty, -} - -struct SubscriberStreams { - streams: SelectAll>>, -} -impl SubscriberStreams { - fn new() -> Self { - Self { - streams: SelectAll::new(), - } + /// Remove one owner's interests, preserving unrelated owner/base interests. + /// + /// The owner's queued backfills are dropped, and source-id/anchor + /// bookkeeping for filters no other owner references is retired. Live + /// streams for retired filters are torn down on the next + /// [`EventSubscriber::next_batch`] call (dropping an Alloy subscription + /// unsubscribes provider-side); events already in flight from them stop + /// matching the merged interest set and are discarded. + pub fn remove_interest_owner(&mut self, owner: &HandlerId) -> Option>> { + let index = self + .owned_interests + .iter() + .position(|entry| &entry.owner == owner)?; + let removed = self.owned_interests.remove(index).interests; + self.pending_backfills + .retain(|backfill| &backfill.owner != owner); + self.rebuild_registered_interests(); + self.retire_unreferenced_filters(); + self.sources_dirty = true; + Some(removed) + } + + /// Borrow the interests currently owned by `owner`. + pub fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest]> { + self.owned_interests + .iter() + .find(|entry| &entry.owner == owner) + .map(|entry| entry.interests.as_slice()) } - fn is_empty(&self) -> bool { - self.streams.is_empty() + fn set_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: Option, + ) -> Result<(), SubscriberError> { + validate_subscriber_config(&self.config)?; + + let mut next_owned = self.clone_owned_interests(); + match next_owned.iter_mut().find(|entry| entry.owner == owner) { + Some(entry) => entry.interests = interests.to_vec(), + None => next_owned.push(OwnedSubscriberInterests { + owner: owner.clone(), + interests: interests.to_vec(), + }), + } + let next_registered = aggregate_interests(&self.base_interests, &next_owned); + validate_supported_interests(self.mode, &self.config, &next_registered)?; + + // Continuity capture, before the mutation lands: the owner's previous + // filter shapes and the oldest delivery anchor among them. A changed + // filter gets a fresh source id with no anchor, so without this + // hand-off, replacing an owner's interests (the normal way to grow a + // pool set) would silently discard the delivery watermark and open a + // gap until some later explicit backfill. + let previous_filters: Vec = self + .owner_interests(&owner) + .map(log_filters) + .unwrap_or_default(); + let continuity_anchor: Option = previous_filters + .iter() + .filter_map(|filter| self.log_anchor(filter)) + .min(); + + self.owned_interests = next_owned; + self.interests = next_registered; + self.retire_unreferenced_filters(); + self.sources_dirty = true; + + // Re-queue this owner's backfills from scratch: previously queued + // entries may reference filter shapes that no longer exist. + self.pending_backfills + .retain(|queued| queued.owner != owner); + for filter in log_filters(interests) { + if let Some(backfill) = backfill { + self.pending_backfills.push_back(QueuedSubscriberBackfill { + owner: owner.clone(), + filter: filter.clone(), + backfill, + }); + } + + // Continuity backfill for changed/new shapes only: an unchanged + // filter kept its anchor and its live stream, and an open-ended + // explicit backfill starting at or below the anchor already covers + // the window. + let unchanged = previous_filters.contains(&filter); + let explicit_covers = backfill.is_some_and(|explicit| { + explicit.end_block().is_none() + && continuity_anchor.is_some_and(|anchor| explicit.start_block() <= anchor) + }); + if let Some(anchor) = continuity_anchor + && !unchanged + && !explicit_covers + { + self.pending_backfills.push_back(QueuedSubscriberBackfill { + owner: owner.clone(), + filter, + backfill: SubscriberBackfill::from_block(anchor), + }); + } + } + Ok(()) + } + + fn clone_owned_interests(&self) -> Vec> { + self.owned_interests + .iter() + .map(|entry| OwnedSubscriberInterests { + owner: entry.owner.clone(), + interests: entry.interests.clone(), + }) + .collect() + } + + fn rebuild_registered_interests(&mut self) { + self.interests = aggregate_interests(&self.base_interests, &self.owned_interests); + } + + /// Delivery anchor (last block known fully delivered) for `filter`, if the + /// filter has a source id and has seen delivery. + fn log_anchor(&self, filter: &Filter) -> Option { + let id = self.log_source_ids.get(filter)?; + self.last_seen_log_blocks.get(id).copied() + } + + /// Every live log filter across base and owner interests — merged within + /// each origin (owner boundaries are preserved so one owner's churn cannot + /// rewrite another's subscription), then deduplicated across origins so an + /// identical shape shared by several owners maps to exactly one stream and + /// one delivery anchor. + // `Filter` derives `Hash`/`Eq` and has no interior mutability; the + // `mutable_key_type` lint is a known false positive for it. + #[allow(clippy::mutable_key_type)] + fn log_stream_filters(&self) -> Vec { + let mut filters = log_filters(&self.base_interests); + for entry in &self.owned_interests { + filters.extend(log_filters(&entry.interests)); + } + let mut seen = HashSet::new(); + filters.retain(|filter| seen.insert(filter.clone())); + filters + } + + /// Drop source-id and anchor bookkeeping for filters no longer referenced + /// by any base or owner interest, so long-lived owner churn cannot grow the + /// maps unboundedly. Live streams for retired filters are pruned by the + /// next reconcile. + // `Filter` derives `Hash`/`Eq` and has no interior mutability; the + // `mutable_key_type` lint is a known false positive for it. + #[allow(clippy::mutable_key_type)] + fn retire_unreferenced_filters(&mut self) { + let live: HashSet = self.log_stream_filters().into_iter().collect(); + self.log_source_ids + .retain(|filter, _| live.contains(filter)); + let live_ids: HashSet = self.log_source_ids.values().copied().collect(); + self.last_seen_log_blocks + .retain(|id, _| live_ids.contains(id)); + } + + fn drain_next_batch(&mut self) -> Option> { + if self.pending_records.is_empty() { + return None; + } + + let len = self.config.max_batch_size.min(self.pending_records.len()); + let records = self.pending_records.drain(..len).collect(); + Some(ReactiveInputBatch::new(records)) + } + + fn reset_delivery_state(&mut self) { + self.pending_records.clear(); + self.last_seen_log_blocks.clear(); + self.recent_input_refs.clear(); + self.recent_input_ref_set.clear(); + self.pending_backfills.clear(); + self.log_source_ids.clear(); + self.next_log_source_id = 0; + self.sources_dirty = true; + } +} + +impl InterestOwnerSubscriber for AlloySubscriber +where + P: Provider + Send + Sync, + N: Network + 'static, + N::HeaderResponse: Send + 'static, +{ + fn add_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + AlloySubscriber::add_interest_owner(self, owner, interests) + } + + fn add_interest_owner_with_backfill( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: SubscriberBackfill, + ) -> Result<(), SubscriberError> { + AlloySubscriber::add_interest_owner_with_backfill(self, owner, interests, backfill) + } + + fn remove_interest_owner(&mut self, owner: &HandlerId) -> Option>> { + AlloySubscriber::remove_interest_owner(self, owner) + } + + fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest]> { + AlloySubscriber::owner_interests(self, owner) + } +} + +enum AlloySubscriberState { + Uninitialized, + Active(SubscriberStreams), + Empty, +} + +struct SubscriberStreams { + entries: Vec>, + next_index: usize, +} + +struct SubscriberStreamEntry { + source: SubscriberStreamSource, + stream: BoxStream<'static, SubscriberEvent>, +} + +impl SubscriberStreams { + fn new() -> Self { + Self { + entries: Vec::new(), + next_index: 0, + } + } + + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn push( + &mut self, + source: SubscriberStreamSource, + stream: BoxStream<'static, SubscriberEvent>, + ) { + self.entries.push(SubscriberStreamEntry { source, stream }); } - fn push(&mut self, stream: BoxStream<'static, SubscriberEvent>) { - self.streams.push(stream); + #[cfg(all(test, feature = "reactive-ws"))] + fn len(&self) -> usize { + self.entries.len() + } + + fn contains_source(&self, source: &SubscriberStreamSource) -> bool { + self.entries + .iter() + .any(|entry| entry.source.same_key(source)) + } + + fn retain_sources(&mut self, sources: &[SubscriberStreamSource]) { + self.entries + .retain(|entry| sources.iter().any(|source| entry.source.same_key(source))); + self.normalize_next_index(); + } + + fn normalize_next_index(&mut self) { + if self.entries.is_empty() { + self.next_index = 0; + } else if self.next_index >= self.entries.len() { + self.next_index %= self.entries.len(); + } } async fn next(&mut self) -> Option> { - self.streams.next().await + poll_fn(|cx| { + self.normalize_next_index(); + if self.entries.is_empty() { + return std::task::Poll::Ready(None); + } + + let mut index = self.next_index; + let mut checked = 0usize; + while checked < self.entries.len() { + if index >= self.entries.len() { + index = 0; + } + match self.entries[index].stream.as_mut().poll_next(cx) { + std::task::Poll::Ready(Some(event)) => { + if matches!(event, SubscriberEvent::StreamTerminated(_)) { + self.entries.remove(index); + self.next_index = if self.entries.is_empty() { + 0 + } else { + index % self.entries.len() + }; + } else { + self.next_index = (index + 1) % self.entries.len(); + } + return std::task::Poll::Ready(Some(event)); + } + std::task::Poll::Ready(None) => { + self.entries.remove(index); + if self.entries.is_empty() { + self.next_index = 0; + return std::task::Poll::Ready(None); + } + } + std::task::Poll::Pending => { + checked += 1; + index += 1; + } + } + } + + if self.entries.is_empty() { + std::task::Poll::Ready(None) + } else { + self.next_index = index % self.entries.len(); + std::task::Poll::Pending + } + }) + .await } } @@ -2802,6 +4752,19 @@ impl SubscriberStreamSource { Self::PubSubLog { .. } | Self::PubSubPendingHashes | Self::PubSubBlockHeaders ) } + + fn same_key(&self, other: &Self) -> bool { + match (self, other) { + (Self::PubSubLog { filter: left, .. }, Self::PubSubLog { filter: right, .. }) + | (Self::PollingLog { filter: left }, Self::PollingLog { filter: right }) => { + left == right + } + (Self::PubSubPendingHashes, Self::PubSubPendingHashes) + | (Self::PubSubBlockHeaders, Self::PubSubBlockHeaders) + | (Self::PollingPendingHashes, Self::PollingPendingHashes) => true, + _ => false, + } + } } #[allow(dead_code)] @@ -2828,7 +4791,9 @@ where validate_subscriber_config(&self.config)?; validate_supported_interests(self.mode, &self.config, interests)?; - self.interests = interests.to_vec(); + self.base_interests = interests.to_vec(); + self.owned_interests.clear(); + self.rebuild_registered_interests(); self.reset_delivery_state(); self.state = AlloySubscriberState::Uninitialized; Ok(()) @@ -2840,17 +4805,18 @@ where return Ok(Some(batch)); } - if self.interests.is_empty() { - return Ok(None); + self.drain_pending_backfills().await?; + if let Some(batch) = self.drain_next_batch() { + return Ok(Some(batch)); } - if matches!(self.state, AlloySubscriberState::Uninitialized) { - let streams = self.init_streams().await?; - self.state = if streams.is_empty() { - AlloySubscriberState::Empty - } else { - AlloySubscriberState::Active(streams) - }; + self.ensure_streams().await?; + if let Some(batch) = self.drain_next_batch() { + return Ok(Some(batch)); + } + + if self.interests.is_empty() { + return Ok(None); } loop { @@ -2873,19 +4839,194 @@ where N: Network + 'static, N::HeaderResponse: Send + 'static, { - async fn init_streams(&mut self) -> Result, SubscriberError> { - let mut streams = SubscriberStreams::new(); - for source in self.stream_sources()? { - streams.push(self.connect_source_stream(source).await?); + /// Bring live streams in line with the current interest set. + /// + /// Runs incrementally: the desired-vs-live diff only happens when interest + /// bookkeeping changed since the last successful pass (`sources_dirty`), so + /// steady-state polling costs nothing here. Missing sources are connected, + /// sources for retired filters are dropped (dropping an Alloy subscription + /// unsubscribes provider-side), and unrelated live streams — with their + /// delivery and anchor state — are left untouched. + /// + /// A newly connected log source whose filter already has a delivery anchor + /// is caught up from that anchor immediately after subscribing (the same + /// subscribe-then-backfill order the reconnect path uses). Together with + /// anchor seeding in [`Self::drain_pending_backfills`], that closes the + /// window between an adoption backfill and live stream start. + async fn ensure_streams(&mut self) -> Result<(), SubscriberError> { + if !self.sources_dirty { + return Ok(()); + } + // An interest-less subscriber stays Uninitialized and never touches the + // provider ([`EventSubscriber::next_batch`] returns `Ok(None)`), + // matching setup-before-interests behavior. + if matches!(self.state, AlloySubscriberState::Uninitialized) && self.interests.is_empty() { + return Ok(()); + } + + let desired = self.stream_sources()?; + let missing: Vec = match &self.state { + AlloySubscriberState::Active(streams) => desired + .iter() + .filter(|source| !streams.contains_source(source)) + .cloned() + .collect(), + AlloySubscriberState::Uninitialized | AlloySubscriberState::Empty => desired.clone(), + }; + + let mut connected = Vec::new(); + for source in missing { + let stream = self.connect_source_stream(source.clone()).await?; + // Anchored catch-up for a source with a known delivery watermark + // (seeded by a drained adoption backfill, or inherited from a + // filter shape that was live before): subscribe first, then fetch + // the gap, so nothing lands between the two. + if let Some(event) = self.backfill_reconnected_source(&source).await? { + self.enqueue_event(event); + } + connected.push((source, stream)); + } + + match &mut self.state { + AlloySubscriberState::Active(streams) => { + streams.retain_sources(&desired); + for (source, stream) in connected { + streams.push(source, stream); + } + if streams.is_empty() { + self.state = AlloySubscriberState::Empty; + } + } + AlloySubscriberState::Uninitialized | AlloySubscriberState::Empty => { + let mut streams = SubscriberStreams::new(); + for (source, stream) in connected { + streams.push(source, stream); + } + self.state = if streams.is_empty() { + AlloySubscriberState::Empty + } else { + AlloySubscriberState::Active(streams) + }; + } } - Ok(streams) + + self.sources_dirty = false; + Ok(()) } - fn stream_sources(&self) -> Result, SubscriberError> { + /// Fetch queued adoption/continuity backfills, oldest first. + /// + /// An entry is consumed only after its `get_logs` fetch succeeds — a + /// transient RPC failure surfaces the error and leaves the entry queued for + /// the next poll, so a flaky request cannot silently discard the missed + /// window the backfill exists to close. Open-ended backfills resolve their + /// upper bound to the provider's current head before fetching, and every + /// drained backfill advances the filter's delivery anchor to that bound — + /// even a zero-log window — so the filter is reconnect-protected from then + /// on. Draining pauses as soon as records are ready for delivery; remaining + /// entries stay queued. + async fn drain_pending_backfills(&mut self) -> Result<(), SubscriberError> { + while let Some(queued) = self.pending_backfills.front() { + // Owner was removed while its backfill was queued. + if self.owner_interests(&queued.owner).is_none() { + self.pending_backfills.pop_front(); + continue; + } + let filter = queued.filter.clone(); + let backfill = queued.backfill; + + let to_block = match backfill.end_block() { + Some(to_block) => to_block, + None => self + .provider + .get_block_number() + .await + .map_err(provider_error)?, + }; + if to_block < backfill.start_block() { + // Anchor already at (or past) the provider head: nothing to + // fetch, and the anchor keeps its current value. + self.pending_backfills.pop_front(); + continue; + } + + let range = filter + .clone() + .from_block(backfill.start_block()) + .to_block(to_block); + let logs = self + .provider + .get_logs(&range) + .await + .map_err(provider_error)?; + + // Fetch succeeded: consume the entry, deliver, and advance the + // anchor through the fetched bound. + self.pending_backfills.pop_front(); + let source_id = self.log_source_id(&filter); + self.enqueue_backfilled_logs(logs, Some(source_id)); + let anchor = self + .last_seen_log_blocks + .entry(source_id) + .or_insert(to_block); + *anchor = (*anchor).max(to_block); + + if !self.pending_records.is_empty() { + break; + } + } + Ok(()) + } + + fn stream_sources(&mut self) -> Result, SubscriberError> { match resolve_subscriber_transport(self.mode)? { - SubscriberTransport::PubSub => Ok(pubsub_stream_sources(&self.interests)), - SubscriberTransport::Polling => Ok(polling_stream_sources(&self.interests)), + SubscriberTransport::PubSub => Ok(self.pubsub_stream_sources()), + SubscriberTransport::Polling => Ok(self.polling_stream_sources()), + } + } + + fn pubsub_stream_sources(&mut self) -> Vec { + let mut sources = Vec::new(); + + for filter in self.log_stream_filters() { + let id = self.log_source_id(&filter); + sources.push(SubscriberStreamSource::PubSubLog { id, filter }); + } + + if needs_pending_hash_stream(&self.interests) { + sources.push(SubscriberStreamSource::PubSubPendingHashes); + } + + if needs_header_block_stream(&self.interests) { + sources.push(SubscriberStreamSource::PubSubBlockHeaders); + } + + sources + } + + fn polling_stream_sources(&self) -> Vec { + let mut sources = Vec::new(); + + for filter in self.log_stream_filters() { + sources.push(SubscriberStreamSource::PollingLog { filter }); + } + + if needs_pending_hash_stream(&self.interests) { + sources.push(SubscriberStreamSource::PollingPendingHashes); + } + + sources + } + + fn log_source_id(&mut self, filter: &Filter) -> usize { + if let Some(id) = self.log_source_ids.get(filter) { + return *id; } + + let id = self.next_log_source_id; + self.next_log_source_id = self.next_log_source_id.saturating_add(1); + self.log_source_ids.insert(filter.clone(), id); + id } async fn connect_source_stream( @@ -3089,13 +5230,7 @@ where } } SubscriberEvent::BackfilledLogs { source_id, logs } => { - for log in logs { - if log_matches_any_interest(&log, &self.interests) { - let record = log_input_record(log, InputSource::Backfill); - self.note_log_block(source_id, &record); - self.enqueue_record(record); - } - } + self.enqueue_backfilled_logs(logs, Some(source_id)); } SubscriberEvent::Logs(logs) => self.pending_records.extend( logs.into_iter() @@ -3121,6 +5256,18 @@ where } } + fn enqueue_backfilled_logs(&mut self, logs: Vec, source_id: Option) { + for log in logs { + if log_matches_any_interest(&log, &self.interests) { + let record = log_input_record(log, InputSource::Backfill); + if let Some(source_id) = source_id { + self.note_log_block(source_id, &record); + } + self.enqueue_record(record); + } + } + } + async fn reconnect_source_stream( &mut self, source: SubscriberStreamSource, @@ -3177,7 +5324,7 @@ where let backfill_event = self.backfill_reconnected_source(&source).await?; match &mut self.state { - AlloySubscriberState::Active(streams) => streams.push(stream), + AlloySubscriberState::Active(streams) => streams.push(source, stream), AlloySubscriberState::Uninitialized | AlloySubscriberState::Empty => { return Err(SubscriberError::Provider( "Alloy subscriber state changed before reconnect completed".to_owned(), @@ -3259,6 +5406,7 @@ where } } +#[cfg(any(feature = "reactive-ws", feature = "reactive-polling", test))] fn stream_with_termination( stream: S, source: SubscriberStreamSource, @@ -3274,40 +5422,18 @@ where .boxed() } -fn pubsub_stream_sources( - interests: &[ReactiveInterest], -) -> Vec { - let mut sources = Vec::new(); - - for (id, filter) in log_filters(interests).into_iter().enumerate() { - sources.push(SubscriberStreamSource::PubSubLog { id, filter }); - } - - if needs_pending_hash_stream(interests) { - sources.push(SubscriberStreamSource::PubSubPendingHashes); - } - - if needs_header_block_stream(interests) { - sources.push(SubscriberStreamSource::PubSubBlockHeaders); - } - - sources -} - -fn polling_stream_sources( - interests: &[ReactiveInterest], -) -> Vec { - let mut sources = Vec::new(); - - for filter in log_filters(interests) { - sources.push(SubscriberStreamSource::PollingLog { filter }); - } - - if needs_pending_hash_stream(interests) { - sources.push(SubscriberStreamSource::PollingPendingHashes); - } - - sources +fn aggregate_interests( + base: &[ReactiveInterest], + owned: &[OwnedSubscriberInterests], +) -> Vec> { + base.iter() + .cloned() + .chain( + owned + .iter() + .flat_map(|entry| entry.interests.iter().cloned()), + ) + .collect() } fn stream_terminated_error(source: &SubscriberStreamSource) -> SubscriberError { @@ -3409,21 +5535,35 @@ mod subscriber_helper_tests { } #[test] + #[cfg(feature = "reactive-ws")] fn pubsub_sources_assign_stable_log_ids_before_shared_streams() { - let sources = pubsub_stream_sources::(&[ - ReactiveInterest::Logs(LogInterest { - provider_filter: Filter::new().address(Address::repeat_byte(0x01)), - local_matcher: None, - route_key: None, - }), - ReactiveInterest::Logs(LogInterest { - provider_filter: Filter::new().address(Address::repeat_byte(0x02)), - local_matcher: None, - route_key: None, - }), - ReactiveInterest::PendingTransactions(PendingTxInterest::default()), - ]); - + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .register_interests(&[ + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(Address::repeat_byte(0x01)), + local_matcher: None, + route_key: None, + }), + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(Address::repeat_byte(0x02)), + local_matcher: None, + route_key: None, + }), + ReactiveInterest::PendingTransactions(PendingTxInterest::default()), + ]) + .expect("register base interests"); + + // The two default-block-option log filters merge into one address + // superset (existing consolidation behavior), so there is one log source + // — assigned id 0, before the pending-hash source. + let sources = subscriber.stream_sources().expect("stream sources"); + assert_eq!(sources.len(), 2); assert!(matches!( &sources[0], SubscriberStreamSource::PubSubLog { id: 0, .. } @@ -3432,6 +5572,10 @@ mod subscriber_helper_tests { sources[1], SubscriberStreamSource::PubSubPendingHashes )); + + // Ids are stable across repeated source construction. + let again = subscriber.stream_sources().expect("stream sources again"); + assert!(again[0].same_key(&sources[0])); } #[tokio::test(flavor = "multi_thread")] @@ -3457,7 +5601,9 @@ mod subscriber_helper_tests { )]; let mut streams = SubscriberStreams::new(); + let source = SubscriberStreamSource::PubSubPendingHashes; streams.push( + source, stream::once(async { SubscriberEvent::::StreamTerminated( SubscriberStreamSource::PubSubPendingHashes, @@ -3540,6 +5686,604 @@ mod subscriber_helper_tests { ); assert_eq!(subscriber.last_seen_log_blocks.get(&0), Some(&7)); } + + #[test] + #[cfg(feature = "reactive-ws")] + fn owner_removal_preserves_delivery_and_dedupe_state() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner( + HandlerId::new("pool-a"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new() + .address(Address::repeat_byte(0x42)) + .event_signature(B256::repeat_byte(0x01)), + local_matcher: None, + route_key: None, + })], + ) + .expect("register pool-a owner"); + subscriber + .add_interest_owner( + HandlerId::new("pool-b"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new() + .address(Address::repeat_byte(0x24)) + .event_signature(B256::repeat_byte(0x02)), + local_matcher: None, + route_key: None, + })], + ) + .expect("register pool-b owner"); + + // Allocate source ids the way live stream setup would (pool-a -> id 0), + // so the injected delivery anchor hangs off a referenced filter. + let _ = subscriber.stream_sources().expect("stream sources"); + subscriber.enqueue_event(SubscriberEvent::Log { + source_id: 0, + log: rpc_log(false), + }); + assert_eq!(subscriber.pending_records.len(), 1); + assert_eq!(subscriber.recent_input_refs.len(), 1); + assert_eq!(subscriber.last_seen_log_blocks.get(&0), Some(&7)); + + let removed = subscriber + .remove_interest_owner(&HandlerId::new("pool-b")) + .expect("pool-b should be removed"); + + assert_eq!(removed.len(), 1); + assert_eq!(subscriber.pending_records.len(), 1); + assert_eq!(subscriber.recent_input_refs.len(), 1); + assert_eq!(subscriber.last_seen_log_blocks.get(&0), Some(&7)); + assert!( + subscriber + .owner_interests(&HandlerId::new("pool-a")) + .is_some() + ); + assert!( + subscriber + .owner_interests(&HandlerId::new("pool-b")) + .is_none() + ); + assert_eq!(subscriber.registered_interests().len(), 1); + } + + #[test] + #[cfg(feature = "reactive-ws")] + fn owner_log_sources_do_not_merge_across_owners() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner( + HandlerId::new("pool-a"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(Address::repeat_byte(0xa1)), + local_matcher: None, + route_key: None, + })], + ) + .expect("register pool-a owner"); + + let initial_sources = subscriber.stream_sources().expect("initial sources"); + assert_eq!(initial_sources.len(), 1); + let pool_a_source = initial_sources[0].clone(); + assert!(matches!( + &pool_a_source, + SubscriberStreamSource::PubSubLog { id: 0, .. } + )); + + subscriber + .add_interest_owner( + HandlerId::new("pool-b"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(Address::repeat_byte(0xb2)), + local_matcher: None, + route_key: None, + })], + ) + .expect("register pool-b owner"); + + let expanded_sources = subscriber.stream_sources().expect("expanded sources"); + assert_eq!(expanded_sources.len(), 2); + assert!( + expanded_sources + .iter() + .any(|source| source.same_key(&pool_a_source)), + "adding pool-b should not rewrite pool-a's stream source" + ); + + subscriber + .remove_interest_owner(&HandlerId::new("pool-b")) + .expect("pool-b should be removed"); + let trimmed_sources = subscriber.stream_sources().expect("trimmed sources"); + assert_eq!(trimmed_sources.len(), 1); + assert!(trimmed_sources[0].same_key(&pool_a_source)); + } + + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn owner_backfill_seeds_reconnect_anchor_before_live_log() { + let asserter = Asserter::new(); + asserter.push_success(&vec![rpc_log(false)]); + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner_with_backfill( + HandlerId::new("pool-a"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new() + .address(Address::repeat_byte(0x42)) + .event_signature(B256::repeat_byte(0x01)), + local_matcher: None, + route_key: None, + })], + SubscriberBackfill::range(1, 7), + ) + .expect("register pool-a with backfill"); + + subscriber + .drain_pending_backfills() + .await + .expect("owner backfill should drain"); + + assert_eq!(subscriber.pending_records.len(), 1); + assert_eq!(subscriber.last_seen_log_blocks.get(&0), Some(&7)); + } + + #[tokio::test(flavor = "multi_thread")] + async fn subscriber_streams_poll_ready_sources_round_robin() { + let first_hash = B256::repeat_byte(0x01); + let second_hash = B256::repeat_byte(0x02); + let mut streams = SubscriberStreams::new(); + streams.push( + SubscriberStreamSource::PubSubPendingHashes, + stream::iter([ + SubscriberEvent::::PendingHash(first_hash), + SubscriberEvent::::PendingHash(first_hash), + ]) + .boxed(), + ); + streams.push( + SubscriberStreamSource::PubSubBlockHeaders, + stream::once(async move { SubscriberEvent::::PendingHash(second_hash) }) + .boxed(), + ); + + assert!(matches!( + streams.next().await, + Some(SubscriberEvent::PendingHash(hash)) if hash == first_hash + )); + assert!(matches!( + streams.next().await, + Some(SubscriberEvent::PendingHash(hash)) if hash == second_hash + )); + } + + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn owner_updates_ensure_streams_without_full_reset() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .register_interests(&[ReactiveInterest::PendingTransactions( + PendingTxInterest::default(), + )]) + .expect("register base pending interest"); + subscriber + .add_interest_owner( + HandlerId::new("headers"), + &[ReactiveInterest::Blocks(BlockInterest::default())], + ) + .expect("register header owner"); + + let mut streams = SubscriberStreams::new(); + streams.push( + SubscriberStreamSource::PubSubPendingHashes, + stream::pending::>().boxed(), + ); + streams.push( + SubscriberStreamSource::PubSubBlockHeaders, + stream::pending::>().boxed(), + ); + subscriber.state = AlloySubscriberState::Active(streams); + + subscriber + .remove_interest_owner(&HandlerId::new("headers")) + .expect("header owner should be removed"); + assert!(matches!( + &subscriber.state, + AlloySubscriberState::Active(streams) if streams.len() == 2 + )); + + subscriber + .ensure_streams() + .await + .expect("pure removal reconciliation should not touch provider"); + + assert!(matches!( + &subscriber.state, + AlloySubscriberState::Active(streams) + if streams.len() == 1 + && streams.contains_source(&SubscriberStreamSource::PubSubPendingHashes) + && !streams.contains_source(&SubscriberStreamSource::PubSubBlockHeaders) + )); + + subscriber + .add_interest_owner( + HandlerId::new("headers"), + &[ReactiveInterest::Blocks(BlockInterest::default())], + ) + .expect("re-add header owner"); + assert!(matches!( + &subscriber.state, + AlloySubscriberState::Active(streams) if streams.len() == 1 + )); + } + + // A log interest matching `rpc_log` (address 0x42, topic0 0x01). + #[cfg(feature = "reactive-ws")] + fn log_interest_matching_rpc_log() -> ReactiveInterest { + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new() + .address(Address::repeat_byte(0x42)) + .event_signature(B256::repeat_byte(0x01)), + local_matcher: None, + route_key: None, + }) + } + + #[cfg(feature = "reactive-ws")] + fn log_interest_for(address: u8) -> ReactiveInterest { + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(Address::repeat_byte(address)), + local_matcher: None, + route_key: None, + }) + } + + // B1: a transient provider error must not consume the queued backfill — the + // missed window has to survive for the next poll to retry. + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn drain_backfill_retains_queue_entry_on_provider_error() { + let asserter = Asserter::new(); + asserter.push_failure_msg("rate limited"); + asserter.push_success(&vec![rpc_log(false)]); + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner_with_backfill( + HandlerId::new("pool"), + &[log_interest_matching_rpc_log()], + SubscriberBackfill::range(1, 7), + ) + .expect("register owner with backfill"); + assert_eq!(subscriber.pending_backfills.len(), 1); + + let first = subscriber.drain_pending_backfills().await; + assert!(first.is_err(), "provider failure should surface"); + assert_eq!( + subscriber.pending_backfills.len(), + 1, + "failed fetch must leave the backfill queued for retry" + ); + assert!(subscriber.pending_records.is_empty()); + + subscriber + .drain_pending_backfills() + .await + .expect("retry should succeed"); + assert!(subscriber.pending_backfills.is_empty()); + assert_eq!(subscriber.pending_records.len(), 1); + } + + // B3: a zero-log backfill window still advances the delivery anchor to its + // upper bound, so a later reconnect catches up from the right block. + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn drain_backfill_seeds_anchor_on_empty_window() { + let asserter = Asserter::new(); + asserter.push_success(&Vec::::new()); + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner_with_backfill( + HandlerId::new("pool"), + &[log_interest_matching_rpc_log()], + SubscriberBackfill::range(1, 42), + ) + .expect("register owner with backfill"); + + subscriber + .drain_pending_backfills() + .await + .expect("empty backfill should drain"); + + assert!(subscriber.pending_records.is_empty()); + let filter = log_filters(subscriber.owner_interests(&HandlerId::new("pool")).unwrap()) + .pop() + .unwrap(); + assert_eq!( + subscriber.log_anchor(&filter), + Some(42), + "empty window must still seed the anchor at its upper bound" + ); + } + + // B3 (open-ended): a `from_block`-only backfill resolves its upper bound to + // the provider head and seeds the anchor there. + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn drain_backfill_open_ended_resolves_head_and_seeds_anchor() { + let asserter = Asserter::new(); + asserter.push_success(&100u64); // get_block_number + asserter.push_success(&Vec::::new()); // get_logs + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner_with_backfill( + HandlerId::new("pool"), + &[log_interest_matching_rpc_log()], + SubscriberBackfill::from_block(10), + ) + .expect("register owner with open-ended backfill"); + + subscriber + .drain_pending_backfills() + .await + .expect("open-ended backfill should drain"); + + let filter = log_filters(subscriber.owner_interests(&HandlerId::new("pool")).unwrap()) + .pop() + .unwrap(); + assert_eq!(subscriber.log_anchor(&filter), Some(100)); + } + + // B2: two owners requesting the same filter shape share exactly one live + // source (and thus one anchor), rather than double-subscribing. + #[test] + #[cfg(feature = "reactive-ws")] + fn duplicate_filters_across_owners_map_to_single_source() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner(HandlerId::new("pool-a"), &[log_interest_for(0xaa)]) + .expect("register pool-a"); + subscriber + .add_interest_owner(HandlerId::new("pool-b"), &[log_interest_for(0xaa)]) + .expect("register pool-b with identical filter"); + + assert_eq!( + subscriber.log_stream_filters().len(), + 1, + "identical filters across owners must collapse to one" + ); + let sources = subscriber.stream_sources().expect("stream sources"); + assert_eq!(sources.len(), 1); + } + + // B4: removing an owner retires the source-id and anchor bookkeeping for + // filters no other owner references, so long-lived churn cannot leak. + #[test] + #[cfg(feature = "reactive-ws")] + fn owner_removal_prunes_source_ids_and_anchors() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner(HandlerId::new("pool-a"), &[log_interest_for(0xaa)]) + .expect("register pool-a"); + subscriber + .add_interest_owner(HandlerId::new("pool-b"), &[log_interest_for(0xbb)]) + .expect("register pool-b"); + + // Allocate ids and simulate delivery anchors on both. + let _ = subscriber.stream_sources().expect("stream sources"); + let filter_a = log_filters(&[log_interest_for(0xaa)]).pop().unwrap(); + let filter_b = log_filters(&[log_interest_for(0xbb)]).pop().unwrap(); + let id_a = subscriber.log_source_id(&filter_a); + let id_b = subscriber.log_source_id(&filter_b); + subscriber.last_seen_log_blocks.insert(id_a, 10); + subscriber.last_seen_log_blocks.insert(id_b, 20); + assert_eq!(subscriber.log_source_ids.len(), 2); + + subscriber + .remove_interest_owner(&HandlerId::new("pool-b")) + .expect("remove pool-b"); + + assert_eq!( + subscriber.log_source_ids.len(), + 1, + "pool-b's filter id should be retired" + ); + assert!(subscriber.log_source_ids.contains_key(&filter_a)); + assert_eq!(subscriber.last_seen_log_blocks.get(&id_a), Some(&10)); + assert_eq!( + subscriber.last_seen_log_blocks.get(&id_b), + None, + "pool-b's anchor should be pruned" + ); + } + + // D1: growing an owner's filter set (a new pool on an existing adapter) + // changes the merged filter shape; the new shape must inherit the old + // anchor via an automatic continuity backfill, or logs between the last + // delivery and the new subscription are silently lost. + #[test] + #[cfg(feature = "reactive-ws")] + fn owner_filter_growth_queues_continuity_backfill_from_prior_anchor() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner(HandlerId::new("amm"), &[log_interest_for(0xaa)]) + .expect("register amm with pool A"); + + // Simulate the owner's single merged filter having delivered up to + // block 50. + let filter_a = log_filters(&[log_interest_for(0xaa)]).pop().unwrap(); + let id_a = subscriber.log_source_id(&filter_a); + subscriber.last_seen_log_blocks.insert(id_a, 50); + + // Grow the owner to also watch pool B (same block option -> merges into + // one {A,B} filter, a new shape). + subscriber + .add_interest_owner( + HandlerId::new("amm"), + &[log_interest_for(0xaa), log_interest_for(0xbb)], + ) + .expect("grow amm to pools A+B"); + + assert_eq!( + subscriber.pending_backfills.len(), + 1, + "the changed merged filter should queue exactly one continuity backfill" + ); + let queued = &subscriber.pending_backfills[0]; + assert_eq!(queued.owner, HandlerId::new("amm")); + assert_eq!(queued.backfill.start_block(), 50); + assert_eq!( + queued.backfill.end_block(), + None, + "continuity backfill runs open-ended to the current head" + ); + } + + // D1 negative: replacing an owner's interests with the identical shape must + // NOT re-fetch — the filter kept its anchor and its live stream. + #[test] + #[cfg(feature = "reactive-ws")] + fn unchanged_owner_filter_does_not_queue_continuity_backfill() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner(HandlerId::new("amm"), &[log_interest_for(0xaa)]) + .expect("register amm"); + let filter_a = log_filters(&[log_interest_for(0xaa)]).pop().unwrap(); + let id_a = subscriber.log_source_id(&filter_a); + subscriber.last_seen_log_blocks.insert(id_a, 50); + + subscriber + .add_interest_owner(HandlerId::new("amm"), &[log_interest_for(0xaa)]) + .expect("re-register identical interests"); + + assert!( + subscriber.pending_backfills.is_empty(), + "an unchanged filter shape must not queue continuity backfill" + ); + } + + // D5 interaction: an explicit open-ended backfill starting at or below the + // owner's prior anchor already covers the continuity window, so no extra + // continuity backfill is queued (no redundant double fetch). + #[test] + #[cfg(feature = "reactive-ws")] + fn explicit_open_ended_backfill_below_anchor_suppresses_continuity() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + subscriber + .add_interest_owner(HandlerId::new("amm"), &[log_interest_for(0xaa)]) + .expect("register amm"); + let filter_a = log_filters(&[log_interest_for(0xaa)]).pop().unwrap(); + let id_a = subscriber.log_source_id(&filter_a); + subscriber.last_seen_log_blocks.insert(id_a, 50); + + // Grow with an explicit deep backfill from block 10 (< anchor 50). + subscriber + .add_interest_owner_with_backfill( + HandlerId::new("amm"), + &[log_interest_for(0xaa), log_interest_for(0xbb)], + SubscriberBackfill::from_block(10), + ) + .expect("grow amm with explicit deep backfill"); + + assert_eq!( + subscriber.pending_backfills.len(), + 1, + "only the explicit backfill should be queued; continuity is subsumed" + ); + assert_eq!(subscriber.pending_backfills[0].backfill.start_block(), 10); + } + + // The dirty flag gates reconciliation: when nothing changed since the last + // reconcile, `ensure_streams` must not touch the provider or the state. + #[tokio::test(flavor = "multi_thread")] + #[cfg(feature = "reactive-ws")] + async fn ensure_streams_is_noop_when_not_dirty() { + let provider = ProviderBuilder::new().connect_mocked_client(Asserter::new()); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig::default(), + ); + // An interest that WOULD require a new block-header source... + subscriber + .add_interest_owner( + HandlerId::new("headers"), + &[ReactiveInterest::Blocks(BlockInterest::default())], + ) + .expect("register header owner"); + // ...but we mark bookkeeping clean and start from Empty. + subscriber.state = AlloySubscriberState::Empty; + subscriber.sources_dirty = false; + + subscriber + .ensure_streams() + .await + .expect("clean reconcile must be a no-op"); + + assert!( + matches!(subscriber.state, AlloySubscriberState::Empty), + "not-dirty ensure_streams must not connect new sources" + ); + } } fn resolve_subscriber_transport( diff --git a/src/tracing.rs b/src/tracing.rs index 8da83e7..04ef23e 100644 --- a/src/tracing.rs +++ b/src/tracing.rs @@ -325,7 +325,7 @@ where /// # use evm_fork_cache::cache::{EvmOverlay, EvmSnapshot, TxConfig}; /// # use evm_fork_cache::inspector::TransferInspector; /// # use evm_fork_cache::{CallTracer, InspectorStack}; -/// # fn run(snapshot: Arc, from: Address, to: Address) -> anyhow::Result<()> { +/// # fn run(snapshot: Arc, from: Address, to: Address) -> Result<(), Box> { /// let mut overlay = EvmOverlay::new(snapshot, None); /// let (_result, stack) = overlay.call_raw_with_inspector( /// from, diff --git a/tests/block_context.rs b/tests/block_context.rs new file mode 100644 index 0000000..2e787d6 --- /dev/null +++ b/tests/block_context.rs @@ -0,0 +1,303 @@ +//! Manager-authored red-green acceptance tests for WS-2 / Phase-8 step 2: +//! strict block-context requirements and engine-driven `advance_block` env +//! refresh. +//! +//! Contract: +//! - `BlockContextRequirements::strict()` rejects a header missing a required +//! block-env field (e.g. EIP-1559 base fee); `lenient()` (the default) accepts +//! it; requirements are per-field so a pre-EIP-1559 chain can opt out of the +//! base-fee requirement. +//! - `EvmCache::advance_block(header)` refreshes the full block env +//! (number / base fee / coinbase / prevrandao / gas limit / timestamp) from a +//! canonical header, and under strict requirements returns an error rather than +//! silently defaulting a missing field. +//! +//! Fully offline: block headers are constructed in memory; no network access. +#![cfg(feature = "reactive")] + +mod common; + +use alloy_consensus::Header; +use alloy_primitives::{Address, B256}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::cache::BlockContextRequirements; + +/// A synthetic canonical header. `basefee = None` models a pre-EIP-1559 block. +fn header(number: u64, basefee: Option) -> Header { + Header { + number, + timestamp: 1_700_000_000 + number, + base_fee_per_gas: basefee, + beneficiary: Address::repeat_byte(0xcb), + gas_limit: 30_000_000, + mix_hash: B256::repeat_byte(0xab), + ..Default::default() + } +} + +/// WS-2: strict requirements reject a header missing the EIP-1559 base fee, and +/// accept a complete one. +#[test] +fn strict_requirements_reject_header_missing_basefee() { + let reqs = BlockContextRequirements::strict(); + + assert!( + reqs.validate_header(&header(100, Some(7))).is_ok(), + "a complete header must satisfy strict requirements" + ); + + let err = reqs + .validate_header(&header(100, None)) + .expect_err("strict must reject a header with no base fee"); + assert!( + err.to_string().to_lowercase().contains("basefee") + || err.to_string().to_lowercase().contains("base fee"), + "the error must name the missing base-fee field, got: {err}" + ); +} + +/// WS-2: the lenient default accepts an incomplete header (today's behavior). +#[test] +fn lenient_requirements_accept_incomplete_header() { + let reqs = BlockContextRequirements::lenient(); + assert!(reqs.validate_header(&header(100, None)).is_ok()); + assert!(reqs.validate_header(&header(100, Some(7))).is_ok()); +} + +/// WS-2: requirements are per-field — a chain without EIP-1559 can turn off the +/// base-fee requirement while still requiring the rest. +#[test] +fn per_field_requirements_allow_opting_out_of_basefee() { + let mut reqs = BlockContextRequirements::strict(); + reqs.require_basefee = false; + assert!( + reqs.validate_header(&header(100, None)).is_ok(), + "opting out of the base-fee requirement must accept a header without one" + ); +} + +/// Phase-8 s2: `advance_block` refreshes every block-env field from the header. +#[tokio::test] +async fn advance_block_refreshes_all_block_env_fields() -> Result<()> { + let mut cache = setup_cache().await?; + let h = header(12_345, Some(42)); + + cache + .advance_block(&h) + .expect("lenient advance_block over a complete header succeeds"); + + assert_eq!(cache.block_number(), Some(12_345)); + assert_eq!(cache.basefee(), Some(42)); + assert_eq!(cache.coinbase(), Some(Address::repeat_byte(0xcb))); + assert_eq!(cache.prevrandao(), Some(B256::repeat_byte(0xab))); + assert_eq!(cache.block_gas_limit(), Some(30_000_000)); + assert_eq!(cache.timestamp(), Some(1_700_000_000 + 12_345)); + // The RPC pin must advance with the env: a lazy miss after the advance must + // fetch state at the NEW block, not the previously pinned one (review + // finding: env said N+1 while the SharedBackend still fetched at N). + assert_eq!( + cache.block(), + alloy_eips::BlockId::number(12_345), + "advance_block must re-pin RPC fetches to the advanced block" + ); + Ok(()) +} + +/// WS-2 / Phase-8 s2: under strict requirements, `advance_block` fails loudly on +/// a header missing a required field instead of silently defaulting it. +#[tokio::test] +async fn advance_block_strict_rejects_incomplete_header() -> Result<()> { + let mut cache = setup_cache().await?; + cache.set_block_context_requirements(BlockContextRequirements::strict()); + + let err = cache + .advance_block(&header(200, None)) + .expect_err("strict advance_block must reject a header with no base fee"); + assert!( + err.to_string().to_lowercase().contains("basefee") + || err.to_string().to_lowercase().contains("base fee"), + "the error must name the missing base-fee field, got: {err}" + ); + + // A complete header still refreshes under strict mode. + cache + .advance_block(&header(200, Some(9))) + .expect("strict advance_block over a complete header succeeds"); + assert_eq!(cache.basefee(), Some(9)); + Ok(()) +} + +// --- Additional coverage (implementation agent, Wave 4) -------------------- + +use std::sync::Arc; + +use alloy_network::Ethereum; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_transport::mock::Asserter; +use evm_fork_cache::EvmCacheBuilder; +use evm_fork_cache::reactive::{ + ChainStatus, InputSource, ReactiveConfig, ReactiveContext, ReactiveInput, ReactiveInputBatch, + ReactiveInputRecord, ReactiveReport, ReactiveRuntime, +}; + +/// Build a mocked provider (no network access) modelled on `common::setup_cache`. +fn mock_provider() -> Arc> { + let client = RpcClient::mocked(Asserter::new()); + Arc::new(RootProvider::::new(client)) +} + +/// WS-2: a strict `try_build` over a provider that yields no header must fail +/// loudly at construction rather than silently defaulting the block env. +#[tokio::test] +async fn try_build_strict_fails_when_header_unavailable() { + let result = EvmCacheBuilder::new(mock_provider()) + .strict_block_context(true) + .try_build() + .await; + // `EvmCache` is not `Debug`, so branch manually rather than `expect_err`. + let err = match result { + Ok(_) => panic!("strict try_build over a header-less mock provider must error"), + Err(err) => err, + }; + // The mock provider returns no block, so the header fetch fails. + assert!( + err.to_string().to_lowercase().contains("fetch failed"), + "expected a fetch-failure error, got: {err}" + ); +} + +/// WS-2: a lenient `try_build` never errors, even when no header is available — +/// preserving the infallible/lenient default construction behavior. +#[tokio::test] +async fn try_build_lenient_succeeds_without_header() -> Result<()> { + // Explicit lenient. + let cache = EvmCacheBuilder::new(mock_provider()) + .strict_block_context(false) + .try_build() + .await?; + // A header-less mock provider leaves the block env unset under lenient mode. + assert_eq!(cache.block_number(), None); + + // Default (no requirements configured) is lenient and also succeeds. + let _cache = EvmCacheBuilder::new(mock_provider()).try_build().await?; + Ok(()) +} + +/// Build the RPC-flavored `HeaderResponse` (`alloy_rpc_types_eth::Header`) that +/// the `Ethereum` reactive runtime ingests, from the in-memory consensus header. +fn rpc_header(number: u64, basefee: Option) -> alloy_rpc_types_eth::Header { + alloy_rpc_types_eth::Header::new(header(number, basefee)) +} + +/// A canonical (`Included`) context for a block header at `number`. +fn included_header_context(number: u64) -> ReactiveContext { + let block = evm_fork_cache::reactive::BlockRef { + number, + hash: B256::repeat_byte(0x11), + parent_hash: Some(B256::repeat_byte(0x10)), + timestamp: Some(1_700_000_000 + number), + }; + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: None, + log_index: None, + } +} + +/// Phase-8 s2: ingesting a canonical `BlockHeader` drives `advance_block`, so +/// the cache's block env is refreshed from the header without any handler. +#[tokio::test] +async fn reactive_ingest_of_canonical_header_refreshes_block_env() -> Result<()> { + let mut cache = setup_cache().await?; + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + + let input = ReactiveInput::BlockHeader(rpc_header(7_777, Some(123))); + let batch = ReactiveInputBatch::new(vec![ReactiveInputRecord::new( + input, + included_header_context(7_777), + )]); + + let report = runtime.ingest_batch(&mut cache, batch)?; + + // The env was refreshed from the ingested header. + assert_eq!(cache.block_number(), Some(7_777)); + assert_eq!(cache.basefee(), Some(123)); + assert_eq!(cache.timestamp(), Some(1_700_000_000 + 7_777)); + // Lenient default: no error report surfaced. + assert!( + !report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::Error(_))), + "a lenient canonical drive must not surface an error report" + ); + Ok(()) +} + +/// Phase-8 s2: a pending (non-canonical) header must NOT drive `advance_block`. +#[tokio::test] +async fn reactive_ingest_of_pending_header_does_not_refresh_block_env() -> Result<()> { + let mut cache = setup_cache().await?; + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + + let ctx = ReactiveContext { + chain_id: Some(1), + source: InputSource::Subscription, + chain_status: ChainStatus::Pending, + block: None, + transaction_index: None, + log_index: None, + }; + let input = ReactiveInput::BlockHeader(rpc_header(9_999, Some(55))); + let batch = ReactiveInputBatch::new(vec![ReactiveInputRecord::new(input, ctx)]); + + runtime.ingest_batch(&mut cache, batch)?; + + // Pending inputs never drive a canonical env refresh. + assert_eq!(cache.block_number(), None); + assert_eq!(cache.basefee(), None); + Ok(()) +} + +/// WS-2 / Phase-8 s2: under strict requirements, a canonical header missing a +/// required field surfaces a non-fatal `ReactiveReport::Error` (the batch is not +/// aborted). +#[tokio::test] +async fn reactive_strict_drive_surfaces_error_report_for_incomplete_header() -> Result<()> { + let mut cache = setup_cache().await?; + cache.set_block_context_requirements(BlockContextRequirements::strict()); + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + + // No base fee -> strict validation fails during the drive. + let input = ReactiveInput::BlockHeader(rpc_header(4_242, None)); + let batch = ReactiveInputBatch::new(vec![ReactiveInputRecord::new( + input, + included_header_context(4_242), + )]); + + let report = runtime.ingest_batch(&mut cache, batch)?; + + let error_message = report + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::Error(e) => Some(e.message.clone()), + _ => None, + }) + .expect("strict drive over an incomplete header must surface an error report"); + assert!( + error_message.to_lowercase().contains("basefee"), + "the error report must name the missing base-fee field, got: {error_message}" + ); + Ok(()) +} diff --git a/tests/builder_chain_id.rs b/tests/builder_chain_id.rs index cea0780..462c9b4 100644 --- a/tests/builder_chain_id.rs +++ b/tests/builder_chain_id.rs @@ -19,7 +19,9 @@ use alloy_provider::network::AnyNetwork; use alloy_rpc_client::RpcClient; use alloy_transport::mock::Asserter; use anyhow::Result; -use evm_fork_cache::cache::{CacheConfig, EvmCache, EvmCacheBuilder}; +use evm_fork_cache::cache::{ + CacheConfig, CacheSpeedMode, EvmCache, EvmCacheBuilder, StorageBatchConfig, +}; fn mock_provider() -> Arc> { Arc::new(RootProvider::::new(RpcClient::mocked( @@ -97,3 +99,28 @@ async fn set_chain_id_updates_after_construction() -> Result<()> { assert_eq!(cache.chain_id(), 137); Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn storage_batch_config_is_per_cache_instance() -> Result<()> { + let custom = StorageBatchConfig::new(321, 9); + let default_cache = EvmCacheBuilder::new(mock_provider()).build().await; + let fast_cache = EvmCacheBuilder::new(mock_provider()) + .speed_mode(CacheSpeedMode::Fast) + .build() + .await; + let custom_cache = EvmCacheBuilder::new(mock_provider()) + .storage_batch_config(custom) + .build() + .await; + + assert_eq!( + default_cache.storage_batch_config(), + StorageBatchConfig::from(CacheSpeedMode::Slow) + ); + assert_eq!( + fast_cache.storage_batch_config(), + StorageBatchConfig::from(CacheSpeedMode::Fast) + ); + assert_eq!(custom_cache.storage_batch_config(), custom); + Ok(()) +} diff --git a/tests/bulk_storage.rs b/tests/bulk_storage.rs new file mode 100644 index 0000000..3f1a05b --- /dev/null +++ b/tests/bulk_storage.rs @@ -0,0 +1,795 @@ +//! Bulk storage extraction: offline correctness tests. +//! +//! Two layers, both fully offline: +//! +//! 1. **EVM-level dogfood tests** — the exact bytes this crate sends over RPC +//! (extractor bytecode, Multicall3 `aggregate3` calldata, state-override +//! code) are executed inside the crate's own revm-backed cache against +//! seeded storage, proving the hand-written bytecode and the +//! encode/decode round-trip behave exactly as the RPC path assumes. +//! 2. **Mocked-transport fetcher tests** — the [`StorageBatchFetchFn`] +//! contract (one result per requested pair, per-slot error mapping, +//! fallback repair, runtime guards) over an [`Asserter`]-mocked provider. + +mod common; + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use alloy_eips::BlockId; +use alloy_primitives::{Address, Bytes, U256}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_sol_types::SolCall; +use alloy_transport::mock::Asserter; +use anyhow::{Result, anyhow}; +use evm_fork_cache::bulk_storage::{ + ACCOUNT_FIELDS_EXTRACTOR_CODE, BLOCK_CONTEXT_EXTRACTOR_CODE, BulkCallConfig, CallDispatch, + STORAGE_EXTRACTOR_CODE, STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI, StorageProgram, + bulk_call_storage_fetcher, bulk_call_storage_fetcher_with_fallback, + decode_multi_target_response, decode_packed_values, encode_multi_target_calldata, + multicall3_runtime_code, pack_slots_calldata, +}; +use evm_fork_cache::cache::{EvmCache, StorageBatchFetchFn}; +use evm_fork_cache::multicall::{IMulticall3, MULTICALL3_ADDRESS}; +use revm::context::result::ExecutionResult; +use revm::state::{AccountInfo, Bytecode}; + +use common::{install_default_account, setup_cache}; + +const CALLER: Address = Address::ZERO; + +fn target(byte: u8) -> Address { + Address::repeat_byte(byte) +} + +/// Install `code` at `addr` with the given seeded storage; unseeded slots read +/// as zero (never falling through to the mocked RPC), mirroring a fully-known +/// forked contract. +fn install_code_with_storage( + cache: &mut EvmCache, + addr: Address, + code: Bytes, + slots: &[(U256, U256)], +) { + let bytecode = Bytecode::new_raw(code); + let code_hash = bytecode.hash_slow(); + let info = AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(bytecode), + code_hash, + account_id: None, + }; + cache.db_mut().insert_account_info(addr, info); + cache + .db_mut() + .replace_account_storage(addr, Default::default()) + .expect("mark storage as fully local"); + for (slot, value) in slots { + cache + .db_mut() + .insert_account_storage(addr, *slot, *value) + .expect("seed storage slot"); + } +} + +fn run_call(cache: &mut EvmCache, to: Address, calldata: Bytes) -> Result { + match cache.call_raw(CALLER, to, calldata, false)? { + ExecutionResult::Success { output, .. } => Ok(output.into_data()), + other => Err(anyhow!("extractor call did not succeed: {other:?}")), + } +} + +// --------------------------------------------------------------------------- +// EVM-level dogfood tests: run the exact override bytecode inside revm. +// --------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn extractor_bytecode_reads_seeded_and_absent_slots() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + + let token = target(0xaa); + let seeded = [ + (U256::from(0u64), U256::from(0x1111u64)), + (U256::from(3u64), U256::MAX), + (U256::from(2u64).pow(U256::from(200u64)), U256::from(42u64)), + ]; + install_code_with_storage( + &mut cache, + token, + Bytes::from_static(STORAGE_EXTRACTOR_CODE), + &seeded, + ); + + // Query the three seeded slots, one absent slot, and a duplicate. + let query = vec![ + seeded[0].0, + seeded[1].0, + seeded[2].0, + U256::from(999u64), + seeded[1].0, + ]; + let out = run_call(&mut cache, token, pack_slots_calldata(&query))?; + let values = decode_packed_values(&out, query.len()).expect("exact word count"); + assert_eq!( + values, + vec![ + seeded[0].1, + seeded[1].1, + seeded[2].1, + U256::ZERO, + seeded[1].1 + ], + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn extractor_bytecode_returns_empty_for_empty_calldata() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + let token = target(0xab); + install_code_with_storage( + &mut cache, + token, + Bytes::from_static(STORAGE_EXTRACTOR_CODE), + &[], + ); + let out = run_call(&mut cache, token, Bytes::new())?; + assert!(out.is_empty()); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn pre_shanghai_extractor_matches_default_variant() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + + let seeded = [ + (U256::from(1u64), U256::from(0xdeadbeefu64)), + (U256::from(7u64), U256::from(1u64) << 255), + ]; + let (shanghai, legacy) = (target(0xa1), target(0xa2)); + install_code_with_storage( + &mut cache, + shanghai, + Bytes::from_static(STORAGE_EXTRACTOR_CODE), + &seeded, + ); + install_code_with_storage( + &mut cache, + legacy, + Bytes::from_static(STORAGE_EXTRACTOR_CODE_PRE_SHANGHAI), + &seeded, + ); + + let query = vec![seeded[0].0, U256::from(5u64), seeded[1].0]; + let calldata = pack_slots_calldata(&query); + let out_shanghai = run_call(&mut cache, shanghai, calldata.clone())?; + let out_legacy = run_call(&mut cache, legacy, calldata)?; + assert_eq!(out_shanghai, out_legacy); + assert_eq!( + decode_packed_values(&out_legacy, 3).expect("exact word count"), + vec![seeded[0].1, U256::ZERO, seeded[1].1], + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn multicall3_dispatch_extracts_across_targets() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + + // The dispatcher, exactly as the RPC path overrides it. + install_code_with_storage( + &mut cache, + MULTICALL3_ADDRESS, + multicall3_runtime_code().clone(), + &[], + ); + + let a = target(0x11); + let b = target(0x22); + install_code_with_storage( + &mut cache, + a, + Bytes::from_static(STORAGE_EXTRACTOR_CODE), + &[ + (U256::from(0u64), U256::from(101u64)), + (U256::from(1u64), U256::from(102u64)), + ], + ); + install_code_with_storage( + &mut cache, + b, + Bytes::from_static(STORAGE_EXTRACTOR_CODE), + &[(U256::from(0u64), U256::from(201u64))], + ); + + let targets = vec![ + ( + a, + vec![U256::from(0u64), U256::from(1u64), U256::from(2u64)], + ), + (b, vec![U256::from(0u64)]), + ]; + let out = run_call( + &mut cache, + MULTICALL3_ADDRESS, + encode_multi_target_calldata(&targets), + )?; + + let mut results = decode_multi_target_response(&targets, &out); + results.sort_by_key(|(addr, slot, _)| (*addr, *slot)); + let values: Vec = results + .iter() + .map(|(_, _, r)| *r.as_ref().expect("all subcalls succeed")) + .collect(); + assert_eq!( + values, + vec![ + U256::from(101u64), + U256::from(102u64), + U256::ZERO, + U256::from(201u64), + ], + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Mocked-transport fetcher tests. +// --------------------------------------------------------------------------- + +fn mocked_provider() -> (Arc>, Asserter) { + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter.clone()); + (Arc::new(RootProvider::::new(client)), asserter) +} + +fn counting_stub_fallback(counter: Arc, value: u64) -> StorageBatchFetchFn { + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + counter.fetch_add(requests.len(), Ordering::Relaxed); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::from(value)))) + .collect() + }) +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_returns_one_result_per_request_from_single_call() { + let (provider, asserter) = mocked_provider(); + let token = target(0x0a); + // Duplicates included: 4 requests, one eth_call, 4 packed words back. + let requests = vec![ + (token, U256::from(1u64)), + (token, U256::from(2u64)), + (token, U256::from(1u64)), + (token, U256::from(3u64)), + ]; + let values = [11u64, 22, 11, 33].map(U256::from); + asserter.push_success(&pack_slots_calldata(&values)); + + let fetcher = bulk_call_storage_fetcher(provider, BulkCallConfig::default()); + let results = fetcher(requests.clone(), BlockId::latest()); + + assert_eq!(results.len(), requests.len()); + assert!( + asserter.read_q().is_empty(), + "exactly one eth_call consumed" + ); + let mut expected: HashMap<(Address, U256), Vec> = HashMap::new(); + for ((addr, slot), value) in requests.iter().zip(values) { + expected.entry((*addr, *slot)).or_default().push(value); + } + for (addr, slot, result) in results { + let bucket = expected + .get_mut(&(addr, slot)) + .expect("result maps to a requested pair"); + let value = result.expect("mocked call succeeds"); + let pos = bucket + .iter() + .position(|v| *v == value) + .expect("value matches a remaining expectation"); + bucket.remove(pos); + } + assert!(expected.values().all(Vec::is_empty), "all pairs consumed"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_maps_transport_error_to_every_slot() { + let (provider, asserter) = mocked_provider(); + asserter.push_failure_msg("state overrides not supported"); + + let fetcher = bulk_call_storage_fetcher(provider, BulkCallConfig::default()); + let results = fetcher( + vec![ + (target(0x0b), U256::from(1u64)), + (target(0x0b), U256::from(2u64)), + ], + BlockId::latest(), + ); + assert_eq!(results.len(), 2); + assert!(results.iter().all(|(_, _, r)| r.is_err())); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_repairs_failed_slots_via_fallback() { + let (provider, asserter) = mocked_provider(); + asserter.push_failure_msg("state overrides not supported"); + + let repaired = Arc::new(AtomicUsize::new(0)); + let fetcher = bulk_call_storage_fetcher_with_fallback( + provider, + BulkCallConfig::default(), + counting_stub_fallback(repaired.clone(), 77), + ); + let requests = vec![ + (target(0x0c), U256::from(1u64)), + (target(0x0c), U256::from(2u64)), + (target(0x0d), U256::from(3u64)), + ]; + let results = fetcher(requests.clone(), BlockId::latest()); + + assert_eq!(results.len(), requests.len()); + assert_eq!(repaired.load(Ordering::Relaxed), requests.len()); + assert!( + results + .iter() + .all(|(_, _, r)| matches!(r, Ok(v) if *v == U256::from(77u64))), + "every failed slot repaired by the fallback" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_routes_tiny_requests_to_fallback_without_rpc() { + let (provider, asserter) = mocked_provider(); + // Nothing queued: any eth_call would surface as an error result. + + let point_reads = Arc::new(AtomicUsize::new(0)); + let fetcher = bulk_call_storage_fetcher_with_fallback( + provider, + BulkCallConfig::default(), // point_read_threshold = 2 + counting_stub_fallback(point_reads.clone(), 5), + ); + let results = fetcher(vec![(target(0x0e), U256::from(9u64))], BlockId::latest()); + + assert_eq!(point_reads.load(Ordering::Relaxed), 1); + assert!(matches!(results[0].2, Ok(v) if v == U256::from(5u64))); + assert!(asserter.read_q().is_empty()); +} + +#[tokio::test] +async fn fetcher_degrades_to_runtime_error_on_current_thread_runtime() { + let (provider, _asserter) = mocked_provider(); + let fetcher = bulk_call_storage_fetcher(provider, BulkCallConfig::default()); + let results = fetcher(vec![(target(0x0f), U256::from(1u64))], BlockId::latest()); + assert_eq!(results.len(), 1); + assert!( + matches!( + &results[0].2, + Err(evm_fork_cache::errors::StorageFetchError::Runtime(_)) + ), + "current-thread runtime must degrade to a typed error, got {:?}", + results[0].2 + ); +} + +#[tokio::test] +async fn fetcher_on_current_thread_runtime_still_repairs_via_fallback() { + let (provider, _asserter) = mocked_provider(); + let repaired = Arc::new(AtomicUsize::new(0)); + let fetcher = bulk_call_storage_fetcher_with_fallback( + provider, + BulkCallConfig::default(), + counting_stub_fallback(repaired.clone(), 3), + ); + // Two slots: above the point-read threshold, so the bulk path is + // attempted, fails the runtime guard, and the (synchronous) fallback + // repairs both pairs. + let results = fetcher( + vec![ + (target(0x1f), U256::from(1u64)), + (target(0x1f), U256::from(2u64)), + ], + BlockId::latest(), + ); + assert_eq!(repaired.load(Ordering::Relaxed), 2); + assert!(results.iter().all(|(_, _, r)| r.is_ok())); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_decodes_multicall_dispatch_response() { + let (provider, asserter) = mocked_provider(); + let (a, b) = (target(0x21), target(0x22)); + + // Two small targets pack into one aggregate3 dispatch; mock its response. + let response = IMulticall3::aggregate3Call::abi_encode_returns(&vec![ + IMulticall3::Result { + success: true, + returnData: pack_slots_calldata(&[U256::from(1001u64), U256::from(1002u64)]), + }, + IMulticall3::Result { + success: true, + returnData: pack_slots_calldata(&[U256::from(2001u64)]), + }, + ]); + asserter.push_success(&Bytes::from(response)); + + let fetcher = bulk_call_storage_fetcher(provider, BulkCallConfig::default()); + let mut results = fetcher( + vec![ + (a, U256::from(0u64)), + (a, U256::from(1u64)), + (b, U256::from(0u64)), + ], + BlockId::latest(), + ); + results.sort_by_key(|(addr, slot, _)| (*addr, *slot)); + let values: Vec = results + .into_iter() + .map(|(_, _, r)| r.expect("mocked dispatch succeeds")) + .collect(); + assert_eq!( + values, + vec![ + U256::from(1001u64), + U256::from(1002u64), + U256::from(2001u64) + ] + ); + assert!( + asserter.read_q().is_empty(), + "one eth_call for both targets" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_empty_request_is_a_no_op() { + let (provider, asserter) = mocked_provider(); + let fetcher = bulk_call_storage_fetcher(provider, BulkCallConfig::default()); + let results = fetcher(Vec::new(), BlockId::latest()); + assert!(results.is_empty()); + assert!(asserter.read_q().is_empty()); +} + +// --------------------------------------------------------------------------- +// Default-on integration, latch, dispatch modes, prewarm. +// --------------------------------------------------------------------------- + +/// Bulk extraction is the cache's DEFAULT batch fetcher: one queued eth_call +/// response serves a whole multi-slot batch. +#[tokio::test(flavor = "multi_thread")] +async fn cache_default_fetcher_is_bulk_extraction() -> Result<()> { + let (cache, asserter) = common::setup_cache_with_asserter().await?; + let token = target(0x31); + let values = [1u64, 2, 3].map(U256::from); + asserter.push_success(&pack_slots_calldata(&values)); + + let fetcher = cache + .storage_batch_fetcher() + .cloned() + .expect("default fetcher installed"); + let results = fetcher( + vec![ + (token, U256::from(0u64)), + (token, U256::from(1u64)), + (token, U256::from(2u64)), + ], + BlockId::latest(), + ); + assert_eq!(results.len(), 3); + assert!(results.iter().all(|(_, _, r)| r.is_ok())); + assert!( + asserter.read_q().is_empty(), + "three slots must consume exactly one eth_call" + ); + Ok(()) +} + +/// A non-default bulk config supplied through the builder is applied. +#[tokio::test(flavor = "multi_thread")] +async fn builder_bulk_call_config_is_applied() -> Result<()> { + use alloy_provider::RootProvider; + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter.clone()); + let provider = Arc::new(RootProvider::::new(client)); + let cache = evm_fork_cache::cache::EvmCache::builder(provider) + .bulk_call_config(BulkCallConfig { + max_slots_per_call: 1, + max_concurrent_calls: 1, + ..BulkCallConfig::default() + }) + .build() + .await; + + // With max_slots_per_call = 1, two slots must consume TWO eth_calls. + let token = target(0x32); + asserter.push_success(&pack_slots_calldata(&[U256::from(7u64)])); + asserter.push_success(&pack_slots_calldata(&[U256::from(8u64)])); + let fetcher = cache.storage_batch_fetcher().cloned().expect("fetcher"); + let results = fetcher( + vec![(token, U256::from(0u64)), (token, U256::from(1u64))], + BlockId::latest(), + ); + assert!(results.iter().all(|(_, _, r)| r.is_ok())); + assert!(asserter.read_q().is_empty(), "two single-slot eth_calls"); + Ok(()) +} + +/// After two consecutive fully-failed batches the fetcher latches to the +/// fallback and stops touching the provider. +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_latches_to_fallback_after_consecutive_provider_failures() { + let (provider, asserter) = mocked_provider(); + asserter.push_failure_msg("state overrides not supported"); + asserter.push_failure_msg("state overrides not supported"); + + let repaired = Arc::new(AtomicUsize::new(0)); + let fetcher = bulk_call_storage_fetcher_with_fallback( + provider, + BulkCallConfig::default(), + counting_stub_fallback(repaired.clone(), 4), + ); + let requests = vec![ + (target(0x33), U256::from(1u64)), + (target(0x33), U256::from(2u64)), + ]; + fetcher(requests.clone(), BlockId::latest()); // failure 1 (repaired) + fetcher(requests.clone(), BlockId::latest()); // failure 2 → latch + + // Queue a would-be success: a latched fetcher must leave it untouched. + asserter.push_success(&pack_slots_calldata(&[U256::from(1u64), U256::from(2u64)])); + let results = fetcher(requests.clone(), BlockId::latest()); + assert!( + !asserter.read_q().is_empty(), + "latched fetcher must not touch the provider" + ); + assert!( + results + .iter() + .all(|(_, _, r)| matches!(r, Ok(v) if *v == U256::from(4u64))), + "latched requests served entirely by the fallback" + ); + assert_eq!(repaired.load(Ordering::Relaxed), 6); +} + +/// CallMany dispatch: one eth_callMany request carries the whole batch. +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_call_many_dispatch_decodes_bundle_response() { + let (provider, asserter) = mocked_provider(); + let token = target(0x34); + let values = [11u64, 22].map(U256::from); + asserter.push_success(&serde_json::json!([[ + { "value": pack_slots_calldata(&values) } + ]])); + + let fetcher = bulk_call_storage_fetcher( + provider, + BulkCallConfig { + dispatch: CallDispatch::CallMany, + ..BulkCallConfig::default() + }, + ); + let results = fetcher( + vec![(token, U256::from(0u64)), (token, U256::from(1u64))], + BlockId::Number(alloy_eips::BlockNumberOrTag::Number(1000)), + ); + assert_eq!(results.len(), 2); + assert!(results.iter().all(|(_, _, r)| r.is_ok())); + assert!(asserter.read_q().is_empty(), "one eth_callMany request"); +} + +/// CallMany on a provider without the method: the same fetch transparently +/// re-dispatches per-call and still succeeds. +#[tokio::test(flavor = "multi_thread")] +async fn fetcher_call_many_failure_redispatches_per_call() { + let (provider, asserter) = mocked_provider(); + let token = target(0x35); + asserter.push_failure_msg("the method eth_callMany does not exist"); + asserter.push_success(&pack_slots_calldata(&[U256::from(5u64), U256::from(6u64)])); + + let fetcher = bulk_call_storage_fetcher( + provider, + BulkCallConfig { + dispatch: CallDispatch::CallMany, + max_concurrent_calls: 1, + ..BulkCallConfig::default() + }, + ); + let results = fetcher( + vec![(token, U256::from(0u64)), (token, U256::from(1u64))], + BlockId::Number(alloy_eips::BlockNumberOrTag::Number(1000)), + ); + assert!( + results.iter().all(|(_, _, r)| r.is_ok()), + "per-call re-dispatch must succeed: {results:?}" + ); + assert!(asserter.read_q().is_empty()); +} + +/// prewarm_slots loads through the (bulk) fetcher and injects into layer 2. +#[tokio::test(flavor = "multi_thread")] +async fn prewarm_slots_bulk_loads_into_cache() -> Result<()> { + let (mut cache, asserter) = common::setup_cache_with_asserter().await?; + let token = target(0x36); + let values = [9u64, 8].map(U256::from); + asserter.push_success(&pack_slots_calldata(&values)); + + let report = cache.prewarm_slots(&[(token, U256::from(0u64)), (token, U256::from(1u64))]); + assert_eq!(report.loaded, 2); + assert!(report.failed.is_empty()); + assert_eq!( + cache.cached_storage_value(token, U256::from(0u64)), + Some(U256::from(9u64)) + ); + assert_eq!( + cache.cached_storage_value(token, U256::from(1u64)), + Some(U256::from(8u64)) + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Custom programs & companion extractors: revm-level dogfood tests. +// --------------------------------------------------------------------------- + +/// The account-fields extractor bytecode, executed in revm: balance + +/// EXTCODEHASH per queried address, host reading *other* accounts. +#[tokio::test(flavor = "multi_thread")] +async fn account_fields_extractor_reads_other_accounts() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + + let host = target(0x41); + install_code_with_storage( + &mut cache, + host, + Bytes::from_static(ACCOUNT_FIELDS_EXTRACTOR_CODE), + &[], + ); + + // A funded contract account and an existing-but-empty account. + let funded = target(0x42); + let bytecode = Bytecode::new_raw(Bytes::from_static(STORAGE_EXTRACTOR_CODE)); + let code_hash = bytecode.hash_slow(); + cache.db_mut().insert_account_info( + funded, + AccountInfo { + balance: U256::from(12_345u64), + nonce: 1, + code: Some(bytecode), + code_hash, + account_id: None, + }, + ); + let empty = target(0x43); + install_default_account(&mut cache, empty); + + let mut calldata = Vec::new(); + for addr in [funded, empty] { + calldata.extend_from_slice(&[0u8; 12]); + calldata.extend_from_slice(addr.as_slice()); + } + let out = run_call(&mut cache, host, calldata.into())?; + assert_eq!(out.len(), 128, "two words per address"); + assert_eq!(U256::from_be_slice(&out[0..32]), U256::from(12_345u64)); + assert_eq!(&out[32..64], code_hash.as_slice()); + assert_eq!(U256::from_be_slice(&out[64..96]), U256::ZERO); + // EIP-1052: an existing empty account hashes to zero. + assert_eq!(U256::from_be_slice(&out[96..128]), U256::ZERO); + Ok(()) +} + +/// The block-context extractor bytecode, executed in revm: seven env words. +#[tokio::test(flavor = "multi_thread")] +async fn block_context_extractor_returns_env_words() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + let host = target(0x44); + install_code_with_storage( + &mut cache, + host, + Bytes::from_static(BLOCK_CONTEXT_EXTRACTOR_CODE), + &[], + ); + cache.set_block(BlockId::Number(alloy_eips::BlockNumberOrTag::Number(42))); + + let out = run_call(&mut cache, host, Bytes::new())?; + assert_eq!(out.len(), 7 * 32); + let word = |i: usize| U256::from_be_slice(&out[i * 32..(i + 1) * 32]); + assert_eq!(word(0), U256::from(42u64), "NUMBER = pinned block"); + assert_eq!(word(6), U256::from(1u64), "CHAINID = cache default"); + Ok(()) +} + +/// A custom storage *program*: the one-shot Uniswap-V3-style observation-ring +/// loader — reads the ring cardinality out of slot0 in-EVM, then returns the +/// whole ring, with zero calldata. Mirrors the program demonstrated live in +/// `examples/bulk_storage_bench.rs`. +const OBSERVATION_RING_PROGRAM: &[u8] = &alloy_primitives::hex!( + "5f5460c81c61ffff165f5b81811460215780600801548160051b52600101600a565b5060051b5ff3" +); + +#[tokio::test(flavor = "multi_thread")] +async fn observation_ring_program_reads_data_dependent_slots() -> Result<()> { + let mut cache = setup_cache().await?; + install_default_account(&mut cache, CALLER); + + let pool = target(0x45); + // slot0 layout puts observationCardinality at bits 200..216; pack junk + // around it to prove the masking. Ring entries live at slots 8+i. + let cardinality = 3u64; + let slot0 = (U256::from(cardinality) << 200usize) + | (U256::from(77u64) << 160usize) + | U256::from(12_345u64); + let ring = [1001u64, 1002, 1003].map(U256::from); + install_code_with_storage( + &mut cache, + pool, + Bytes::from_static(OBSERVATION_RING_PROGRAM), + &[ + (U256::from(0u64), slot0), + (U256::from(8u64), ring[0]), + (U256::from(9u64), ring[1]), + (U256::from(10u64), ring[2]), + ], + ); + + let out = run_call(&mut cache, pool, Bytes::new())?; + let values = decode_packed_values(&out, cardinality as usize).expect("cardinality words"); + assert_eq!(values, ring.to_vec()); + Ok(()) +} + +/// run_storage_programs batches distinct targets into one dispatch (mocked). +#[tokio::test(flavor = "multi_thread")] +async fn storage_programs_batch_distinct_targets() { + let (provider, asserter) = mocked_provider(); + let programs = vec![ + StorageProgram { + target: target(0x46), + code: Bytes::from_static(OBSERVATION_RING_PROGRAM), + calldata: Bytes::new(), + }, + StorageProgram { + target: target(0x47), + code: Bytes::from_static(STORAGE_EXTRACTOR_CODE), + calldata: pack_slots_calldata(&[U256::from(1u64)]), + }, + ]; + let response = IMulticall3::aggregate3Call::abi_encode_returns(&vec![ + IMulticall3::Result { + success: true, + returnData: Bytes::from(vec![0xAA; 32]), + }, + IMulticall3::Result { + success: false, + returnData: Bytes::new(), + }, + ]); + asserter.push_success(&Bytes::from(response)); + + let rt_provider = provider.clone(); + let results = evm_fork_cache::bulk_storage::run_storage_programs( + rt_provider.as_ref(), + BlockId::latest(), + &programs, + ) + .await; + assert_eq!(results.len(), 2); + assert_eq!( + results[0].as_ref().expect("first program"), + &Bytes::from(vec![0xAA; 32]) + ); + assert!(results[1].is_err(), "failed subcall surfaces per-program"); + assert!( + asserter.read_q().is_empty(), + "one dispatch for both programs" + ); +} diff --git a/tests/bundle_simulation.rs b/tests/bundle_simulation.rs index 438dda8..21e7200 100644 --- a/tests/bundle_simulation.rs +++ b/tests/bundle_simulation.rs @@ -74,7 +74,7 @@ async fn token_with_funded_alice() #[tokio::test(flavor = "multi_thread")] async fn bundle_applies_txs_over_cumulative_state() -> Result<()> { let (mut cache, token, alice, bob, carol) = token_with_funded_alice().await?; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let txs = vec![ BundleTx::new(alice, token, transfer_calldata(bob, 100)), @@ -113,7 +113,7 @@ async fn bundle_applies_txs_over_cumulative_state() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn atomic_bundle_reverts_whole_on_failure() -> Result<()> { let (mut cache, token, alice, bob, _carol) = token_with_funded_alice().await?; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let txs = vec![ BundleTx::new(alice, token, transfer_calldata(bob, 100)), @@ -147,7 +147,7 @@ async fn atomic_bundle_reverts_whole_on_failure() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn allow_reverts_keeps_prior_effects() -> Result<()> { let (mut cache, token, alice, bob, _carol) = token_with_funded_alice().await?; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let txs = vec![ BundleTx::new(alice, token, transfer_calldata(bob, 100)), @@ -207,7 +207,7 @@ async fn direct_coinbase_payment_is_captured() -> Result<()> { }, ); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let result = overlay.simulate_bundle( std::slice::from_ref(&tx), &BundleOptions { @@ -254,7 +254,7 @@ async fn coinbase_payment_is_priority_fee_only() -> Result<()> { }, ); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let result = overlay.simulate_bundle(std::slice::from_ref(&tx), &BundleOptions::default())?; assert!(result.gas_used > 0); @@ -272,7 +272,7 @@ async fn coinbase_payment_is_priority_fee_only() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn commit_flag_controls_overlay_persistence() -> Result<()> { let (mut cache, token, alice, bob, _carol) = token_with_funded_alice().await?; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); // commit = false: overlay unchanged afterward. let mut overlay = EvmOverlay::new(snapshot.clone(), None); @@ -300,3 +300,78 @@ async fn commit_flag_controls_overlay_persistence() -> Result<()> { ); Ok(()) } + +/// WS-7 (manager-authored red-green): cost-accounting breakdown. After an +/// `AllowReverts` bundle whose whitelisted tx reverts, the reverted tx's gas is +/// excluded from `coinbase_payment` (the honest miner receipt) but is exposed via +/// `reverted_tx_gas`, and `successful_tx_gas + reverted_tx_gas == gas_used`. This +/// lets a searcher compute net cost (`coinbase_payment + reverted_tx_gas`) without +/// manually iterating `per_tx`. +#[tokio::test(flavor = "multi_thread")] +async fn allow_reverts_exposes_reverted_and_successful_gas() -> Result<()> { + let (mut cache, token, alice, bob, _carol) = token_with_funded_alice().await?; + let mut overlay = EvmOverlay::new(cache.snapshot(), None); + + let txs = vec![ + BundleTx::new(alice, token, transfer_calldata(bob, 100)), + BundleTx::new(alice, token, Bytes::from(vec![0xde, 0xad, 0xbe, 0xef])), + ]; + let result = overlay.simulate_bundle( + &txs, + &BundleOptions { + revert_policy: RevertPolicy::AllowReverts(vec![1]), + commit: true, + }, + )?; + + assert!(result.succeeded); + assert!(!result.per_tx[0].reverted); + assert!(result.per_tx[1].reverted); + + assert_eq!( + result.successful_tx_gas, result.per_tx[0].gas_used, + "successful bucket = kept tx gas" + ); + assert_eq!( + result.reverted_tx_gas, result.per_tx[1].gas_used, + "reverted bucket = reverted tx gas" + ); + assert!( + result.reverted_tx_gas > 0, + "the reverted whitelisted tx still consumed gas" + ); + assert_eq!( + result.successful_tx_gas + result.reverted_tx_gas, + result.gas_used, + "the two buckets must reconstruct total gas_used" + ); + Ok(()) +} + +/// WS-7 (manager-authored red-green): a fully successful bundle reports zero +/// reverted gas and all gas in the successful bucket. +#[tokio::test(flavor = "multi_thread")] +async fn successful_bundle_reports_zero_reverted_gas() -> Result<()> { + let (mut cache, token, alice, bob, carol) = token_with_funded_alice().await?; + let mut overlay = EvmOverlay::new(cache.snapshot(), None); + + let txs = vec![ + BundleTx::new(alice, token, transfer_calldata(bob, 100)), + BundleTx::new(bob, token, transfer_calldata(carol, 30)), + ]; + let result = overlay.simulate_bundle( + &txs, + &BundleOptions { + commit: true, + ..Default::default() + }, + )?; + + assert!(result.succeeded); + assert_eq!(result.reverted_tx_gas, 0, "no tx reverted"); + assert_eq!( + result.successful_tx_gas, result.gas_used, + "all gas is in the successful bucket" + ); + Ok(()) +} diff --git a/tests/bundle_simulation_extra.rs b/tests/bundle_simulation_extra.rs index 2a8a968..0cc2703 100644 --- a/tests/bundle_simulation_extra.rs +++ b/tests/bundle_simulation_extra.rs @@ -64,7 +64,7 @@ async fn token_with_funded_alice() -> Result<(EvmCache, Address, Address, Addres async fn empty_bundle_succeeds_as_noop() -> Result<()> { let mut cache = setup_cache().await?; install_default_account(&mut cache, Address::ZERO); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let result = overlay.simulate_bundle(&[], &BundleOptions::default())?; assert!(result.succeeded); @@ -79,7 +79,7 @@ async fn empty_bundle_succeeds_as_noop() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn allow_reverts_non_whitelisted_index_aborts_atomically() -> Result<()> { let (mut cache, token, alice, bob) = token_with_funded_alice().await?; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let txs = vec![ BundleTx::new(alice, token, transfer_calldata(bob, 100)), @@ -111,7 +111,7 @@ async fn allow_reverts_non_whitelisted_index_aborts_atomically() -> Result<()> { #[tokio::test(flavor = "multi_thread")] async fn allow_reverts_commit_false_is_isolated() -> Result<()> { let (mut cache, token, alice, bob) = token_with_funded_alice().await?; - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let txs = vec![ BundleTx::new(alice, token, transfer_calldata(bob, 100)), @@ -154,7 +154,7 @@ async fn cache_simulate_bundle_does_not_mutate_cache() -> Result<()> { assert_eq!(result.per_tx.len(), 1); // The cache is unchanged: a fresh snapshot still shows alice's full balance. - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); assert_eq!( overlay_balance(&mut overlay, token, alice)?, U256::from(1_000u64) diff --git a/tests/cache_state.rs b/tests/cache_state.rs index b01c6fa..2149729 100644 --- a/tests/cache_state.rs +++ b/tests/cache_state.rs @@ -1,5 +1,5 @@ //! Offline integration tests for `EvmCache` state manipulation: balance -//! overrides via storage-slot scanning, snapshot/restore, two-layer cache +//! overrides via storage-slot scanning, checkpoint/restore, two-layer cache //! purging, and contract deployment/etching. //! //! All state is injected directly over a mocked provider, so these tests run @@ -22,6 +22,7 @@ use common::{ mock_erc20_creation_code, mock_erc20_runtime, setup_cache, transfer, }; use evm_fork_cache::cache::{EvmCache, TxConfig}; +use evm_fork_cache::errors::CacheError; /// Deterministic CREATE address for `Address::ZERO` at nonce 0: /// `keccak256(rlp([ZERO, 0]))[12..]`. @@ -67,7 +68,7 @@ async fn snapshot_restore_reverts_token_state() -> Result<()> { assert_eq!(balance_of(&mut cache, token, owner)?, initial_balance); - let snapshot = cache.snapshot(); + let checkpoint = cache.checkpoint(); transfer(&mut cache, token, owner, recipient, U256::from(123u64))?; assert_eq!( @@ -75,7 +76,7 @@ async fn snapshot_restore_reverts_token_state() -> Result<()> { initial_balance - U256::from(123u64) ); - cache.restore(snapshot); + cache.restore(checkpoint); assert_eq!(balance_of(&mut cache, token, owner)?, initial_balance); Ok(()) @@ -388,7 +389,7 @@ async fn balance_delta_post_read_error_reverts_target_checkpoint() -> Result<()> .simulate_call_with_balance_deltas(owner, token, Bytes::new(), owner, [token], true) .expect_err("post balanceOf failure must surface as an error"); assert!( - err.to_string().contains("balanceOf call failed"), + matches!(err, CacheError::CallNotSuccessful { .. }), "unexpected error: {err:#}" ); assert_eq!( diff --git a/tests/call_tracer.rs b/tests/call_tracer.rs index 8c1f1ea..e725433 100644 --- a/tests/call_tracer.rs +++ b/tests/call_tracer.rs @@ -62,7 +62,7 @@ async fn tracer_captures_single_top_level_frame() -> Result<()> { install_default_account(&mut cache, caller); install_mock_erc20(&mut cache, token); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let calldata = balance_of_calldata(caller); let (_result, tracer) = overlay.call_raw_with_inspector( caller, @@ -103,7 +103,7 @@ async fn tracer_captures_nested_subcalls() -> Result<()> { }]; let calldata = Bytes::from(IMulticall3::aggregate3Call { calls }.abi_encode()); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let (_result, tracer) = overlay.call_raw_with_inspector( caller, MULTICALL3_ADDRESS, @@ -134,7 +134,7 @@ async fn tracer_attributes_reverts() -> Result<()> { install_default_account(&mut cache, caller); install_mock_erc20(&mut cache, token); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let (_result, tracer) = overlay.call_raw_with_inspector( caller, token, @@ -176,7 +176,7 @@ async fn inspector_stack_composes_tracer_and_transfer() -> Result<()> { .abi_encode(), ); - let mut overlay = EvmOverlay::new(cache.create_snapshot(), None); + let mut overlay = EvmOverlay::new(cache.snapshot(), None); let (_result, stack) = overlay.call_raw_with_inspector( alice, token, diff --git a/tests/code_seeding.rs b/tests/code_seeding.rs new file mode 100644 index 0000000..7a0193a --- /dev/null +++ b/tests/code_seeding.rs @@ -0,0 +1,662 @@ +//! Verified code seeding & local etch — acceptance tests. +//! +//! Covers the spec's acceptance items 1–8 and 10 +//! (docs/verified-code-seeding-spec.md §7): verification outcomes and their +//! fail-closed/fail-safe split, conflict rules, etch unification, persistence +//! round-trips, snapshot-generation semantics, and the no-`basic_ref` +//! guarantee for seeded accounts. Every test runs fully offline over a mocked +//! provider; verification reads go through stubbed [`AccountFieldsFetchFn`]s. + +mod common; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use alloy_primitives::{Address, B256, Bytes, U256}; +use alloy_provider::RootProvider; +use alloy_provider::network::AnyNetwork; +use alloy_rpc_client::RpcClient; +use alloy_transport::mock::Asserter; +use anyhow::Result; +use common::{ + failing_fields_fetcher, install_default_account, install_mock_erc20, mock_erc20_runtime, + setup_cache, stub_fields_fetcher, +}; +use evm_fork_cache::cache::{CacheConfig, CodeSeedState, EvmCache}; +use evm_fork_cache::errors::CacheError; +use evm_fork_cache::multicall::MULTICALL3_ADDRESS; +use revm::context::result::ExecutionResult; + +/// Minimal runtime: `PUSH1 01 PUSH1 00 MSTORE PUSH1 20 PUSH1 00 RETURN` — +/// returns one 32-byte word (value 1) and touches no storage or env, so a +/// call against it can never fall through to the (mocked) RPC backend. +const RETURN_ONE_RUNTIME: [u8; 10] = [0x60, 0x01, 0x60, 0x00, 0x52, 0x60, 0x20, 0x60, 0x00, 0xf3]; + +/// Creation code deploying [`RETURN_ONE_RUNTIME`]: +/// `PUSH1 0a PUSH1 0c PUSH1 00 CODECOPY PUSH1 0a PUSH1 00 RETURN` + runtime. +fn return_one_creation_code() -> Vec { + let mut creation = vec![ + 0x60, 0x0a, 0x60, 0x0c, 0x60, 0x00, 0x39, 0x60, 0x0a, 0x60, 0x00, 0xf3, + ]; + creation.extend_from_slice(&RETURN_ONE_RUNTIME); + creation +} + +fn return_one_bytes() -> Bytes { + Bytes::from(RETURN_ONE_RUNTIME.to_vec()) +} + +fn mock_erc20_bytes() -> Bytes { + mock_erc20_runtime().original_bytes() +} + +/// A per-test temp cache directory, keyed by pid so concurrent `cargo test` +/// processes never share (or delete) each other's directory. +fn temp_cache_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "evm_fork_cache_code_seeding_{tag}_{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +/// Build a cache over a mocked provider with disk persistence at `dir`. +async fn setup_cache_with_config(dir: &PathBuf) -> EvmCache { + let asserter = Asserter::new(); + let client = RpcClient::mocked(asserter); + let provider = RootProvider::::new(client); + let cfg = CacheConfig::new(dir, 1, Default::default(), Default::default()); + EvmCache::builder(Arc::new(provider)) + .cache_config(cfg) + .build() + .await +} + +/// Spec item 5a: seeding an unmarked (RPC-origin) account whose code hash +/// already matches is an instant `Verified` — zero RPC, zero writes. +#[tokio::test(flavor = "multi_thread")] +async fn seed_over_rpc_origin_equal_hash_is_instant_verified() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x10); + install_mock_erc20(&mut cache, token); + + let generation_before = cache.snapshot_generation(); + let hash = cache.seed_account_code(token, mock_erc20_bytes())?; + + match cache.code_seed_state(&token) { + Some(CodeSeedState::Verified { code_hash, .. }) => assert_eq!(*code_hash, hash), + other => panic!("expected instant Verified, got {other:?}"), + } + assert!( + cache.pending_code_seeds().is_empty(), + "instant verification must not leave a Pending claim" + ); + assert_eq!( + cache.snapshot_generation(), + generation_before, + "the zero-write instant-Verified path must not bump the generation" + ); + Ok(()) +} + +/// Spec item 5b: a seed contradicting RPC-origin code (or an EOA) is a +/// `CodeSeedConflict`, and the cached code stays untouched. +#[tokio::test(flavor = "multi_thread")] +async fn seed_conflicting_with_rpc_origin_errors_and_keeps_cached_code() -> Result<()> { + let mut cache = setup_cache().await?; + let token = Address::repeat_byte(0x11); + install_mock_erc20(&mut cache, token); + let original_hash = mock_erc20_runtime().hash_slow(); + + let err = cache + .seed_account_code(token, return_one_bytes()) + .expect_err("conflicting seed must be rejected"); + match err { + CacheError::CodeSeedConflict { + address, cached, .. + } => { + assert_eq!(address, token); + assert_eq!(cached, original_hash); + } + other => panic!("expected CodeSeedConflict, got {other:?}"), + } + assert!( + cache.code_seed_state(&token).is_none(), + "a rejected seed must not leave a mark" + ); + + // A code-less EOA is chain knowledge too: seeding over it conflicts. + let eoa = Address::repeat_byte(0x12); + install_default_account(&mut cache, eoa); + assert!(matches!( + cache.seed_account_code(eoa, return_one_bytes()), + Err(CacheError::CodeSeedConflict { .. }) + )); + + // Empty bytes are not a seedable claim at all. + let fresh = Address::repeat_byte(0x13); + assert!(matches!( + cache.seed_account_code(fresh, Bytes::new()), + Err(CacheError::CodeSeedEmpty { .. }) + )); + Ok(()) +} + +/// Spec conflict-table rows 1 + 4: an absent address seeds as `Pending`, and +/// re-seeding a marked address overwrites and restarts the claim. +#[tokio::test(flavor = "multi_thread")] +async fn seed_absent_is_pending_and_reseed_restarts_claim() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x20); + + let first_hash = cache.seed_account_code(pool, mock_erc20_bytes())?; + assert!(matches!( + cache.code_seed_state(&pool), + Some(CodeSeedState::Pending { code_hash }) if *code_hash == first_hash + )); + assert_eq!(cache.pending_code_seeds(), vec![pool]); + + // Re-seed with different bytes: the claim restarts as Pending under the + // new hash — never a conflict for an already-marked address. + let second_hash = cache.seed_account_code(pool, return_one_bytes())?; + assert_ne!(first_hash, second_hash); + assert!(matches!( + cache.code_seed_state(&pool), + Some(CodeSeedState::Pending { code_hash }) if *code_hash == second_hash + )); + Ok(()) +} + +/// Spec item 6 (first half): etched code is marked, excluded from the pending +/// set, and actually executed by simulations. +#[tokio::test(flavor = "multi_thread")] +async fn etch_marks_account_and_sims_read_etched_code() -> Result<()> { + let mut cache = setup_cache().await?; + let target = Address::repeat_byte(0x30); + let caller = Address::repeat_byte(0x31); + install_default_account(&mut cache, caller); + // The default beneficiary; revm touches it post-execution. + install_default_account(&mut cache, Address::ZERO); + + let hash = cache.etch_account_code(target, return_one_bytes())?; + assert!(matches!( + cache.code_seed_state(&target), + Some(CodeSeedState::Etched { code_hash }) if *code_hash == hash + )); + assert_eq!(cache.etched_accounts(), vec![target]); + assert!( + cache.pending_code_seeds().is_empty(), + "etched accounts are never part of the canonical verify set" + ); + + let result = cache.call_raw(caller, target, Bytes::new(), false)?; + match result { + ExecutionResult::Success { output, .. } => { + assert_eq!( + output.into_data(), + Bytes::from(U256::from(1).to_be_bytes::<32>().to_vec()), + "the simulation must execute the etched runtime" + ); + } + other => panic!("call against etched code failed: {other:?}"), + } + Ok(()) +} + +/// Spec item 6 (second half): every locally-divergent code site joins the +/// etched set — `override_account_code` targets and `deploy_contract` +/// creations included. +#[tokio::test(flavor = "multi_thread")] +async fn override_and_deploy_targets_join_the_etched_set() -> Result<()> { + let mut cache = setup_cache().await?; + let source = Address::repeat_byte(0x40); + let target = Address::repeat_byte(0x41); + install_mock_erc20(&mut cache, source); + install_mock_erc20(&mut cache, target); + + cache.override_account_code(source, target)?; + assert!( + matches!( + cache.code_seed_state(&target), + Some(CodeSeedState::Etched { .. }) + ), + "an override target is local divergence and must be etched-marked" + ); + assert!( + cache.code_seed_state(&source).is_none(), + "the override source is untouched chain state" + ); + + let deployer = Address::repeat_byte(0x42); + install_default_account(&mut cache, deployer); + // Pre-materialize the accounts revm touches during the create so the + // mocked backend is never consulted: the beneficiary and the (nonce-0) + // deterministic create target. + install_default_account(&mut cache, Address::ZERO); + install_default_account(&mut cache, deployer.create(0)); + let deployed = cache.deploy_contract(deployer, Bytes::from(return_one_creation_code()))?; + match cache.code_seed_state(&deployed) { + Some(CodeSeedState::Etched { code_hash }) => { + assert_eq!( + *code_hash, + revm::state::Bytecode::new_raw(return_one_bytes()).hash_slow(), + "the etched mark must record the deployed runtime's hash" + ); + } + other => panic!("deployed contract must be etched-marked, got {other:?}"), + } + + let etched = cache.etched_accounts(); + assert!(etched.contains(&target) && etched.contains(&deployed)); + Ok(()) +} + +/// An account-scope purge clears the mark along with both state layers (the +/// re-seed escape hatch after a believed redeploy). +#[tokio::test(flavor = "multi_thread")] +async fn purge_account_clears_the_mark() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x50); + cache.seed_account_code(pool, return_one_bytes())?; + assert!(cache.code_seed_state(&pool).is_some()); + + cache.purge_account(pool); + assert!( + cache.code_seed_state(&pool).is_none(), + "purge must clear the code-seed mark" + ); + assert!(cache.pending_code_seeds().is_empty()); + Ok(()) +} + +/// Spec item 8 (C1 portion): seed and etch bump the snapshot generation; the +/// rejected-conflict path does not. +#[tokio::test(flavor = "multi_thread")] +async fn generation_bumps_on_seed_and_etch_not_on_rejects() -> Result<()> { + let mut cache = setup_cache().await?; + + let g0 = cache.snapshot_generation(); + cache.seed_account_code(Address::repeat_byte(0x60), return_one_bytes())?; + let g1 = cache.snapshot_generation(); + assert_ne!(g0, g1, "a fresh seed writes executable state and must bump"); + + cache.etch_account_code(Address::repeat_byte(0x61), return_one_bytes())?; + let g2 = cache.snapshot_generation(); + assert_ne!(g1, g2, "an etch mutates executable state and must bump"); + + let token = Address::repeat_byte(0x62); + install_mock_erc20(&mut cache, token); + let g3 = cache.snapshot_generation(); + let _ = cache.seed_account_code(token, return_one_bytes()); + assert_eq!( + cache.snapshot_generation(), + g3, + "a rejected (conflicting) seed writes nothing and must not bump" + ); + Ok(()) +} + +/// Spec item 7: all three mark kinds survive a flush + reload; a fresh cache +/// directory has no marks (missing `code_seeds.bin` is a clean miss). +#[tokio::test(flavor = "multi_thread")] +async fn persistence_round_trip_preserves_marks() -> Result<()> { + let dir = temp_cache_dir("roundtrip"); + let pending_addr = Address::repeat_byte(0x70); + let verified_addr = Address::repeat_byte(0x71); + let etched_addr = Address::repeat_byte(0x72); + + let (pending_hash, verified_hash, etched_hash) = { + let mut cache = setup_cache_with_config(&dir).await; + let pending_hash = cache.seed_account_code(pending_addr, mock_erc20_bytes())?; + // Instant-Verified via the equal-hash fast path over an RPC-origin + // account. Real RPC-origin accounts land in layer 2 (the backend + // map, which is what persists) — mirror that here. + let bytecode = mock_erc20_runtime(); + let info = revm::state::AccountInfo { + balance: U256::ZERO, + nonce: 1, + code_hash: bytecode.hash_slow(), + code: Some(bytecode), + account_id: None, + }; + cache.with_blockchain_db_mut(|db| { + db.accounts().write().insert(verified_addr, info); + }); + let verified_hash = cache.seed_account_code(verified_addr, mock_erc20_bytes())?; + let etched_hash = cache.etch_account_code(etched_addr, return_one_bytes())?; + cache.flush()?; + (pending_hash, verified_hash, etched_hash) + }; + + let reloaded = setup_cache_with_config(&dir).await; + assert!( + matches!( + reloaded.code_seed_state(&pending_addr), + Some(CodeSeedState::Pending { code_hash }) if *code_hash == pending_hash + ), + "Pending must reload as Pending — never masquerading as RPC-origin" + ); + assert!(matches!( + reloaded.code_seed_state(&verified_addr), + Some(CodeSeedState::Verified { code_hash, .. }) if *code_hash == verified_hash + )); + assert!(matches!( + reloaded.code_seed_state(&etched_addr), + Some(CodeSeedState::Etched { code_hash }) if *code_hash == etched_hash + )); + assert_eq!(reloaded.pending_code_seeds(), vec![pending_addr]); + assert_eq!(reloaded.etched_accounts(), vec![etched_addr]); + + // A brand-new cache directory has no code_seeds.bin: clean empty miss. + let fresh_dir = temp_cache_dir("fresh"); + let fresh = setup_cache_with_config(&fresh_dir).await; + assert!(fresh.pending_code_seeds().is_empty()); + assert!(fresh.etched_accounts().is_empty()); + + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&fresh_dir); + Ok(()) +} + +/// Spec item 7 (pruning): a mark whose code did not survive the reload is +/// dropped rather than describing state that no longer exists. +#[tokio::test(flavor = "multi_thread")] +async fn marks_without_surviving_code_are_pruned_on_load() -> Result<()> { + let dir = temp_cache_dir("prune"); + let pool = Address::repeat_byte(0x80); + + { + let mut cache = setup_cache_with_config(&dir).await; + cache.seed_account_code(pool, return_one_bytes())?; + cache.flush()?; + } + + // Wipe the state + bytecode files, keeping only code_seeds.bin: on + // reload the marked account has no code, so the mark must be pruned. + let chain_dir = dir.join("chain_1"); + std::fs::remove_file(chain_dir.join("evm_state.bin"))?; + std::fs::remove_file(chain_dir.join("bytecodes.bin"))?; + assert!(chain_dir.join("code_seeds.bin").exists()); + + let reloaded = setup_cache_with_config(&dir).await; + assert!( + reloaded.code_seed_state(&pool).is_none(), + "a mark without surviving code must be pruned on load" + ); + + let _ = std::fs::remove_dir_all(&dir); + Ok(()) +} + +/// Spec item 10: a seeded account is fully materialized, so neither +/// `ensure_account` nor an EVM call ever reaches the RPC backend for it. The +/// mocked provider has an empty response queue — any RPC attempt would error. +#[tokio::test(flavor = "multi_thread")] +async fn seeded_account_never_triggers_the_backend_triple() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0x90); + let caller = Address::repeat_byte(0x91); + install_default_account(&mut cache, caller); + // The default beneficiary; revm touches it post-execution. Installing it + // keeps this test's assertion focused on the *seeded* account. + install_default_account(&mut cache, Address::ZERO); + + cache.seed_account_code(pool, return_one_bytes())?; + + // ensure_account early-returns for a present account; a backend fetch + // against the empty mock queue would fail loudly. + cache.ensure_account(pool).await?; + + let result = cache.call_raw(caller, pool, Bytes::new(), false)?; + assert!( + matches!(result, ExecutionResult::Success { .. }), + "call against a seeded account must succeed without any RPC" + ); + Ok(()) +} + +/// Spec item 1: a matching verification marks `Verified`, patches the real +/// balance from the same response without a generation bump, and the fetcher +/// is called exactly once ever — a settled set costs nothing. +#[tokio::test(flavor = "multi_thread")] +async fn verify_match_marks_verified_and_patches_balance_once() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0xa0); + let expected = cache.seed_account_code(pool, return_one_bytes())?; + + let calls = Arc::new(AtomicUsize::new(0)); + let balance = U256::from(7_777u64); + cache.set_account_fields_fetcher(stub_fields_fetcher( + HashMap::from([(pool, (balance, expected))]), + calls.clone(), + )); + + let generation_before = cache.snapshot_generation(); + let report = cache.verify_code_seeds()?; + assert_eq!(report.verified, vec![pool]); + assert!(report.mismatched.is_empty() && report.unverifiable.is_empty()); + assert!(matches!( + cache.code_seed_state(&pool), + Some(CodeSeedState::Verified { code_hash, .. }) if *code_hash == expected + )); + assert_eq!( + cache.snapshot_generation(), + generation_before, + "a verify-match materializes pinned-block truth and must not bump" + ); + assert_eq!( + cache + .db_mut() + .cache + .accounts + .get(&pool) + .expect("seeded account present") + .info + .balance, + balance, + "the on-chain balance from the verification sample must be patched in" + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + // A second sweep has nothing pending: the fetcher is never re-consulted. + let second = cache.verify_code_seeds()?; + assert!(second.verified.is_empty()); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "Verified seeds are never re-verified" + ); + Ok(()) +} + +/// Spec item 1 (persistence half): `Verified` survives a save/load and is +/// still never re-queried — the reloaded cache issues zero fields calls. +#[tokio::test(flavor = "multi_thread")] +async fn verified_seed_is_not_requeried_across_save_load() -> Result<()> { + let dir = temp_cache_dir("verified_reload"); + let pool = Address::repeat_byte(0xa1); + + { + let mut cache = setup_cache_with_config(&dir).await; + let expected = cache.seed_account_code(pool, return_one_bytes())?; + let calls = Arc::new(AtomicUsize::new(0)); + cache.set_account_fields_fetcher(stub_fields_fetcher( + HashMap::from([(pool, (U256::ZERO, expected))]), + calls.clone(), + )); + cache.verify_code_seeds()?; + assert_eq!(calls.load(Ordering::SeqCst), 1); + cache.flush()?; + } + + let mut reloaded = setup_cache_with_config(&dir).await; + assert!(matches!( + reloaded.code_seed_state(&pool), + Some(CodeSeedState::Verified { .. }) + )); + // A fetcher that would fail loudly if consulted: with nothing pending, + // verify_code_seeds must not touch it. + let calls = Arc::new(AtomicUsize::new(0)); + reloaded.set_account_fields_fetcher(failing_fields_fetcher(calls.clone())); + let report = reloaded.verify_code_seeds()?; + assert!(report.verified.is_empty() && report.unverifiable.is_empty()); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "a settled (Verified) set must cost zero fields calls after reload" + ); + + let _ = std::fs::remove_dir_all(&dir); + Ok(()) +} + +/// Spec item 2: a mismatch purges the seed from both layers and the mark +/// (bumping the generation via the purge path), and the next touch refetches +/// authoritative chain state through the ordinary backend. +#[tokio::test(flavor = "multi_thread")] +async fn verify_mismatch_purges_and_next_touch_refetches() -> Result<()> { + let (mut cache, asserter) = common::setup_cache_with_asserter().await?; + let pool = Address::repeat_byte(0xa2); + let expected = cache.seed_account_code(pool, return_one_bytes())?; + + let actual = B256::repeat_byte(0xdd); + let calls = Arc::new(AtomicUsize::new(0)); + cache.set_account_fields_fetcher(stub_fields_fetcher( + HashMap::from([(pool, (U256::ZERO, actual))]), + calls.clone(), + )); + + let generation_before = cache.snapshot_generation(); + let report = cache.verify_code_seeds()?; + assert_eq!(report.mismatched.len(), 1); + assert_eq!(report.mismatched[0].address, pool); + assert_eq!(report.mismatched[0].expected, expected); + assert_eq!(report.mismatched[0].actual, actual); + assert!(report.verified.is_empty()); + + assert!( + cache.code_seed_state(&pool).is_none(), + "a contradicted claim must not leave a mark" + ); + assert!( + !cache.db_mut().cache.accounts.contains_key(&pool), + "the purge must clear the overlay layer" + ); + assert_ne!( + cache.snapshot_generation(), + generation_before, + "a mismatch purge changes executable state and must bump" + ); + + // The next touch goes back to the ordinary lazy backend: with the mock + // queue empty, `ensure_account` must now *attempt* an RPC fetch and fail + // loudly — the exact inverse of the seeded-account no-RPC guarantee. + // (Queued-response ordering is not exercised here because the backend + // issues the balance/nonce/code triple concurrently.) + assert!(asserter.read_q().is_empty()); + assert!( + cache.ensure_account(pool).await.is_err(), + "a purged seed must fall through to the backend on the next touch" + ); + Ok(()) +} + +/// Spec item 3: `EXTCODEHASH == 0` (no account) and `keccak256("")` (EOA) +/// are classified into their own buckets — both purged, since the claim is +/// contradicted either way. +#[tokio::test(flavor = "multi_thread")] +async fn verify_classifies_not_deployed_and_codeless() -> Result<()> { + let mut cache = setup_cache().await?; + let undeployed = Address::repeat_byte(0xa3); + let eoa = Address::repeat_byte(0xa4); + cache.seed_account_code(undeployed, return_one_bytes())?; + cache.seed_account_code(eoa, return_one_bytes())?; + + let calls = Arc::new(AtomicUsize::new(0)); + cache.set_account_fields_fetcher(stub_fields_fetcher( + HashMap::from([ + (undeployed, (U256::ZERO, B256::ZERO)), + (eoa, (U256::from(5u64), revm::primitives::KECCAK_EMPTY)), + ]), + calls.clone(), + )); + + let report = cache.verify_code_seeds()?; + assert_eq!(report.not_deployed, vec![undeployed]); + assert_eq!(report.codeless, vec![eoa]); + assert!(report.verified.is_empty() && report.mismatched.is_empty()); + assert!(cache.code_seed_state(&undeployed).is_none()); + assert!(cache.code_seed_state(&eoa).is_none()); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "both classifications come from the single bulk call" + ); + Ok(()) +} + +/// Spec item 4: a transport failure is fail-safe — every seed stays +/// `Pending`, nothing is purged, nothing bumps, and the report says why. +#[tokio::test(flavor = "multi_thread")] +async fn verify_transport_failure_keeps_seeds_pending() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0xa5); + cache.seed_account_code(pool, return_one_bytes())?; + + let calls = Arc::new(AtomicUsize::new(0)); + cache.set_account_fields_fetcher(failing_fields_fetcher(calls.clone())); + + let generation_before = cache.snapshot_generation(); + let report = cache.verify_code_seeds()?; + assert_eq!(report.unverifiable.len(), 1); + assert_eq!(report.unverifiable[0].0, pool); + assert!(report.unverifiable[0].1.contains("stub transport failure")); + assert!(report.verified.is_empty() && report.mismatched.is_empty()); + + assert!( + matches!( + cache.code_seed_state(&pool), + Some(CodeSeedState::Pending { .. }) + ), + "a failed read proves nothing: the seed must stay Pending" + ); + assert!( + cache.db_mut().cache.accounts.contains_key(&pool), + "nothing may be purged on a transport failure" + ); + assert_eq!(cache.snapshot_generation(), generation_before); + assert_eq!(cache.pending_code_seeds(), vec![pool]); + assert_eq!(calls.load(Ordering::SeqCst), 1); + Ok(()) +} + +/// The extractor-host caveat: a seed at `MULTICALL3_ADDRESS` cannot be +/// verified by the fields path (the extractor is hosted there under the +/// override) — reported unverifiable without consulting the fetcher. +#[tokio::test(flavor = "multi_thread")] +async fn verify_reports_extractor_host_seed_unverifiable() -> Result<()> { + let mut cache = setup_cache().await?; + cache.seed_account_code(MULTICALL3_ADDRESS, return_one_bytes())?; + + let calls = Arc::new(AtomicUsize::new(0)); + cache.set_account_fields_fetcher(stub_fields_fetcher(HashMap::new(), calls.clone())); + + let report = cache.verify_code_seeds()?; + assert_eq!(report.unverifiable.len(), 1); + assert_eq!(report.unverifiable[0].0, MULTICALL3_ADDRESS); + assert!(report.unverifiable[0].1.contains("eth_getProof")); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "a host-only pending set must not issue a fields call at all" + ); + assert!(matches!( + cache.code_seed_state(&MULTICALL3_ADDRESS), + Some(CodeSeedState::Pending { .. }) + )); + Ok(()) +} diff --git a/tests/cold_start.rs b/tests/cold_start.rs index 2c39210..19e8111 100644 --- a/tests/cold_start.rs +++ b/tests/cold_start.rs @@ -25,7 +25,7 @@ use std::sync::{Arc, Mutex}; use alloy_eips::BlockId; use alloy_primitives::{Address, B256, Bytes, U256, keccak256}; use alloy_sol_types::{SolCall, SolValue}; -use anyhow::{Result, anyhow}; +use anyhow::Result; use revm::primitives::hardfork::SpecId; use evm_fork_cache::cache::{EvmCache, StorageBatchFetchFn}; @@ -33,6 +33,7 @@ use evm_fork_cache::cold_start::{ ColdStartCall, ColdStartConfig, ColdStartError, ColdStartPin, ColdStartPlan, ColdStartPlanner, ColdStartResults, ColdStartStep, SlotFetch, }; +use evm_fork_cache::errors::StorageFetchError; use evm_fork_cache::events::StateView; use common::{ @@ -148,22 +149,20 @@ fn mixed_fetcher( value: U256, fail_slot: (Address, U256), ) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { - requests - .into_iter() - .map(|(addr, slot)| { - if (addr, slot) == fail_slot { - (addr, slot, Err(anyhow!("archive miss"))) - } else if (addr, slot) == value_slot { - (addr, slot, Ok(value)) - } else { - (addr, slot, Ok(U256::ZERO)) - } - }) - .collect() - }, - ) + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + requests + .into_iter() + .map(|(addr, slot)| { + if (addr, slot) == fail_slot { + (addr, slot, Err(StorageFetchError::custom("archive miss"))) + } else if (addr, slot) == value_slot { + (addr, slot, Ok(value)) + } else { + (addr, slot, Ok(U256::ZERO)) + } + }) + .collect() + }) } /// Build a cache with NO storage batch fetcher (mirrors the canonical pattern in @@ -949,18 +948,16 @@ async fn probe_and_verify_in_one_round_are_independent() -> Result<()> { // S4 acceptance tests: ColdStartPin::Hash pins the run and restores afterward // --------------------------------------------------------------------------- -/// A fetcher that records the `Option` it is called with, so a test can +/// A fetcher that records the `BlockId` it is called with, so a test can /// assert which block the run's reads were pinned to. -fn block_recording_fetcher(seen: Arc>>>) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, block: Option| { - seen.lock().unwrap().push(block); - requests - .into_iter() - .map(|(addr, slot)| (addr, slot, Ok(U256::ZERO))) - .collect() - }, - ) +fn block_recording_fetcher(seen: Arc>>) -> StorageBatchFetchFn { + Arc::new(move |requests: Vec<(Address, U256)>, block: BlockId| { + seen.lock().unwrap().push(block); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::ZERO))) + .collect() + }) } /// `ColdStartPin::Hash { require_canonical }` pins every round's reads to @@ -1002,7 +999,7 @@ async fn hash_pin_reads_at_hash_and_restores_prior_block() -> Result<()> { let seen = seen.lock().unwrap(); assert!(!seen.is_empty(), "the verify phase issued a pinned read"); assert!( - seen.iter().all(|b| *b == Some(expected)), + seen.iter().all(|b| *b == expected), "every read was pinned to the hash (with require_canonical): {seen:?}" ); assert_eq!( @@ -1049,3 +1046,124 @@ async fn hash_pin_restores_prior_block_on_error() -> Result<()> { ); Ok(()) } + +// --------------------------------------------------------------------------- +// verify_code phase (verified code seeding, spec §3.5 / acceptance item 9) +// --------------------------------------------------------------------------- + +/// Runtime bytes for code-seed driver tests: returns one 32-byte word and +/// touches no storage, so nothing here ever reaches the mocked backend. +const SEED_RUNTIME: [u8; 10] = [0x60, 0x01, 0x60, 0x00, 0x52, 0x60, 0x20, 0x60, 0x00, 0xf3]; + +/// The `NoAccountFieldsFetcher` guard fires only for pending-bearing rounds: +/// the same fetcher-less cache runs an empty round cleanly, then errors once +/// a pending seed exists. +#[tokio::test(flavor = "multi_thread")] +async fn verify_code_guard_fires_only_for_pending_bearing_rounds() -> Result<()> { + let mut cache = no_fetcher_cache().await?; + assert!(cache.account_fields_fetcher().is_none()); + + // No pending seeds: the round runs without any fetcher and the phase is + // a no-op. + let outcome = cache.execute_cold_start_round(&ColdStartPlan::default()); + assert!(outcome.error.is_none(), "got {:?}", outcome.error); + assert!(outcome.results.code_verifications.is_none()); + + // A pending seed with no fields fetcher short-circuits before any read. + cache.seed_account_code( + Address::repeat_byte(0xc0), + Bytes::from(SEED_RUNTIME.to_vec()), + )?; + let outcome = cache.execute_cold_start_round(&ColdStartPlan::default()); + assert!( + matches!(outcome.error, Some(ColdStartError::NoAccountFieldsFetcher)), + "got {:?}", + outcome.error + ); + assert!(outcome.results.code_verifications.is_none()); + Ok(()) +} + +/// verify_code runs before accounts, and its report survives an +/// accounts-phase hard error: the mismatched seed is purged first, then the +/// accounts phase's refetch of that same (now cold) address fails against the +/// empty mock queue — proving both the ordering and the partial-outcome +/// contract. +#[tokio::test(flavor = "multi_thread")] +async fn verify_code_report_survives_accounts_hard_error() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0xc1); + let expected = cache.seed_account_code(pool, Bytes::from(SEED_RUNTIME.to_vec()))?; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let actual = B256::repeat_byte(0xdd); + cache.set_account_fields_fetcher(common::stub_fields_fetcher( + HashMap::from([(pool, (U256::ZERO, actual))]), + calls.clone(), + )); + + let plan = ColdStartPlan { + accounts: vec![pool], + ..ColdStartPlan::default() + }; + let outcome = cache.execute_cold_start_round(&plan); + + // The accounts phase fails: verify_code purged the mismatched seed, so + // ensure_account falls through to the (empty) mocked backend. + assert!( + matches!(outcome.error, Some(ColdStartError::Fetch(_))), + "got {:?}", + outcome.error + ); + // ...but the verify_code report, computed first, is preserved. + let report = outcome + .results + .code_verifications + .expect("the verify_code report must survive an accounts-phase hard error"); + assert_eq!(report.mismatched.len(), 1); + assert_eq!(report.mismatched[0].address, pool); + assert_eq!(report.mismatched[0].expected, expected); + assert_eq!(report.mismatched[0].actual, actual); + assert!( + cache.code_seed_state(&pool).is_none(), + "the contradicted claim was purged before the accounts phase ran" + ); + Ok(()) +} + +/// Happy path: a matching seed settles in round one (report recorded, mark +/// Verified) and the phase is a no-op in round two — the fetcher is consulted +/// exactly once across both rounds. +#[tokio::test(flavor = "multi_thread")] +async fn verify_code_settles_in_one_round_then_noops() -> Result<()> { + let mut cache = setup_cache().await?; + let pool = Address::repeat_byte(0xc2); + let expected = cache.seed_account_code(pool, Bytes::from(SEED_RUNTIME.to_vec()))?; + + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + cache.set_account_fields_fetcher(common::stub_fields_fetcher( + HashMap::from([(pool, (U256::from(9u64), expected))]), + calls.clone(), + )); + + let first = cache.execute_cold_start_round(&ColdStartPlan::default()); + assert!(first.error.is_none(), "got {:?}", first.error); + let report = first + .results + .code_verifications + .expect("a pending-bearing round records a report"); + assert_eq!(report.verified, vec![pool]); + + let second = cache.execute_cold_start_round(&ColdStartPlan::default()); + assert!(second.error.is_none()); + assert!( + second.results.code_verifications.is_none(), + "a settled set makes the phase a no-op" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the fields fetcher is consulted exactly once across both rounds" + ); + Ok(()) +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c977e07..ea47c94 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,17 +6,20 @@ #![allow(dead_code)] use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use alloy_eips::BlockId; -use alloy_primitives::{Address, Bytes, U256, hex}; +use alloy_primitives::{Address, B256, Bytes, U256, hex}; use alloy_provider::RootProvider; use alloy_provider::network::AnyNetwork; use alloy_rpc_client::RpcClient; use alloy_sol_types::{SolCall, sol}; use alloy_transport::mock::Asserter; use anyhow::{Result, anyhow}; -use evm_fork_cache::cache::{EvmCache, StorageBatchFetchFn}; +use evm_fork_cache::bulk_storage::AccountFieldsSample; +use evm_fork_cache::cache::{AccountFieldsFetchFn, EvmCache, StorageBatchFetchFn}; +use evm_fork_cache::errors::{StorageFetchError, StorageFetchResult}; use revm::context::result::ExecutionResult; use revm::state::{AccountInfo, Bytecode}; @@ -115,27 +118,66 @@ pub fn balance_of(cache: &mut EvmCache, token: Address, owner: Address) -> Resul /// how an unseen slot reads in a simulation). This is the offline stand-in for /// the real RPC batch fetcher. pub fn stub_fetcher(values: HashMap<(Address, U256), U256>) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { - requests - .into_iter() - .map(|(addr, slot)| { - let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); - (addr, slot, Ok(value)) + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }) +} + +/// Stub [`AccountFieldsFetchFn`] reporting fixed `(balance, code_hash)` +/// samples, counting invocations. Addresses absent from `samples` are omitted +/// from the response (the "fetcher omitted this address" path). +pub fn stub_fields_fetcher( + samples: HashMap, + calls: Arc, +) -> AccountFieldsFetchFn { + Arc::new(move |addresses: Vec
, _block: BlockId| { + calls.fetch_add(1, Ordering::SeqCst); + Ok(addresses + .into_iter() + .filter_map(|address| { + samples.get(&address).map(|(balance, code_hash)| { + ( + address, + AccountFieldsSample { + balance: *balance, + code_hash: *code_hash, + }, + ) }) - .collect() - }, - ) + }) + .collect()) + }) +} + +/// Stub [`AccountFieldsFetchFn`] that fails the whole call (the transport +/// fail-safe path), counting invocations. +pub fn failing_fields_fetcher(calls: Arc) -> AccountFieldsFetchFn { + Arc::new(move |_addresses: Vec
, _block: BlockId| { + calls.fetch_add(1, Ordering::SeqCst); + Err(StorageFetchError::custom("stub transport failure")) + }) } /// Build a stub [`StorageBatchFetchFn`] that fails every request. /// /// Used to exercise the `Unverified` / error paths offline. pub fn failing_fetcher() -> StorageBatchFetchFn { - Arc::new(|requests: Vec<(Address, U256)>, _block: Option| { + Arc::new(|requests: Vec<(Address, U256)>, _block: BlockId| { requests .into_iter() - .map(|(addr, slot)| (addr, slot, Err(anyhow!("stub fetcher error")))) + .map(|(addr, slot)| { + ( + addr, + slot, + Err(StorageFetchError::custom("stub fetcher error")), + ) + }) .collect() }) } @@ -195,18 +237,16 @@ pub fn gated_tracking_fetcher( values: HashMap<(Address, U256), U256>, gate: Gate, ) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { - gate.wait(); - requests - .into_iter() - .map(|(addr, slot)| { - let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); - (addr, slot, Ok(value)) - }) - .collect() - }, - ) + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + gate.wait(); + requests + .into_iter() + .map(|(addr, slot)| { + let value = values.get(&(addr, slot)).copied().unwrap_or(U256::ZERO); + (addr, slot, Ok(value)) + }) + .collect() + }) } /// Build a stub [`StorageBatchFetchFn`] that panics, to exercise the validator's @@ -214,8 +254,8 @@ pub fn gated_tracking_fetcher( pub fn panicking_fetcher() -> StorageBatchFetchFn { Arc::new( |_requests: Vec<(Address, U256)>, - _block: Option| - -> Vec<(Address, U256, Result)> { + _block: BlockId| + -> Vec<(Address, U256, StorageFetchResult)> { panic!("panicking fetcher: deliberate failure for the Unverified test") }, ) @@ -230,5 +270,5 @@ pub fn transfer( amount: U256, ) -> Result { let call = MockERC20::transferCall { to, amount }; - cache.call_raw(from, token, Bytes::from(call.abi_encode()), true) + Ok(cache.call_raw(from, token, Bytes::from(call.abi_encode()), true)?) } diff --git a/tests/cow_snapshot.rs b/tests/cow_snapshot.rs index b7bb1d6..49cfb7f 100644 --- a/tests/cow_snapshot.rs +++ b/tests/cow_snapshot.rs @@ -2,8 +2,8 @@ //! snapshots, authored before implementation. //! //! The gate is a *differential-equivalence* property: the new, memoized -//! [`EvmCache::create_snapshot`] must be **read-indistinguishable** from the -//! retained reference [`EvmCache::create_snapshot_deep_clone`] after every kind of +//! [`EvmCache::snapshot`] must be **read-indistinguishable** from the +//! retained reference [`EvmCache::snapshot_deep_clone`] after every kind of //! cache mutation. Because the two use different internal representations (the COW //! snapshot shares an `Arc`-ed cold base; the reference is a full flatten), they are //! compared *through reads only* — `storage_value`, overlay `basic`/`storage`, and a @@ -15,7 +15,7 @@ //! //! All state is injected over a mocked provider — no test touches the network. //! -//! These reference `create_snapshot_deep_clone` and `EvmOverlay::reset`, which do +//! These reference `snapshot_deep_clone` and `EvmOverlay::reset`, which do //! not exist until the Phase 5 implementation lands; until then this file fails to //! compile (red), exactly as intended. @@ -73,8 +73,8 @@ fn overlay_balance_of(overlay: &mut EvmOverlay, token: Address, owner: Address) /// Assert the COW snapshot and the deep-clone reference are read-indistinguishable /// across a probe set of addresses and slots. `label` identifies the mutation step. fn assert_equivalent(cache: &mut EvmCache, addrs: &[Address], slots: &[U256], label: &str) { - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); @@ -287,7 +287,7 @@ async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result< let slot = U256::from(0u64); cache.inject_storage_batch(&[(pool, slot, U256::from(111u64))]); - let _warm = cache.create_snapshot(); // memoize the base at 111 + let _warm = cache.snapshot(); // memoize the base at 111 // Out-of-band overwrite at unchanged length (bypasses the write funnel). { @@ -301,8 +301,8 @@ async fn invalidate_snapshot_base_rehonest_after_escape_hatch_write() -> Result< // The documented re-honest hook must make the next snapshot reflect the write. cache.invalidate_snapshot_base(); - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); assert_eq!( cow.storage_value(pool, slot), deep.storage_value(pool, slot), @@ -345,7 +345,7 @@ async fn invalidate_snapshot_base_rehonest_after_existing_account_write() -> Res let bdb = cache.unchecked_blockchain_db(); bdb.accounts().write().insert(account, original.clone()); } - let warm = cache.create_snapshot(); // memoize the base with `original`. + let warm = cache.snapshot(); // memoize the base with `original`. let mut ov_warm = EvmOverlay::new(Arc::clone(&warm), None); let warm_info = ov_warm .basic(account) @@ -381,8 +381,8 @@ async fn invalidate_snapshot_base_rehonest_after_existing_account_write() -> Res } cache.invalidate_snapshot_base(); - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); @@ -424,7 +424,7 @@ async fn with_blockchain_db_mut_rehonest_after_storage_overwrite() -> Result<()> let slot = U256::from(0u64); cache.inject_storage_batch(&[(pool, slot, U256::from(111u64))]); - let _warm = cache.create_snapshot(); + let _warm = cache.snapshot(); cache.with_blockchain_db_mut(|bdb| { bdb.storage() @@ -434,8 +434,8 @@ async fn with_blockchain_db_mut_rehonest_after_storage_overwrite() -> Result<()> .insert(slot, U256::from(222u64)); }); - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); assert_eq!( cow.storage_value(pool, slot), deep.storage_value(pool, slot), @@ -471,7 +471,7 @@ async fn with_blockchain_db_mut_rehonest_after_account_overwrite() -> Result<()> cache.with_blockchain_db_mut(|bdb| { bdb.accounts().write().insert(account, original); }); - let _warm = cache.create_snapshot(); + let _warm = cache.snapshot(); cache.with_blockchain_db_mut(|bdb| { let mut accounts = bdb.accounts().write(); @@ -480,8 +480,8 @@ async fn with_blockchain_db_mut_rehonest_after_account_overwrite() -> Result<()> assert_eq!(accounts.len(), len_before); }); - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); let cow_basic = ov_cow.basic(account).expect("cow basic"); @@ -533,7 +533,7 @@ async fn cow_code_index_matches_deep_clone_after_base_account_recoded() -> Resul }; put_account(&cache, &code_v1, h1); cache.invalidate_snapshot_base(); - let warm = cache.create_snapshot(); // base now indexes h1 -> code_v1 + let warm = cache.snapshot(); // base now indexes h1 -> code_v1 let mut ov_warm = EvmOverlay::new(Arc::clone(&warm), None); assert_eq!( ov_warm @@ -554,8 +554,8 @@ async fn cow_code_index_matches_deep_clone_after_base_account_recoded() -> Resul U256::from(9u64), )]); - let cow = cache.create_snapshot(); - let deep = cache.create_snapshot_deep_clone(); + let cow = cache.snapshot(); + let deep = cache.snapshot_deep_clone(); let mut ov_cow = EvmOverlay::new(Arc::clone(&cow), None); let mut ov_deep = EvmOverlay::new(Arc::clone(&deep), None); @@ -588,12 +588,12 @@ async fn earlier_snapshot_unaffected_by_later_base_mutation() -> Result<()> { let slot = U256::from(3u64); cache.inject_storage_batch(&[(pool, slot, U256::from(100u64))]); - let early = cache.create_snapshot(); + let early = cache.snapshot(); assert_eq!(early.storage_value(pool, slot), Some(U256::from(100u64))); // Mutate the same base slot, then take a second snapshot. cache.inject_storage_batch(&[(pool, slot, U256::from(200u64))]); - let late = cache.create_snapshot(); + let late = cache.snapshot(); assert_eq!( early.storage_value(pool, slot), @@ -634,7 +634,7 @@ async fn overlay_reset_restores_pristine_snapshot_reads() -> Result<()> { U256::ZERO, )?; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); // Mutate the dirty layer with a committing transfer through the overlay. @@ -694,7 +694,7 @@ async fn overlay_buffer_reuse_is_result_stable() -> Result<()> { U256::from(7_777u64), )?; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); let first = overlay_balance_of(&mut overlay, token, owner)?; diff --git a/tests/errors.rs b/tests/errors.rs index 32e2585..f448bbb 100644 --- a/tests/errors.rs +++ b/tests/errors.rs @@ -10,9 +10,8 @@ use std::sync::Arc; use alloy_primitives::{Bytes, U256}; use alloy_sol_types::{Panic, SolError, sol}; -use anyhow::anyhow; use evm_fork_cache::errors::{ - PANIC_SELECTOR, RevertDecoder, RevertReason, SimError, SimulationError, + PANIC_SELECTOR, RevertDecoder, RevertReason, SimError, SimHostError, SimulationError, }; sol! { @@ -35,7 +34,10 @@ fn sim_error_classification() { assert!(!halt.is_revert()); assert!(halt.as_revert().is_none()); - let other: SimError = anyhow!("rpc exploded").into(); + let other: SimError = SimHostError::Database { + details: "rpc exploded".to_string(), + } + .into(); assert!(!other.is_revert()); assert!(!other.is_halt()); assert!(other.as_revert().is_none()); @@ -54,8 +56,11 @@ fn sim_error_display_distinguishes_variants() { assert!(shown.contains("halted"), "{shown}"); assert!(shown.contains("StackOverflow"), "{shown}"); - let other: SimError = anyhow!("boom").into(); - assert_eq!(other.to_string(), "boom"); + let other: SimError = SimHostError::Database { + details: "boom".to_string(), + } + .into(); + assert_eq!(other.to_string(), "database operation failed: boom"); } #[test] diff --git a/tests/event_pipeline.rs b/tests/event_pipeline.rs index bea625e..e4dfb2c 100644 --- a/tests/event_pipeline.rs +++ b/tests/event_pipeline.rs @@ -564,13 +564,12 @@ async fn ingest_keeps_state_fresh_with_zero_fetches() -> Result<()> { // A counting fetcher: `ingest_logs` must never invoke it. let fetches = Arc::new(AtomicUsize::new(0)); let counter = fetches.clone(); - let f: StorageBatchFetchFn = - Arc::new(move |reqs: Vec<(Address, U256)>, _b: Option| { - counter.fetch_add(reqs.len(), Ordering::Relaxed); - reqs.into_iter() - .map(|(a, s)| (a, s, Ok(U256::from(1_000)))) - .collect() - }); + let f: StorageBatchFetchFn = Arc::new(move |reqs: Vec<(Address, U256)>, _b: BlockId| { + counter.fetch_add(reqs.len(), Ordering::Relaxed); + reqs.into_iter() + .map(|(a, s)| (a, s, Ok(U256::from(1_000)))) + .collect() + }); cache.set_storage_batch_fetcher(f); let mut registry = DecoderRegistry::new(); @@ -608,3 +607,56 @@ async fn ingest_keeps_state_fresh_with_zero_fetches() -> Result<()> { } Ok(()) } + +/// WS-3 (manager-authored red-green): `derived_slots` must be bounded to the +/// reorg horizon (`ReorgConfig::depth`), mirroring the `touched` ring, rather +/// than growing unbounded across steady-state ingestion. With `depth = 3`, +/// after ingesting 6 blocks that each touch a distinct `(address, slot)`, only +/// the 3 most-recent blocks' derived slots may be retained; the aged-out ones +/// must be evicted. +#[tokio::test] +async fn derived_slots_is_bounded_to_reorg_horizon() -> Result<()> { + let slot = U256::from(0); + let tokens: Vec
= (0u8..6).map(|i| Address::repeat_byte(0x50 + i)).collect(); + + let mut cache = setup_cache().await?; + for t in &tokens { + install_mock_erc20(&mut cache, *t); + } + + let mut registry = DecoderRegistry::new(); + registry.register(Arc::new(MarkDecoder { + slot, + value: U256::from(77), + })); + let mut pipeline = EventPipeline::new(registry).with_reorg_config(ReorgConfig { + depth: 3, + scope: PurgeScope::AllStorage, + }); + + for (i, t) in tokens.iter().enumerate() { + pipeline.ingest_logs(&mut cache, 100 + i as u64, &[bare_log(*t, vec![])]); + } + + let derived: std::collections::HashSet<(Address, U256)> = pipeline.derived_slots().collect(); + + assert_eq!( + derived.len(), + 3, + "derived_slots must be bounded to the reorg horizon (depth = 3), was {}", + derived.len() + ); + for t in &tokens[3..] { + assert!( + derived.contains(&(*t, slot)), + "recent block's derived slot must be retained" + ); + } + for t in &tokens[..3] { + assert!( + !derived.contains(&(*t, slot)), + "aged-out derived slot must be evicted from the horizon" + ); + } + Ok(()) +} diff --git a/tests/fetch_minimization.rs b/tests/fetch_minimization.rs index b622bcb..95f7029 100644 --- a/tests/fetch_minimization.rs +++ b/tests/fetch_minimization.rs @@ -33,15 +33,13 @@ fn balance_slot(owner: Address) -> U256 { } fn counting_fetcher(counter: Arc) -> StorageBatchFetchFn { - Arc::new( - move |requests: Vec<(Address, U256)>, _block: Option| { - counter.fetch_add(requests.len(), Ordering::Relaxed); - requests - .into_iter() - .map(|(addr, slot)| (addr, slot, Ok(U256::from(SEEDED_BALANCE)))) - .collect() - }, - ) + Arc::new(move |requests: Vec<(Address, U256)>, _block: BlockId| { + counter.fetch_add(requests.len(), Ordering::Relaxed); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::from(SEEDED_BALANCE)))) + .collect() + }) } async fn cache_with_counter(token: Address) -> Result<(EvmCache, Arc)> { @@ -90,7 +88,7 @@ async fn fan_out_reuses_one_warmup_fetch_across_all_candidates() -> Result<()> { "warm-up fetches each working-set slot exactly once" ); - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); counter.store(0, Ordering::Relaxed); // Fan N candidates out; each reads the whole shared hot set from the snapshot. diff --git a/tests/freshness.rs b/tests/freshness.rs index 79a79fb..54c8d9a 100644 --- a/tests/freshness.rs +++ b/tests/freshness.rs @@ -22,10 +22,22 @@ use common::{ use evm_fork_cache::cache::{ EvmCache, EvmOverlay, SimStatus, SlotObservationTracker, StorageBatchFetchFn, }; +use evm_fork_cache::errors::StorageFetchResult; use evm_fork_cache::freshness::{ AlwaysVerify, BlockClock, FreshnessController, FreshnessParams, FreshnessRegistry, NeverVerify, ObservationDriven, SimRequest, Validation, WallClock, }; +use revm::state::{AccountInfo, Bytecode}; + +/// Runtime bytecode that returns `blockhash(0)`: +/// `PUSH1 0 BLOCKHASH PUSH1 0 MSTORE PUSH1 32 PUSH1 0 RETURN`. +/// +/// Control flow doesn't branch on the hash, but the *result* embeds it — the +/// exact shape the validator cannot vouch for, since its overlays resolve +/// `BLOCKHASH` to ZERO. +const BLOCKHASH_READER_RUNTIME: &[u8] = &[ + 0x60, 0x00, 0x40, 0x60, 0x00, 0x52, 0x60, 0x20, 0x60, 0x00, 0xF3, +]; /// Hashed storage slot of `balanceOf[owner]` for the MockERC20 fixture. fn balance_slot_for(owner: Address) -> U256 { @@ -271,7 +283,7 @@ async fn overlay_call_raw_with_access_list_captures_read_set() -> Result<()> { let balance_slot = U256::from(MOCK_ERC20_BALANCE_SLOT); cache.insert_mapping_storage_slot(token, balance_slot, owner, U256::from(1000))?; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); // balanceOf(owner) reads the token's balance mapping slot. @@ -303,7 +315,7 @@ async fn overlay_override_slot_takes_precedence() -> Result<()> { let slot = U256::from(3); cache.inject_storage_batch(&[(contract, slot, U256::from(1))]); - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); overlay.override_slot(contract, slot, U256::from(999)); @@ -369,14 +381,75 @@ async fn run_match_path_confirmed() -> Result<()> { let optimistic_gas = sim.optimistic()[0].gas_used; assert!(optimistic_gas > 0); - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( - matches!(validation, Validation::Confirmed), + matches!(validation, Validation::ConfirmedStorage), "unchanged values should confirm: {validation:?}" ); Ok(()) } +/// G5 / WS-9: a sim that reads `BLOCKHASH` must fail closed as `Unverified`. +/// +/// The controller's overlays carry no block hashes, so the opcode resolves to +/// ZERO — storage verification cannot vouch for the result. Before 0.2.0 this +/// sim (which touches no volatile storage at all) sailed through the +/// empty-verify-set early path and was silently `ConfirmedStorage`. +#[tokio::test(flavor = "multi_thread")] +async fn run_blockhash_reading_sim_fails_closed_as_unverified() -> Result<()> { + let caller = Address::repeat_byte(0x0c); + let target = Address::repeat_byte(0x0d); + + let mut cache = setup_cache().await?; + install_default_account(&mut cache, caller); + let bytecode = Bytecode::new_raw(Bytes::from_static(BLOCKHASH_READER_RUNTIME)); + let code_hash = bytecode.hash_slow(); + cache.db_mut().insert_account_info( + target, + AccountInfo { + balance: U256::ZERO, + nonce: 0, + code: Some(bytecode), + code_hash, + account_id: None, + }, + ); + cache + .db_mut() + .replace_account_storage(target, Default::default())?; + // Pin a concrete height so `blockhash(0)` is IN the EVM's valid lookback + // range and revm actually consults the database. (Out-of-range requests + // return spec-mandated ZERO without a DB call — correct on-chain too, so + // they are deliberately not flagged.) + cache.set_block(BlockId::number(100)); + // A fetcher that would happily "confirm" anything — it must never get the + // chance to vouch for a hash-dependent result. + cache.set_storage_batch_fetcher(stub_fetcher(HashMap::new())); + + let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); + let sim = controller.run( + &mut cache, + vec![SimRequest::new(caller, target, Bytes::new())], + )?; + assert_eq!( + sim.optimistic().len(), + 1, + "the optimistic run still executes" + ); + + let validation = sim.validate().await?; + match validation { + Validation::Unverified { reason } => { + assert!( + reason.contains("BLOCKHASH"), + "the reason must name the unverifiable read: {reason}" + ); + } + other => panic!("BLOCKHASH-reading sim must fail closed, got {other:?}"), + } + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { let token = Address::repeat_byte(0x44); @@ -444,19 +517,23 @@ async fn run_mismatch_path_corrected_only_affected_rerun() -> Result<()> { "optimistic token_deltas empty" ); - let validation = sim.validate().await; + let validation = sim.validate().await?; match validation { - Validation::Corrected { results, changed } => { + Validation::Corrected { + results, + changed_slots, + .. + } => { // Exactly owner's balance slot changed. assert_eq!( - changed.len(), + changed_slots.len(), 1, - "only owner's balance changed: {changed:?}" + "only owner's balance changed: {changed_slots:?}" ); - assert_eq!(changed[0].address, token); - assert_eq!(changed[0].slot, balance_slot_for(owner)); - assert_eq!(changed[0].old, U256::from(1000)); - assert_eq!(changed[0].new, U256::from(50)); + assert_eq!(changed_slots[0].address, token); + assert_eq!(changed_slots[0].slot, balance_slot_for(owner)); + assert_eq!(changed_slots[0].old, U256::from(1000)); + assert_eq!(changed_slots[0].new, U256::from(50)); // req1 was re-run with the reduced balance → now reverts (no log) and // differs from its optimistic (successful) result. @@ -526,14 +603,18 @@ async fn run_view_call_corrected_success_to_different_success() -> Result<()> { "optimistic returns the old balance" ); - let validation = sim.validate().await; + let validation = sim.validate().await?; match validation { - Validation::Corrected { results, changed } => { - assert_eq!(changed.len(), 1, "exactly the balance slot changed"); - assert_eq!(changed[0].address, token); - assert_eq!(changed[0].slot, balance_slot_for(owner)); - assert_eq!(changed[0].old, U256::from(1000)); - assert_eq!(changed[0].new, U256::from(250)); + Validation::Corrected { + results, + changed_slots, + .. + } => { + assert_eq!(changed_slots.len(), 1, "exactly the balance slot changed"); + assert_eq!(changed_slots[0].address, token); + assert_eq!(changed_slots[0].slot, balance_slot_for(owner)); + assert_eq!(changed_slots[0].old, U256::from(1000)); + assert_eq!(changed_slots[0].new, U256::from(250)); // The corrected re-run STILL succeeds (a balanceOf view never reverts) // but its return data reflects the NEW balance. @@ -595,7 +676,7 @@ async fn run_drains_pending_on_next_run() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!(matches!(validation, Validation::Corrected { .. })); assert_eq!(controller.pending_len(), 1, "a correction was queued"); @@ -627,9 +708,9 @@ async fn run_drains_pending_on_next_run() -> Result<()> { !sim.optimistic()[0].logs.is_empty(), "optimistic still succeeds" ); - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( - matches!(validation, Validation::Confirmed), + matches!(validation, Validation::ConfirmedStorage), "{validation:?}" ); Ok(()) @@ -681,7 +762,10 @@ async fn run_converges_when_corrected_slot_is_overlay_resident() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - assert!(matches!(sim.validate().await, Validation::Corrected { .. })); + assert!(matches!( + sim.validate().await?, + Validation::Corrected { .. } + )); assert_eq!(controller.pending_len(), 1); // Second run: drains the correction. It must overwrite the overlay-resident @@ -702,9 +786,9 @@ async fn run_converges_when_corrected_slot_is_overlay_resident() -> Result<()> { ); // The snapshot now matches the fetcher → Confirmed, proving convergence. - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( - matches!(validation, Validation::Confirmed), + matches!(validation, Validation::ConfirmedStorage), "must converge, got {validation:?}" ); // No background re-run happened on the converged second cycle. @@ -849,7 +933,7 @@ async fn dropping_after_fetch_started_suppresses_correction() -> Result<()> { let barrier = Arc::new(std::sync::Barrier::new(2)); let fb = Arc::clone(&barrier); let fetcher: StorageBatchFetchFn = - Arc::new(move |reqs: Vec<(Address, U256)>, _block: Option| { + Arc::new(move |reqs: Vec<(Address, U256)>, _block: BlockId| { fb.wait(); // R1 fb.wait(); // R2 reqs.into_iter() @@ -943,10 +1027,14 @@ async fn run_corrected_rerun_verifies_newly_read_volatile_slot() -> Result<()> { U256::from(5) ); - match sim.validate().await { - Validation::Corrected { results, changed } => { + match sim.validate().await? { + Validation::Corrected { + results, + changed_slots, + .. + } => { let keys: std::collections::HashSet<(Address, U256)> = - changed.iter().map(|c| (c.address, c.slot)).collect(); + changed_slots.iter().map(|c| (c.address, c.slot)).collect(); assert!(keys.contains(&(contract, slot_a)), "A reported as changed"); assert!( keys.contains(&(contract, slot_b)), @@ -999,7 +1087,7 @@ async fn overlay_call_with_tx_config_threads_value() -> Result<()> { .replace_account_storage(callee, Default::default()) .unwrap(); - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); fn returned_value(res: ExecutionResult) -> U256 { @@ -1086,14 +1174,14 @@ async fn validator_fetches_at_captured_latest_pin_despite_repin() -> Result<()> let barrier = Arc::new(std::sync::Barrier::new(2)); let fb = Arc::clone(&barrier); - let seen_block: Arc>>> = Arc::new(Mutex::new(None)); + let seen_block: Arc>> = Arc::new(Mutex::new(None)); let seen = Arc::clone(&seen_block); let fetcher: StorageBatchFetchFn = - Arc::new(move |reqs: Vec<(Address, U256)>, block: Option| { + Arc::new(move |reqs: Vec<(Address, U256)>, block: BlockId| { *seen.lock().unwrap() = Some(block); fb.wait(); fb.wait(); - let at_snapshot_pin = block == Some(BlockId::latest()); + let at_snapshot_pin = block == BlockId::latest(); reqs.into_iter() .map(|(a, s)| { let v = if s == slot { @@ -1125,14 +1213,14 @@ async fn validator_fetches_at_captured_latest_pin_despite_repin() -> Result<()> cache.set_block(BlockId::Number(BlockNumberOrTag::Number(101))); barrier.wait(); - let verdict = sim.validate().await; + let verdict = sim.validate().await?; assert!( - matches!(verdict, Validation::Confirmed), + matches!(verdict, Validation::ConfirmedStorage), "validator must fetch at the captured latest pin, not the later numeric repin; got {verdict:?}" ); assert_eq!( *seen_block.lock().unwrap(), - Some(Some(BlockId::latest())), + Some(BlockId::latest()), "the fetch must receive the concrete snapshot pin" ); Ok(()) @@ -1176,14 +1264,14 @@ async fn validator_fetches_at_snapshot_block_despite_repin() -> Result<()> { // a barrier so the test can repin before the fetch resolves. let barrier = Arc::new(std::sync::Barrier::new(2)); let fb = Arc::clone(&barrier); - let seen_block: Arc>>> = Arc::new(Mutex::new(None)); + let seen_block: Arc>> = Arc::new(Mutex::new(None)); let seen = Arc::clone(&seen_block); let fetcher: StorageBatchFetchFn = - Arc::new(move |reqs: Vec<(Address, U256)>, block: Option| { + Arc::new(move |reqs: Vec<(Address, U256)>, block: BlockId| { *seen.lock().unwrap() = Some(block); fb.wait(); // R1: fetch entered fb.wait(); // R2: released after the test repins - let at_n = block == Some(block_n); + let at_n = block == block_n; reqs.into_iter() .map(|(a, s)| { // At block N every slot matches the snapshot (sender = 1000, @@ -1219,14 +1307,14 @@ async fn validator_fetches_at_snapshot_block_despite_repin() -> Result<()> { cache.set_block(BlockId::Number(BlockNumberOrTag::Number(n + 1))); barrier.wait(); // R2: release the fetcher. - let verdict = sim.validate().await; + let verdict = sim.validate().await?; assert!( - matches!(verdict, Validation::Confirmed), + matches!(verdict, Validation::ConfirmedStorage), "validator must fetch at the snapshot's block N, not the re-pinned N+1; got {verdict:?}" ); assert_eq!( *seen_block.lock().unwrap(), - Some(Some(block_n)), + Some(block_n), "the fetch must be pinned to the snapshot block N" ); Ok(()) @@ -1250,7 +1338,7 @@ async fn run_unverified_on_fetcher_error() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( matches!(validation, Validation::Unverified { .. }), "fetcher error should yield Unverified: {validation:?}" @@ -1387,7 +1475,7 @@ async fn never_verify_skips_predicted_but_reconciles_read_set() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( matches!(validation, Validation::Corrected { .. }), "actual-read-set reconcile should still catch the change: {validation:?}" @@ -1420,9 +1508,9 @@ async fn pinned_slot_is_not_verified() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( - matches!(validation, Validation::Confirmed), + matches!(validation, Validation::ConfirmedStorage), "pinned slot must not be verified: {validation:?}" ); Ok(()) @@ -1451,9 +1539,9 @@ async fn wall_clock_controller_runs() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( - matches!(validation, Validation::Confirmed), + matches!(validation, Validation::ConfirmedStorage), "{validation:?}" ); Ok(()) @@ -1487,7 +1575,10 @@ async fn valid_through_becomes_volatile_after_boundary() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - assert!(matches!(sim.validate().await, Validation::Confirmed)); + assert!(matches!( + sim.validate().await?, + Validation::ConfirmedStorage + )); // Advance past the boundary: now volatile → the change is caught. clock.set_block(101); @@ -1500,7 +1591,7 @@ async fn valid_through_becomes_volatile_after_boundary() -> Result<()> { )], )?; assert!( - matches!(sim.validate().await, Validation::Corrected { .. }), + matches!(sim.validate().await?, Validation::Corrected { .. }), "past ValidThrough boundary the slot is volatile and the change is caught" ); Ok(()) @@ -1541,9 +1632,13 @@ async fn run_missing_slot_treated_as_zero_is_corrected() -> Result<()> { "unseen slot reads as zero optimistically" ); - match sim.validate().await { - Validation::Corrected { results, changed } => { - let change = changed + match sim.validate().await? { + Validation::Corrected { + results, + changed_slots, + .. + } => { + let change = changed_slots .iter() .find(|c| c.address == token && c.slot == slot) .expect("the missing balance slot should be reported as changed"); @@ -1594,7 +1689,10 @@ async fn pending_drain_alters_subsequent_result() -> Result<()> { !sim.optimistic()[0].logs.is_empty(), "first-run optimistic transfer succeeds against cached 1000" ); - assert!(matches!(sim.validate().await, Validation::Corrected { .. })); + assert!(matches!( + sim.validate().await?, + Validation::Corrected { .. } + )); assert_eq!(controller.pending_len(), 1, "a correction (→50) is queued"); // Second run drains the correction (balance := 50) BEFORE snapshotting, so @@ -1616,9 +1714,9 @@ async fn pending_drain_alters_subsequent_result() -> Result<()> { Ok(()) } -// T6a: a panicking fetcher → the validator task panics → JoinError → Unverified. +// T6a: a panicking fetcher → the validator task panics → JoinError → validate Err. #[tokio::test(flavor = "multi_thread")] -async fn run_unverified_on_fetcher_panic() -> Result<()> { +async fn validate_returns_err_on_fetcher_panic() -> Result<()> { let token = Address::repeat_byte(0x44); let owner = Address::repeat_byte(0x55); let recipient = Address::repeat_byte(0x66); @@ -1635,10 +1733,13 @@ async fn run_unverified_on_fetcher_panic() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - let validation = sim.validate().await; + let err = sim + .validate() + .await + .expect_err("a panicking fetcher should make validate return Err"); assert!( - matches!(validation, Validation::Unverified { .. }), - "a panicking fetcher (JoinError) should yield Unverified: {validation:?}" + err.to_string().contains("validation task failed"), + "a panicking fetcher should surface the JoinError: {err}" ); Ok(()) } @@ -1683,7 +1784,7 @@ async fn run_unverified_without_fetcher() -> Result<()> { transfer_calldata(recipient, U256::from(100)), )], )?; - match sim.validate().await { + match sim.validate().await? { Validation::Unverified { reason } => { assert_eq!(reason, "no storage batch fetcher available", "{reason}"); } @@ -1745,10 +1846,12 @@ async fn observation_driven_controller_end_to_end() -> Result<()> { // Even though the policy declined to *predictively* verify the stable slot, // the validator's actual-read-set reconcile catches the real change. - match sim.validate().await { - Validation::Corrected { changed, .. } => { + match sim.validate().await? { + Validation::Corrected { changed_slots, .. } => { assert!( - changed.iter().any(|c| c.address == token && c.slot == slot), + changed_slots + .iter() + .any(|c| c.address == token && c.slot == slot), "the actual-read-set reconcile catches the balance change" ); } @@ -1789,7 +1892,7 @@ async fn on_new_block_ages_valid_through() -> Result<()> { )], )?; assert!( - matches!(sim.validate().await, Validation::Confirmed), + matches!(sim.validate().await?, Validation::ConfirmedStorage), "at block 100 the ValidThrough slot is still pinned" ); @@ -1805,7 +1908,7 @@ async fn on_new_block_ages_valid_through() -> Result<()> { )], )?; assert!( - matches!(sim.validate().await, Validation::Corrected { .. }), + matches!(sim.validate().await?, Validation::Corrected { .. }), "after on_new_block(101) the slot is volatile and the change is caught" ); Ok(()) @@ -1826,17 +1929,15 @@ async fn run_unverified_when_fetcher_omits_requested_slot() -> Result<()> { let mut cache = cache_with_balance(token, owner, U256::from(1000)).await?; // A fetcher that returns NOTHING — it omits every requested slot. - cache.set_storage_batch_fetcher(Arc::new( - |_req: Vec<(Address, U256)>, _block: Option| { - Vec::<(Address, U256, Result)>::new() - }, - )); + cache.set_storage_batch_fetcher(Arc::new(|_req: Vec<(Address, U256)>, _block: BlockId| { + Vec::<(Address, U256, StorageFetchResult)>::new() + })); let mut controller = FreshnessController::new(FreshnessRegistry::new(), AlwaysVerify); let req = SimRequest::new(owner, token, transfer_calldata(recipient, U256::from(100))); let sim = controller.run(&mut cache, vec![req])?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( matches!(validation, Validation::Unverified { .. }), "a fetcher that omits a requested slot must yield Unverified, not a false \ @@ -1916,7 +2017,7 @@ async fn run_unverified_when_fixed_point_round_cap_exceeded() -> Result<()> { let req = SimRequest::new(caller, contract, Bytes::new()); let sim = controller.run(&mut cache, vec![req])?; - let validation = sim.validate().await; + let validation = sim.validate().await?; assert!( matches!(validation, Validation::Unverified { .. }), "exceeding the fixed-point round cap must yield Unverified, not a trusted \ @@ -1929,3 +2030,30 @@ async fn run_unverified_when_fixed_point_round_cap_exceeded() -> Result<()> { ); Ok(()) } + +/// WS-1c (manager-authored red-green): the verdict taxonomy distinguishes a +/// storage-only confirmation from a full (storage + account) one, so callers can +/// no longer mistake "no volatile storage slot changed" for "account state +/// verified". The storage-only success verdict is renamed `Confirmed -> +/// ConfirmedStorage`; a new `ConfirmedFull` means storage AND account fields were +/// verified; and `Corrected` carries `changed_accounts` alongside `changed_slots`. +#[test] +fn verdict_taxonomy_separates_storage_only_from_full_confirmation() { + // Renamed storage-only success verdict (was `Confirmed`). + assert!(matches!( + Validation::ConfirmedStorage, + Validation::ConfirmedStorage + )); + // New: storage AND account fields verified. + assert!(matches!( + Validation::ConfirmedFull, + Validation::ConfirmedFull + )); + // Corrected now reports account changes alongside slot changes. + let corrected = Validation::Corrected { + results: vec![], + changed_slots: vec![], + changed_accounts: vec![], + }; + assert!(matches!(corrected, Validation::Corrected { .. })); +} diff --git a/tests/liveness_cold_start.rs b/tests/liveness_cold_start.rs new file mode 100644 index 0000000..68a0a1c --- /dev/null +++ b/tests/liveness_cold_start.rs @@ -0,0 +1,388 @@ +//! Manager-authored red-green acceptance tests for Phase-8 step 5: the +//! cold-start root baseline (`roots.bin`). +//! +//! A process restarting after downtime should not blindly re-read its whole +//! working set. It persists each tracked account's observed storage root as a +//! baseline; on restart it probes the root *now* and, where it equals the +//! baseline, the cached tracked slots are provably current — **skip re-reading** +//! ("if no divergence, we're already synced"). Where it diverges (or no baseline +//! exists, or the probe fails), re-read the tracked slots and adopt the new root. +//! This is a *currency* gate, not a completeness gate (spec §6). +//! +//! Fully offline: proof/storage fetchers are stubbed; persistence uses a +//! pid-keyed temp dir. +#![cfg(feature = "reactive")] + +mod common; + +use std::path::PathBuf; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use alloy_primitives::{Address, B256, U256}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::cache::AccountProof; +use evm_fork_cache::errors::StorageFetchError; +use evm_fork_cache::{ColdStartConfig, RootBaseline, RootBaselinePlanner}; + +/// A pid-keyed temp dir so concurrent `cargo test` processes never collide. +fn temp_dir(tag: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("evm_fork_cache_roots_{tag}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +/// Install an account-proof fetcher that always reports `root` for any address. +fn install_const_root_fetcher(cache: &mut evm_fork_cache::cache::EvmCache, root: B256) { + cache.set_account_proof_fetcher(Arc::new(move |requests, _block| { + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: root, + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + })); +} + +/// Install a storage batch fetcher that serves `value` for every slot and counts +/// how many slots were fetched. +fn install_counting_storage_fetcher( + cache: &mut evm_fork_cache::cache::EvmCache, + value: U256, +) -> Arc { + let count = Arc::new(AtomicUsize::new(0)); + let seen = count.clone(); + cache.set_storage_batch_fetcher(Arc::new(move |requests, _block| { + seen.fetch_add(requests.len(), Ordering::SeqCst); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(value))) + .collect() + })); + count +} + +/// Phase-8 s5: `roots.bin` round-trips through the versioned envelope, and a +/// legacy/unknown-magic file is a cache miss (never an error, never trusted). +#[test] +fn root_baseline_round_trips_and_rejects_unknown_magic() -> Result<()> { + let dir = temp_dir("roundtrip"); + let path = dir.join("roots.bin"); + let addr = Address::repeat_byte(0x21); + let root = B256::repeat_byte(0xaa); + + let mut baseline = RootBaseline::default(); + baseline.insert(addr, root); + baseline.save(&path)?; + + let loaded = RootBaseline::load(&path).expect("a just-saved baseline loads"); + assert_eq!(loaded.get(&addr), Some(root), "the baseline round-trips"); + + // A missing file is a miss. + assert!( + RootBaseline::load(&dir.join("absent.bin")).is_none(), + "a missing file is a cache miss" + ); + + // A legacy / unknown-magic payload is a miss, not an error and not data. + std::fs::write(&path, b"not a versioned roots baseline")?; + assert!( + RootBaseline::load(&path).is_none(), + "unknown magic must be treated as a cache miss" + ); + + let _ = std::fs::remove_dir_all(&dir); + Ok(()) +} + +/// Phase-8 s5: when the probed root equals the persisted baseline, the tracked +/// slots are provably current — the planner finishes without a single storage +/// re-read ("if no divergence, we're already synced"). +#[tokio::test] +async fn equal_baseline_skips_rereading_tracked_slots() -> Result<()> { + let tracked = Address::repeat_byte(0x31); + let slot = U256::from(4); + let root = B256::repeat_byte(0xaa); + let mut cache = setup_cache().await?; + + install_const_root_fetcher(&mut cache, root); + let fetches = install_counting_storage_fetcher(&mut cache, U256::from(1)); + + // The persisted baseline already matches what the chain reports now. + let mut baseline = RootBaseline::default(); + baseline.insert(tracked, root); + + let mut planner = RootBaselinePlanner::new(vec![(tracked, vec![slot])], baseline); + let report = cache.run_cold_start(&mut planner, ColdStartConfig::default())?; + + assert_eq!( + fetches.load(Ordering::SeqCst), + 0, + "an unchanged root must not re-read any tracked slot" + ); + assert_eq!(report.rounds, 1, "the probe round is the only round"); + assert_eq!( + planner.updated_baseline().get(&tracked), + Some(root), + "the (unchanged) observed root is retained in the updated baseline" + ); + Ok(()) +} + +/// Phase-8 s5: a diverged (or missing) baseline re-reads the tracked slots and +/// adopts the newly observed root into the updated baseline. +#[tokio::test] +async fn diverged_baseline_rereads_and_adopts_new_root() -> Result<()> { + let tracked = Address::repeat_byte(0x32); + let slot = U256::from(7); + let old_root = B256::repeat_byte(0xaa); + let new_root = B256::repeat_byte(0xbb); + let fresh_value = U256::from(777); + let mut cache = setup_cache().await?; + + install_const_root_fetcher(&mut cache, new_root); + let fetches = install_counting_storage_fetcher(&mut cache, fresh_value); + + // The persisted baseline predates the (moved) on-chain root. + let mut baseline = RootBaseline::default(); + baseline.insert(tracked, old_root); + + let mut planner = RootBaselinePlanner::new(vec![(tracked, vec![slot])], baseline); + let report = cache.run_cold_start(&mut planner, ColdStartConfig::default())?; + + assert_eq!( + fetches.load(Ordering::SeqCst), + 1, + "a diverged root must re-read exactly the tracked slots" + ); + assert_eq!(report.rounds, 2, "probe round + re-read round"); + assert_eq!( + cache.cached_storage_value(tracked, slot), + Some(fresh_value), + "the re-read value is injected into the cache" + ); + assert_eq!( + planner.updated_baseline().get(&tracked), + Some(new_root), + "the newly observed root is adopted" + ); + Ok(()) +} + +/// Phase-8 s5: with NO baseline entry for a tracked account (first run), the +/// planner conservatively re-reads and adopts — absence of evidence is not +/// currency. +#[tokio::test] +async fn missing_baseline_entry_rereads_and_adopts() -> Result<()> { + let tracked = Address::repeat_byte(0x33); + let slot = U256::from(9); + let root = B256::repeat_byte(0xcc); + let mut cache = setup_cache().await?; + + install_const_root_fetcher(&mut cache, root); + let fetches = install_counting_storage_fetcher(&mut cache, U256::from(5)); + + let mut planner = RootBaselinePlanner::new( + vec![(tracked, vec![slot])], + RootBaseline::default(), // empty: nothing persisted yet + ); + cache.run_cold_start(&mut planner, ColdStartConfig::default())?; + + assert_eq!( + fetches.load(Ordering::SeqCst), + 1, + "no baseline entry means the tracked slot must be read" + ); + assert_eq!( + planner.updated_baseline().get(&tracked), + Some(root), + "the first observed root is adopted as the new baseline" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Implementation-agent tests (Wave 8): guard, probe-failure no-clobber, mixed run +// --------------------------------------------------------------------------- + +/// Phase-8 s5: a probe_roots-bearing round over a cache with no account-proof +/// fetcher errors with `NoAccountProofFetcher` before issuing any read — +/// mirroring the storage-fetcher `NoBatchFetcher` guard. +#[tokio::test(flavor = "multi_thread")] +async fn probe_roots_without_account_proof_fetcher_errors() -> Result<()> { + // `from_backend` builds a cache with no fetchers installed (the same + // pattern the NoBatchFetcher tests use in tests/cold_start.rs). + let base = setup_cache().await?; + let mut cache = evm_fork_cache::cache::EvmCache::from_backend( + base.unchecked_backend().clone(), + base.unchecked_blockchain_db().clone(), + base.block(), + base.chain_id(), + None, + None, + revm::primitives::hardfork::SpecId::CANCUN, + ); + assert!( + cache.account_proof_fetcher().is_none(), + "from_backend cache has no account-proof fetcher" + ); + + let mut planner = RootBaselinePlanner::new( + vec![(Address::repeat_byte(0x41), vec![U256::from(1)])], + RootBaseline::default(), + ); + let err = cache + .run_cold_start(&mut planner, ColdStartConfig::default()) + .expect_err("a probe_roots round with no account-proof fetcher must error"); + assert!( + matches!(err, evm_fork_cache::ColdStartError::NoAccountProofFetcher), + "got {err:?}" + ); + Ok(()) +} + +/// Phase-8 s5: a failed probe (`root: None`) is conservative — the tracked +/// slots ARE re-read, but no root is adopted: the updated baseline keeps the +/// OLD persisted root for that address (an unobserved root must not clobber a +/// real one). +#[tokio::test] +async fn failed_probe_rereads_but_never_clobbers_baseline_root() -> Result<()> { + let tracked = Address::repeat_byte(0x42); + let slot = U256::from(11); + let old_root = B256::repeat_byte(0xdd); + let mut cache = setup_cache().await?; + + // Every probe fails: the fetcher returns Err for each requested address. + cache.set_account_proof_fetcher(Arc::new(|requests, _block| { + requests + .into_iter() + .map(|(addr, _keys)| (addr, Err(StorageFetchError::custom("proof endpoint down")))) + .collect() + })); + let fetches = install_counting_storage_fetcher(&mut cache, U256::from(3)); + + let mut baseline = RootBaseline::default(); + baseline.insert(tracked, old_root); + + let mut planner = RootBaselinePlanner::new(vec![(tracked, vec![slot])], baseline); + let report = cache.run_cold_start(&mut planner, ColdStartConfig::default())?; + + assert_eq!( + fetches.load(Ordering::SeqCst), + 1, + "an unobservable root must conservatively re-read the tracked slots" + ); + assert_eq!(report.rounds, 2, "probe round + conservative re-read round"); + assert_eq!( + report.per_round[0].probe_roots_requested, 1, + "the probe round declared one root probe" + ); + assert_eq!( + report.per_round[0].probe_roots_failed, 1, + "the failed probe is visible in the round summary" + ); + assert_eq!( + planner.updated_baseline().get(&tracked), + Some(old_root), + "an unobserved root must not clobber the persisted baseline entry" + ); + Ok(()) +} + +/// Phase-8 s5: a mixed run — one tracked account's root equals the baseline and +/// another's diverged — re-reads ONLY the diverged account's tracked slots and +/// adopts per-account. +#[tokio::test] +async fn mixed_run_rereads_only_the_diverged_account() -> Result<()> { + let stable = Address::repeat_byte(0x51); + let moved = Address::repeat_byte(0x52); + let stable_slot = U256::from(1); + let moved_slot = U256::from(2); + let stable_root = B256::repeat_byte(0xa1); + let old_moved_root = B256::repeat_byte(0xb1); + let new_moved_root = B256::repeat_byte(0xb2); + let mut cache = setup_cache().await?; + + // Per-address roots: `stable` still matches its baseline; `moved` reports a + // new root. + cache.set_account_proof_fetcher(Arc::new(move |requests, _block| { + requests + .into_iter() + .map(|(addr, _keys)| { + let root = if addr == stable { + stable_root + } else { + new_moved_root + }; + ( + addr, + Ok(AccountProof { + storage_hash: root, + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + })); + + // Record exactly which slots get re-read. + let seen: Arc>> = + Arc::new(std::sync::Mutex::new(Vec::new())); + let record = seen.clone(); + cache.set_storage_batch_fetcher(Arc::new(move |requests, _block| { + record + .lock() + .unwrap() + .extend(requests.iter().map(|&(a, s)| (a, s))); + requests + .into_iter() + .map(|(a, s)| (a, s, Ok(U256::from(9)))) + .collect() + })); + + let mut baseline = RootBaseline::default(); + baseline.insert(stable, stable_root); + baseline.insert(moved, old_moved_root); + + let mut planner = RootBaselinePlanner::new( + vec![(stable, vec![stable_slot]), (moved, vec![moved_slot])], + baseline, + ); + let report = cache.run_cold_start(&mut planner, ColdStartConfig::default())?; + + assert_eq!(report.rounds, 2, "one probe round + one re-read round"); + assert_eq!( + *seen.lock().unwrap(), + vec![(moved, moved_slot)], + "only the diverged account's tracked slots are re-read" + ); + assert_eq!( + planner.updated_baseline().get(&stable), + Some(stable_root), + "the unchanged account retains its (re-observed) root" + ); + assert_eq!( + planner.updated_baseline().get(&moved), + Some(new_moved_root), + "the diverged account adopts the newly observed root" + ); + Ok(()) +} diff --git a/tests/liveness_root_gate.rs b/tests/liveness_root_gate.rs new file mode 100644 index 0000000..b77facb --- /dev/null +++ b/tests/liveness_root_gate.rs @@ -0,0 +1,796 @@ +//! Manager-authored red-green acceptance tests for Phase-8 step 4: the +//! `storageHash` root gate. +//! +//! A `WholeAccount`-tracked contract's storage root is a sound per-account change +//! oracle. Each canonical block the runtime probes tracked accounts' roots via the +//! account-proof seam and compares them to the baseline it adopted. If a tracked +//! account's root MOVED but no decoder touched it this block, that is a coverage +//! gap: state changed through a path no decoder covers. The runtime emits a +//! `ReactiveReport::CoverageGap`, schedules a `ResyncReason::RootMoved` repair, and +//! counts it — turning the decoder blind spot into an explicit, cheap signal. When +//! the root is unchanged, there is no gap and no resync. +//! +//! Fully offline: the account-proof seam is stubbed with an in-memory table keyed +//! on the probed block, so no test reaches the network. +#![cfg(feature = "reactive")] + +mod common; + +use std::sync::Arc; + +use alloy_consensus::Header; +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, U256}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::cache::AccountProof; +use evm_fork_cache::reactive::{ + ChainStatus, InputSource, ReactiveConfig, ReactiveContext, ReactiveInput, ReactiveInputBatch, + ReactiveInputRecord, ReactiveReport, ReactiveRuntime, ResyncReason, RootGateCadence, + TrackingPolicy, +}; + +/// A canonical block-header input for block `number`. +fn header_input(number: u64) -> ReactiveInput { + let consensus = Header { + number, + timestamp: 1_700_000_000 + number, + base_fee_per_gas: Some(7), + beneficiary: Address::repeat_byte(0xcb), + gas_limit: 30_000_000, + mix_hash: B256::repeat_byte(0xab), + ..Default::default() + }; + ReactiveInput::BlockHeader(alloy_rpc_types_eth::Header::new(consensus)) +} + +fn canonical_context(number: u64) -> ReactiveContext { + let block = evm_fork_cache::reactive::BlockRef { + number, + hash: B256::repeat_byte(number as u8), + parent_hash: Some(B256::repeat_byte((number.saturating_sub(1)) as u8)), + timestamp: Some(1_700_000_000 + number), + }; + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(0), + } +} + +fn header_batch(number: u64) -> ReactiveInputBatch { + ReactiveInputBatch::new(vec![ReactiveInputRecord::new( + header_input(number), + canonical_context(number), + )]) +} + +/// Install an account-proof fetcher whose `storage_hash` for any address is a +/// function of the probed block: blocks `<= pivot` return `root_a`, blocks +/// `> pivot` return `root_b`. Deterministic regardless of probe cadence. +fn install_block_keyed_root_fetcher( + cache: &mut evm_fork_cache::cache::EvmCache, + pivot: u64, + root_a: B256, + root_b: B256, +) { + use alloy_eips::BlockId; + cache.set_account_proof_fetcher(Arc::new( + move |requests: Vec<(Address, Vec)>, block: BlockId| { + let probed = match block { + BlockId::Number(n) => n.as_number().unwrap_or(u64::MAX), + _ => u64::MAX, + }; + let root = if probed <= pivot { root_a } else { root_b }; + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: root, + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }, + )); +} + +/// Phase-8 s4: a `WholeAccount`-tracked account whose root MOVES on a block that +/// touched no decoder emits a `CoverageGap` report + a `RootMoved` resync, and +/// increments the `coverage_gaps` counter. +#[tokio::test] +async fn whole_account_root_move_untouched_emits_coverage_gap_and_resync() -> Result<()> { + let tracked = Address::repeat_byte(0x77); + let mut cache = setup_cache().await?; + // Baseline root through block 10; a moved root from block 11 onward. + install_block_keyed_root_fetcher( + &mut cache, + 10, + B256::repeat_byte(0xa1), + B256::repeat_byte(0xb2), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + runtime.track_account(tracked, TrackingPolicy::WholeAccount); + + // Block 10: adopt the baseline root (no decoder touches `tracked`). + let report10 = runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + assert!( + !report10 + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "adopting the baseline is not a coverage gap" + ); + + // Block 11: the tracked account's root moved but nothing touched it. + let report11 = runtime.ingest_batch_with_resync(&mut cache, header_batch(11))?; + + let gap = report11 + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::CoverageGap(report) => Some(report), + _ => None, + }) + .expect("a moved root with no covering decoder must emit a CoverageGap"); + assert_eq!(gap.address, tracked, "the gap names the tracked account"); + + // A RootMoved resync was scheduled for the tracked account. + let root_moved = report11 + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved); + assert!( + root_moved, + "a moved root must schedule a RootMoved resync, got {:?}", + report11.resyncs + ); + + assert_eq!(runtime.metrics().coverage_gaps, 1); + Ok(()) +} + +/// Phase-8 s4: when a tracked account's root is unchanged, there is no coverage +/// gap and no resync — the tight, cheap steady-state path. +#[tokio::test] +async fn whole_account_root_unchanged_no_gap_or_resync() -> Result<()> { + let tracked = Address::repeat_byte(0x78); + let mut cache = setup_cache().await?; + // Same root for all blocks (pivot far in the future). + install_block_keyed_root_fetcher( + &mut cache, + u64::MAX, + B256::repeat_byte(0xa1), + B256::repeat_byte(0xb2), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + runtime.track_account(tracked, TrackingPolicy::WholeAccount); + + runtime.ingest_batch_with_resync(&mut cache, header_batch(20))?; + let report = runtime.ingest_batch_with_resync(&mut cache, header_batch(21))?; + + assert!( + !report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "an unchanged root is not a coverage gap" + ); + assert!( + !report + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved), + "an unchanged root schedules no RootMoved resync" + ); + assert_eq!(runtime.metrics().coverage_gaps, 0); + Ok(()) +} + +// ---------------------------------------------------------------------------- +// Wave-7 (implementation-agent) tests: decoder-covered move, Scalars field move, +// and the Slots opt-out. +// ---------------------------------------------------------------------------- + +use alloy_primitives::{Bytes, Log as PrimitiveLog}; +use alloy_rpc_types_eth::{Filter, Log}; +use evm_fork_cache::StateUpdate; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + HandlerError, HandlerId, HandlerOutcome, LogInterest, ReactiveEffect, ReactiveHandler, + ReactiveInterest, RouteKeySpec, StateEffectQuality, +}; + +/// A handler that writes a single absolute storage slot on `address` for every +/// matching log — enough to place `address` in the batch's touched set. +struct WriteSlotOnLog { + address: Address, + slot: U256, +} + +impl ReactiveHandler for WriteSlotOnLog { + fn id(&self) -> HandlerId { + HandlerId::new("write-slot-on-log") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + // Write the block number as the slot value so each block produces a real + // `SlotChange` (a repeated identical write records no change, which would + // leave the account out of the touched set). + let value = U256::from(ctx.block.as_ref().map(|b| b.number).unwrap_or_default()); + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + self.address, + self.slot, + value, + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +/// A canonical log input emitted by `address` at `number`. +fn log_batch(address: Address, number: u64) -> ReactiveInputBatch { + let log = Log { + inner: PrimitiveLog::new_unchecked(address, vec![B256::repeat_byte(0xee)], Bytes::new()), + block_hash: Some(B256::repeat_byte(number as u8)), + block_number: Some(number), + block_timestamp: Some(1_700_000_000 + number), + transaction_hash: Some(B256::repeat_byte(0x44)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + }; + ReactiveInputBatch::new(vec![ReactiveInputRecord::new( + ReactiveInput::Log(log), + canonical_context(number), + )]) +} + +/// Phase-8 s4: a `WholeAccount`-tracked account whose root moved but which a +/// decoder wrote this block (addr ∈ touched) is covered — no `CoverageGap`, no +/// `RootMoved` resync, and the counter stays put. +#[tokio::test] +async fn whole_account_root_move_touched_no_coverage_gap() -> Result<()> { + let tracked = Address::repeat_byte(0x79); + let mut cache = setup_cache().await?; + // Baseline root through block 10; a moved root from block 11 onward. + install_block_keyed_root_fetcher( + &mut cache, + 10, + B256::repeat_byte(0xa1), + B256::repeat_byte(0xb2), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + // A decoder that writes a slot on `tracked` for every log it emits. + runtime.register_handler(Arc::new(WriteSlotOnLog { + address: tracked, + slot: U256::from(7), + }))?; + runtime.track_account(tracked, TrackingPolicy::WholeAccount); + + // Block 10: adopt the baseline (the log write covers the account, but there is + // no baseline yet, so adoption — not a gap — is the outcome regardless). + runtime.ingest_batch_with_resync(&mut cache, log_batch(tracked, 10))?; + + // Block 11: the root moved AND a decoder wrote `tracked` this block. + let report = runtime.ingest_batch_with_resync(&mut cache, log_batch(tracked, 11))?; + + assert!( + !report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "a decoder covered the move; there is no coverage gap" + ); + assert!( + !report + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved), + "a decoder-covered move schedules no RootMoved resync, got {:?}", + report.resyncs + ); + assert_eq!(runtime.metrics().coverage_gaps, 0); + Ok(()) +} + +/// Install an account-proof fetcher whose balance for any address is a function +/// of the probed block: blocks `<= pivot` return `balance_a`, later blocks return +/// `balance_b`. The storage root is held constant (Scalars does not root-gate). +fn install_block_keyed_balance_fetcher( + cache: &mut evm_fork_cache::cache::EvmCache, + pivot: u64, + root: B256, + balance_a: U256, + balance_b: U256, +) { + use alloy_eips::BlockId; + cache.set_account_proof_fetcher(Arc::new( + move |requests: Vec<(Address, Vec)>, block: BlockId| { + let probed = match block { + BlockId::Number(n) => n.as_number().unwrap_or(u64::MAX), + _ => u64::MAX, + }; + let balance = if probed <= pivot { + balance_a + } else { + balance_b + }; + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: root, + balance, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }, + )); +} + +/// Phase-8 s4: a `Scalars`-tracked account whose native balance moves (baseline +/// vs probe, storage root held constant) schedules an account-field resync — the +/// balance/nonce freshness path. Root is irrelevant for `Scalars`. +#[tokio::test] +async fn scalars_balance_move_schedules_account_resync() -> Result<()> { + let tracked = Address::repeat_byte(0x7a); + let mut cache = setup_cache().await?; + // Root constant across all blocks; balance moves after block 10. + install_block_keyed_balance_fetcher( + &mut cache, + 10, + B256::repeat_byte(0xa1), + U256::from(100), + U256::from(200), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + runtime.track_account(tracked, TrackingPolicy::Scalars); + + // Block 10: adopt the baseline balance (100). + let report10 = runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + assert!( + !report10 + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved), + "adopting the Scalars baseline schedules no resync" + ); + + // Block 11: the balance moved (100 -> 200) with the root unchanged. + let report11 = runtime.ingest_batch_with_resync(&mut cache, header_batch(11))?; + + let resync = report11 + .resyncs + .iter() + .find(|req| req.reason == ResyncReason::RootMoved) + .expect("a moved balance must schedule an account-field resync"); + let balances_target = resync.targets.iter().any(|target| { + matches!( + target, + evm_fork_cache::reactive::ResyncTarget::Account { address, fields } + if *address == tracked && fields.balance + ) + }); + assert!( + balances_target, + "the Scalars resync targets the account's balance field, got {:?}", + resync.targets + ); + Ok(()) +} + +/// Phase-8 s4 / spec Decision 3: a `Slots`-tracked account is never root-gated — +/// even when its storage root moves, no `CoverageGap` is emitted and no +/// `RootMoved` resync is scheduled. +#[tokio::test] +async fn slots_policy_is_never_root_gated() -> Result<()> { + let tracked = Address::repeat_byte(0x7b); + let mut cache = setup_cache().await?; + // Root moves after block 10 — which a Slots policy must ignore. + install_block_keyed_root_fetcher( + &mut cache, + 10, + B256::repeat_byte(0xa1), + B256::repeat_byte(0xb2), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + runtime.track_account( + tracked, + TrackingPolicy::Slots { + slots: vec![U256::from(1), U256::from(2)], + }, + ); + + runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + let report = runtime.ingest_batch_with_resync(&mut cache, header_batch(11))?; + + assert!( + !report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "a Slots-tracked account is never root-gated: no coverage gap" + ); + assert!( + !report + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved), + "a Slots-tracked account is never root-gated: no RootMoved resync" + ); + assert_eq!(runtime.metrics().coverage_gaps, 0); + Ok(()) +} + +/// §6.1 (spec item 13): the root gate probes ALL root-gated targets through +/// ONE seam invocation per firing — the fetcher can then fan the requests out +/// concurrently — instead of one invocation per tracked account. +#[tokio::test] +async fn root_gate_issues_one_batched_seam_invocation_per_firing() -> Result<()> { + use std::sync::Mutex; + + let mut cache = setup_cache().await?; + // Record (batch size) per seam invocation; every probe returns a stable + // root so no gaps/resyncs distract the assertion. + let invocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + cache.set_account_proof_fetcher({ + let invocations = invocations.clone(); + Arc::new( + move |requests: Vec<(Address, Vec)>, _block: alloy_eips::BlockId| { + invocations.lock().unwrap().push(requests.len()); + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: B256::repeat_byte(0x11), + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }, + ) + }); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + // These tests exercise per-block gate semantics on consecutive blocks. + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(1)); + // Three WholeAccount + one Scalars = four root-gated targets (Slots never + // gates). + runtime.track_account(Address::repeat_byte(0x01), TrackingPolicy::WholeAccount); + runtime.track_account(Address::repeat_byte(0x02), TrackingPolicy::WholeAccount); + runtime.track_account(Address::repeat_byte(0x03), TrackingPolicy::WholeAccount); + runtime.track_account(Address::repeat_byte(0x04), TrackingPolicy::Scalars); + runtime.track_account( + Address::repeat_byte(0x05), + TrackingPolicy::Slots { + slots: vec![U256::ZERO], + }, + ); + + runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + assert_eq!( + invocations.lock().unwrap().clone(), + vec![4], + "one firing = one seam invocation carrying every root-gated target" + ); + + runtime.ingest_batch_with_resync(&mut cache, header_batch(11))?; + assert_eq!( + invocations.lock().unwrap().clone(), + vec![4, 4], + "each firing batches all targets; none are probed individually" + ); + Ok(()) +} + +/// §6.1 (spec item 15, RPC-gated): a ~50-address probe through the DEFAULT +/// proof fetcher completes in far less than 50 × single-proof latency, +/// demonstrating the bounded concurrent fan-out. Needs a live endpoint: +/// `E2E_RPC_URL=... cargo test --test liveness_root_gate -- --ignored`. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs E2E_RPC_URL (live provider); latency bound is environment-dependent"] +async fn default_proof_fetcher_fans_out_concurrently() -> Result<()> { + use alloy_rpc_client::RpcClient; + use alloy_transport_http::Http; + use evm_fork_cache::cache::EvmCache; + + let Ok(rpc_url) = std::env::var("E2E_RPC_URL") else { + eprintln!("E2E_RPC_URL not set; skipping"); + return Ok(()); + }; + let client = reqwest::Client::builder().gzip(true).build()?; + let http = Http::with_client(client, rpc_url.parse()?); + let provider = Arc::new(alloy_provider::RootProvider::< + alloy_provider::network::AnyNetwork, + >::new(RpcClient::new(http, false))); + + let cache = EvmCache::builder(provider).build().await; + let fetcher = cache + .account_proof_fetcher() + .expect("provider-backed cache has a default proof fetcher") + .clone(); + let block = cache.block(); + + // 50 distinct addresses (existence is irrelevant: absent accounts also + // return proofs). + let batch: Vec<(Address, Vec)> = (1..=50u8) + .map(|i| (Address::repeat_byte(i), vec![])) + .collect(); + + let single_start = std::time::Instant::now(); + let single = (fetcher)(vec![(Address::repeat_byte(0xee), vec![])], block); + let single_elapsed = single_start.elapsed(); + assert_eq!(single.len(), 1); + + let batch_start = std::time::Instant::now(); + let results = (fetcher)(batch, block); + let batch_elapsed = batch_start.elapsed(); + assert_eq!(results.len(), 50); + + eprintln!("single proof: {single_elapsed:?}; 50-address batch: {batch_elapsed:?}"); + assert!( + batch_elapsed < single_elapsed * 25, + "a 50-address batch must complete well under 50x a single proof \ + (got batch={batch_elapsed:?}, single={single_elapsed:?})" + ); + Ok(()) +} + +/// §6.2 (spec item 11): the gate fires on the first canonical block ever +/// seen, then only on cadence boundaries; `Disabled` never invokes the seam; +/// `every_n_blocks(1)` reproduces per-block probing. +#[tokio::test] +async fn root_gate_fires_on_cadence_boundaries_only() -> Result<()> { + use std::sync::Mutex; + + async fn firings_for(cadence: RootGateCadence, blocks: &[u64]) -> Result> { + let mut cache = setup_cache().await?; + let invocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + cache.set_account_proof_fetcher({ + let invocations = invocations.clone(); + Arc::new( + move |requests: Vec<(Address, Vec)>, _block: alloy_eips::BlockId| { + invocations.lock().unwrap().push(requests.len()); + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: B256::repeat_byte(0x11), + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }, + ) + }); + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.set_root_gate_cadence(cadence); + runtime.track_account(Address::repeat_byte(0x31), TrackingPolicy::WholeAccount); + for &block in blocks { + runtime.ingest_batch_with_resync(&mut cache, header_batch(block))?; + } + let invocations = invocations.lock().unwrap().clone(); + Ok(invocations) + } + + // Cadence 4 over blocks 10..=14: fires at 10 (first canonical block ever + // seen) and at 14 (>= 10 + 4); 11, 12, 13 are skipped. + assert_eq!( + firings_for(RootGateCadence::every_n_blocks(4), &[10, 11, 12, 13, 14]).await?, + vec![1, 1], + "cadence 4 fires exactly at the first block and the boundary" + ); + + // Per-block cadence reproduces the old behavior: every block probes. + assert_eq!( + firings_for(RootGateCadence::every_n_blocks(1), &[10, 11, 12]).await?, + vec![1, 1, 1] + ); + + // Disabled never consults the seam. + assert_eq!( + firings_for(RootGateCadence::Disabled, &[10, 11, 12]).await?, + Vec::::new() + ); + Ok(()) +} + +/// §6.2 (spec item 12): the decoder-touched set accumulates across skipped +/// blocks — a covered write in a skipped block does NOT report a +/// `CoverageGap` at the next firing — and the accumulator drains per firing, +/// so a later uncovered move still does. +#[tokio::test] +async fn touched_accumulates_across_skipped_blocks_and_drains_per_firing() -> Result<()> { + let tracked = Address::repeat_byte(0x7c); + let mut cache = setup_cache().await?; + // Three-phase roots keyed on the probed block: baseline through 10, a + // first move visible at the block-14 firing, a second at the block-18 one. + cache.set_account_proof_fetcher(Arc::new( + move |requests: Vec<(Address, Vec)>, block: alloy_eips::BlockId| { + let probed = match block { + alloy_eips::BlockId::Number(n) => n.as_number().unwrap_or(u64::MAX), + _ => u64::MAX, + }; + let root = if probed <= 10 { + B256::repeat_byte(0xa1) + } else if probed <= 14 { + B256::repeat_byte(0xb2) + } else { + B256::repeat_byte(0xc3) + }; + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: root, + balance: U256::ZERO, + nonce: 0, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }, + )); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(4)); + runtime.register_handler(Arc::new(WriteSlotOnLog { + address: tracked, + slot: U256::from(7u64), + }))?; + runtime.track_account(tracked, TrackingPolicy::WholeAccount); + + // Block 10: first firing adopts the baseline. + runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + + // Block 12 (skipped window): a decoder covers a write on the tracked + // account. The gate does not fire here; the touch must be remembered. + runtime.ingest_batch_with_resync(&mut cache, log_batch(tracked, 12))?; + + // Block 14: the gate fires; the root moved, but the accumulated touched + // set covers it — no gap, re-adopt. + let report14 = runtime.ingest_batch_with_resync(&mut cache, header_batch(14))?; + assert!( + !report14 + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "a decoder-covered write in a skipped block must not gap at the next firing" + ); + assert_eq!(runtime.metrics().coverage_gaps, 0); + + // Block 18: the gate fires again; the root moved again with NO covering + // touch in this window (the accumulator drained at 14) — a genuine gap. + let report18 = runtime.ingest_batch_with_resync(&mut cache, header_batch(18))?; + let gap = report18 + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::CoverageGap(report) => Some(report), + _ => None, + }) + .expect("an uncovered move after the accumulator drained must gap"); + assert_eq!(gap.address, tracked); + assert!( + report18 + .resyncs + .iter() + .any(|req| req.reason == ResyncReason::RootMoved) + ); + assert_eq!(runtime.metrics().coverage_gaps, 1); + Ok(()) +} + +/// §6.2 (spec item 14): a root move occurring inside a skipped window is +/// detected at the next firing — cadence delays detection (bounded by the +/// window), it never loses it. The gate diffs against the persisted baseline, +/// not block-over-block. +#[tokio::test] +async fn root_move_in_skipped_window_is_detected_at_next_firing() -> Result<()> { + let tracked = Address::repeat_byte(0x7d); + let mut cache = setup_cache().await?; + // The root moves at block 12 — strictly inside the skipped window + // (firings happen at 10 and 14). + install_block_keyed_root_fetcher( + &mut cache, + 11, + B256::repeat_byte(0xa1), + B256::repeat_byte(0xb2), + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.set_root_gate_cadence(RootGateCadence::every_n_blocks(4)); + runtime.track_account(tracked, TrackingPolicy::WholeAccount); + + runtime.ingest_batch_with_resync(&mut cache, header_batch(10))?; + // Skipped blocks: no probes, no reports. + let report12 = runtime.ingest_batch_with_resync(&mut cache, header_batch(12))?; + assert!( + !report12 + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::CoverageGap(_))), + "no probe happens inside the skipped window" + ); + + // The next firing sees the moved root and reports the gap: detection was + // delayed from block 12 to block 14, never lost. + let report14 = runtime.ingest_batch_with_resync(&mut cache, header_batch(14))?; + let gap = report14 + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::CoverageGap(report) => Some(report), + _ => None, + }) + .expect("a move inside a skipped window must be detected at the next firing"); + assert_eq!(gap.address, tracked); + assert_eq!(runtime.metrics().coverage_gaps, 1); + Ok(()) +} diff --git a/tests/public_release_surface.rs b/tests/public_release_surface.rs index e311943..5be4f29 100644 --- a/tests/public_release_surface.rs +++ b/tests/public_release_surface.rs @@ -102,11 +102,16 @@ fn release_docs_and_ci_do_not_advertise_removed_protocol_feature() { "docs/ROADMAP.md", ] { let text = read(path); + // NOTE: `--no-default-features` was previously forbidden here — in the + // protocols-feature era it was the "exclude protocol adapters" + // incantation. That feature is gone; the flag now legitimately drives the + // reactive feature matrix in CI (polling-only / no-reactive core), so it + // is no longer a protocol-surface tell. The specific protocol strings + // below remain the real guard. for forbidden in [ "protocols feature", "feature = \"protocols\"", "default = [\"protocols\"]", - "--no-default-features", "non-`protocols`", "UniswapV3Decoder", "inject_v3_", @@ -190,3 +195,107 @@ fn generic_storage_purge_api_uses_contract_terminology() { } } } + +#[test] +fn pre_1_0_api_renames_are_clean_breaks_not_aliases() { + let cache = read("src/cache/mod.rs"); + + for required in [ + "pub fn checkpoint(&self) -> revm::database::Cache", + "pub fn snapshot(&mut self) -> Arc", + "pub struct StorageBatchConfig", + "impl From for StorageBatchConfig", + "pub fn storage_batch_config(mut self, config: impl Into) -> Self", + "pub fn speed_mode(self, mode: CacheSpeedMode) -> Self", + "pub fn storage_batch_config(&self) -> StorageBatchConfig", + "StorageFetchResult", + "StorageFetchResult", + ] { + assert!( + cache.contains(required), + "cache API should expose the renamed surface: {required}" + ); + } + + for forbidden in [ + "pub fn create_snapshot(", + "pub fn set_cache_speed_mode", + "pub fn cache_speed_mode", + "static CACHE_SPEED_MODE", + "validation handle taken twice", + "dyn Fn(Vec<(Address, U256)>, Option)", + "dyn Fn(Vec<(Address, Vec)>, Option)", + "batch_block_id", + ] { + assert!( + !cache.contains(forbidden), + "cache API should not keep pre-0.2.0 aliases/state: {forbidden}" + ); + } + + let freshness = read("src/freshness.rs"); + assert!( + freshness.contains("pub async fn validate(mut self) -> Result"), + "SpeculativeSim::validate should return Result" + ); + assert!( + !freshness.contains("validation handle taken twice"), + "SpeculativeSim::validate should return Err instead of panicking on a consumed handle" + ); +} + +#[test] +fn library_error_surface_is_typed_not_anyhow() { + let manifest = read("Cargo.toml"); + let dependencies = manifest + .split("[dev-dependencies]") + .next() + .expect("manifest has dependency section"); + assert!( + !dependencies.contains("anyhow"), + "anyhow should not be a normal library dependency" + ); + + let errors = read("src/errors.rs"); + for required in [ + "pub enum CacheError", + "pub enum StorageFetchError", + "pub enum RpcError", + "pub enum FreshnessError", + "pub enum DeployError", + "pub enum MulticallError", + "pub enum AccessListError", + "pub enum SimHostError", + "Other(#[source] SimHostError)", + ] { + assert!( + errors.contains(required), + "typed error surface should expose {required}" + ); + } + + for path in [ + "src/access_list.rs", + "src/cache/mod.rs", + "src/cache/overlay.rs", + "src/deploy.rs", + "src/errors.rs", + "src/events/mod.rs", + "src/freshness.rs", + "src/multicall.rs", + ] { + let text = read(path); + for forbidden in [ + "use anyhow", + "anyhow::Result", + "anyhow::Error", + "anyhow!(", + "From", + ] { + assert!( + !text.contains(forbidden), + "{path} should not expose crate-owned anyhow errors: {forbidden}" + ); + } + } +} diff --git a/tests/reactive_alloy_subscriber.rs b/tests/reactive_alloy_subscriber.rs index 8148cfb..91d30b3 100644 --- a/tests/reactive_alloy_subscriber.rs +++ b/tests/reactive_alloy_subscriber.rs @@ -7,28 +7,36 @@ use std::time::Duration; use alloy_network::Ethereum; -use alloy_primitives::{Address, keccak256}; #[cfg(feature = "reactive-polling")] -use alloy_primitives::{B256, Bytes, Log as PrimitiveLog, U256}; +use alloy_primitives::U256; +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] +use alloy_primitives::{Address, keccak256}; +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] +use alloy_primitives::{B256, Bytes, Log as PrimitiveLog}; use alloy_provider::ProviderBuilder; +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] use alloy_rpc_types_eth::Filter; -#[cfg(feature = "reactive-polling")] +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] use alloy_rpc_types_eth::Log; use alloy_transport::mock::Asserter; use anyhow::Result; -#[cfg(feature = "reactive-polling")] +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] use anyhow::bail; #[cfg(feature = "reactive-ws")] use evm_fork_cache::reactive::BlockInterestMode; use evm_fork_cache::reactive::{ - AlloySubscriber, BlockInterest, EventSubscriber, LogInterest, PendingTxInterest, - ReactiveInterest, SubscriberConfig, SubscriberError, SubscriberMode, SubscriberReconnectConfig, + AlloySubscriber, EventSubscriber, PendingTxInterest, ReactiveInterest, SubscriberConfig, + SubscriberError, SubscriberMode, SubscriberReconnectConfig, }; -#[cfg(feature = "reactive-polling")] +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] +use evm_fork_cache::reactive::{BlockInterest, LogInterest}; +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] use evm_fork_cache::reactive::{ChainStatus, InputSource, ReactiveInput}; +#[cfg(feature = "reactive-ws")] +use evm_fork_cache::reactive::{HandlerId, SubscriberBackfill}; -#[cfg(feature = "reactive-polling")] +#[cfg(any(feature = "reactive-polling", feature = "reactive-ws"))] fn rpc_log(address: Address, topic0: B256, block_number: u64, log_index: u64) -> Log { Log { inner: PrimitiveLog::new_unchecked(address, vec![topic0], Bytes::new()), @@ -120,6 +128,188 @@ fn alloy_subscriber_pubsub_accepts_logs_pending_hashes_and_block_headers() -> Re Ok(()) } +#[test] +#[cfg(feature = "reactive-ws")] +fn alloy_subscriber_owner_interests_add_and_remove_incrementally() -> Result<()> { + let pool_a = Address::repeat_byte(0xa1); + let pool_b = Address::repeat_byte(0xb2); + let topic_a = keccak256(b"PoolA(uint256)"); + let topic_b = keccak256(b"PoolB(uint256)"); + + let mut subscriber = mock_subscriber(SubscriberMode::PubSub); + subscriber.add_interest_owner( + HandlerId::new("pool-a"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool_a).event_signature(topic_a), + local_matcher: None, + route_key: None, + })], + )?; + subscriber.add_interest_owner( + HandlerId::new("pool-b"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool_b).event_signature(topic_b), + local_matcher: None, + route_key: None, + })], + )?; + + assert_eq!(subscriber.registered_interests().len(), 2); + assert_eq!( + subscriber + .owner_interests(&HandlerId::new("pool-a")) + .expect("pool-a interests") + .len(), + 1 + ); + + let removed = subscriber + .remove_interest_owner(&HandlerId::new("pool-a")) + .expect("pool-a should be removed"); + assert_eq!(removed.len(), 1); + assert!( + subscriber + .owner_interests(&HandlerId::new("pool-a")) + .is_none() + ); + assert_eq!(subscriber.registered_interests().len(), 1); + assert_eq!( + subscriber + .owner_interests(&HandlerId::new("pool-b")) + .expect("pool-b interests") + .len(), + 1 + ); + assert!( + subscriber + .remove_interest_owner(&HandlerId::new("missing")) + .is_none() + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +#[cfg(feature = "reactive-ws")] +async fn alloy_subscriber_owner_backfill_yields_before_live_streams() -> Result<()> { + let asserter = Asserter::new(); + let pool = Address::repeat_byte(0xcd); + let topic = keccak256(b"DiscoveredPool(uint256)"); + let log = rpc_log(pool, topic, 42, 3); + asserter.push_success(&vec![log.clone()]); + + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig { + hydrate_pending_transactions: false, + max_batch_size: 16, + ..SubscriberConfig::default() + }, + ); + subscriber.add_interest_owner_with_backfill( + HandlerId::new("pool-cd"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool).event_signature(topic), + local_matcher: None, + route_key: None, + })], + SubscriberBackfill::range(40, 42), + )?; + + let Some(batch) = subscriber.next_batch().await? else { + bail!("expected owner-scoped backfill batch before live subscription"); + }; + let records = batch.records(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].context.source, InputSource::Backfill); + assert!( + matches!(records[0].context.chain_status, ChainStatus::Included { ref block, confirmations: 0 } if block.number == 42) + ); + assert!(matches!(&records[0].input, ReactiveInput::Log(actual) if actual == &log)); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +#[cfg(feature = "reactive-ws")] +async fn alloy_subscriber_owner_growth_backfills_continuity_gap_end_to_end() -> Result<()> { + // An established owner (pool A) has delivered up to block 100. Growing it to + // also watch pool B changes the merged filter shape; the subscriber must + // automatically backfill the new shape from the prior anchor so nothing is + // lost across the change. Driven entirely through the public API. + let pool_a = Address::repeat_byte(0xaa); + let pool_b = Address::repeat_byte(0xbb); + + let asserter = Asserter::new(); + // (1) Establish pool A's anchor at 100 via an explicit range backfill that + // returns a log — the returned record makes next_batch yield before it + // ever reaches live-stream init. + let anchor_log = rpc_log(pool_a, keccak256(b"Swap()"), 100, 0); + asserter.push_success(&vec![anchor_log.clone()]); + // (2) Continuity backfill for the grown {A,B} shape is open-ended from the + // prior anchor: get_block_number, then get_logs over [100, 105]. + asserter.push_success(&105u64); + let gap_log = rpc_log(pool_b, keccak256(b"Swap()"), 103, 0); + asserter.push_success(&vec![gap_log.clone()]); + + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig { + hydrate_pending_transactions: false, + max_batch_size: 16, + ..SubscriberConfig::default() + }, + ); + + // Establish pool A with an explicit backfill through block 100. + subscriber.add_interest_owner_with_backfill( + HandlerId::new("amm"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool_a), + local_matcher: None, + route_key: None, + })], + SubscriberBackfill::range(90, 100), + )?; + let Some(first) = subscriber.next_batch().await? else { + bail!("expected the establishing backfill batch"); + }; + assert!( + matches!(&first.records()[0].input, ReactiveInput::Log(actual) if actual == &anchor_log) + ); + + // Grow the owner to A+B (same block option -> merged shape changes). + subscriber.add_interest_owner( + HandlerId::new("amm"), + &[ + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool_a), + local_matcher: None, + route_key: None, + }), + ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool_b), + local_matcher: None, + route_key: None, + }), + ], + )?; + + let Some(batch) = subscriber.next_batch().await? else { + bail!("expected a continuity backfill batch for the grown owner"); + }; + let records = batch.records(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].context.source, InputSource::Backfill); + assert!(matches!(&records[0].input, ReactiveInput::Log(actual) if actual == &gap_log)); + + Ok(()) +} + #[test] #[cfg(feature = "reactive-ws")] fn alloy_subscriber_pubsub_rejects_full_body_modes() -> Result<()> { diff --git a/tests/reactive_engine.rs b/tests/reactive_engine.rs new file mode 100644 index 0000000..25b4864 --- /dev/null +++ b/tests/reactive_engine.rs @@ -0,0 +1,627 @@ +//! Implementation-owned tests for the runtime/subscriber binding layer. +#![cfg(feature = "reactive")] + +mod common; + +use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, +}; + +use alloy_network::{Ethereum, Network}; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, keccak256}; +use alloy_rpc_types_eth::{Filter, Log}; + +use common::{install_mock_erc20, setup_cache}; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + AccountFieldMask, BlockRef, ChainStatus, EventSubscriber, HandlerError, HandlerId, + HandlerOutcome, InputSource, InterestOwnerSubscriber, LogInterest, ReactiveConfig, + ReactiveContext, ReactiveEffect, ReactiveEngine, ReactiveEngineRegisterError, ReactiveHandler, + ReactiveInput, ReactiveInputBatch, ReactiveInputRecord, ReactiveInterest, ReactiveRegistry, + ReactiveReport, RegisterError, ResyncBlock, ResyncId, ResyncPriority, ResyncReason, + ResyncRequest, ResyncTarget, RouteKeySpec, StateEffectQuality, SubscriberBackfill, + SubscriberError, SubscriberNextBatch, +}; + +fn rpc_log(address: Address, topic0: B256, block_number: u64) -> Log { + Log { + inner: PrimitiveLog::new_unchecked(address, vec![topic0], Bytes::new()), + block_hash: Some(B256::repeat_byte(block_number as u8)), + block_number: Some(block_number), + block_timestamp: Some(1_700_000_000 + block_number), + transaction_hash: Some(B256::repeat_byte(0xcc)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + } +} + +fn included_context(block_number: u64) -> ReactiveContext { + let block = BlockRef { + number: block_number, + hash: B256::repeat_byte(block_number as u8), + parent_hash: Some(B256::repeat_byte(block_number.saturating_sub(1) as u8)), + timestamp: Some(1_700_000_000 + block_number), + }; + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(0), + } +} + +fn canonical_log_batch(address: Address, block_number: u64) -> ReactiveInputBatch { + ReactiveInputBatch::new(vec![ReactiveInputRecord::new( + ReactiveInput::Log(rpc_log(address, keccak256(b"Event()"), block_number)), + included_context(block_number), + )]) +} + +struct RecordingSubscriber { + full_replace_interests: Vec>, + owners: HashMap>>, + backfills: Vec<(HandlerId, SubscriberBackfill)>, + batches: VecDeque>, + fail_owner: Option, +} + +impl Default for RecordingSubscriber { + fn default() -> Self { + Self { + full_replace_interests: Vec::new(), + owners: HashMap::new(), + backfills: Vec::new(), + batches: VecDeque::new(), + fail_owner: None, + } + } +} + +impl RecordingSubscriber { + fn fail_owner(owner: HandlerId) -> Self { + Self { + fail_owner: Some(owner), + ..Self::default() + } + } +} + +impl EventSubscriber for RecordingSubscriber +where + N: Network + Send + 'static, +{ + fn register_interests( + &mut self, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + self.full_replace_interests = interests.to_vec(); + self.owners.clear(); + Ok(()) + } + + fn next_batch(&mut self) -> SubscriberNextBatch<'_, N> { + Box::pin(async move { Ok(self.batches.pop_front()) }) + } +} + +impl InterestOwnerSubscriber for RecordingSubscriber +where + N: Network + Send + 'static, +{ + fn add_interest_owner( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + ) -> Result<(), SubscriberError> { + if self.fail_owner.as_ref() == Some(&owner) { + return Err(SubscriberError::InvalidConfig("forced owner failure")); + } + self.owners.insert(owner, interests.to_vec()); + Ok(()) + } + + fn add_interest_owner_with_backfill( + &mut self, + owner: HandlerId, + interests: &[ReactiveInterest], + backfill: SubscriberBackfill, + ) -> Result<(), SubscriberError> { + self.add_interest_owner(owner.clone(), interests)?; + self.backfills.push((owner, backfill)); + Ok(()) + } + + fn remove_interest_owner(&mut self, owner: &HandlerId) -> Option>> { + self.owners.remove(owner) + } + + fn owner_interests(&self, owner: &HandlerId) -> Option<&[ReactiveInterest]> { + self.owners.get(owner).map(Vec::as_slice) + } +} + +struct NoopHandler { + id: HandlerId, + address: Address, +} + +impl NoopHandler { + fn new(id: &'static str, address: Address) -> Self { + Self { + id: HandlerId::new(id), + address, + } + } +} + +impl ReactiveHandler for NoopHandler { + fn id(&self) -> HandlerId { + self.id.clone() + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome::empty(StateEffectQuality::NoStateEffect)) + } +} + +#[test] +fn engine_register_handler_updates_runtime_and_subscriber() { + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("engine registration should succeed"); + + assert!(engine.runtime().contains_handler(&HandlerId::new("pool-a"))); + assert_eq!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-a")) + .expect("subscriber owner interests") + .len(), + 1 + ); +} + +#[test] +fn engine_duplicate_handler_does_not_mutate_subscriber() { + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("initial registration should succeed"); + + let err = engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xb2), + ))) + .expect_err("duplicate id should fail before subscriber mutation"); + + assert!(matches!( + err, + ReactiveEngineRegisterError::Register(RegisterError::DuplicateHandler(id)) + if id == HandlerId::new("pool-a") + )); + assert_eq!(engine.subscriber().owners.len(), 1); + assert_eq!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-a")) + .expect("pool-a owner") + .len(), + 1 + ); +} + +#[test] +fn engine_rolls_back_runtime_when_subscriber_registration_fails() { + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::fail_owner(HandlerId::new("pool-a")), + ); + + let err = engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect_err("subscriber failure should fail engine registration"); + + assert!(matches!( + err, + ReactiveEngineRegisterError::Subscriber(SubscriberError::InvalidConfig(_)) + )); + assert!(!engine.runtime().contains_handler(&HandlerId::new("pool-a"))); + assert!(engine.subscriber().owners.is_empty()); +} + +#[test] +fn engine_unregister_handler_updates_subscriber_then_runtime() { + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("engine registration should succeed"); + + let removed = engine + .unregister_handler(&HandlerId::new("pool-a")) + .expect("runtime handler should be removed"); + + assert_eq!(removed.id(), HandlerId::new("pool-a")); + assert!(!engine.runtime().contains_handler(&HandlerId::new("pool-a"))); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-a")) + .is_none() + ); +} + +#[test] +fn engine_register_handler_with_backfill_records_owner_backfill() { + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + let backfill = SubscriberBackfill::range(100, 120); + + engine + .register_handler_with_backfill( + Arc::new(NoopHandler::new("pool-a", Address::repeat_byte(0xa1))), + backfill, + ) + .expect("engine registration with backfill should succeed"); + + assert!(engine.runtime().contains_handler(&HandlerId::new("pool-a"))); + assert_eq!( + engine.subscriber().backfills, + vec![(HandlerId::new("pool-a"), backfill)] + ); +} + +#[test] +fn engine_sync_handler_interests_bootstraps_owner_per_handler() { + // A runtime pre-populated with handlers (registered directly, not through + // the engine) is bootstrapped onto a fresh subscriber via + // `sync_handler_interests`: each handler becomes its own owner, and the + // full-replacement blob is never touched. + let mut runtime = evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()); + runtime + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("register pool-a on runtime"); + runtime + .register_handler(Arc::new(NoopHandler::new( + "pool-b", + Address::repeat_byte(0xb2), + ))) + .expect("register pool-b on runtime"); + + let mut engine = ReactiveEngine::new(runtime, RecordingSubscriber::default()); + engine + .sync_handler_interests() + .expect("bootstrap sync should succeed"); + + assert_eq!(engine.subscriber().owners.len(), 2); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-a")) + .is_some() + ); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-b")) + .is_some() + ); + // Bootstrap uses the owner-scoped path, not the full-replacement blob. + assert!(engine.subscriber().full_replace_interests.is_empty()); + + // Re-running is idempotent (upsert semantics). + engine + .sync_handler_interests() + .expect("re-sync should succeed"); + assert_eq!(engine.subscriber().owners.len(), 2); +} + +#[test] +fn registry_and_engine_expose_same_interests_after_registration() { + let mut registry = ReactiveRegistry::::new(); + registry + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("registry registration should succeed"); + + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("engine registration should succeed"); + + assert_eq!( + engine.runtime().interests().len(), + registry.interests().len() + ); + assert_eq!(engine.subscriber().owners.len(), 1); +} + +// A handler that emits a storage resync for its account on every matching log — +// used to drive the resync-execution and teardown paths. +struct ResyncHandler { + id: HandlerId, + address: Address, +} + +impl ReactiveHandler for ResyncHandler { + fn id(&self) -> HandlerId { + self.id.clone() + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("acct-repair"), + reason: ResyncReason::HandlerRequested, + block: ResyncBlock::Latest, + targets: vec![ResyncTarget::Account { + address: self.address, + fields: AccountFieldMask { + balance: true, + ..Default::default() + }, + }], + priority: ResyncPriority::Normal, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} + +#[tokio::test] +async fn engine_register_handler_auto_anchors_from_last_canonical_block() { + let emitter = Address::repeat_byte(0xa1); + let mut cache = setup_cache().await.expect("cache"); + install_mock_erc20(&mut cache, emitter); + + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new("seed", emitter))) + .expect("register seed handler"); + + // Drive the runtime's canonical position to block 500. + engine + .ingest_batch(&mut cache, canonical_log_batch(emitter, 500)) + .expect("ingest canonical block"); + assert_eq!( + engine.runtime().last_canonical_block().map(|b| b.number), + Some(500) + ); + + // A handler registered now should be backfilled from the canonical head. + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-late", + Address::repeat_byte(0xb2), + ))) + .expect("register late handler"); + + assert_eq!( + engine.subscriber().backfills, + vec![( + HandlerId::new("pool-late"), + SubscriberBackfill::from_block(500) + )], + "mid-lifecycle registration must anchor to the last canonical block" + ); +} + +#[test] +fn engine_register_handler_on_fresh_runtime_is_live_only() { + // With no canonical block journaled yet, default registration requests no + // backfill (bootstrap before ingestion). + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new( + "pool-a", + Address::repeat_byte(0xa1), + ))) + .expect("register on fresh runtime"); + assert!( + engine.subscriber().backfills.is_empty(), + "no canonical block => no backfill" + ); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-a")) + .is_some() + ); +} + +#[tokio::test] +async fn engine_register_handler_live_only_never_backfills() { + let emitter = Address::repeat_byte(0xa1); + let mut cache = setup_cache().await.expect("cache"); + install_mock_erc20(&mut cache, emitter); + + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(NoopHandler::new("seed", emitter))) + .expect("register seed"); + engine + .ingest_batch(&mut cache, canonical_log_batch(emitter, 500)) + .expect("ingest"); + + // Even past a canonical block, live-only registration opts out of backfill. + engine + .register_handler_live_only(Arc::new(NoopHandler::new( + "pool-live", + Address::repeat_byte(0xb2), + ))) + .expect("register live-only"); + assert!(engine.subscriber().backfills.is_empty()); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool-live")) + .is_some() + ); +} + +#[tokio::test] +async fn engine_next_ingest_with_resync_executes_surfaced_resyncs() { + let emitter = Address::repeat_byte(0xd1); + let mut cache = setup_cache().await.expect("cache"); + install_mock_erc20(&mut cache, emitter); + + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(ResyncHandler { + id: HandlerId::new("repair"), + address: emitter, + })) + .expect("register resync handler"); + + // Queue a batch for the subscriber to hand back, then drive the + // resync-executing loop. + engine + .subscriber_mut() + .batches + .push_back(canonical_log_batch(emitter, 700)); + + let report = engine + .next_ingest_with_resync(&mut cache) + .await + .expect("next_ingest_with_resync") + .expect("a batch was queued"); + + // The surfaced resync was executed (a Resynced report is present) and the + // pending ledger was drained by the with-resync path. + assert!( + report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::Resynced(_))), + "resync execution should produce a Resynced report" + ); + assert!(engine.runtime().pending_resyncs().is_empty()); +} + +#[tokio::test] +async fn engine_teardown_recipe_clears_routing_tracking_and_pending_resyncs() { + use evm_fork_cache::reactive::TrackingPolicy; + + let emitter = Address::repeat_byte(0xe1); + let mut cache = setup_cache().await.expect("cache"); + install_mock_erc20(&mut cache, emitter); + + let mut engine = ReactiveEngine::new( + evm_fork_cache::ReactiveRuntime::::new(ReactiveConfig::default()), + RecordingSubscriber::default(), + ); + engine + .register_handler(Arc::new(ResyncHandler { + id: HandlerId::new("pool"), + address: emitter, + })) + .expect("register"); + engine + .runtime_mut() + .track_account(emitter, TrackingPolicy::WholeAccount); + + // Ingest (no resync execution) so a pending resync accumulates. + engine + .ingest_batch(&mut cache, canonical_log_batch(emitter, 800)) + .expect("ingest"); + assert_eq!(engine.runtime().pending_resyncs().len(), 1); + + // Full teardown recipe. + let removed = engine.unregister_handler(&HandlerId::new("pool")); + assert!(removed.is_some()); + assert!(engine.runtime_mut().untrack_account(emitter)); + let cancelled = engine.runtime_mut().cancel_pending_resyncs(emitter); + assert_eq!(cancelled.len(), 1); + + // Routing, subscriber ownership, and the pending ledger are all clear. + assert!(!engine.runtime().contains_handler(&HandlerId::new("pool"))); + assert!( + engine + .subscriber() + .owner_interests(&HandlerId::new("pool")) + .is_none() + ); + assert!(engine.runtime().pending_resyncs().is_empty()); +} diff --git a/tests/reactive_freshness.rs b/tests/reactive_freshness.rs new file mode 100644 index 0000000..df4d9d0 --- /dev/null +++ b/tests/reactive_freshness.rs @@ -0,0 +1,313 @@ +//! Manager-authored red-green acceptance test for Phase-8 step 3: `Validity` +//! stamping of reactive/event-derived writes. +//! +//! With freshness stamping enabled, applying a canonical event write stamps the +//! touched `(address, slot)` as `Validity::ValidThrough(N)` in the runtime's +//! `FreshnessRegistry` (N = the canonical block number). It is therefore NOT +//! volatile at block N (event-maintained, no need to re-verify), but ages to +//! volatile once the clock advances past N. Stamping is opt-in; a runtime that +//! never enables it exposes no registry and behaves exactly as before. +//! +//! Fully offline (mocked provider, injected state). +#![cfg(feature = "reactive")] + +mod common; + +use std::sync::Arc; + +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256}; +use alloy_rpc_types_eth::{Filter, Log}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::StateUpdate; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + BlockRef, ChainStatus, HandlerError, HandlerId, HandlerOutcome, InputSource, LogInterest, + ReactiveConfig, ReactiveContext, ReactiveEffect, ReactiveHandler, ReactiveInput, + ReactiveInputBatch, ReactiveInputRecord, ReactiveInterest, ReactiveRuntime, RouteKeySpec, + StateEffectQuality, +}; + +fn block(number: u64, hash: B256, parent_hash: B256) -> BlockRef { + BlockRef { + number, + hash, + parent_hash: Some(parent_hash), + timestamp: Some(1_700_000_000 + number), + } +} + +fn rpc_log(address: Address, block: &BlockRef, log_index: u64) -> Log { + Log { + inner: PrimitiveLog::new_unchecked(address, vec![B256::repeat_byte(0xee)], Bytes::new()), + block_hash: Some(block.hash), + block_number: Some(block.number), + block_timestamp: block.timestamp, + transaction_hash: Some(B256::repeat_byte(0x01)), + transaction_index: Some(0), + log_index: Some(log_index), + removed: false, + } +} + +fn included_context(block: BlockRef, log_index: u64) -> ReactiveContext { + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(log_index), + } +} + +fn batch(input: ReactiveInput, ctx: ReactiveContext) -> ReactiveInputBatch { + ReactiveInputBatch::new(vec![ReactiveInputRecord::new(input, ctx)]) +} + +/// A handler that writes a fixed slot on every matching log. +struct SlotWriter { + address: Address, + slot: U256, +} + +impl ReactiveHandler for SlotWriter { + fn id(&self) -> HandlerId { + HandlerId::new("freshness-slot-writer") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + self.address, + self.slot, + U256::from(1u64), + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +/// A handler that writes the log's block number into a fixed slot, so each +/// canonical block produces a genuinely-changed value (and therefore a +/// `SlotChange` in the applied diff, which is what freshness stamping keys off). +struct BlockValueWriter { + address: Address, + slot: U256, +} + +impl ReactiveHandler for BlockValueWriter { + fn id(&self) -> HandlerId { + HandlerId::new("freshness-block-value-writer") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + let number = ctx.block.as_ref().map(|b| b.number).unwrap_or_default(); + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + self.address, + self.slot, + U256::from(number), + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +/// Phase-8 s3: an enabled runtime stamps a canonical event write +/// `ValidThrough(N)` — not volatile at N, volatile once past N. +#[tokio::test] +async fn reactive_write_stamps_validity_through_block() -> Result<()> { + let address = Address::repeat_byte(0xf1); + let slot = U256::from(3); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.enable_freshness_stamping(); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + included_context(b10.clone(), 10), + ), + )?; + + let freshness = runtime + .freshness() + .expect("stamping was enabled, so a registry is present"); + // Stamped ValidThrough(10): event-maintained, so still valid *at* block 10, + // but ages to volatile once the clock moves past it. + assert!( + !freshness.is_volatile(address, slot, 10), + "an event-stamped slot is valid through its write block" + ); + assert!( + freshness.is_volatile(address, slot, 11), + "the stamp ages to volatile after its block" + ); + Ok(()) +} + +/// Phase-8 s3: stamping is opt-in — a runtime that never enables it exposes no +/// registry (behavior unchanged from before the coupling). +#[tokio::test] +async fn freshness_registry_absent_unless_enabled() -> Result<()> { + let runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + assert!( + runtime.freshness().is_none(), + "freshness stamping is opt-in; disabled by default" + ); + Ok(()) +} + +/// A pending (mempool-only) context: it must never mutate canonical cache state, +/// and so must never stamp canonical freshness. +fn pending_context(block: BlockRef, log_index: u64) -> ReactiveContext { + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Pending, + block: Some(block), + transaction_index: Some(0), + log_index: Some(log_index), + } +} + +/// Phase-8 s3: a pending input never stamps freshness — pending/mempool writes +/// are not canonical, so the stamped slot stays volatile at its block number +/// (the registry never records a `ValidThrough`). +/// +/// A pending input that emits a canonical state effect is itself rejected by the +/// runtime (`InvalidPendingEffect`); either way the freshness registry must be +/// left untouched, so the slot resolves to the default (Volatile). +#[tokio::test] +async fn pending_write_does_not_stamp_validity() -> Result<()> { + let address = Address::repeat_byte(0xf2); + let slot = U256::from(7); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.enable_freshness_stamping(); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + // The pending canonical write is rejected by the runtime; the important + // invariant for this wave is that nothing was stamped either way. + let _ = runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + pending_context(b10.clone(), 10), + ), + ); + + // The registry exists (stamping was enabled) but was never stamped: a + // pending write is not canonical, so the slot resolves to the registry + // default (Volatile) and is volatile even *at* its block number. + let freshness = runtime + .freshness() + .expect("stamping was enabled, so a registry is present"); + assert!( + freshness.is_volatile(address, slot, 10), + "a pending write must not stamp canonical freshness" + ); + Ok(()) +} + +/// Phase-8 s3: re-writing a slot at a later canonical block re-stamps it — the +/// later `ValidThrough(N)` wins, so it is valid through the newer block, not the +/// older one. +#[tokio::test] +async fn later_canonical_stamp_wins() -> Result<()> { + let address = Address::repeat_byte(0xf3); + let slot = U256::from(9); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.enable_freshness_stamping(); + // Writes the block number as the value, so the block-11 write is a genuine + // change (a no-op re-write of the same value produces no `SlotChange`, and + // stamping keys off the applied diff's changed slots). + runtime.register_handler(Arc::new(BlockValueWriter { address, slot }))?; + + // First canonical write at block 10 -> ValidThrough(10). + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + included_context(b10.clone(), 10), + ), + )?; + assert!( + !runtime.freshness().unwrap().is_volatile(address, slot, 10), + "valid through its first write block" + ); + // At this point the older stamp is already volatile past block 10. + assert!( + runtime.freshness().unwrap().is_volatile(address, slot, 11), + "the first stamp is volatile once past block 10" + ); + + // Second canonical write at the next block 11 (child of block 10) re-stamps + // the same slot -> ValidThrough(11). + let b11 = block(11, B256::repeat_byte(0x11), b10.hash); + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b11, 11)), + included_context(b11.clone(), 11), + ), + )?; + + let freshness = runtime + .freshness() + .expect("stamping was enabled, so a registry is present"); + // The later stamp wins: valid through block 11 (no longer volatile at 11), + // and ages to volatile only once the clock passes 11. + assert!( + !freshness.is_volatile(address, slot, 11), + "the later stamp wins: valid through block 11" + ); + assert!( + freshness.is_volatile(address, slot, 12), + "the later stamp ages to volatile after block 11" + ); + Ok(()) +} diff --git a/tests/reactive_health.rs b/tests/reactive_health.rs new file mode 100644 index 0000000..0980e78 --- /dev/null +++ b/tests/reactive_health.rs @@ -0,0 +1,657 @@ +//! Manager-authored red-green acceptance tests for WS-4/WS-5: the queryable +//! `CacheHealth` state and `CacheMetrics` counters on `ReactiveRuntime`. +//! +//! These describe the public contract before the implementation exists: +//! - health defaults to `Healthy`; +//! - a reorg deeper than the journal (aged-out / parent-not-in-journal) degrades +//! health to `Degraded`, increments the `deep_reorgs` counter, and emits a +//! `ReactiveReport::Health` transition; +//! - an in-journal reorg is fully recovered, increments `reorgs_recovered`, and +//! does NOT degrade health. +//! +//! Fully offline (mocked provider, injected state). +#![cfg(feature = "reactive")] + +mod common; + +use std::sync::Arc; + +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256}; +use alloy_rpc_types_eth::{Filter, Log}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + AccountFieldMask, BlockRef, CacheHealth, ChainStatus, HandlerError, HandlerId, HandlerOutcome, + InputSource, LogInterest, PendingTxInterest, ReactiveConfig, ReactiveContext, ReactiveEffect, + ReactiveError, ReactiveHandler, ReactiveInput, ReactiveInputBatch, ReactiveInputRecord, + ReactiveInterest, ReactiveReport, ReactiveRuntime, ResyncBlock, ResyncId, ResyncPriority, + ResyncReason, ResyncRequest, ResyncTarget, RouteKeySpec, StateEffectQuality, +}; +use evm_fork_cache::{PurgeScope, StateUpdate}; + +fn block(number: u64, hash: B256, parent_hash: B256) -> BlockRef { + BlockRef { + number, + hash, + parent_hash: Some(parent_hash), + timestamp: Some(1_700_000_000 + number), + } +} + +fn rpc_log(address: Address, block: &BlockRef, log_index: u64) -> Log { + Log { + inner: PrimitiveLog::new_unchecked(address, vec![B256::repeat_byte(0xee)], Bytes::new()), + block_hash: Some(block.hash), + block_number: Some(block.number), + block_timestamp: block.timestamp, + transaction_hash: Some(B256::repeat_byte(0x01)), + transaction_index: Some(0), + log_index: Some(log_index), + removed: false, + } +} + +fn included_context(block: BlockRef, log_index: u64) -> ReactiveContext { + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(log_index), + } +} + +fn batch(input: ReactiveInput, ctx: ReactiveContext) -> ReactiveInputBatch { + ReactiveInputBatch::new(vec![ReactiveInputRecord::new(input, ctx)]) +} + +/// A handler that writes `log_index` to a fixed slot, so each canonical block +/// leaves a distinct reversible storage effect in the journal. +struct SlotWriter { + address: Address, + slot: U256, +} + +impl ReactiveHandler for SlotWriter { + fn id(&self) -> HandlerId { + HandlerId::new("health-slot-writer") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + self.address, + self.slot, + U256::from(ctx.log_index.expect("test context carries a log index")), + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +/// WS-4/WS-5: a fresh runtime is `Healthy` with zeroed counters. +#[tokio::test] +async fn health_defaults_to_healthy() -> Result<()> { + let runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + assert_eq!(runtime.health(), CacheHealth::Healthy); + let m = runtime.metrics(); + assert_eq!(m.deep_reorgs, 0); + assert_eq!(m.reorgs_recovered, 0); + Ok(()) +} + +/// WS-4/WS-5: a reorg that references a block no longer in the journal (here +/// forced via `journal_depth = 1`) degrades health to `Degraded`, increments the +/// `deep_reorgs` counter, and emits a `ReactiveReport::Health` transition. +#[tokio::test] +async fn deep_reorg_beyond_journal_degrades_health() -> Result<()> { + let address = Address::repeat_byte(0xd1); + let slot = U256::from(7); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig { + journal_depth: 1, + ..ReactiveConfig::default() + }); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + let b11 = block(11, B256::repeat_byte(0x11), b10.hash); + // Replacement for block 11 whose parent is NOT in the (depth-1) journal. + let b11_alt = block(11, B256::repeat_byte(0x1b), B256::repeat_byte(0x1a)); + + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + included_context(b10.clone(), 10), + ), + )?; + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b11, 11)), + included_context(b11.clone(), 11), + ), + )?; + assert_eq!(runtime.health(), CacheHealth::Healthy, "healthy so far"); + + // Ingesting the replacement block references a parent aged out of the + // depth-1 journal: under-recovery => degrade. + let report = runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b11_alt, 12)), + included_context(b11_alt.clone(), 12), + ), + )?; + + assert!( + matches!(runtime.health(), CacheHealth::Degraded { .. }), + "a reorg beyond the journal must degrade health, got {:?}", + runtime.health() + ); + assert_eq!( + runtime.metrics().deep_reorgs, + 1, + "the deep-reorg counter must increment" + ); + assert!( + report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::Health(_))), + "a Health transition report must be emitted" + ); + Ok(()) +} + +/// WS-4/WS-5: an in-journal reorg (parent present in the default-depth journal) +/// is fully recovered — it increments `reorgs_recovered` and does NOT degrade +/// health. +#[tokio::test] +async fn in_journal_reorg_recovers_without_degrading() -> Result<()> { + let address = Address::repeat_byte(0xd2); + let slot = U256::from(8); + let parent = block(79, B256::repeat_byte(0x79), B256::repeat_byte(0x78)); + let dropped = block(80, B256::repeat_byte(0x80), parent.hash); + let replacement = block(80, B256::repeat_byte(0x81), parent.hash); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &parent, 10)), + included_context(parent.clone(), 10), + ), + )?; + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &dropped, 20)), + included_context(dropped.clone(), 20), + ), + )?; + let report = runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &replacement, 30)), + included_context(replacement.clone(), 30), + ), + )?; + + // An in-journal parent-mismatch reorg was recovered. + assert!( + report + .reports + .iter() + .any(|r| matches!(r.as_ref(), ReactiveReport::Reorg(_))), + "replacement block emits a reorg report" + ); + assert_eq!( + runtime.metrics().reorgs_recovered, + 1, + "one in-journal reorg increments reorgs_recovered exactly once" + ); + assert_eq!( + runtime.health(), + CacheHealth::Healthy, + "a fully-recovered in-journal reorg must NOT degrade health" + ); + Ok(()) +} + +/// A handler that requests an account-field resync, which the provider-neutral +/// cache seam does not support — the resync execution pass reports it as a +/// failure. Used to drive both `resync_requests` and `resync_failures`. +struct AccountResyncHandler { + address: Address, + block: ResyncBlock, +} + +impl ReactiveHandler for AccountResyncHandler { + fn id(&self) -> HandlerId { + HandlerId::new("health-account-resync") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("account-repair"), + reason: ResyncReason::HandlerRequested, + block: self.block.clone(), + targets: vec![ResyncTarget::Account { + address: self.address, + fields: AccountFieldMask { + balance: true, + nonce: false, + code: false, + }, + }], + priority: ResyncPriority::High, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} + +/// WS-5: `ingest_batch_with_resync` counts the resync targets it considers and +/// the ones that fail. An unsupported account-field target moves both counters. +#[tokio::test] +async fn resync_requests_and_failures_increment() -> Result<()> { + let address = Address::repeat_byte(0xa9); + let b5 = block(5, B256::repeat_byte(0x05), B256::repeat_byte(0x04)); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(AccountResyncHandler { + address, + block: ResyncBlock::Hash { + number: b5.number, + hash: b5.hash, + require_canonical: true, + }, + }))?; + + assert_eq!(runtime.metrics().resync_requests, 0); + assert_eq!(runtime.metrics().resync_failures, 0); + + runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b5, 5)), + included_context(b5.clone(), 5), + ), + )?; + + let m = runtime.metrics(); + assert_eq!( + m.resync_requests, 1, + "one considered resync target must be counted" + ); + assert_eq!( + m.resync_failures, 1, + "the unsupported account target must be counted as a failure" + ); + Ok(()) +} + +/// A handler that, on a pending input, emits a canonical cache effect — which +/// the runtime rejects as `InvalidPendingEffect`. +struct PendingCanonicalWriter; + +impl ReactiveHandler for PendingCanonicalWriter { + fn id(&self) -> HandlerId { + HandlerId::new("health-pending-writer") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::PendingTransactions( + PendingTxInterest::default(), + )] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::purge( + Address::repeat_byte(0x75), + PurgeScope::AllStorage, + ))], + quality: StateEffectQuality::RequiresRepair, + tags: vec![], + }) + } +} + +/// WS-5: a pending-source input that attempts a canonical effect increments +/// `pending_contamination` (without changing the error's public behavior). +#[tokio::test] +async fn pending_contamination_increments_on_invalid_pending_effect() -> Result<()> { + let mut cache = setup_cache().await?; + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(PendingCanonicalWriter))?; + + assert_eq!(runtime.metrics().pending_contamination, 0); + + let err = runtime + .ingest_batch( + &mut cache, + batch( + ReactiveInput::PendingTxHash(B256::repeat_byte(0x99)), + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Pending, + block: None, + transaction_index: None, + log_index: None, + }, + ), + ) + .expect_err("pending inputs must not mutate canonical cache state"); + + assert!(matches!(err, ReactiveError::InvalidPendingEffect { .. })); + assert_eq!( + runtime.metrics().pending_contamination, + 1, + "pending contamination must be counted" + ); + Ok(()) +} + +/// WS-5: a fresh runtime's metrics snapshot is all-zero. +#[tokio::test] +async fn metrics_snapshot_starts_all_zero() -> Result<()> { + let runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + let m = runtime.metrics(); + assert_eq!(m.deep_reorgs, 0); + assert_eq!(m.reorgs_recovered, 0); + assert_eq!(m.resync_requests, 0); + assert_eq!(m.resync_failures, 0); + assert_eq!(m.missed_ranges, 0); + assert_eq!(m.coverage_gaps, 0); + assert_eq!(m.pending_contamination, 0); + assert_eq!(m.stale_verdicts, 0); + Ok(()) +} + +/// WS-4 (manager-authored red-green): a forward gap in the canonical block +/// sequence (block N followed by N+k, k>1) is no longer silently accepted. The +/// runtime emits a `ReactiveReport::MissedBlockRange { from, to }` for the skipped +/// span, increments `missed_ranges`, and degrades health — while STILL accepting +/// and applying the arriving block (the chain extends). +#[tokio::test] +async fn forward_block_gap_is_detected_and_degrades() -> Result<()> { + let address = Address::repeat_byte(0xd3); + let slot = U256::from(9); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + // Block 15 arrives after 10, skipping 11..=14 (e.g. a disconnect gap). + let b15 = block(15, B256::repeat_byte(0x15), B256::repeat_byte(0x14)); + + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + included_context(b10.clone(), 10), + ), + )?; + assert_eq!(runtime.health(), CacheHealth::Healthy); + + let report = runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b15, 15)), + included_context(b15.clone(), 15), + ), + )?; + + // A missed-range report identifies the skipped span 11..=14. + let gap = report + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::MissedBlockRange(report) => Some(report), + _ => None, + }) + .expect("a forward gap must emit a MissedBlockRange report"); + assert_eq!(gap.from, 11, "gap starts one past the last-seen block"); + assert_eq!(gap.to, 14, "gap ends one before the arriving block"); + + assert_eq!(runtime.metrics().missed_ranges, 1); + assert!( + matches!(runtime.health(), CacheHealth::Degraded { .. }), + "a missed range must degrade health, got {:?}", + runtime.health() + ); + // The arriving block is still accepted and applied (chain extends). + assert_eq!( + cache.cached_storage_value(address, slot), + Some(U256::from(15u64)), + "the gap block's effects must still be applied" + ); + Ok(()) +} + +/// WS-4 (manager-authored red-green): repeated trust-loss events escalate the +/// health state — the first degrades to `Degraded`, a second (here a second gap) +/// escalates to `Unhealthy` (the "stop until rebuilt" signal). +#[tokio::test] +async fn repeated_trust_loss_escalates_to_unhealthy() -> Result<()> { + let address = Address::repeat_byte(0xd4); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + let b15 = block(15, B256::repeat_byte(0x15), B256::repeat_byte(0x14)); + let b20 = block(20, B256::repeat_byte(0x20), B256::repeat_byte(0x1e)); + + for b in [&b10, &b15, &b20] { + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, b, b.number)), + included_context(b.clone(), b.number), + ), + )?; + } + + // First gap (10 -> 15) degraded; second gap (15 -> 20) escalates. + assert!( + matches!(runtime.health(), CacheHealth::Unhealthy { .. }), + "repeated gaps must escalate to Unhealthy, got {:?}", + runtime.health() + ); + assert_eq!(runtime.metrics().missed_ranges, 2); + Ok(()) +} + +/// WS-4 (manager-authored red-green): after the caller has repaired/resynced, +/// `reset_health` returns the runtime to `Healthy` (the self-heal completion). +#[tokio::test] +async fn reset_health_restores_healthy() -> Result<()> { + let address = Address::repeat_byte(0xd5); + let slot = U256::from(1); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + let b15 = block(15, B256::repeat_byte(0x15), B256::repeat_byte(0x14)); + for b in [&b10, &b15] { + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, b, b.number)), + included_context(b.clone(), b.number), + ), + )?; + } + assert!(matches!(runtime.health(), CacheHealth::Degraded { .. })); + + runtime.reset_health(); + assert_eq!(runtime.health(), CacheHealth::Healthy); + Ok(()) +} + +/// WS-4 (implementation agent): mixed trust-loss event types share the same +/// escalation ladder. A deep reorg (journal_depth=1, parent aged out) degrades to +/// `Degraded`, then a subsequent forward gap escalates to `Unhealthy`. +#[tokio::test] +async fn mixed_trust_loss_events_escalate_to_unhealthy() -> Result<()> { + let address = Address::repeat_byte(0xd6); + let slot = U256::from(3); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig { + journal_depth: 1, + ..ReactiveConfig::default() + }); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + let b11 = block(11, B256::repeat_byte(0x11), b10.hash); + // Replacement for block 11 whose parent is NOT in the (depth-1) journal: + // a deep reorg -> first trust-loss event -> Degraded. + let b11_alt = block(11, B256::repeat_byte(0x1b), B256::repeat_byte(0x1a)); + // A forward gap after 11: 11 -> 16 skips 12..=15 -> second trust-loss event. + let b16 = block(16, B256::repeat_byte(0x16), B256::repeat_byte(0x15)); + + for (b, log_index) in [(&b10, 10u64), (&b11, 11)] { + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, b, log_index)), + included_context(b.clone(), log_index), + ), + )?; + } + assert_eq!(runtime.health(), CacheHealth::Healthy, "healthy so far"); + + // Deep reorg: parent aged out of the depth-1 journal -> Degraded. + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b11_alt, 12)), + included_context(b11_alt.clone(), 12), + ), + )?; + assert!( + matches!(runtime.health(), CacheHealth::Degraded { .. }), + "the deep reorg degrades health, got {:?}", + runtime.health() + ); + + // Forward gap: escalates to the terminal Unhealthy stop signal. + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b16, 16)), + included_context(b16.clone(), 16), + ), + )?; + assert!( + matches!(runtime.health(), CacheHealth::Unhealthy { .. }), + "a second trust-loss event of a different type escalates to Unhealthy, got {:?}", + runtime.health() + ); + assert_eq!(runtime.metrics().deep_reorgs, 1); + assert_eq!(runtime.metrics().missed_ranges, 1); + Ok(()) +} + +/// WS-4 (implementation agent): the `MissedBlockRange` report's `block` field +/// equals the arriving block number that revealed the gap. +#[tokio::test] +async fn missed_range_report_block_equals_arriving_block() -> Result<()> { + let address = Address::repeat_byte(0xd7); + let slot = U256::from(2); + let mut cache = setup_cache().await?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter { address, slot }))?; + + let b10 = block(10, B256::repeat_byte(0x10), B256::repeat_byte(0x0f)); + let b15 = block(15, B256::repeat_byte(0x15), B256::repeat_byte(0x14)); + + runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b10, 10)), + included_context(b10.clone(), 10), + ), + )?; + + let report = runtime.ingest_batch( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, &b15, 15)), + included_context(b15.clone(), 15), + ), + )?; + + let gap = report + .reports + .iter() + .find_map(|r| match r.as_ref() { + ReactiveReport::MissedBlockRange(report) => Some(report), + _ => None, + }) + .expect("a forward gap must emit a MissedBlockRange report"); + assert_eq!( + gap.block, 15, + "the report's block field equals the arriving block number" + ); + Ok(()) +} diff --git a/tests/reactive_registry.rs b/tests/reactive_registry.rs index 509d40d..8e0c1a8 100644 --- a/tests/reactive_registry.rs +++ b/tests/reactive_registry.rs @@ -73,3 +73,102 @@ fn reactive_registry_rejects_duplicate_handler_ids() { RegisterError::DuplicateHandler(id) if id == HandlerId::new("duplicate") )); } + +#[test] +fn reactive_registry_unregisters_one_handler_without_rebuilding_others() { + let pool_a = Address::repeat_byte(0xa1); + let pool_b = Address::repeat_byte(0xb2); + let mut registry = ReactiveRegistry::::new(); + registry + .register_handler(Arc::new(NoopHandler::new("pool-a", pool_a))) + .expect("register pool a"); + registry + .register_handler(Arc::new(NoopHandler::new("pool-b", pool_b))) + .expect("register pool b"); + + assert!(registry.contains_handler(&HandlerId::new("pool-a"))); + assert!(registry.contains_handler(&HandlerId::new("pool-b"))); + assert_eq!(registry.interests().len(), 2); + assert_eq!( + registry + .handler_interests(&HandlerId::new("pool-a")) + .expect("pool a interests") + .len(), + 1 + ); + + let removed = registry + .unregister_handler(&HandlerId::new("pool-a")) + .expect("pool a should be removed"); + assert_eq!(removed.id(), HandlerId::new("pool-a")); + + assert!(!registry.contains_handler(&HandlerId::new("pool-a"))); + assert!(registry.contains_handler(&HandlerId::new("pool-b"))); + assert_eq!(registry.interests().len(), 1); + assert!(registry.route_log(&rpc_log(pool_a)).is_empty()); + assert_eq!(registry.route_log(&rpc_log(pool_b)).len(), 1); + + registry + .register_handler(Arc::new(NoopHandler::new("pool-a", pool_a))) + .expect("unregistered id may be reused"); + assert!(registry.contains_handler(&HandlerId::new("pool-a"))); + assert_eq!(registry.interests().len(), 2); +} + +fn rpc_log(address: Address) -> alloy_rpc_types_eth::Log { + alloy_rpc_types_eth::Log { + inner: alloy_primitives::Log::new_unchecked( + address, + vec![], + alloy_primitives::Bytes::new(), + ), + block_hash: Some(alloy_primitives::B256::repeat_byte(0x10)), + block_number: Some(10), + block_timestamp: Some(1_700_000_010), + transaction_hash: Some(alloy_primitives::B256::repeat_byte(0x20)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + } +} + +#[test] +fn reactive_registry_handler_ids_preserve_registration_order() { + let mut registry = ReactiveRegistry::::new(); + registry + .register_handler(Arc::new(NoopHandler::new( + "first", + Address::repeat_byte(0x01), + ))) + .expect("register first"); + registry + .register_handler(Arc::new(NoopHandler::new( + "second", + Address::repeat_byte(0x02), + ))) + .expect("register second"); + registry + .register_handler(Arc::new(NoopHandler::new( + "third", + Address::repeat_byte(0x03), + ))) + .expect("register third"); + + assert_eq!( + registry.handler_ids(), + vec![ + HandlerId::new("first"), + HandlerId::new("second"), + HandlerId::new("third"), + ] + ); + + // Removing the middle handler preserves the order of the rest. + registry + .unregister_handler(&HandlerId::new("second")) + .expect("remove second"); + assert_eq!( + registry.handler_ids(), + vec![HandlerId::new("first"), HandlerId::new("third")] + ); +} diff --git a/tests/reactive_resync.rs b/tests/reactive_resync.rs index bba1918..7803790 100644 --- a/tests/reactive_resync.rs +++ b/tests/reactive_resync.rs @@ -14,10 +14,12 @@ use alloy_eips::BlockId; use alloy_network::Ethereum; use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256, keccak256}; use alloy_rpc_types_eth::{Filter, Log}; -use anyhow::{Result, anyhow}; +use anyhow::Result; use common::setup_cache; use evm_fork_cache::StateUpdate; +use evm_fork_cache::cache::{AccountProof, EvmCache}; +use evm_fork_cache::errors::StorageFetchError; use evm_fork_cache::events::StateView; use evm_fork_cache::reactive::{ AccountFieldMask, BlockRef, ChainStatus, HandlerError, HandlerId, HandlerOutcome, InputSource, @@ -26,6 +28,10 @@ use evm_fork_cache::reactive::{ ResyncBlock, ResyncFailureKind, ResyncId, ResyncPriority, ResyncReason, ResyncRequest, ResyncTarget, RouteKeySpec, StateEffectQuality, }; +use revm::primitives::hardfork::SpecId; + +/// Recorded `(requests, block)` calls captured by a mock `AccountProofFetchFn`. +type ProofFetchCalls = Vec<(Vec<(Address, Vec)>, BlockId)>; fn rpc_log(address: Address, topics: Vec, block_number: u64) -> Log { Log { @@ -266,7 +272,7 @@ async fn reactive_runtime_executes_storage_resync_after_direct_effects() -> Resu assert_eq!(fetches.len(), 1); assert_eq!(fetches[0].0, vec![(address, slot)]); match &fetches[0].1 { - Some(BlockId::Hash(hash)) => { + BlockId::Hash(hash) => { assert_eq!(hash.block_hash, block_hash); assert_eq!(hash.require_canonical, Some(true)); } @@ -307,7 +313,11 @@ async fn reactive_runtime_batches_resync_slots_and_reports_failed_targets() -> R .into_iter() .map(|(addr, slot)| { if slot == slot_b { - (addr, slot, Err(anyhow!("stub slot failure"))) + ( + addr, + slot, + Err(StorageFetchError::custom("stub slot failure")), + ) } else { (addr, slot, Ok(U256::from(777))) } @@ -340,7 +350,7 @@ async fn reactive_runtime_batches_resync_slots_and_reports_failed_targets() -> R let fetches = seen_fetches.lock().unwrap(); assert_eq!(fetches.len(), 1); assert_eq!(fetches[0].0, vec![(address, slot_a), (address, slot_b)]); - assert_eq!(fetches[0].1, Some(BlockId::number(61))); + assert_eq!(fetches[0].1, BlockId::number(61)); let resynced: Vec<_> = report .reports @@ -366,16 +376,289 @@ async fn reactive_runtime_batches_resync_slots_and_reports_failed_targets() -> R ) && failure.kind == ResyncFailureKind::StorageFetchFailed && failure.message.contains("stub slot failure"))); + // This cache is built via `setup_cache()` (a `new()` cache), so a real + // account proof fetcher IS installed — it just fails against the offline + // mock provider (here, because the test runs on a current-thread runtime the + // fetcher's `block_in_place` bridge degrades to an error). The account target + // therefore fails as `AccountFetchFailed`, not `MissingAccountFetcher`. assert!(resynced[0].failed.iter().any(|failure| matches!( failure.target, ResyncTarget::Account { .. } ) && failure.kind - == ResyncFailureKind::UnsupportedAccountTarget - && failure.message.contains("account"))); + == ResyncFailureKind::AccountFetchFailed)); Ok(()) } +/// A handler that emits only an `Account`-target resync (balance + nonce). +struct AccountOnlyResync { + address: Address, +} + +impl ReactiveHandler for AccountOnlyResync { + fn id(&self) -> HandlerId { + HandlerId::new("account-only-resync") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("account-repair"), + reason: ResyncReason::HandlerRequested, + block: ResyncBlock::Number(61), + targets: vec![ResyncTarget::Account { + address: self.address, + fields: AccountFieldMask { + balance: true, + nonce: true, + code: false, + }, + }], + priority: ResyncPriority::Normal, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} + +/// WS-1a / Phase-8 s1 (manager-authored red-green): with an `AccountProofFetchFn` +/// installed, an `Account`-target resync now SUCCEEDS via the `eth_getProof` seam +/// — it no longer fails as `UnsupportedAccountTarget`. The fetched account fields +/// are applied through the cache (materialized, so a cold account is not silently +/// skipped) and no target fails. +#[tokio::test] +async fn account_target_resync_succeeds_via_account_proof_seam() -> Result<()> { + let address = Address::repeat_byte(0x93); + let seen: Arc> = Arc::new(Mutex::new(Vec::new())); + let mut cache = setup_cache().await?; + cache.set_account_proof_fetcher({ + let seen = seen.clone(); + Arc::new(move |requests: Vec<(Address, Vec)>, block: BlockId| { + seen.lock().unwrap().push((requests.clone(), block)); + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: B256::ZERO, + balance: U256::from(999u64), + nonce: 7, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }) + }); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(AccountOnlyResync { address }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"AccountRepair()")], 61)), + included_context(61), + ), + )?; + + // The seam was invoked for the account. + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + assert!(seen[0].0.iter().any(|(addr, _)| *addr == address)); + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + // No target failed — the account target is now supported. + assert!( + resynced[0].failed.is_empty(), + "account target must not fail once the seam is installed, got {:?}", + resynced[0].failed + ); + // An authoritative account update was built and applied to the cache. + assert!(!resynced[0].state_updates.is_empty()); + assert!( + resynced[0] + .diff + .accounts + .iter() + .any(|change| change.address == address), + "the resynced account fields must be applied (materialized) to the cache" + ); + assert_eq!(runtime.metrics().resync_failures, 0); + Ok(()) +} + +/// With NO account proof fetcher installed, an `Account`-target resync fails with +/// `MissingAccountFetcher`. A `from_backend` cache captures no provider, so it +/// exposes no account proof fetcher (unlike a `new()` cache, whose real fetcher +/// would instead surface `AccountFetchFailed`). +#[tokio::test] +async fn account_target_resync_without_fetcher_reports_missing_account_fetcher() -> Result<()> { + let address = Address::repeat_byte(0x94); + let base = setup_cache().await?; + let mut cache = EvmCache::from_backend( + base.unchecked_backend().clone(), + base.unchecked_blockchain_db().clone(), + base.block(), + base.chain_id(), + None, + None, + SpecId::CANCUN, + ); + assert!( + cache.account_proof_fetcher().is_none(), + "from_backend cache has no account proof fetcher" + ); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(AccountOnlyResync { address }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"AccountRepair()")], 61)), + included_context(61), + ), + )?; + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert!(resynced[0].state_updates.is_empty()); + assert_eq!(resynced[0].failed.len(), 1); + assert!(matches!( + resynced[0].failed[0].target, + ResyncTarget::Account { .. } + )); + assert_eq!( + resynced[0].failed[0].kind, + ResyncFailureKind::MissingAccountFetcher + ); + assert_eq!(runtime.metrics().resync_failures, 1); + Ok(()) +} + +/// An account proof fetcher that returns `Err` for the address produces an +/// `AccountFetchFailed` failure carrying the error message. +#[tokio::test] +async fn account_target_resync_fetch_error_reports_account_fetch_failed() -> Result<()> { + let address = Address::repeat_byte(0x95); + let mut cache = setup_cache().await?; + cache.set_account_proof_fetcher(Arc::new( + move |requests: Vec<(Address, Vec)>, _block: BlockId| { + requests + .into_iter() + .map(|(addr, _keys)| (addr, Err(StorageFetchError::custom("stub account failure")))) + .collect() + }, + )); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(AccountOnlyResync { address }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"AccountRepair()")], 61)), + included_context(61), + ), + )?; + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert!(resynced[0].state_updates.is_empty()); + assert_eq!(resynced[0].failed.len(), 1); + assert_eq!( + resynced[0].failed[0].kind, + ResyncFailureKind::AccountFetchFailed + ); + assert!( + resynced[0].failed[0] + .message + .contains("stub account failure") + ); + assert_eq!(runtime.metrics().resync_failures, 1); + Ok(()) +} + +/// An account proof fetcher that omits the requested address (returns no entry) +/// produces an `AccountFetchOmitted` failure. +#[tokio::test] +async fn account_target_resync_omitted_address_reports_account_fetch_omitted() -> Result<()> { + let address = Address::repeat_byte(0x96); + let mut cache = setup_cache().await?; + cache.set_account_proof_fetcher(Arc::new( + // Return no entries at all — the requested address is omitted. + move |_requests: Vec<(Address, Vec)>, _block: BlockId| Vec::new(), + )); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(AccountOnlyResync { address }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"AccountRepair()")], 61)), + included_context(61), + ), + )?; + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert!(resynced[0].state_updates.is_empty()); + assert_eq!(resynced[0].failed.len(), 1); + assert_eq!( + resynced[0].failed[0].kind, + ResyncFailureKind::AccountFetchOmitted + ); + assert_eq!(runtime.metrics().resync_failures, 1); + Ok(()) +} + #[tokio::test] async fn reactive_runtime_fans_out_duplicate_resync_failures_to_all_request_origins() -> Result<()> { @@ -389,7 +672,13 @@ async fn reactive_runtime_fans_out_duplicate_resync_failures_to_all_request_orig seen_fetches.lock().unwrap().push((requests.clone(), block)); requests .into_iter() - .map(|(addr, slot)| (addr, slot, Err(anyhow!("shared fetch failure")))) + .map(|(addr, slot)| { + ( + addr, + slot, + Err(StorageFetchError::custom("shared fetch failure")), + ) + }) .collect() }) }); @@ -442,3 +731,127 @@ async fn reactive_runtime_fans_out_duplicate_resync_failures_to_all_request_orig Ok(()) } + +/// A handler that emits ONE resync request carrying TWO account targets. +struct TwoAccountResync { + first: Address, + second: Address, +} + +impl ReactiveHandler for TwoAccountResync { + fn id(&self) -> HandlerId { + HandlerId::new("two-account-resync") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.first), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + let fields = AccountFieldMask { + balance: true, + nonce: true, + code: false, + }; + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("two-account-repair"), + reason: ResyncReason::HandlerRequested, + block: ResyncBlock::Number(61), + targets: vec![ + ResyncTarget::Account { + address: self.first, + fields, + }, + ResyncTarget::Account { + address: self.second, + fields, + }, + ], + priority: ResyncPriority::Normal, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} + +/// §6.1 (spec item 13, resync side): multiple account targets pinned to the +/// same block resolve through ONE seam invocation carrying both addresses — +/// not one `eth_getProof` seam call per target. +#[tokio::test] +async fn account_target_resync_batches_targets_in_one_seam_call() -> Result<()> { + let first = Address::repeat_byte(0x95); + let second = Address::repeat_byte(0x96); + let seen: Arc> = Arc::new(Mutex::new(Vec::new())); + let mut cache = setup_cache().await?; + cache.set_account_proof_fetcher({ + let seen = seen.clone(); + Arc::new(move |requests: Vec<(Address, Vec)>, block: BlockId| { + seen.lock().unwrap().push((requests.clone(), block)); + requests + .into_iter() + .map(|(addr, _keys)| { + ( + addr, + Ok(AccountProof { + storage_hash: B256::ZERO, + balance: U256::from(123u64), + nonce: 1, + code_hash: B256::ZERO, + slots: vec![], + }), + ) + }) + .collect() + }) + }); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(TwoAccountResync { first, second }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(first, vec![keccak256(b"TwoAccountRepair()")], 61)), + included_context(61), + ), + )?; + + // ONE invocation, carrying BOTH targets. + let seen = seen.lock().unwrap(); + assert_eq!( + seen.len(), + 1, + "same-block account targets must share one seam invocation" + ); + assert_eq!(seen[0].0.len(), 2); + assert!(seen[0].0.iter().any(|(addr, _)| *addr == first)); + assert!(seen[0].0.iter().any(|(addr, _)| *addr == second)); + + // Both targets succeeded. + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert!( + resynced[0].failed.is_empty(), + "both batched account targets must resolve, got {:?}", + resynced[0].failed + ); + Ok(()) +} diff --git a/tests/reactive_runtime.rs b/tests/reactive_runtime.rs index 6fdf74e..772ff11 100644 --- a/tests/reactive_runtime.rs +++ b/tests/reactive_runtime.rs @@ -607,3 +607,171 @@ async fn reactive_runtime_allows_speculative_pending_effects_without_cache_mutat ); Ok(()) } + +// --- Handler lifecycle accessors (0.2.0 register/unregister support) --- + +#[tokio::test] +async fn reactive_runtime_last_canonical_block_tracks_journal_head() -> Result<()> { + let emitter = Address::repeat_byte(0x91); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, emitter); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(SlotWriter::any_log_from( + "writer", + emitter, + U256::from(1), + U256::from(2), + )))?; + + // Nothing ingested yet: no canonical position. + assert!(runtime.last_canonical_block().is_none()); + + runtime.ingest_batch( + &mut cache, + batch(vec![( + ReactiveInput::Log(rpc_log(emitter, vec![keccak256(b"Write()")], 100, 0, 0)), + included_context(100, 0), + )]), + )?; + let head = runtime + .last_canonical_block() + .expect("a canonical block should be journaled"); + assert_eq!(head.number, 100); + + // A later canonical block advances the head. + runtime.ingest_batch( + &mut cache, + batch(vec![( + ReactiveInput::Log(rpc_log(emitter, vec![keccak256(b"Write()")], 105, 0, 0)), + included_context(105, 0), + )]), + )?; + assert_eq!(runtime.last_canonical_block().map(|b| b.number), Some(105)); + Ok(()) +} + +#[tokio::test] +async fn reactive_runtime_cancel_pending_resyncs_drops_targeted_account() -> Result<()> { + let address = Address::repeat_byte(0x76); + let slot = U256::from(11); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, address); + cache + .db_mut() + .insert_account_storage(address, slot, U256::from(123))?; + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(InvalidateAndResyncHandler { address, slot }))?; + + // ingest_batch (no resync execution) leaves the request queued in the + // pending ledger. + runtime.ingest_batch( + &mut cache, + batch(vec![( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"Repair()")], 50, 0, 0)), + included_context(50, 0), + )]), + )?; + assert_eq!(runtime.pending_resyncs().len(), 1); + + // An unrelated address cancels nothing. + let none = runtime.cancel_pending_resyncs(Address::repeat_byte(0xff)); + assert!(none.is_empty()); + assert_eq!(runtime.pending_resyncs().len(), 1); + + // The targeted address is cancelled and returned. + let cancelled = runtime.cancel_pending_resyncs(address); + assert_eq!(cancelled.len(), 1); + assert_eq!(cancelled[0].id, ResyncId::new("resync-slot")); + assert_eq!(cancelled[0].targets.len(), 1); + assert!(runtime.pending_resyncs().is_empty()); + Ok(()) +} + +#[tokio::test] +async fn reactive_runtime_cancel_pending_resyncs_preserves_other_targets() -> Result<()> { + let keep = Address::repeat_byte(0x33); + let drop = Address::repeat_byte(0x44); + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, keep); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(MultiTargetResyncHandler { + emitter: keep, + keep, + drop, + }))?; + + runtime.ingest_batch( + &mut cache, + batch(vec![( + ReactiveInput::Log(rpc_log(keep, vec![keccak256(b"Multi()")], 60, 0, 0)), + included_context(60, 0), + )]), + )?; + assert_eq!(runtime.pending_resyncs().len(), 1); + assert_eq!(runtime.pending_resyncs()[0].targets.len(), 2); + + // Cancelling one account keeps the request alive with its other target. + let cancelled = runtime.cancel_pending_resyncs(drop); + assert_eq!(cancelled.len(), 1); + assert_eq!(cancelled[0].targets.len(), 1); + assert_eq!(runtime.pending_resyncs().len(), 1); + assert_eq!(runtime.pending_resyncs()[0].targets.len(), 1); + Ok(()) +} + +struct MultiTargetResyncHandler { + emitter: Address, + keep: Address, + drop: Address, +} + +impl ReactiveHandler for MultiTargetResyncHandler { + fn id(&self) -> HandlerId { + HandlerId::new("multi-target-resync") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.emitter), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("multi"), + reason: ResyncReason::HandlerRequested, + block: ResyncBlock::Latest, + targets: vec![ + ResyncTarget::Account { + address: self.keep, + fields: evm_fork_cache::reactive::AccountFieldMask { + balance: true, + ..Default::default() + }, + }, + ResyncTarget::Account { + address: self.drop, + fields: evm_fork_cache::reactive::AccountFieldMask { + balance: true, + ..Default::default() + }, + }, + ], + priority: ResyncPriority::Normal, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} diff --git a/tests/reactive_subscriber_ingest.rs b/tests/reactive_subscriber_ingest.rs new file mode 100644 index 0000000..61eef11 --- /dev/null +++ b/tests/reactive_subscriber_ingest.rs @@ -0,0 +1,152 @@ +//! End-to-end composition: real `AlloySubscriber` output → `ReactiveRuntime::ingest_batch`. +//! +//! Closes the integration gap tracked in `docs/KNOWN_ISSUES.md`: prior reactive +//! tests exercised the subscriber and the runtime separately (or the runtime with +//! a mock subscriber). This drives an actual [`AlloySubscriber`]-produced batch +//! into an actual [`ReactiveRuntime`] and asserts the handler write lands in the +//! cache — the full seam a consumer relies on. +//! +//! It runs offline by fetching the batch through the subscriber's `get_logs` +//! backfill path (mockable), which produces exactly the same +//! `ReactiveInputBatch` shape the live pubsub path emits. The live WebSocket +//! transport plumbing itself is covered by the reconnect/termination unit tests +//! in `tests/reactive_alloy_subscriber.rs`. +#![cfg(feature = "reactive-ws")] + +mod common; + +use std::sync::Arc; + +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256, keccak256}; +use alloy_provider::ProviderBuilder; +use alloy_rpc_types_eth::{Filter, Log}; +use alloy_transport::mock::Asserter; +use anyhow::{Result, bail}; + +use common::{install_mock_erc20, setup_cache}; +use evm_fork_cache::StateUpdate; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + AlloySubscriber, EventSubscriber, HandlerError, HandlerId, HandlerOutcome, InputSource, + LogInterest, ReactiveConfig, ReactiveContext, ReactiveEffect, ReactiveHandler, ReactiveInput, + ReactiveInterest, ReactiveRuntime, RouteKeySpec, StateEffectQuality, SubscriberBackfill, + SubscriberConfig, SubscriberMode, +}; + +const POOL_SLOT: u64 = 0; + +/// Writes the log's data word into `POOL_SLOT` of the emitting contract. +struct PoolHandler { + pool: Address, +} + +impl ReactiveHandler for PoolHandler { + fn id(&self) -> HandlerId { + HandlerId::new("pool") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.pool), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + let ReactiveInput::Log(log) = input else { + return Ok(HandlerOutcome::empty(StateEffectQuality::NoStateEffect)); + }; + let value = U256::from_be_slice(log.data().data.as_ref()); + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::StateUpdate(StateUpdate::slot( + log.address(), + U256::from(POOL_SLOT), + value, + ))], + quality: StateEffectQuality::ExactFromInput, + tags: vec![], + }) + } +} + +fn swap_log(pool: Address, topic: B256, block_number: u64, value: u64) -> Log { + Log { + inner: PrimitiveLog::new_unchecked( + pool, + vec![topic], + Bytes::from(U256::from(value).to_be_bytes::<32>().to_vec()), + ), + block_hash: Some(B256::repeat_byte(block_number as u8)), + block_number: Some(block_number), + block_timestamp: Some(1_700_000_000 + block_number), + transaction_hash: Some(B256::repeat_byte(0xcc)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn alloy_subscriber_batch_feeds_reactive_runtime_ingest_end_to_end() -> Result<()> { + let pool = Address::repeat_byte(0xcd); + let topic = keccak256(b"Swap()"); + let new_value = 4242u64; + + // Cache + tracked pool contract. + let mut cache = setup_cache().await?; + install_mock_erc20(&mut cache, pool); + + // Real AlloySubscriber over a mocked provider; the backfill get_logs returns + // one swap log carrying the post-state value. + let asserter = Asserter::new(); + asserter.push_success(&vec![swap_log(pool, topic, 100, new_value)]); + let provider = ProviderBuilder::new().connect_mocked_client(asserter); + let mut subscriber = AlloySubscriber::<_, Ethereum>::new( + provider, + SubscriberMode::PubSub, + SubscriberConfig { + hydrate_pending_transactions: false, + ..SubscriberConfig::default() + }, + ); + subscriber.add_interest_owner_with_backfill( + HandlerId::new("pool"), + &[ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(pool).event_signature(topic), + local_matcher: None, + route_key: None, + })], + SubscriberBackfill::range(90, 100), + )?; + + // Real runtime with the matching handler. + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(PoolHandler { pool }))?; + + // Pull the batch the subscriber produced and feed it straight into ingest. + let Some(batch) = subscriber.next_batch().await? else { + bail!("expected a backfilled batch from the subscriber"); + }; + assert_eq!(batch.records().len(), 1); + assert_eq!(batch.records()[0].context.source, InputSource::Backfill); + + let report = runtime.ingest_batch(&mut cache, batch)?; + + // The handler ran on the subscriber-produced record and its write landed. + assert_eq!(report.applied.len(), 1); + assert_eq!(report.applied[0].handler_id, HandlerId::new("pool")); + assert_eq!( + cache.cached_storage_value(pool, U256::from(POOL_SLOT)), + Some(U256::from(new_value)), + "the subscriber-delivered swap should have updated the pool slot with no RPC" + ); + + Ok(()) +} diff --git a/tests/reactive_trace_resync.rs b/tests/reactive_trace_resync.rs new file mode 100644 index 0000000..e8e4599 --- /dev/null +++ b/tests/reactive_trace_resync.rs @@ -0,0 +1,394 @@ +//! Manager-authored acceptance tests for trace-backed reactive resync execution. +//! +//! These tests pin the Tier-3 liveness/resync path: when handlers request sync +//! for a block, the runtime should be able to satisfy matching targets from one +//! block-level state diff before falling back to per-slot RPC reads. +#![cfg(feature = "reactive")] + +mod common; + +use std::sync::{Arc, Mutex}; + +use alloy_eips::BlockId; +use alloy_network::Ethereum; +use alloy_primitives::{Address, B256, Bytes, Log as PrimitiveLog, U256, keccak256}; +use alloy_rpc_types_eth::{Filter, Log}; +use anyhow::Result; + +use common::setup_cache; +use evm_fork_cache::cache::{ + BlockStateAccountDiff, BlockStateDiff, BlockStateStorageDiff, EvmCache, +}; +use evm_fork_cache::events::StateView; +use evm_fork_cache::reactive::{ + BlockRef, ChainStatus, HandlerError, HandlerId, HandlerOutcome, InputSource, LogInterest, + ReactiveConfig, ReactiveContext, ReactiveEffect, ReactiveHandler, ReactiveInput, + ReactiveInputBatch, ReactiveInputRecord, ReactiveInterest, ReactiveReport, ReactiveRuntime, + ResyncBlock, ResyncId, ResyncPriority, ResyncReason, ResyncRequest, ResyncTarget, RouteKeySpec, + StateEffectQuality, +}; + +fn rpc_log(address: Address, topics: Vec, block_number: u64) -> Log { + Log { + inner: PrimitiveLog::new_unchecked(address, topics, Bytes::new()), + block_hash: Some(B256::repeat_byte(block_number as u8)), + block_number: Some(block_number), + block_timestamp: Some(1_700_000_000 + block_number), + transaction_hash: Some(B256::repeat_byte(0x44)), + transaction_index: Some(0), + log_index: Some(0), + removed: false, + } +} + +fn included_context(block_number: u64) -> ReactiveContext { + let block = BlockRef { + number: block_number, + hash: B256::repeat_byte(block_number as u8), + parent_hash: Some(B256::repeat_byte(block_number.saturating_sub(1) as u8)), + timestamp: Some(1_700_000_000 + block_number), + }; + + ReactiveContext { + chain_id: Some(1), + source: InputSource::Batch, + chain_status: ChainStatus::Included { + block: block.clone(), + confirmations: 0, + }, + block: Some(block), + transaction_index: Some(0), + log_index: Some(0), + } +} + +fn batch(input: ReactiveInput, ctx: ReactiveContext) -> ReactiveInputBatch { + ReactiveInputBatch::new(vec![ReactiveInputRecord::new(input, ctx)]) +} + +fn diff_for_slots( + address: Address, + slots: impl IntoIterator, +) -> BlockStateDiff { + BlockStateDiff { + accounts: vec![BlockStateAccountDiff { + address, + balance: None, + nonce: None, + code: None, + storage: slots + .into_iter() + .map(|(slot, value)| BlockStateStorageDiff { slot, value }) + .collect(), + }], + } +} + +struct TraceMultiSlotResync { + address: Address, + slots: Vec, + block: ResyncBlock, +} + +impl ReactiveHandler for TraceMultiSlotResync { + fn id(&self) -> HandlerId { + HandlerId::new("trace-multi-slot-resync") + } + + fn interests(&self) -> Vec { + vec![ReactiveInterest::Logs(LogInterest { + provider_filter: Filter::new().address(self.address), + local_matcher: None, + route_key: Some(RouteKeySpec::EmitterAddress), + })] + } + + fn handle( + &self, + _ctx: &ReactiveContext, + _input: &ReactiveInput, + _state: &dyn StateView, + ) -> Result { + Ok(HandlerOutcome { + effects: vec![ReactiveEffect::Resync(ResyncRequest { + id: ResyncId::new("trace-slot-repair"), + reason: ResyncReason::HandlerRequested, + block: self.block.clone(), + targets: vec![ResyncTarget::StorageSlots { + address: self.address, + slots: self.slots.clone(), + }], + priority: ResyncPriority::High, + })], + quality: StateEffectQuality::AppliedWithPendingResync, + tags: vec![], + }) + } +} + +fn install_panic_storage_fetcher(cache: &mut EvmCache) { + cache.set_storage_batch_fetcher(Arc::new(|requests, _block| { + panic!("storage fetcher should not be called; requests={requests:?}") + })); +} + +#[tokio::test] +async fn trace_resync_coalesces_block_fetch_and_applies_matching_slots() -> Result<()> { + let address = Address::repeat_byte(0xa1); + let slot_a = U256::from(10); + let slot_b = U256::from(11); + let block_hash = B256::repeat_byte(0x70); + let seen_trace_blocks = Arc::new(Mutex::new(Vec::new())); + let mut cache = setup_cache().await?; + cache.set_block_state_diff_fetcher({ + let seen_trace_blocks = seen_trace_blocks.clone(); + Arc::new(move |block| { + seen_trace_blocks.lock().unwrap().push(block); + Ok(diff_for_slots( + address, + [(slot_a, U256::from(700)), (slot_b, U256::from(800))], + )) + }) + }); + install_panic_storage_fetcher(&mut cache); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(TraceMultiSlotResync { + address, + slots: vec![slot_a, slot_b], + block: ResyncBlock::Hash { + number: 70, + hash: block_hash, + require_canonical: true, + }, + }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"TraceRepair()")], 70)), + included_context(70), + ), + )?; + + assert_eq!( + cache.cached_storage_value(address, slot_a), + Some(U256::from(700)) + ); + assert_eq!( + cache.cached_storage_value(address, slot_b), + Some(U256::from(800)) + ); + + let trace_blocks = seen_trace_blocks.lock().unwrap(); + assert_eq!( + trace_blocks.len(), + 1, + "all targets in one block should share one trace request" + ); + match &trace_blocks[0] { + BlockId::Hash(hash) => { + assert_eq!(hash.block_hash, block_hash); + assert_eq!(hash.require_canonical, Some(true)); + } + other => panic!("expected hash-pinned trace request, got {other:?}"), + } + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert_eq!(resynced[0].state_updates.len(), 2); + assert!(resynced[0].failed.is_empty()); + assert_eq!(runtime.metrics().resync_failures, 0); + + Ok(()) +} + +#[tokio::test] +async fn trace_resync_falls_back_to_storage_for_unresolved_cold_slot() -> Result<()> { + let address = Address::repeat_byte(0xa2); + let slot = U256::from(21); + let seen_trace_blocks = Arc::new(Mutex::new(Vec::new())); + let seen_storage_fetches = Arc::new(Mutex::new(Vec::new())); + let mut cache = setup_cache().await?; + cache.set_block_state_diff_fetcher({ + let seen_trace_blocks = seen_trace_blocks.clone(); + Arc::new(move |block| { + seen_trace_blocks.lock().unwrap().push(block); + Ok(BlockStateDiff { accounts: vec![] }) + }) + }); + cache.set_storage_batch_fetcher({ + let seen_storage_fetches = seen_storage_fetches.clone(); + Arc::new(move |requests, block| { + seen_storage_fetches + .lock() + .unwrap() + .push((requests.clone(), block)); + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::from(900)))) + .collect() + }) + }); + + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(TraceMultiSlotResync { + address, + slots: vec![slot], + block: ResyncBlock::Number(71), + }))?; + + let report = runtime.ingest_batch_with_resync( + &mut cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"TraceRepair()")], 71)), + included_context(71), + ), + )?; + + assert_eq!( + cache.cached_storage_value(address, slot), + Some(U256::from(900)) + ); + assert_eq!(seen_trace_blocks.lock().unwrap().len(), 1); + let storage_fetches = seen_storage_fetches.lock().unwrap(); + assert_eq!(storage_fetches.len(), 1); + assert_eq!(storage_fetches[0].0, vec![(address, slot)]); + assert_eq!(storage_fetches[0].1, BlockId::number(71)); + + let resynced: Vec<_> = report + .reports + .iter() + .filter_map(|report| match report.as_ref() { + ReactiveReport::Resynced(report) => Some(report), + _ => None, + }) + .collect(); + assert_eq!(resynced.len(), 1); + assert_eq!(resynced[0].state_updates.len(), 1); + assert!(resynced[0].failed.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn trace_resync_matches_storage_resync_state_with_fewer_rpc_units() -> Result<()> { + let address = Address::repeat_byte(0xa3); + let slot_a = U256::from(31); + let slot_b = U256::from(32); + + let mut trace_cache = setup_cache().await?; + let trace_blocks = Arc::new(Mutex::new(Vec::new())); + let trace_storage_fetches = Arc::new(Mutex::new(0usize)); + trace_cache.set_block_state_diff_fetcher({ + let trace_blocks = trace_blocks.clone(); + Arc::new(move |block| { + trace_blocks.lock().unwrap().push(block); + Ok(diff_for_slots( + address, + [(slot_a, U256::from(3100)), (slot_b, U256::from(3200))], + )) + }) + }); + trace_cache.set_storage_batch_fetcher({ + let trace_storage_fetches = trace_storage_fetches.clone(); + Arc::new(move |requests, _block| { + *trace_storage_fetches.lock().unwrap() += 1; + requests + .into_iter() + .map(|(addr, slot)| (addr, slot, Ok(U256::ZERO))) + .collect() + }) + }); + + let storage_fetches = Arc::new(Mutex::new(Vec::new())); + let mut storage_cache = setup_cache().await?; + storage_cache.set_storage_batch_fetcher({ + let storage_fetches = storage_fetches.clone(); + Arc::new(move |requests, block| { + storage_fetches + .lock() + .unwrap() + .push((requests.clone(), block)); + requests + .into_iter() + .map(|(addr, slot)| { + let value = if slot == slot_a { + U256::from(3100) + } else if slot == slot_b { + U256::from(3200) + } else { + U256::ZERO + }; + (addr, slot, Ok(value)) + }) + .collect() + }) + }); + + let run = |cache: &mut EvmCache| -> Result<_> { + let mut runtime = ReactiveRuntime::::new(ReactiveConfig::default()); + runtime.register_handler(Arc::new(TraceMultiSlotResync { + address, + slots: vec![slot_a, slot_b], + block: ResyncBlock::Number(72), + }))?; + Ok(runtime.ingest_batch_with_resync( + cache, + batch( + ReactiveInput::Log(rpc_log(address, vec![keccak256(b"TraceRepair()")], 72)), + included_context(72), + ), + )?) + }; + + let trace_report = run(&mut trace_cache)?; + let storage_report = run(&mut storage_cache)?; + + assert_eq!( + trace_cache.cached_storage_value(address, slot_a), + storage_cache.cached_storage_value(address, slot_a) + ); + assert_eq!( + trace_cache.cached_storage_value(address, slot_b), + storage_cache.cached_storage_value(address, slot_b) + ); + assert_eq!( + trace_cache.cached_storage_value(address, slot_a), + Some(U256::from(3100)) + ); + assert_eq!( + trace_cache.cached_storage_value(address, slot_b), + Some(U256::from(3200)) + ); + + assert_eq!( + trace_report.applied.len(), + storage_report.applied.len(), + "direct handler behavior should be identical" + ); + assert_eq!( + trace_blocks.lock().unwrap().len(), + 1, + "trace path should need one block-level request" + ); + assert_eq!( + *trace_storage_fetches.lock().unwrap(), + 0, + "trace-covered targets should not fall back to per-slot storage RPC" + ); + assert_eq!( + storage_fetches.lock().unwrap().len(), + 1, + "storage-only path still uses the existing batched storage fetcher" + ); + + Ok(()) +} diff --git a/tests/serialization_roundtrip.rs b/tests/serialization_roundtrip.rs index 3d79fbd..fd54696 100644 --- a/tests/serialization_roundtrip.rs +++ b/tests/serialization_roundtrip.rs @@ -14,7 +14,12 @@ struct TempDir(PathBuf); impl TempDir { fn new(tag: &str) -> Self { - let dir = std::env::temp_dir().join(format!("evm_fork_cache_roundtrip_{tag}")); + // Keyed by pid so concurrent `cargo test` processes never share (and + // never `remove_dir_all`) each other's directory. + let dir = std::env::temp_dir().join(format!( + "evm_fork_cache_roundtrip_{tag}_{}", + std::process::id() + )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).expect("create temp dir"); TempDir(dir) diff --git a/tests/snapshot_overlay.rs b/tests/snapshot_overlay.rs index 10ae6d4..b03390f 100644 --- a/tests/snapshot_overlay.rs +++ b/tests/snapshot_overlay.rs @@ -2,7 +2,7 @@ //! underpin the crate's parallel fan-out model. //! //! These pin the invariants a search loop relies on: -//! - [`EvmCache::create_snapshot`] yields an immutable, point-in-time view that +//! - [`EvmCache::snapshot`] yields an immutable, point-in-time view that //! later cache mutations cannot perturb. //! - Overlays derived from one snapshot are isolated from each other and from the //! live cache. @@ -65,7 +65,7 @@ async fn snapshot_is_immutable_after_later_cache_mutation() -> Result<()> { cache.insert_mapping_storage_slot(token, balance_slot, recipient, U256::ZERO)?; // Freeze the state, then mutate the live cache with a committed transfer. - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); transfer(&mut cache, token, owner, recipient, U256::from(250u64))?; // The live cache reflects the transfer... @@ -106,12 +106,12 @@ async fn overlays_from_one_snapshot_are_isolated() -> Result<()> { // a StorageCleared account reads as ZERO via `cached_storage_value` (mirroring // the EVM SLOAD), so the live-cache assertion below would observe 0. Seeding // the overlay (the winning layer) is what the test means by "the cache holds - // `original`" and is captured by `create_snapshot`. + // `original`" and is captured by `snapshot`. cache .db_mut() .insert_account_storage(contract, slot, original)?; - let snapshot = cache.create_snapshot(); + let snapshot = cache.snapshot(); let mut overlay_a = EvmOverlay::new(Arc::clone(&snapshot), None); let mut overlay_b = EvmOverlay::new(Arc::clone(&snapshot), None); @@ -159,7 +159,7 @@ async fn overlay_reads_reflect_snapshot_state() -> Result<()> { U256::from(42_000u64), )?; - let snapshot: Arc = cache.create_snapshot(); + let snapshot: Arc = cache.snapshot(); let mut overlay = EvmOverlay::new(snapshot, None); assert_eq!( @@ -178,7 +178,7 @@ async fn overlay_reads_reflect_snapshot_state() -> Result<()> { Ok(()) } -/// Regression (§16 fix-review HIGH): `create_snapshot` must mirror the live +/// Regression (§16 fix-review HIGH): `snapshot` must mirror the live /// account-state-aware read. A `StorageCleared` account with a backend-only /// (shadowed) slot reads ZERO live; the snapshot, `storage_value`, and a /// snapshot-backed overlay must all agree — not the shadowed backend value. Pre- @@ -194,7 +194,7 @@ async fn snapshot_mirrors_live_read_for_cleared_account() -> Result<()> { // Live read is ZERO (the §16.0 fix). assert_eq!(cache.cached_storage_value(token, slot), Some(U256::ZERO)); - let snapshot: Arc = cache.create_snapshot(); + let snapshot: Arc = cache.snapshot(); assert_eq!( snapshot.storage_value(token, slot), Some(U256::ZERO), @@ -215,7 +215,7 @@ async fn snapshot_mirrors_live_read_for_cleared_account() -> Result<()> { Ok(()) } -/// Regression (round-2 HIGH, account axis): `create_snapshot` / `EvmOverlay::basic` +/// Regression (round-2 HIGH, account axis): `snapshot` / `EvmOverlay::basic` /// must mirror the live account read for a `NotExisting` account. revm treats such /// an account as absent (`DbAccount::info()` → None), and `loaded_account_info` /// already does; the snapshot/parallel path must agree — not surface a phantom @@ -246,7 +246,7 @@ async fn snapshot_basic_returns_none_for_notexisting_account() -> Result<()> { .expect("overlay account present") .account_state = AccountState::NotExisting; - let snapshot: Arc = cache.create_snapshot(); + let snapshot: Arc = cache.snapshot(); let mut overlay = EvmOverlay::new(Arc::clone(&snapshot), None); let basic = overlay .basic(acct)