From 52a06451dee8f8d9301558876dd4fedf5216ecb9 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 20 Mar 2026 08:59:22 -0300 Subject: [PATCH 01/29] interface & tests --- .../stdlib/src/interfaces/world_state.ts | 7 ++ .../txe/src/state_machine/synchronizer.ts | 5 + .../src/native/native_world_state.test.ts | 96 +++++++++++++++++++ .../src/native/native_world_state.ts | 4 + .../server_world_state_synchronizer.ts | 4 + 5 files changed, 116 insertions(+) diff --git a/yarn-project/stdlib/src/interfaces/world_state.ts b/yarn-project/stdlib/src/interfaces/world_state.ts index a99bcc1fa35f..6c933be92f7f 100644 --- a/yarn-project/stdlib/src/interfaces/world_state.ts +++ b/yarn-project/stdlib/src/interfaces/world_state.ts @@ -51,6 +51,13 @@ export interface ForkMerkleTreeOperations { */ fork(block?: BlockNumber, opts?: { closeDelayMs?: number }): Promise; + /** + * Commits a fork as the current "committed" view of the world state. + * Only succeeds if the canonical tip hasn't moved past the given block number. + * Ownership of the fork transfers to the world state — caller must not close it. + */ + commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise; + /** Backups the db to the target path. */ backupTo(dstPath: string, compact?: boolean): Promise, string>>; } diff --git a/yarn-project/txe/src/state_machine/synchronizer.ts b/yarn-project/txe/src/state_machine/synchronizer.ts index 8e217f4785d1..bf3fbdca9499 100644 --- a/yarn-project/txe/src/state_machine/synchronizer.ts +++ b/yarn-project/txe/src/state_machine/synchronizer.ts @@ -47,6 +47,11 @@ export class TXESynchronizer implements WorldStateSynchronizer { return this.nativeWorldStateService.getCommitted(); } + /** Commits a fork as the current "committed" view of the world state. */ + public commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { + return this.nativeWorldStateService.commitFork(fork, blockNumber); + } + /** Forks the world state at the given block number, defaulting to the latest one. */ public fork(block?: number): Promise { return this.nativeWorldStateService.fork(block ? BlockNumber(block) : undefined); diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index ea52aa4a20b3..5239c7ee4bb7 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -1995,4 +1995,100 @@ describe('NativeWorldState', () => { await fork.close(); }); }); + + describe('commitFork', () => { + let ws: NativeWorldStateService; + + beforeEach(async () => { + ws = await NativeWorldStateService.tmp(); + }); + + afterEach(async () => { + await ws.close(); + }); + + it('makes getCommitted() return the fork state', async () => { + const fork = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork); + + await ws.commitFork(fork, BlockNumber(0)); + + await assertSameState(ws.getCommitted(), fork); + }); + + it('fails if tip has moved', async () => { + // Build and sync block 1 via handleL2BlockAndMessages + const setupFork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, setupFork); + await ws.handleL2BlockAndMessages(block, messages); + await setupFork.close(); + + // Create a fork at block 0 (now stale) + const staleFork = await ws.fork(BlockNumber(0)); + await mockBlock(BlockNumber(1), 1, staleFork); + + // commitFork should fail because tip moved from 0 to 1 + await expect(ws.commitFork(staleFork, BlockNumber(0))).rejects.toThrow(); + }); + + it('handleL2BlockAndMessages clears the committed fork', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + + await ws.commitFork(fork, BlockNumber(0)); + + // getCommitted() should return the fork's state + await assertSameState(ws.getCommitted(), fork); + + // Now sync the same block via handleL2BlockAndMessages + await ws.handleL2BlockAndMessages(block, messages); + + // getCommitted() should now return LMDB state (which should match since same block was synced) + const committed = ws.getCommitted(); + await assertSameState(committed, fork); + + // Verify the committed fork was cleared by mutating the fork and checking getCommitted() doesn't change + const committedInfoBefore = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); + await fork.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [Fr.random()]); + const committedInfoAfter = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); + expect(committedInfoAfter).toEqual(committedInfoBefore); + }); + + it('unwindBlocks clears the committed fork', async () => { + // Sync blocks 1..3 + const setupFork = await ws.fork(); + for (let i = 1; i <= 3; i++) { + const { block, messages } = await mockBlock(BlockNumber(i), 1, setupFork); + await ws.handleL2BlockAndMessages(block, messages); + } + await setupFork.close(); + + // Create fork at block 3, build block 4, commitFork + const fork = await ws.fork(); + await mockBlock(BlockNumber(4), 1, fork); + await ws.commitFork(fork, BlockNumber(3)); + + // Reorg back to block 2 + await ws.unwindBlocks(BlockNumber(2)); + + // getCommitted() should return LMDB state at block 2, not the fork + const snapshot2 = ws.getSnapshot(BlockNumber(2)); + await assertSameState(ws.getCommitted(), snapshot2); + }); + + it('replaces a previous committed fork', async () => { + // Build fork1 with block 1 + const fork1 = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork1); + await ws.commitFork(fork1, BlockNumber(0)); + + // Build fork2 with different state for block 1 + const fork2 = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork2); + await ws.commitFork(fork2, BlockNumber(0)); + + // getCommitted() should match fork2, not fork1 + await assertSameState(ws.getCommitted(), fork2); + }); + }); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 966d5787f8e3..5be7c8186b3a 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -192,6 +192,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } + public async commitFork(_fork: MerkleTreeWriteOperations, _blockNumber: BlockNumber): Promise { + // TODO: implement + } + public getInitialHeader(): BlockHeader { return this.initialHeader!; } diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index b99c1869b2f3..fe33c26bc8c9 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -85,6 +85,10 @@ export class ServerWorldStateSynchronizer return this.merkleTreeDb.fork(blockNumber, opts); } + public commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { + return this.merkleTreeDb.commitFork(fork, blockNumber); + } + public backupTo(dstPath: string, compact?: boolean): Promise, string>> { return this.merkleTreeDb.backupTo(dstPath, compact); } From 69a8f00822002feed922fe573d979e935bba0363 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 20 Mar 2026 09:30:06 -0300 Subject: [PATCH 02/29] implementation --- .../src/native/native_world_state.test.ts | 30 +++++++++++------ .../src/native/native_world_state.ts | 33 ++++++++++++++++--- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 5239c7ee4bb7..af6fa1c934c3 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2035,23 +2035,30 @@ describe('NativeWorldState', () => { const fork = await ws.fork(); const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + // Snapshot the fork's state before committing (fork will be closed by handleL2BlockAndMessages) + const forkStateRef = await fork.getStateReference(); + await ws.commitFork(fork, BlockNumber(0)); // getCommitted() should return the fork's state - await assertSameState(ws.getCommitted(), fork); + const committedStateRef = await ws.getCommitted().getStateReference(); + expect(committedStateRef).toEqual(forkStateRef); - // Now sync the same block via handleL2BlockAndMessages + // Now sync the same block via handleL2BlockAndMessages (this closes the committed fork) await ws.handleL2BlockAndMessages(block, messages); // getCommitted() should now return LMDB state (which should match since same block was synced) - const committed = ws.getCommitted(); - await assertSameState(committed, fork); + const lmdbStateRef = await ws.getCommitted().getStateReference(); + expect(lmdbStateRef).toEqual(forkStateRef); - // Verify the committed fork was cleared by mutating the fork and checking getCommitted() doesn't change + // Verify getCommitted() is now a fresh LMDB facade, not the fork, by creating a new fork + // and confirming getCommitted() doesn't track new fork mutations + const newFork = await ws.fork(); const committedInfoBefore = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - await fork.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [Fr.random()]); + await newFork.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [Fr.random()]); const committedInfoAfter = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); expect(committedInfoAfter).toEqual(committedInfoBefore); + await newFork.close(); }); it('unwindBlocks clears the committed fork', async () => { @@ -2063,17 +2070,20 @@ describe('NativeWorldState', () => { } await setupFork.close(); + // Snapshot state at block 2 before the reorg + const snapshot2StateRef = await ws.getSnapshot(BlockNumber(2)).getStateReference(); + // Create fork at block 3, build block 4, commitFork const fork = await ws.fork(); await mockBlock(BlockNumber(4), 1, fork); await ws.commitFork(fork, BlockNumber(3)); - // Reorg back to block 2 + // Reorg back to block 2 (this closes the committed fork) await ws.unwindBlocks(BlockNumber(2)); - // getCommitted() should return LMDB state at block 2, not the fork - const snapshot2 = ws.getSnapshot(BlockNumber(2)); - await assertSameState(ws.getCommitted(), snapshot2); + // getCommitted() should return LMDB state at block 2, not the fork's state + const committedStateRef = await ws.getCommitted().getStateReference(); + expect(committedStateRef).toEqual(snapshot2StateRef); }); it('replaces a previous committed fork', async () => { diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 5be7c8186b3a..f3855c4ce708 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -48,6 +48,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: BlockHeader | undefined; // This is read heavily and only changes when data is persisted, so we cache it private cachedStatusSummary: WorldStateStatusSummary | undefined; + private committedFork: MerkleTreeWriteOperations | undefined; protected constructor( protected instance: NativeWorldState, @@ -153,6 +154,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async clear() { + await this.closeCommittedFork(); await this.instance.close(); this.cachedStatusSummary = undefined; await tryRmDir(this.instance.getDataDir(), this.log); @@ -160,6 +162,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public getCommitted(): MerkleTreeReadOperations { + if (this.committedFork) { + return this.committedFork; + } return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty()); } @@ -192,8 +197,15 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } - public async commitFork(_fork: MerkleTreeWriteOperations, _blockNumber: BlockNumber): Promise { - // TODO: implement + public async commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { + const status = await this.getStatusSummary(); + if (status.unfinalizedBlockNumber !== blockNumber) { + throw new Error( + `Can't commit fork: expected tip at block ${blockNumber}, but canonical tip is at ${status.unfinalizedBlockNumber}`, + ); + } + await this.closeCommittedFork(); + this.committedFork = fork; } public getInitialHeader(): BlockHeader { @@ -232,7 +244,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { }); try { - return await this.instance.call( + const result = await this.instance.call( WorldStateMessageType.SYNC_BLOCK, { blockNumber: l2Block.number, @@ -247,6 +259,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); + await this.closeCommittedFork(); + return result; } catch (err) { this.worldStateInstrumentation.incCriticalErrors('synch_pending_block'); throw err; @@ -279,6 +293,14 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.cachedStatusSummary = undefined; } + private async closeCommittedFork(): Promise { + const fork = this.committedFork; + if (fork) { + this.committedFork = undefined; + await fork.close(); + } + } + /** * Advances the finalized block number to be the number provided * @param toBlockNumber The block number that is now the tip of the finalized chain @@ -331,7 +353,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { */ public async unwindBlocks(toBlockNumber: BlockNumber) { try { - return await this.instance.call( + const result = await this.instance.call( WorldStateMessageType.UNWIND_BLOCKS, { toBlockNumber, @@ -340,6 +362,9 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); + // Just null the reference — native UNWIND_BLOCKS already deletes forks past the unwind point. + this.committedFork = undefined; + return result; } catch (err) { this.worldStateInstrumentation.incCriticalErrors('prune_pending_block'); throw err; From 641d871cc91f6f8e0bedf81fbb345cbddb17459d Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 20 Mar 2026 09:37:13 -0300 Subject: [PATCH 03/29] fix close --- .../src/native/native_world_state.test.ts | 57 +++++++++++++++++++ .../src/native/native_world_state.ts | 1 + 2 files changed, 58 insertions(+) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index af6fa1c934c3..57e1188f6f18 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2100,5 +2100,62 @@ describe('NativeWorldState', () => { // getCommitted() should match fork2, not fork1 await assertSameState(ws.getCommitted(), fork2); }); + + const expectForkClosed = async (fork: MerkleTreeWriteOperations) => { + await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); + }; + + it('closes the previous fork when replacing', async () => { + const fork1 = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork1); + await ws.commitFork(fork1, BlockNumber(0)); + + const fork2 = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork2); + await ws.commitFork(fork2, BlockNumber(0)); + + await expectForkClosed(fork1); + // fork2 is still the committed fork and should be usable + await expect(fork2.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).resolves.toBeDefined(); + }); + + it('handleL2BlockAndMessages closes the committed fork', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + await ws.commitFork(fork, BlockNumber(0)); + + await ws.handleL2BlockAndMessages(block, messages); + + await expectForkClosed(fork); + }); + + it('unwindBlocks disposes the committed fork', async () => { + const setupFork = await ws.fork(); + for (let i = 1; i <= 3; i++) { + const { block, messages } = await mockBlock(BlockNumber(i), 1, setupFork); + await ws.handleL2BlockAndMessages(block, messages); + } + await setupFork.close(); + + const fork = await ws.fork(); + await mockBlock(BlockNumber(4), 1, fork); + await ws.commitFork(fork, BlockNumber(3)); + + await ws.unwindBlocks(BlockNumber(2)); + + await expectForkClosed(fork); + }); + + it('close() closes the committed fork', async () => { + const fork = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork); + await ws.commitFork(fork, BlockNumber(0)); + + await ws.close(); + + await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow(); + // Recreate ws so afterEach doesn't double-close + ws = await NativeWorldStateService.tmp(); + }); }); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index f3855c4ce708..98fc75efe6b4 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -268,6 +268,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async close(): Promise { + await this.closeCommittedFork(); await this.instance.close(); await this.cleanup(); } From a6f177d3d97ce8b4f03676c673ee94c3cb5988e7 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 20 Mar 2026 10:04:52 -0300 Subject: [PATCH 04/29] detach --- .../simulator/src/public/hinting_db_sources.ts | 4 ++++ .../public/public_processor/guarded_merkle_tree.ts | 4 ++++ .../src/interfaces/merkle_tree_operations.ts | 3 +++ .../world-state/src/native/merkle_trees_facade.ts | 9 +++++++++ .../src/native/native_world_state.test.ts | 14 ++++++++++++++ .../world-state/src/native/native_world_state.ts | 1 + 6 files changed, 35 insertions(+) diff --git a/yarn-project/simulator/src/public/hinting_db_sources.ts b/yarn-project/simulator/src/public/hinting_db_sources.ts index 85f8ab422ccf..9bff8b8a40cd 100644 --- a/yarn-project/simulator/src/public/hinting_db_sources.ts +++ b/yarn-project/simulator/src/public/hinting_db_sources.ts @@ -574,6 +574,10 @@ export class HintingMerkleWriteOperations implements MerkleTreeWriteOperations { return await this.db.close(); } + public detach(): void { + this.db.detach(); + } + async [Symbol.asyncDispose](): Promise { await this.close(); } diff --git a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts index 71133c4a2ebf..7295b5b4c204 100644 --- a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts +++ b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts @@ -82,6 +82,10 @@ export class GuardedMerkleTreeOperations implements MerkleTreeWriteOperations { return this.guardAndPush(() => this.target.close()); } + detach(): void { + this.target.detach(); + } + async [Symbol.asyncDispose](): Promise { await this.close(); } diff --git a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts index 29625e9d4c43..13095d4b3cf7 100644 --- a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts @@ -288,6 +288,9 @@ export interface MerkleTreeWriteOperations * Closes the database, discarding any uncommitted changes. */ close(): Promise; + + /** Prevents auto-dispose from closing this fork. Used when ownership is transferred via commitFork. */ + detach(): void; } /** diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index b8d4ca92b3e0..7de3b07bce8c 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -207,6 +207,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { private log = createLogger('world-state:merkle-trees-fork-facade'); + private detached = false; constructor( instance: NativeWorldStateInstance, @@ -218,6 +219,11 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr assert.equal(revision.includeUncommitted, true, 'Fork must include uncommitted data'); super(instance, initialHeader, revision); } + + /** Prevents auto-dispose from closing this fork. Used when ownership is transferred via commitFork. */ + detach(): void { + this.detached = true; + } async updateArchive(header: BlockHeader): Promise { await this.instance.call(WorldStateMessageType.UPDATE_ARCHIVE, { forkId: this.revision.forkId, @@ -305,6 +311,9 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr } async [Symbol.asyncDispose](): Promise { + if (this.detached) { + return; + } if (this.opts.closeDelayMs) { void sleep(this.opts.closeDelayMs) .then(() => this.close()) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 57e1188f6f18..139ff9fe678d 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2157,5 +2157,19 @@ describe('NativeWorldState', () => { // Recreate ws so afterEach doesn't double-close ws = await NativeWorldStateService.tmp(); }); + + it('committed fork survives await using dispose', async () => { + // Simulate the sequencer pattern: fork created with await using, then committed + { + await using fork = await ws.fork(); + await mockBlock(BlockNumber(1), 1, fork); + await ws.commitFork(fork, BlockNumber(0)); + // Scope exit triggers [Symbol.asyncDispose], which should no-op due to detach + } + + // The committed fork should still be alive and usable via getCommitted() + await expect(ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).resolves.toBeDefined(); + await expect(ws.getCommitted().getStateReference()).resolves.toBeDefined(); + }); }); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 98fc75efe6b4..e1ceff0ec7c1 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -205,6 +205,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } await this.closeCommittedFork(); + fork.detach(); this.committedFork = fork; } From d9510f9256a7f847214528b9b5997ed23f94540b Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 20 Mar 2026 18:22:42 -0300 Subject: [PATCH 05/29] simplify --- .../src/sequencer/checkpoint_proposal_job.ts | 21 +++++++++ .../src/native/merkle_trees_facade.ts | 5 +++ .../world-state/src/native/message.ts | 5 +++ .../src/native/native_world_state.ts | 43 +++++++++++-------- .../src/native/native_world_state_instance.ts | 4 +- 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 17703d4a016d..5c0a6818e48d 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -37,6 +37,7 @@ import { Gas } from '@aztec/stdlib/gas'; import { type BlockBuilderOptions, InsufficientValidTxsError, + type MerkleTreeWriteOperations, type ResolvedSequencerConfig, type WorldStateSynchronizer, } from '@aztec/stdlib/interfaces/server'; @@ -354,6 +355,7 @@ export class CheckpointProposalJob implements Traceable { checkpointGlobalVariables.timestamp, inHash, blockProposalOptions, + fork, ); blocksInCheckpoint = result.blocksInCheckpoint; blockPendingBroadcast = result.blockPendingBroadcast; @@ -498,6 +500,7 @@ export class CheckpointProposalJob implements Traceable { timestamp: bigint, inHash: Fr, blockProposalOptions: BlockProposalOptions, + fork: MerkleTreeWriteOperations, ): Promise<{ blocksInCheckpoint: L2Block[]; blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined; @@ -505,6 +508,7 @@ export class CheckpointProposalJob implements Traceable { const blocksInCheckpoint: L2Block[] = []; const txHashesAlreadyIncluded = new Set(); const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1); + let forkCommitted = false; // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined; @@ -570,6 +574,10 @@ export class CheckpointProposalJob implements Traceable { // If this is the last block, sync it to the archiver and exit the loop // so we can build the checkpoint and start collecting attestations. if (timingInfo.isLastBlock) { + if (!forkCommitted) { + forkCommitted = true; + await this.tryCommitFork(fork); + } await this.syncProposedBlockToArchiver(block); this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, { slot: this.targetSlot, @@ -589,6 +597,10 @@ export class CheckpointProposalJob implements Traceable { // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal. // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building. // If this throws, we abort the entire checkpoint. + if (!forkCommitted) { + forkCommitted = true; + await this.tryCommitFork(fork); + } await this.syncProposedBlockToArchiver(block); // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it. @@ -1008,6 +1020,15 @@ export class CheckpointProposalJob implements Traceable { await this.p2pClient.handleFailedExecution(failedTxHashes); } + /** Commits the fork so getCommitted() immediately reflects the built blocks. */ + private async tryCommitFork(fork: MerkleTreeWriteOperations): Promise { + try { + await this.worldState.commitFork(fork, this.syncedToBlockNumber); + } catch (err) { + this.log.debug(`Could not commit fork (block stream may have synced first)`, { err }); + } + } + /** * Adds the proposed block to the archiver so it's available via P2P. * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 7de3b07bce8c..94adbaa26fba 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -224,6 +224,11 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr detach(): void { this.detached = true; } + + /** Returns the native fork ID. */ + get forkId(): number { + return this.revision.forkId; + } async updateArchive(header: BlockHeader): Promise { await this.instance.call(WorldStateMessageType.UPDATE_ARCHIVE, { forkId: this.revision.forkId, diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index edceed40e4b3..ce44b79e87a8 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -34,6 +34,7 @@ export enum WorldStateMessageType { CREATE_FORK, DELETE_FORK, + COMMIT_FORK, FINALIZE_BLOCKS, UNWIND_BLOCKS, @@ -441,6 +442,8 @@ interface CreateForkResponse { interface DeleteForkRequest extends WithForkId {} +interface CommitForkRequest extends WithForkId {} + interface CopyStoresRequest extends WithCanonicalForkId { dstPath: string; compact: boolean; @@ -487,6 +490,7 @@ export type WorldStateRequest = { [WorldStateMessageType.CREATE_FORK]: CreateForkRequest; [WorldStateMessageType.DELETE_FORK]: DeleteForkRequest; + [WorldStateMessageType.COMMIT_FORK]: CommitForkRequest; [WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: BlockShiftRequest; [WorldStateMessageType.UNWIND_BLOCKS]: BlockShiftRequest; @@ -532,6 +536,7 @@ export type WorldStateResponse = { [WorldStateMessageType.CREATE_FORK]: CreateForkResponse; [WorldStateMessageType.DELETE_FORK]: void; + [WorldStateMessageType.COMMIT_FORK]: WorldStateStatusFull; [WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]: WorldStateStatusFull; [WorldStateMessageType.UNWIND_BLOCKS]: WorldStateStatusFull; diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index e1ceff0ec7c1..727bef0922b5 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -32,6 +32,7 @@ import { type WorldStateStatusFull, type WorldStateStatusSummary, blockStateReference, + buildEmptyWorldStateStatusFull, sanitizeFullStatus, sanitizeSummary, treeStateReferenceToSnapshot, @@ -48,7 +49,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: BlockHeader | undefined; // This is read heavily and only changes when data is persisted, so we cache it private cachedStatusSummary: WorldStateStatusSummary | undefined; - private committedFork: MerkleTreeWriteOperations | undefined; + private committedForkStatus: WorldStateStatusFull | undefined; protected constructor( protected instance: NativeWorldState, @@ -154,7 +155,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async clear() { - await this.closeCommittedFork(); await this.instance.close(); this.cachedStatusSummary = undefined; await tryRmDir(this.instance.getDataDir(), this.log); @@ -162,9 +162,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public getCommitted(): MerkleTreeReadOperations { - if (this.committedFork) { - return this.committedFork; - } return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty()); } @@ -204,9 +201,17 @@ export class NativeWorldStateService implements MerkleTreeDatabase { `Can't commit fork: expected tip at block ${blockNumber}, but canonical tip is at ${status.unfinalizedBlockNumber}`, ); } - await this.closeCommittedFork(); fork.detach(); - this.committedFork = fork; + + // Promote fork's tree caches to canonical LMDB. + // After this call, the fork is consumed by the native layer and canonical LMDB has the fork's state. + const forkFacade = fork as MerkleTreesForkFacade; + this.committedForkStatus = await this.instance.call( + WorldStateMessageType.COMMIT_FORK, + { forkId: forkFacade.forkId }, + this.sanitizeAndCacheSummaryFromFull.bind(this), + this.deleteCachedSummary.bind(this), + ); } public getInitialHeader(): BlockHeader { @@ -214,6 +219,18 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { + // Skip if this block is already persisted (e.g. via COMMIT_FORK) + const currentStatus = await this.getStatusSummary(); + if (l2Block.number <= currentStatus.unfinalizedBlockNumber) { + this.log.debug( + `Skipping SYNC_BLOCK for block ${l2Block.number} — already at tip ${currentStatus.unfinalizedBlockNumber}`, + ); + if (this.committedForkStatus) { + return this.committedForkStatus; + } + return buildEmptyWorldStateStatusFull(); + } + const isFirstBlock = l2Block.indexWithinCheckpoint === 0; if (!isFirstBlock && l1ToL2Messages.length > 0) { throw new Error( @@ -260,7 +277,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); - await this.closeCommittedFork(); return result; } catch (err) { this.worldStateInstrumentation.incCriticalErrors('synch_pending_block'); @@ -269,7 +285,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async close(): Promise { - await this.closeCommittedFork(); await this.instance.close(); await this.cleanup(); } @@ -295,14 +310,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.cachedStatusSummary = undefined; } - private async closeCommittedFork(): Promise { - const fork = this.committedFork; - if (fork) { - this.committedFork = undefined; - await fork.close(); - } - } - /** * Advances the finalized block number to be the number provided * @param toBlockNumber The block number that is now the tip of the finalized chain @@ -364,8 +371,6 @@ export class NativeWorldStateService implements MerkleTreeDatabase { this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); - // Just null the reference — native UNWIND_BLOCKS already deletes forks past the unwind point. - this.committedFork = undefined; return result; } catch (err) { this.worldStateInstrumentation.incCriticalErrors('prune_pending_block'); diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index ebd3d330faf2..441586d013f0 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -198,8 +198,8 @@ export class NativeWorldState implements NativeWorldStateInstance { committedOnly, ); - // If the request was to delete the fork then we clean it up here - if (messageType === WorldStateMessageType.DELETE_FORK) { + // If the request was to delete or commit the fork then we clean it up here + if (messageType === WorldStateMessageType.DELETE_FORK || messageType === WorldStateMessageType.COMMIT_FORK) { await requestQueue.stop(); this.queues.delete(forkId); } From 5eb39c96bbccde7e13016ea71092e9f67af6abd1 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 11:02:41 -0300 Subject: [PATCH 06/29] interface barrateberg --- .../content_addressed_append_only_tree.hpp | 2 ++ .../cached_content_addressed_tree_store.hpp | 2 ++ .../nodejs_module/world_state/world_state.cpp | 18 ++++++++++++++++++ .../nodejs_module/world_state/world_state.hpp | 1 + .../world_state/world_state_message.hpp | 1 + .../barretenberg/world_state/world_state.cpp | 5 +++++ .../barretenberg/world_state/world_state.hpp | 1 + 7 files changed, 30 insertions(+) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index a3e101135878..6c02efd2a940 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -75,6 +75,8 @@ template class ContentAddressedAppendOn ContentAddressedAppendOnlyTree& operator=(ContentAddressedAppendOnlyTree const&& other) = delete; virtual ~ContentAddressedAppendOnlyTree() = default; + void clear_initialized_from_block() { store_->clear_initialized_from_block(); } + /** * @brief Adds a single value to the end of the tree * @param value The value to be added diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 49a4dc1110cd..7fe848465dec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -64,6 +64,8 @@ template class ContentAddressedCachedTreeStore { ContentAddressedCachedTreeStore& operator=(ContentAddressedCachedTreeStore const& other) = delete; ContentAddressedCachedTreeStore& operator=(ContentAddressedCachedTreeStore const&& other) = delete; + void clear_initialized_from_block() { forkConstantData_.initialized_from_block_.reset(); } + /** * @brief Returns the index of the leaf with a value immediately lower than the value provided */ diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp index 9087aceeb19e..4a41cbe9f4ce 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp @@ -233,6 +233,10 @@ WorldStateWrapper::WorldStateWrapper(const Napi::CallbackInfo& info) WorldStateMessageType::DELETE_FORK, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return delete_fork(obj, buffer); }); + _dispatcher.register_target( + WorldStateMessageType::COMMIT_FORK, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return commit_fork(obj, buffer); }); + _dispatcher.register_target( WorldStateMessageType::FINALIZE_BLOCKS, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return set_finalized(obj, buffer); }); @@ -787,6 +791,20 @@ bool WorldStateWrapper::delete_fork(msgpack::object& obj, msgpack::sbuffer& buf) return true; } +bool WorldStateWrapper::commit_fork(msgpack::object& obj, msgpack::sbuffer& buf) +{ + TypedMessage request; + obj.convert(request); + + WorldStateStatusFull status = _ws->commit_fork(request.value.forkId); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg(WorldStateMessageType::COMMIT_FORK, header, { status }); + msgpack::pack(buf, resp_msg); + + return true; +} + bool WorldStateWrapper::close(msgpack::object& obj, msgpack::sbuffer& buf) { HeaderOnlyMessage request; diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp index cd4f0d02e8e1..f9e523a8da8a 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp @@ -63,6 +63,7 @@ class WorldStateWrapper : public Napi::ObjectWrap { bool create_fork(msgpack::object& obj, msgpack::sbuffer& buffer); bool delete_fork(msgpack::object& obj, msgpack::sbuffer& buffer); + bool commit_fork(msgpack::object& obj, msgpack::sbuffer& buffer); bool close(msgpack::object& obj, msgpack::sbuffer& buffer); diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp index 8f6b481ad41a..406b72e14cad 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp @@ -44,6 +44,7 @@ enum WorldStateMessageType { CREATE_FORK, DELETE_FORK, + COMMIT_FORK, FINALIZE_BLOCKS, UNWIND_BLOCKS, diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index f221e93fcf6f..cc614776f7f2 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -243,6 +243,11 @@ void WorldState::delete_fork(const uint64_t& forkId) } } +WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) +{ + throw std::runtime_error("commit_fork not implemented"); +} + Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index d7d8f99d46f5..1efe874add3a 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -274,6 +274,7 @@ class WorldState { uint64_t create_fork(const std::optional& blockNumber); void delete_fork(const uint64_t& forkId); + WorldStateStatusFull commit_fork(const uint64_t& forkId); WorldStateStatusSummary set_finalized_blocks(const block_number_t& toBlockNumber); WorldStateStatusFull unwind_blocks(const block_number_t& toBlockNumber); From 90140d658d7931ae657f320eaa553a3993d2adad Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 13:54:22 -0300 Subject: [PATCH 07/29] implementation --- .../barretenberg/world_state/world_state.cpp | 48 +++++- .../barretenberg/world_state/world_state.hpp | 2 + .../world_state/world_state.test.cpp | 160 ++++++++++++++++++ 3 files changed, 208 insertions(+), 2 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index cc614776f7f2..893f090bc36f 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -245,7 +245,47 @@ void WorldState::delete_fork(const uint64_t& forkId) WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) { - throw std::runtime_error("commit_fork not implemented"); + if (forkId == CANONICAL_FORK_ID) { + throw std::runtime_error("Cannot commit the canonical fork"); + } + validate_trees_are_equally_synched(); + + Fork::SharedPtr fork = retrieve_fork(forkId); + + // Validate tip hasn't moved since fork was created + auto archiveMeta = get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + if (archiveMeta.meta.unfinalizedBlockHeight != fork->_blockNumber) { + throw std::runtime_error("Can't commit fork: canonical tip has moved from " + + std::to_string(fork->_blockNumber) + " to " + + std::to_string(archiveMeta.meta.unfinalizedBlockHeight)); + } + + // Rollback canonical to clear any uncommitted state + rollback(); + + // Clear fork flags so commit_block() is allowed on fork stores + for (auto& [id, tree] : fork->_trees) { + std::visit([](auto&& wrapper) { wrapper.tree->clear_initialized_from_block(); }, tree); + } + + // Commit fork trees to LMDB + WorldStateStatusFull status; + auto [success, message] = commit(fork, status); + if (!success) { + throw std::runtime_error("Failed to commit fork: " + message); + } + + // Rollback canonical so it re-reads the updated LMDB state + rollback(); + + // Destroy the fork + { + std::unique_lock lock(mtx); + _forks.erase(forkId); + } + + populate_status_summary(status); + return status; } Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) @@ -525,9 +565,13 @@ void WorldState::update_archive(const StateReference& block_state_ref, } std::pair WorldState::commit(WorldStateStatusFull& status) +{ + return commit(retrieve_fork(CANONICAL_FORK_ID), status); +} + +std::pair WorldState::commit(Fork::SharedPtr fork, WorldStateStatusFull& status) { // NOTE: the calling code is expected to ensure no other reads or writes happen during commit - Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); std::atomic_bool success = true; std::string message; Signal signal(static_cast(fork->_trees.size())); diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 1efe874add3a..4492fb8d53d2 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -344,6 +344,8 @@ class WorldState { static void populate_status_summary(WorldStateStatusFull& status); + std::pair commit(Fork::SharedPtr fork, WorldStateStatusFull& status); + template void commit_tree(TreeDBStats& dbStats, Signal& signal, diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index e28a3d4d9549..8116ec84cfdf 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -896,6 +896,166 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); } +TEST_F(WorldStateTest, CommitForkHappyPath) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + auto fork_id = ws.create_fork(0); + + // Build a block on the fork (same pattern as BuildsABlockInAFork) + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork_id); + + auto fork_state_ref = ws.get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); + ws.update_archive(fork_state_ref, { 1 }, fork_id); + + // Commit the fork + WorldStateStatusFull status = ws.commit_fork(fork_id); + EXPECT_EQ(status.summary.unfinalizedBlockNumber, 1); + + // Verify canonical committed state has the new leaves + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); + assert_leaf_value( + ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(129)); + assert_leaf_value( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 128, PublicDataLeafValue(129, 1)); + + // Verify state reference matches + EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); +} + +TEST_F(WorldStateTest, CommitForkThenCreateNewForkAtAdvancedTip) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + + // Build and commit block 1 + auto fork1 = ws.create_fork(0); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork1); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork1); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork1); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork1); + auto state_ref1 = ws.get_state_reference(WorldStateRevision{ .forkId = fork1, .includeUncommitted = true }); + ws.update_archive(state_ref1, { 1 }, fork1); + WorldStateStatusFull status1 = ws.commit_fork(fork1); + EXPECT_EQ(status1.summary.unfinalizedBlockNumber, 1); + + // Create new fork at latest — should be at block 1 + auto fork2 = ws.create_fork(std::nullopt); + + // Build and commit block 2 + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 44 }, fork2); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 45 }, fork2); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 130 } }, 0, fork2); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 130, 2 } }, 0, fork2); + auto state_ref2 = ws.get_state_reference(WorldStateRevision{ .forkId = fork2, .includeUncommitted = true }); + ws.update_archive(state_ref2, { 2 }, fork2); + WorldStateStatusFull status2 = ws.commit_fork(fork2); + EXPECT_EQ(status2.summary.unfinalizedBlockNumber, 2); + + // Verify both blocks' leaves are visible in canonical + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 1, fr(44)); +} + +TEST_F(WorldStateTest, CommitForkRejectsWhenTipMoved) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + + // Build block 1 using a temporary fork to get correct state references + auto tmp_fork = ws.create_fork(0); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, tmp_fork); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, tmp_fork); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, tmp_fork); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, tmp_fork); + auto state_ref1 = + ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork, .includeUncommitted = true }); + ws.delete_fork(tmp_fork); + ws.sync_block(state_ref1, fr(1), { 42 }, { 43 }, { NullifierLeafValue(129) }, { { PublicDataLeafValue(129, 1) } }); + + // Create fork at block 1 — this is the fork we'll try to commit later + auto fork_id = ws.create_fork(1); + + // Build and sync block 2 to advance canonical tip + auto tmp_fork2 = ws.create_fork(1); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 44 }, tmp_fork2); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 45 }, tmp_fork2); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 130 } }, 0, tmp_fork2); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 130, 2 } }, 0, tmp_fork2); + auto state_ref2 = + ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork2, .includeUncommitted = true }); + ws.delete_fork(tmp_fork2); + ws.sync_block(state_ref2, fr(2), { 44 }, { 45 }, { NullifierLeafValue(130) }, { { PublicDataLeafValue(130, 2) } }); + + // commit_fork should reject because tip moved from 1 to 2 + EXPECT_THROW(ws.commit_fork(fork_id), std::runtime_error); + + // Verify canonical is still at block 2, not corrupted + auto archive_info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + EXPECT_EQ(archive_info.meta.unfinalizedBlockHeight, 2); +} + +TEST_F(WorldStateTest, CommitForkRejectsCanonicalForkId) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + EXPECT_THROW(ws.commit_fork(CANONICAL_FORK_ID), std::runtime_error); +} + +TEST_F(WorldStateTest, CommitForkRejectsInvalidForkId) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + EXPECT_THROW(ws.commit_fork(99999), std::runtime_error); +} + +TEST_F(WorldStateTest, CommitForkCanonicalReadsReflectData) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + auto fork_id = ws.create_fork(0); + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork_id); + auto fork_state_ref = ws.get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); + ws.update_archive(fork_state_ref, { 1 }, fork_id); + + ws.commit_fork(fork_id); + + // Verify leaves are searchable from canonical + assert_leaf_exists(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, fr(42), true); + assert_leaf_exists(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, fr(43), true); + assert_leaf_exists( + ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, NullifierLeafValue(129), true); + assert_leaf_exists( + ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, PublicDataLeafValue(129, 1), true); + + // Verify tree sizes advanced + assert_tree_size(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 1); + assert_tree_size(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 1); + assert_tree_size(ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 129); + assert_tree_size(ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 129); +} + +TEST_F(WorldStateTest, CommitForkDestroysFork) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + auto fork_id = ws.create_fork(0); + + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork_id); + auto fork_state_ref = ws.get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); + ws.update_archive(fork_state_ref, { 1 }, fork_id); + + ws.commit_fork(fork_id); + + // Fork should be destroyed — operations on it should fail + EXPECT_THROW(ws.delete_fork(fork_id), std::runtime_error); + EXPECT_THROW(ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 99 }, fork_id), std::runtime_error); +} + TEST_F(WorldStateTest, GetBlockForIndex) { WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); From 72b34d48431323edd6b7c38744322d541c35ad82 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 14:10:34 -0300 Subject: [PATCH 08/29] ts part --- .../src/sequencer/checkpoint_proposal_job.ts | 35 +++--- .../src/public/hinting_db_sources.ts | 4 - .../public_processor/guarded_merkle_tree.ts | 4 - .../src/interfaces/merkle_tree_operations.ts | 3 - .../stdlib/src/interfaces/world_state.ts | 8 +- .../txe/src/state_machine/synchronizer.ts | 4 +- .../src/checkpoint_builder.ts | 5 + .../src/native/merkle_trees_facade.ts | 19 +-- .../world-state/src/native/message.ts | 4 +- .../src/native/native_world_state.test.ts | 116 ++++++------------ .../src/native/native_world_state.ts | 14 +-- .../src/native/native_world_state_instance.ts | 12 +- .../server_world_state_synchronizer.ts | 4 +- 13 files changed, 85 insertions(+), 147 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 5c0a6818e48d..22e2fcf5518d 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -319,8 +319,9 @@ export class CheckpointProposalJob implements Traceable { // Get the fee asset price modifier from the oracle const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(); - // Create a long-lived forked world state for the checkpoint builder - await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); + // Create a forked world state for the checkpoint builder. + // Fork lifecycle is managed manually: each block commits and destroys the fork, then creates a new one. + let fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); // Create checkpoint builder for the entire slot const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint( @@ -508,7 +509,6 @@ export class CheckpointProposalJob implements Traceable { const blocksInCheckpoint: L2Block[] = []; const txHashesAlreadyIncluded = new Set(); const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1); - let forkCommitted = false; // Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal let blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined = undefined; @@ -571,14 +571,12 @@ export class CheckpointProposalJob implements Traceable { blocksInCheckpoint.push(block); usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString())); - // If this is the last block, sync it to the archiver and exit the loop - // so we can build the checkpoint and start collecting attestations. + // Commit the fork to persist this block to LMDB. The fork is destroyed after this call. + await this.tryCommitFork(fork); + await this.syncProposedBlockToArchiver(block); + + // If this is the last block, exit the loop so we can build the checkpoint and start collecting attestations. if (timingInfo.isLastBlock) { - if (!forkCommitted) { - forkCommitted = true; - await this.tryCommitFork(fork); - } - await this.syncProposedBlockToArchiver(block); this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, { slot: this.targetSlot, blockNumber, @@ -594,18 +592,13 @@ export class CheckpointProposalJob implements Traceable { // a HA error we don't pollute our archiver with a block that won't make it to the chain. const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions); - // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal. - // We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building. - // If this throws, we abort the entire checkpoint. - if (!forkCommitted) { - forkCommitted = true; - await this.tryCommitFork(fork); - } - await this.syncProposedBlockToArchiver(block); - // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it. proposal && (await this.p2pClient.broadcastProposal(proposal)); + // Create a new fork at the advanced tip for the next block + fork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); + checkpointBuilder.setFork(fork); + // Wait until the next block's start time await this.waitUntilNextSubslot(timingInfo.deadline); } @@ -1020,10 +1013,10 @@ export class CheckpointProposalJob implements Traceable { await this.p2pClient.handleFailedExecution(failedTxHashes); } - /** Commits the fork so getCommitted() immediately reflects the built blocks. */ + /** Commits the fork so getCommitted() immediately reflects the built blocks. The fork is destroyed after this call. */ private async tryCommitFork(fork: MerkleTreeWriteOperations): Promise { try { - await this.worldState.commitFork(fork, this.syncedToBlockNumber); + await this.worldState.commitFork(fork); } catch (err) { this.log.debug(`Could not commit fork (block stream may have synced first)`, { err }); } diff --git a/yarn-project/simulator/src/public/hinting_db_sources.ts b/yarn-project/simulator/src/public/hinting_db_sources.ts index 9bff8b8a40cd..85f8ab422ccf 100644 --- a/yarn-project/simulator/src/public/hinting_db_sources.ts +++ b/yarn-project/simulator/src/public/hinting_db_sources.ts @@ -574,10 +574,6 @@ export class HintingMerkleWriteOperations implements MerkleTreeWriteOperations { return await this.db.close(); } - public detach(): void { - this.db.detach(); - } - async [Symbol.asyncDispose](): Promise { await this.close(); } diff --git a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts index 7295b5b4c204..71133c4a2ebf 100644 --- a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts +++ b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts @@ -82,10 +82,6 @@ export class GuardedMerkleTreeOperations implements MerkleTreeWriteOperations { return this.guardAndPush(() => this.target.close()); } - detach(): void { - this.target.detach(); - } - async [Symbol.asyncDispose](): Promise { await this.close(); } diff --git a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts index 13095d4b3cf7..29625e9d4c43 100644 --- a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts @@ -288,9 +288,6 @@ export interface MerkleTreeWriteOperations * Closes the database, discarding any uncommitted changes. */ close(): Promise; - - /** Prevents auto-dispose from closing this fork. Used when ownership is transferred via commitFork. */ - detach(): void; } /** diff --git a/yarn-project/stdlib/src/interfaces/world_state.ts b/yarn-project/stdlib/src/interfaces/world_state.ts index 6c933be92f7f..7f32a32e0876 100644 --- a/yarn-project/stdlib/src/interfaces/world_state.ts +++ b/yarn-project/stdlib/src/interfaces/world_state.ts @@ -52,11 +52,11 @@ export interface ForkMerkleTreeOperations { fork(block?: BlockNumber, opts?: { closeDelayMs?: number }): Promise; /** - * Commits a fork as the current "committed" view of the world state. - * Only succeeds if the canonical tip hasn't moved past the given block number. - * Ownership of the fork transfers to the world state — caller must not close it. + * Commits a fork's state to canonical LMDB and destroys the fork. + * Only succeeds if the canonical tip hasn't moved since the fork was created. + * The fork is invalid after this call — caller must not use it. */ - commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise; + commitFork(fork: MerkleTreeWriteOperations): Promise; /** Backups the db to the target path. */ backupTo(dstPath: string, compact?: boolean): Promise, string>>; diff --git a/yarn-project/txe/src/state_machine/synchronizer.ts b/yarn-project/txe/src/state_machine/synchronizer.ts index bf3fbdca9499..13d2129ecfd7 100644 --- a/yarn-project/txe/src/state_machine/synchronizer.ts +++ b/yarn-project/txe/src/state_machine/synchronizer.ts @@ -48,8 +48,8 @@ export class TXESynchronizer implements WorldStateSynchronizer { } /** Commits a fork as the current "committed" view of the world state. */ - public commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { - return this.nativeWorldStateService.commitFork(fork, blockNumber); + public commitFork(fork: MerkleTreeWriteOperations): Promise { + return this.nativeWorldStateService.commitFork(fork); } /** Forks the world state at the given block number, defaulting to the latest one. */ diff --git a/yarn-project/validator-client/src/checkpoint_builder.ts b/yarn-project/validator-client/src/checkpoint_builder.ts index 05489c21e809..6200927a4db4 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.ts @@ -50,6 +50,11 @@ export class CheckpointBuilder implements ICheckpointBlockBuilder { /** Persistent contracts DB shared across all blocks in this checkpoint. */ protected contractsDB: PublicContractsDB; + /** Replaces the fork used for subsequent block builds. */ + public setFork(fork: MerkleTreeWriteOperations): void { + this.fork = fork; + } + constructor( private checkpointBuilder: LightweightCheckpointBuilder, private fork: MerkleTreeWriteOperations, diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 94adbaa26fba..9f611b0f3199 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -207,7 +207,6 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { private log = createLogger('world-state:merkle-trees-fork-facade'); - private detached = false; constructor( instance: NativeWorldStateInstance, @@ -220,11 +219,6 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr super(instance, initialHeader, revision); } - /** Prevents auto-dispose from closing this fork. Used when ownership is transferred via commitFork. */ - detach(): void { - this.detached = true; - } - /** Returns the native fork ID. */ get forkId(): number { return this.revision.forkId; @@ -306,9 +300,9 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr try { await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId }); } catch (err: any) { - // Ignore errors due to native instance being closed during shutdown. - // This can happen when validators are still processing block proposals while the node is stopping. - if (err?.message === 'Native instance is closed') { + // Ignore errors due to native instance being closed during shutdown, or fork already + // destroyed (e.g. via commitFork). + if (err?.message === 'Native instance is closed' || err?.message === 'Fork not found') { return; } throw err; @@ -316,15 +310,12 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr } async [Symbol.asyncDispose](): Promise { - if (this.detached) { - return; - } if (this.opts.closeDelayMs) { void sleep(this.opts.closeDelayMs) .then(() => this.close()) .catch(err => { - if (err && 'message' in err && err.message === 'Native instance is closed') { - return; // Ignore errors due to native instance being closed + if (err && 'message' in err && (err.message === 'Native instance is closed' || err.message === 'Fork not found')) { + return; } this.log.warn('Error closing MerkleTreesForkFacade after delay', { err }); }); diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index ce44b79e87a8..6191b5aa977f 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -442,7 +442,9 @@ interface CreateForkResponse { interface DeleteForkRequest extends WithForkId {} -interface CommitForkRequest extends WithForkId {} +interface CommitForkRequest extends WithCanonicalForkId { + forkId: number; +} interface CopyStoresRequest extends WithCanonicalForkId { dstPath: string; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 139ff9fe678d..3c0b5254ca7a 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2011,9 +2011,13 @@ describe('NativeWorldState', () => { const fork = await ws.fork(); await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork, BlockNumber(0)); + // Snapshot the fork's state before committing (fork is destroyed after commit) + const forkStateRef = await fork.getStateReference(); + + await ws.commitFork(fork); - await assertSameState(ws.getCommitted(), fork); + const committedStateRef = await ws.getCommitted().getStateReference(); + expect(committedStateRef).toEqual(forkStateRef); }); it('fails if tip has moved', async () => { @@ -2028,40 +2032,31 @@ describe('NativeWorldState', () => { await mockBlock(BlockNumber(1), 1, staleFork); // commitFork should fail because tip moved from 0 to 1 - await expect(ws.commitFork(staleFork, BlockNumber(0))).rejects.toThrow(); + await expect(ws.commitFork(staleFork)).rejects.toThrow(); }); - it('handleL2BlockAndMessages clears the committed fork', async () => { + it('handleL2BlockAndMessages skips already committed block', async () => { const fork = await ws.fork(); const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); - // Snapshot the fork's state before committing (fork will be closed by handleL2BlockAndMessages) + // Snapshot the fork's state before committing const forkStateRef = await fork.getStateReference(); - await ws.commitFork(fork, BlockNumber(0)); + await ws.commitFork(fork); - // getCommitted() should return the fork's state + // getCommitted() should return the committed state const committedStateRef = await ws.getCommitted().getStateReference(); expect(committedStateRef).toEqual(forkStateRef); - // Now sync the same block via handleL2BlockAndMessages (this closes the committed fork) + // Sync the same block via handleL2BlockAndMessages (should be skipped since already committed) await ws.handleL2BlockAndMessages(block, messages); - // getCommitted() should now return LMDB state (which should match since same block was synced) + // State should still match const lmdbStateRef = await ws.getCommitted().getStateReference(); expect(lmdbStateRef).toEqual(forkStateRef); - - // Verify getCommitted() is now a fresh LMDB facade, not the fork, by creating a new fork - // and confirming getCommitted() doesn't track new fork mutations - const newFork = await ws.fork(); - const committedInfoBefore = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - await newFork.appendLeaves(MerkleTreeId.NOTE_HASH_TREE, [Fr.random()]); - const committedInfoAfter = await ws.getCommitted().getTreeInfo(MerkleTreeId.NOTE_HASH_TREE); - expect(committedInfoAfter).toEqual(committedInfoBefore); - await newFork.close(); }); - it('unwindBlocks clears the committed fork', async () => { + it('unwindBlocks after commitFork', async () => { // Sync blocks 1..3 const setupFork = await ws.fork(); for (let i = 1; i <= 3; i++) { @@ -2076,80 +2071,45 @@ describe('NativeWorldState', () => { // Create fork at block 3, build block 4, commitFork const fork = await ws.fork(); await mockBlock(BlockNumber(4), 1, fork); - await ws.commitFork(fork, BlockNumber(3)); + await ws.commitFork(fork); - // Reorg back to block 2 (this closes the committed fork) + // Reorg back to block 2 await ws.unwindBlocks(BlockNumber(2)); - // getCommitted() should return LMDB state at block 2, not the fork's state + // getCommitted() should return LMDB state at block 2 const committedStateRef = await ws.getCommitted().getStateReference(); expect(committedStateRef).toEqual(snapshot2StateRef); }); - it('replaces a previous committed fork', async () => { - // Build fork1 with block 1 - const fork1 = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork1); - await ws.commitFork(fork1, BlockNumber(0)); - - // Build fork2 with different state for block 1 - const fork2 = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork2); - await ws.commitFork(fork2, BlockNumber(0)); - - // getCommitted() should match fork2, not fork1 - await assertSameState(ws.getCommitted(), fork2); - }); - - const expectForkClosed = async (fork: MerkleTreeWriteOperations) => { - await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); - }; - - it('closes the previous fork when replacing', async () => { + it('commit then create new fork at advanced tip', async () => { + // Build and commit block 1 const fork1 = await ws.fork(); await mockBlock(BlockNumber(1), 1, fork1); - await ws.commitFork(fork1, BlockNumber(0)); + await ws.commitFork(fork1); + // Create new fork at latest (should be at block 1) const fork2 = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork2); - await ws.commitFork(fork2, BlockNumber(0)); - - await expectForkClosed(fork1); - // fork2 is still the committed fork and should be usable - await expect(fork2.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).resolves.toBeDefined(); - }); + await mockBlock(BlockNumber(2), 1, fork2); + await ws.commitFork(fork2); - it('handleL2BlockAndMessages closes the committed fork', async () => { - const fork = await ws.fork(); - const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork, BlockNumber(0)); - - await ws.handleL2BlockAndMessages(block, messages); - - await expectForkClosed(fork); + // Verify canonical is at block 2 + const status = await ws.getStatusSummary(); + expect(status.unfinalizedBlockNumber).toEqual(2); }); - it('unwindBlocks disposes the committed fork', async () => { - const setupFork = await ws.fork(); - for (let i = 1; i <= 3; i++) { - const { block, messages } = await mockBlock(BlockNumber(i), 1, setupFork); - await ws.handleL2BlockAndMessages(block, messages); - } - await setupFork.close(); - + it('fork is destroyed after commitFork', async () => { const fork = await ws.fork(); - await mockBlock(BlockNumber(4), 1, fork); - await ws.commitFork(fork, BlockNumber(3)); - - await ws.unwindBlocks(BlockNumber(2)); + await mockBlock(BlockNumber(1), 1, fork); + await ws.commitFork(fork); - await expectForkClosed(fork); + // Fork should be destroyed — operations on it should fail + await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); }); - it('close() closes the committed fork', async () => { + it('close() after commitFork', async () => { const fork = await ws.fork(); await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork, BlockNumber(0)); + await ws.commitFork(fork); await ws.close(); @@ -2158,16 +2118,16 @@ describe('NativeWorldState', () => { ws = await NativeWorldStateService.tmp(); }); - it('committed fork survives await using dispose', async () => { - // Simulate the sequencer pattern: fork created with await using, then committed + it('committed state survives await using dispose', async () => { + // Simulate: fork created with await using, then committed. + // asyncDispose calls close() which tolerates "Fork not found" since C++ already destroyed the fork. { await using fork = await ws.fork(); await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork, BlockNumber(0)); - // Scope exit triggers [Symbol.asyncDispose], which should no-op due to detach + await ws.commitFork(fork); } - // The committed fork should still be alive and usable via getCommitted() + // The committed state should be accessible via getCommitted() await expect(ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).resolves.toBeDefined(); await expect(ws.getCommitted().getStateReference()).resolves.toBeDefined(); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 727bef0922b5..d5a6cb73222c 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -194,21 +194,11 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } - public async commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { - const status = await this.getStatusSummary(); - if (status.unfinalizedBlockNumber !== blockNumber) { - throw new Error( - `Can't commit fork: expected tip at block ${blockNumber}, but canonical tip is at ${status.unfinalizedBlockNumber}`, - ); - } - fork.detach(); - - // Promote fork's tree caches to canonical LMDB. - // After this call, the fork is consumed by the native layer and canonical LMDB has the fork's state. + public async commitFork(fork: MerkleTreeWriteOperations): Promise { const forkFacade = fork as MerkleTreesForkFacade; this.committedForkStatus = await this.instance.call( WorldStateMessageType.COMMIT_FORK, - { forkId: forkFacade.forkId }, + { forkId: forkFacade.forkId, canonical: true as const }, this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index 441586d013f0..32ef77acfb4b 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -198,10 +198,18 @@ export class NativeWorldState implements NativeWorldStateInstance { committedOnly, ); - // If the request was to delete or commit the fork then we clean it up here - if (messageType === WorldStateMessageType.DELETE_FORK || messageType === WorldStateMessageType.COMMIT_FORK) { + // If the request was to delete the fork then we clean it up here + if (messageType === WorldStateMessageType.DELETE_FORK) { await requestQueue.stop(); this.queues.delete(forkId); + } else if (messageType === WorldStateMessageType.COMMIT_FORK) { + // COMMIT_FORK runs on the canonical queue, but we need to clean up the fork's queue + const actualForkId = (body as { forkId: number }).forkId; + const forkQueue = this.queues.get(actualForkId); + if (forkQueue) { + await forkQueue.stop(); + this.queues.delete(actualForkId); + } } return response; } diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index fe33c26bc8c9..390f5f4ffbb6 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -85,8 +85,8 @@ export class ServerWorldStateSynchronizer return this.merkleTreeDb.fork(blockNumber, opts); } - public commitFork(fork: MerkleTreeWriteOperations, blockNumber: BlockNumber): Promise { - return this.merkleTreeDb.commitFork(fork, blockNumber); + public commitFork(fork: MerkleTreeWriteOperations): Promise { + return this.merkleTreeDb.commitFork(fork); } public backupTo(dstPath: string, compact?: boolean): Promise, string>> { From 9b9948a600837e8a5f5882cc47d6c6cbb2d4ef79 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 15:21:20 -0300 Subject: [PATCH 09/29] format --- .../src/barretenberg/world_state/world_state.test.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 8116ec84cfdf..8a6bc58f20f5 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -917,8 +917,7 @@ TEST_F(WorldStateTest, CommitForkHappyPath) // Verify canonical committed state has the new leaves assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, 0, fr(42)); assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE, 0, fr(43)); - assert_leaf_value( - ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(129)); + assert_leaf_value(ws, WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE, 128, NullifierLeafValue(129)); assert_leaf_value( ws, WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE, 128, PublicDataLeafValue(129, 1)); @@ -969,8 +968,7 @@ TEST_F(WorldStateTest, CommitForkRejectsWhenTipMoved) ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, tmp_fork); ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, tmp_fork); ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, tmp_fork); - auto state_ref1 = - ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork, .includeUncommitted = true }); + auto state_ref1 = ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork, .includeUncommitted = true }); ws.delete_fork(tmp_fork); ws.sync_block(state_ref1, fr(1), { 42 }, { 43 }, { NullifierLeafValue(129) }, { { PublicDataLeafValue(129, 1) } }); @@ -983,8 +981,7 @@ TEST_F(WorldStateTest, CommitForkRejectsWhenTipMoved) ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 45 }, tmp_fork2); ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 130 } }, 0, tmp_fork2); ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 130, 2 } }, 0, tmp_fork2); - auto state_ref2 = - ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork2, .includeUncommitted = true }); + auto state_ref2 = ws.get_state_reference(WorldStateRevision{ .forkId = tmp_fork2, .includeUncommitted = true }); ws.delete_fork(tmp_fork2); ws.sync_block(state_ref2, fr(2), { 44 }, { 45 }, { NullifierLeafValue(130) }, { { PublicDataLeafValue(130, 2) } }); From aed9ac5218b97729348b31917e5dfd827f63de2a Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 15:41:00 -0300 Subject: [PATCH 10/29] format ts --- .../src/sequencer/checkpoint_proposal_job.ts | 2 +- yarn-project/world-state/src/native/merkle_trees_facade.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 22e2fcf5518d..dbfe2a035437 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -321,7 +321,7 @@ export class CheckpointProposalJob implements Traceable { // Create a forked world state for the checkpoint builder. // Fork lifecycle is managed manually: each block commits and destroys the fork, then creates a new one. - let fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); + const fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); // Create checkpoint builder for the entire slot const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint( diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 9f611b0f3199..0eb844fb5460 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -314,7 +314,11 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr void sleep(this.opts.closeDelayMs) .then(() => this.close()) .catch(err => { - if (err && 'message' in err && (err.message === 'Native instance is closed' || err.message === 'Fork not found')) { + if ( + err && + 'message' in err && + (err.message === 'Native instance is closed' || err.message === 'Fork not found') + ) { return; } this.log.warn('Error closing MerkleTreesForkFacade after delay', { err }); From 1b1b27b59a8b6a4f2bf82de3970d8e437a12ef56 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 23 Mar 2026 20:30:49 -0300 Subject: [PATCH 11/29] fix test --- .../prover-client/src/light/lightweight_checkpoint_builder.ts | 2 +- yarn-project/validator-client/src/checkpoint_builder.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts index 1e72a75d7684..278952365a66 100644 --- a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts +++ b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts @@ -48,7 +48,7 @@ export class LightweightCheckpointBuilder { public feeAssetPriceModifier: bigint, public readonly l1ToL2Messages: Fr[], private readonly previousCheckpointOutHashes: Fr[], - public readonly db: MerkleTreeWriteOperations, + public db: MerkleTreeWriteOperations, bindings?: LoggerBindings, ) { this.logger = createLogger('checkpoint-builder', { diff --git a/yarn-project/validator-client/src/checkpoint_builder.ts b/yarn-project/validator-client/src/checkpoint_builder.ts index 6200927a4db4..3ecb3d5365fa 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.ts @@ -53,6 +53,7 @@ export class CheckpointBuilder implements ICheckpointBlockBuilder { /** Replaces the fork used for subsequent block builds. */ public setFork(fork: MerkleTreeWriteOperations): void { this.fork = fork; + this.checkpointBuilder.db = fork; } constructor( From c5de7cb00b1d17b69d076a29dbdf4c92a55d5142 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 24 Mar 2026 08:52:56 -0300 Subject: [PATCH 12/29] fix mock --- .../sequencer-client/src/test/mock_checkpoint_builder.ts | 4 ++++ yarn-project/stdlib/src/interfaces/block-builder.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts index f0a6afca82cc..5cbde03178fb 100644 --- a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts +++ b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts @@ -66,6 +66,10 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder { return this; } + setFork(_fork: MerkleTreeWriteOperations): void { + // No-op in mock — the mock doesn't use the fork directly + } + getConstantData(): CheckpointGlobalVariables { return this.constants; } diff --git a/yarn-project/stdlib/src/interfaces/block-builder.ts b/yarn-project/stdlib/src/interfaces/block-builder.ts index 6a2f49bb4209..b038e7c47f10 100644 --- a/yarn-project/stdlib/src/interfaces/block-builder.ts +++ b/yarn-project/stdlib/src/interfaces/block-builder.ts @@ -135,6 +135,9 @@ export interface ICheckpointBlockBuilder { timestamp: bigint, opts: BlockBuilderOptions, ): Promise; + + /** Replaces the fork used for subsequent block builds. */ + setFork(fork: MerkleTreeWriteOperations): void; } /** Interface for creating checkpoint builders. */ From de9de2fb84497b25812f97c5922be3de49ef4923 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 24 Mar 2026 12:54:35 -0300 Subject: [PATCH 13/29] do not reuse fork --- .../world-state/src/native/native_bench.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/yarn-project/world-state/src/native/native_bench.test.ts b/yarn-project/world-state/src/native/native_bench.test.ts index 1d498024f7ec..51faf20ac6f8 100644 --- a/yarn-project/world-state/src/native/native_bench.test.ts +++ b/yarn-project/world-state/src/native/native_bench.test.ts @@ -64,16 +64,16 @@ describe('Native World State: benchmarks', () => { effectsPerTx: number, worldState: NativeWorldStateService, ) => { - const blocks = []; - const fork = await worldState.fork(); - for (let i = 0; i < numBlocks; i++) { - const { block, messages } = await mockBlock(BlockNumber(i + 1), txsPerBlock, fork, effectsPerTx); - blocks.push({ block, messages }); - } - + // Build each block on a separate fork and sync it before building the next. + // Each fork starts from the latest committed state (one fork per block). const startTime = performance.now(); - for (const { block, messages } of blocks) { + for (let i = 0; i < numBlocks; i++) { + const status = await worldState.getStatusSummary(); + const blockNumber = BlockNumber(status.unfinalizedBlockNumber + 1); + const fork = await worldState.fork(); + const { block, messages } = await mockBlock(blockNumber, txsPerBlock, fork, effectsPerTx); + await fork.close(); await worldState.handleL2BlockAndMessages(block, messages); } From 4619e4de4c835c1672f9c9493ef5576ae3b05787 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 24 Mar 2026 14:44:04 -0300 Subject: [PATCH 14/29] fix some tests --- .../src/native/native_world_state.test.ts | 26 ++++++++++++------- yarn-project/world-state/src/test/utils.ts | 6 ++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 3c0b5254ca7a..9930d579d370 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -219,8 +219,8 @@ describe('NativeWorldState', () => { await timesAsync(5, async i => { const fork = await ws.fork(); - const { block, messages } = await mockBlock(BlockNumber(i + 1), 2, fork); - await ws.handleL2BlockAndMessages(block, messages); + const { block: b, messages: m } = await mockBlock(BlockNumber(i + 2), 2, fork); + await ws.handleL2BlockAndMessages(b, m); await fork.close(); }); @@ -296,15 +296,23 @@ describe('NativeWorldState', () => { publicDataTreeMapSizeKb: 1024, }; const ws = await NativeWorldStateService.new(rollupAddress, dataDir, wsTreeMapSizes); - const initialFork = await ws.fork(); - const { block: block1, messages: messages1 } = await mockBlock(BlockNumber(1), 8, initialFork); - const { block: block2, messages: messages2 } = await mockBlock(BlockNumber(2), 8, initialFork); - const { block: block3, messages: messages3 } = await mockBlock(BlockNumber(3), 8, initialFork); + const fork1 = await ws.fork(); + const { block: block1, messages: messages1 } = await mockBlock(BlockNumber(1), 8, fork1); + await fork1.close(); // The first block should succeed await expect(ws.handleL2BlockAndMessages(block1, messages1)).resolves.toBeDefined(); + // Build blocks 2 and 3 on separate forks at the advanced tip + const fork2 = await ws.fork(); + const { block: block2, messages: messages2 } = await mockBlock(BlockNumber(2), 16, fork2); + await fork2.close(); + + const fork3 = await ws.fork(); + const { block: block3, messages: messages3 } = await mockBlock(BlockNumber(3), 16, fork3); + await fork3.close(); + // The trees should be synched at block 1 const goodSummary = await ws.getStatusSummary(); expect(goodSummary).toEqual({ @@ -1087,7 +1095,7 @@ describe('NativeWorldState', () => { const publicWrites: Buffer[] = []; for (let i = 0; i < numBlocks; i++) { const fork = await ws.fork(); - ({ block, messages } = await mockBlock(BlockNumber(1), txsPerBlock, fork)); + ({ block, messages } = await mockBlock(BlockNumber(i + 1), txsPerBlock, fork)); noteHashes.push(...block.body.txEffects.flatMap(x => x.noteHashes.flatMap(x => x))); nullifiers.push(...block.body.txEffects.flatMap(x => x.nullifiers.flatMap(x => x.toBuffer()))); publicWrites.push(...block.body.txEffects.flatMap(x => x.publicDataWrites.flatMap(x => x.toBuffer()))); @@ -1150,7 +1158,7 @@ describe('NativeWorldState', () => { const txsPerBlock = 2; for (let i = 0; i < numBlocks; i++) { const fork = await ws.fork(); - ({ block, messages } = await mockBlock(BlockNumber(1), txsPerBlock, fork)); + ({ block, messages } = await mockBlock(BlockNumber(i + 1), txsPerBlock, fork)); noteHashes = block.body.txEffects[0].noteHashes.length; nullifiers = block.body.txEffects[0].nullifiers.length; publicTree = block.body.txEffects[0].publicDataWrites.length; @@ -1206,7 +1214,7 @@ describe('NativeWorldState', () => { const statuses = []; for (let i = 0; i < 2; i++) { const fork = await ws.fork(); - ({ block, messages } = await mockBlock(BlockNumber(1), 2, fork)); + ({ block, messages } = await mockBlock(BlockNumber(i + 1), 2, fork)); await fork.close(); const status = await ws.handleL2BlockAndMessages(block, messages); statuses.push(status); diff --git a/yarn-project/world-state/src/test/utils.ts b/yarn-project/world-state/src/test/utils.ts index 7a9a2fd3a9fd..1ae038308efd 100644 --- a/yarn-project/world-state/src/test/utils.ts +++ b/yarn-project/world-state/src/test/utils.ts @@ -111,18 +111,16 @@ export async function mockBlocks( numTxs: number, worldState: NativeWorldStateService, ) { - const tempFork = await worldState.fork(BlockNumber(from - 1)); - const blocks = []; const messagesArray = []; for (let blockNumber = from; blockNumber < from + count; blockNumber++) { + const tempFork = await worldState.fork(); const { block, messages } = await mockBlock(BlockNumber(blockNumber), numTxs, tempFork); blocks.push(block); messagesArray.push(messages); + await worldState.commitFork(tempFork); } - await tempFork.close(); - return { blocks, messages: messagesArray }; } From 7c5a9c4e78edd1551ca45ce0ec076ac8cb545e6a Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 24 Mar 2026 17:18:04 -0300 Subject: [PATCH 15/29] make commit atomic --- .../src/barretenberg/world_state/world_state.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 893f090bc36f..2b8f38ab39ba 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -717,7 +717,18 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re throw std::runtime_error(result.second); } } catch (const std::exception& e) { - // We failed, rollback any uncommitted state before leaving + // Clear uncommitted state first (required by unwind_block) + rollback(); + // If commit partially succeeded, some trees may have advanced their block height. + // Unwind to restore consistency. unwind_block is a no-op on trees that didn't advance. + try { + auto archiveMeta = get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + if (archiveMeta.meta.unfinalizedBlockHeight > 0) { + WorldStateStatusFull unwindStatus; + unwind_block(archiveMeta.meta.unfinalizedBlockHeight, unwindStatus); + } + } catch (...) { + } rollback(); throw; } From 87dca1383ab548394717386d61a1d93ee1affe69 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 24 Mar 2026 19:09:59 -0300 Subject: [PATCH 16/29] rollback healling --- .../barretenberg/world_state/world_state.cpp | 13 +---------- .../src/native/native_world_state.test.ts | 22 ++++++++++--------- .../src/native/native_world_state.ts | 5 +++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 2b8f38ab39ba..893f090bc36f 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -717,18 +717,7 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re throw std::runtime_error(result.second); } } catch (const std::exception& e) { - // Clear uncommitted state first (required by unwind_block) - rollback(); - // If commit partially succeeded, some trees may have advanced their block height. - // Unwind to restore consistency. unwind_block is a no-op on trees that didn't advance. - try { - auto archiveMeta = get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); - if (archiveMeta.meta.unfinalizedBlockHeight > 0) { - WorldStateStatusFull unwindStatus; - unwind_block(archiveMeta.meta.unfinalizedBlockHeight, unwindStatus); - } - } catch (...) { - } + // We failed, rollback any uncommitted state before leaving rollback(); throw; } diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 9930d579d370..a8d82d528885 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -322,11 +322,10 @@ describe('NativeWorldState', () => { treesAreSynched: true, } as WorldStateStatusSummary); - // The second block should fail + // The second block should fail (DB too small) await expect(ws.handleL2BlockAndMessages(block2, messages2)).rejects.toThrow(); - // The summary should indicate that the unfinalized block number (that of the archive tree) is 2 - // But it should also tell us that the trees are not synched + // The archive tree committed (small) but other trees failed → permanently out of sync const badSummary = await ws.getStatusSummary(); expect(badSummary).toEqual({ unfinalizedBlockNumber: BlockNumber(2), @@ -335,11 +334,10 @@ describe('NativeWorldState', () => { treesAreSynched: false, } as WorldStateStatusSummary); - // Commits should always fail now, the trees are in an inconsistent state + // Further syncs fail because trees are out of sync await expect(ws.handleL2BlockAndMessages(block2, messages2)).rejects.toThrow('World state trees are out of sync'); await expect(ws.handleL2BlockAndMessages(block3, messages3)).rejects.toThrow('World state trees are out of sync'); - // Creating another world state instance should fail await ws.close(); }); @@ -960,13 +958,13 @@ describe('NativeWorldState', () => { }); it('handles invalid blocks', async () => { - const fork = await ws.fork(); - - // Insert a few blocks + // Insert a few blocks, each on its own fork for (let i = 0; i < 4; i++) { const blockNumber = i + 1; const provenBlock = blockNumber - 2; + const fork = await ws.fork(); const { block, messages } = await mockBlock(BlockNumber(blockNumber), 1, fork); + await fork.close(); const status = await ws.handleL2BlockAndMessages(block, messages); expect(status.summary.unfinalizedBlockNumber).toBe(blockNumber); @@ -984,7 +982,9 @@ describe('NativeWorldState', () => { // Now build an invalid block, see that it is rejected and that we can then insert the correct block { - const { block: block, messages } = await mockBlock(BlockNumber(5), 1, fork); + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(5), 1, fork); + await fork.close(); const invalidBlock = L2Block.fromBuffer(block.toBuffer()); invalidBlock.header.state.partial.nullifierTree.root = Fr.random(); @@ -1003,7 +1003,9 @@ describe('NativeWorldState', () => { // Now we push another invalid block, see that it is rejected and check we can unwind to the last proven block { - const { block: block, messages } = await mockBlock(BlockNumber(6), 1, fork); + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(6), 1, fork); + await fork.close(); const invalidBlock = L2Block.fromBuffer(block.toBuffer()); invalidBlock.header.state.partial.nullifierTree.root = Fr.random(); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index d5a6cb73222c..d36f4dda1503 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -209,9 +209,10 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { - // Skip if this block is already persisted (e.g. via COMMIT_FORK) + // Skip if this block is already persisted (e.g. via COMMIT_FORK). + // Don't skip if trees are out of sync — partial commits need to be detected. const currentStatus = await this.getStatusSummary(); - if (l2Block.number <= currentStatus.unfinalizedBlockNumber) { + if (l2Block.number <= currentStatus.unfinalizedBlockNumber && currentStatus.treesAreSynched) { this.log.debug( `Skipping SYNC_BLOCK for block ${l2Block.number} — already at tip ${currentStatus.unfinalizedBlockNumber}`, ); From a4140ead9005f39ad32c58c495e522b183a23429 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 25 Mar 2026 10:49:59 -0300 Subject: [PATCH 17/29] fix flake --- yarn-project/world-state/src/native/native_world_state.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index a8d82d528885..469e04d3b681 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -1089,7 +1089,7 @@ describe('NativeWorldState', () => { let messages: Fr[]; it('retrieves leaf sibling paths', async () => { - const ws = await NativeWorldStateService.new(rollupAddress, dataDir, wsTreeMapSizes); + const ws = await NativeWorldStateService.tmp(); const numBlocks = 2; const txsPerBlock = 2; const noteHashes: Fr[] = []; From 4363bf07bfe5e41dd5016afd294f2868b258611f Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Fri, 27 Mar 2026 11:15:00 -0300 Subject: [PATCH 18/29] fix reorgs --- .../block/l2_block_stream/l2_block_stream.ts | 6 ++++++ .../src/native/native_world_state.test.ts | 18 ++++++++++++++++++ .../src/native/world_state_ops_queue.ts | 1 + 3 files changed, 25 insertions(+) diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts index 9aac6c45b143..c669fdd067ae 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts @@ -285,6 +285,12 @@ export class L2BlockStream { args.sourceCache.add({ number: blockNumber, hash: sourceBlockHash }); } + // If local has a block the source doesn't know about yet, local is ahead (e.g. via commitFork). + // This is not a reorg — the source will catch up. + if (localBlockHash && !sourceBlockHash) { + return true; + } + this.log.trace(`Comparing block hashes for block ${blockNumber}`, { localBlockHash, sourceBlockHash }); return localBlockHash === sourceBlockHash; } diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 469e04d3b681..2880c091831c 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2030,6 +2030,24 @@ describe('NativeWorldState', () => { expect(committedStateRef).toEqual(forkStateRef); }); + it('produces same state as sync_block', async () => { + // Build block on a fork + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 2, fork); + await ws.commitFork(fork); + + // Get state after commitFork + const commitForkState = await ws.getCommitted().getStateReference(); + + // Create a fresh world state and sync the same block via handleL2BlockAndMessages + const ws2 = await NativeWorldStateService.tmp(); + await ws2.handleL2BlockAndMessages(block, messages); + const syncBlockState = await ws2.getCommitted().getStateReference(); + await ws2.close(); + + expect(commitForkState).toEqual(syncBlockState); + }); + it('fails if tip has moved', async () => { // Build and sync block 1 via handleL2BlockAndMessages const setupFork = await ws.fork(); diff --git a/yarn-project/world-state/src/native/world_state_ops_queue.ts b/yarn-project/world-state/src/native/world_state_ops_queue.ts index ac47e9f40c87..26ab74eeff6a 100644 --- a/yarn-project/world-state/src/native/world_state_ops_queue.ts +++ b/yarn-project/world-state/src/native/world_state_ops_queue.ts @@ -35,6 +35,7 @@ export const MUTATING_MSG_TYPES = new Set([ WorldStateMessageType.SYNC_BLOCK, WorldStateMessageType.CREATE_FORK, WorldStateMessageType.DELETE_FORK, + WorldStateMessageType.COMMIT_FORK, WorldStateMessageType.FINALIZE_BLOCKS, WorldStateMessageType.UNWIND_BLOCKS, WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, From d72c1c32f65c9389951b536beaaf0e3e0eb5a8fc Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 30 Mar 2026 11:55:09 -0300 Subject: [PATCH 19/29] revert ForkOperations --- .../src/sequencer/checkpoint_proposal_job.ts | 28 ++-- .../src/test/mock_checkpoint_builder.ts | 8 + .../src/public/hinting_db_sources.ts | 4 + .../public_processor/guarded_merkle_tree.ts | 4 + .../block/l2_block_stream/l2_block_stream.ts | 6 - .../stdlib/src/interfaces/block-builder.ts | 6 + .../src/interfaces/merkle_tree_operations.ts | 3 + .../stdlib/src/interfaces/world_state.ts | 10 +- .../txe/src/state_machine/synchronizer.ts | 6 +- .../src/checkpoint_builder.ts | 10 ++ .../src/native/native_world_state.test.ts | 140 ++++-------------- .../src/native/native_world_state.ts | 35 ++--- .../server_world_state_synchronizer.ts | 4 +- yarn-project/world-state/src/test/utils.ts | 3 +- 14 files changed, 99 insertions(+), 168 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index dbfe2a035437..14cfe0e1eb27 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -37,7 +37,6 @@ import { Gas } from '@aztec/stdlib/gas'; import { type BlockBuilderOptions, InsufficientValidTxsError, - type MerkleTreeWriteOperations, type ResolvedSequencerConfig, type WorldStateSynchronizer, } from '@aztec/stdlib/interfaces/server'; @@ -320,7 +319,8 @@ export class CheckpointProposalJob implements Traceable { const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(); // Create a forked world state for the checkpoint builder. - // Fork lifecycle is managed manually: each block commits and destroys the fork, then creates a new one. + // The fork is registered for each built block so SYNC_BLOCK can commit it. + // After each block, a new fork is created at the advanced tip. const fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); // Create checkpoint builder for the entire slot @@ -356,7 +356,6 @@ export class CheckpointProposalJob implements Traceable { checkpointGlobalVariables.timestamp, inHash, blockProposalOptions, - fork, ); blocksInCheckpoint = result.blocksInCheckpoint; blockPendingBroadcast = result.blockPendingBroadcast; @@ -501,7 +500,6 @@ export class CheckpointProposalJob implements Traceable { timestamp: bigint, inHash: Fr, blockProposalOptions: BlockProposalOptions, - fork: MerkleTreeWriteOperations, ): Promise<{ blocksInCheckpoint: L2Block[]; blockPendingBroadcast: { block: L2Block; txs: Tx[] } | undefined; @@ -571,9 +569,11 @@ export class CheckpointProposalJob implements Traceable { blocksInCheckpoint.push(block); usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString())); - // Commit the fork to persist this block to LMDB. The fork is destroyed after this call. - await this.tryCommitFork(fork); + // Register the fork so SYNC_BLOCK can commit it instead of recalculating. + this.worldState.registerForkForBlock(block.archive.root, checkpointBuilder.getForkId()); await this.syncProposedBlockToArchiver(block); + // Force the block stream to pick up the block and commit the fork via SYNC_BLOCK. + await this.worldState.syncImmediate(); // If this is the last block, exit the loop so we can build the checkpoint and start collecting attestations. if (timingInfo.isLastBlock) { @@ -595,9 +595,10 @@ export class CheckpointProposalJob implements Traceable { // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it. proposal && (await this.p2pClient.broadcastProposal(proposal)); - // Create a new fork at the advanced tip for the next block - fork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); - checkpointBuilder.setFork(fork); + // Create a new fork at the advanced tip for the next block. + // syncImmediate committed the previous fork via SYNC_BLOCK, so LMDB is up to date. + const newFork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); + checkpointBuilder.setFork(newFork); // Wait until the next block's start time await this.waitUntilNextSubslot(timingInfo.deadline); @@ -1013,15 +1014,6 @@ export class CheckpointProposalJob implements Traceable { await this.p2pClient.handleFailedExecution(failedTxHashes); } - /** Commits the fork so getCommitted() immediately reflects the built blocks. The fork is destroyed after this call. */ - private async tryCommitFork(fork: MerkleTreeWriteOperations): Promise { - try { - await this.worldState.commitFork(fork); - } catch (err) { - this.log.debug(`Could not commit fork (block stream may have synced first)`, { err }); - } - } - /** * Adds the proposed block to the archiver so it's available via P2P. * Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state diff --git a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts index 5cbde03178fb..bf635de4d87c 100644 --- a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts +++ b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts @@ -66,6 +66,14 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder { return this; } + getFork(): MerkleTreeWriteOperations { + return {} as MerkleTreeWriteOperations; + } + + getForkId(): number { + return 0; + } + setFork(_fork: MerkleTreeWriteOperations): void { // No-op in mock — the mock doesn't use the fork directly } diff --git a/yarn-project/simulator/src/public/hinting_db_sources.ts b/yarn-project/simulator/src/public/hinting_db_sources.ts index 85f8ab422ccf..1cb2b75a0fb7 100644 --- a/yarn-project/simulator/src/public/hinting_db_sources.ts +++ b/yarn-project/simulator/src/public/hinting_db_sources.ts @@ -237,6 +237,10 @@ export class HintingMerkleWriteOperations implements MerkleTreeWriteOperations { } // Use create() to instantiate. + get forkId(): number { + return this.db.forkId; + } + private constructor( private db: MerkleTreeWriteOperations, private hints: AvmExecutionHints, diff --git a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts index 71133c4a2ebf..b28c852b5926 100644 --- a/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts +++ b/yarn-project/simulator/src/public/public_processor/guarded_merkle_tree.ts @@ -25,6 +25,10 @@ export class GuardedMerkleTreeOperations implements MerkleTreeWriteOperations { private isStopped = false; private serialQueue = new SerialQueue(); + get forkId(): number { + return this.target.forkId; + } + constructor(private target: MerkleTreeWriteOperations) { this.serialQueue.start(); } diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts index c669fdd067ae..9aac6c45b143 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts @@ -285,12 +285,6 @@ export class L2BlockStream { args.sourceCache.add({ number: blockNumber, hash: sourceBlockHash }); } - // If local has a block the source doesn't know about yet, local is ahead (e.g. via commitFork). - // This is not a reorg — the source will catch up. - if (localBlockHash && !sourceBlockHash) { - return true; - } - this.log.trace(`Comparing block hashes for block ${blockNumber}`, { localBlockHash, sourceBlockHash }); return localBlockHash === sourceBlockHash; } diff --git a/yarn-project/stdlib/src/interfaces/block-builder.ts b/yarn-project/stdlib/src/interfaces/block-builder.ts index b038e7c47f10..108581d76cea 100644 --- a/yarn-project/stdlib/src/interfaces/block-builder.ts +++ b/yarn-project/stdlib/src/interfaces/block-builder.ts @@ -136,6 +136,12 @@ export interface ICheckpointBlockBuilder { opts: BlockBuilderOptions, ): Promise; + /** Returns the current fork. */ + getFork(): MerkleTreeWriteOperations; + + /** Returns the native fork ID of the current fork. */ + getForkId(): number; + /** Replaces the fork used for subsequent block builds. */ setFork(fork: MerkleTreeWriteOperations): void; } diff --git a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts index 29625e9d4c43..7f1edc398f7c 100644 --- a/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/stdlib/src/interfaces/merkle_tree_operations.ts @@ -245,6 +245,9 @@ export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations, MerkleTreeCheckpointOperations, AsyncDisposable { + /** The native fork ID assigned by the world state. */ + readonly forkId: number; + /** * Appends leaves to a given tree. * @param treeId - The tree to be updated. diff --git a/yarn-project/stdlib/src/interfaces/world_state.ts b/yarn-project/stdlib/src/interfaces/world_state.ts index 7f32a32e0876..564be5ad7c13 100644 --- a/yarn-project/stdlib/src/interfaces/world_state.ts +++ b/yarn-project/stdlib/src/interfaces/world_state.ts @@ -1,4 +1,5 @@ import { BlockNumber, BlockNumberSchema } from '@aztec/foundation/branded-types'; +import type { Fr } from '@aztec/foundation/curves/bn254'; import type { PromiseWithResolvers } from '@aztec/foundation/promise'; import { z } from 'zod'; @@ -52,11 +53,12 @@ export interface ForkMerkleTreeOperations { fork(block?: BlockNumber, opts?: { closeDelayMs?: number }): Promise; /** - * Commits a fork's state to canonical LMDB and destroys the fork. - * Only succeeds if the canonical tip hasn't moved since the fork was created. - * The fork is invalid after this call — caller must not use it. + * Registers a fork that has built a block. When SYNC_BLOCK is later called for a block + * with the same archive root, the fork will be committed instead of recalculating from scratch. + * @param archiveRoot - The archive root of the block built on the fork. + * @param forkId - The native fork ID. */ - commitFork(fork: MerkleTreeWriteOperations): Promise; + registerForkForBlock(archiveRoot: Fr, forkId: number): void; /** Backups the db to the target path. */ backupTo(dstPath: string, compact?: boolean): Promise, string>>; diff --git a/yarn-project/txe/src/state_machine/synchronizer.ts b/yarn-project/txe/src/state_machine/synchronizer.ts index 13d2129ecfd7..6298ca9cd8ca 100644 --- a/yarn-project/txe/src/state_machine/synchronizer.ts +++ b/yarn-project/txe/src/state_machine/synchronizer.ts @@ -47,9 +47,9 @@ export class TXESynchronizer implements WorldStateSynchronizer { return this.nativeWorldStateService.getCommitted(); } - /** Commits a fork as the current "committed" view of the world state. */ - public commitFork(fork: MerkleTreeWriteOperations): Promise { - return this.nativeWorldStateService.commitFork(fork); + /** Registers a fork for a block (no-op in TXE). */ + public registerForkForBlock(_archiveRoot: Fr, _forkId: number): void { + // No-op — TXE doesn't use the block stream pipeline } /** Forks the world state at the given block number, defaulting to the latest one. */ diff --git a/yarn-project/validator-client/src/checkpoint_builder.ts b/yarn-project/validator-client/src/checkpoint_builder.ts index 3ecb3d5365fa..b93b88d605e2 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.ts @@ -50,6 +50,16 @@ export class CheckpointBuilder implements ICheckpointBlockBuilder { /** Persistent contracts DB shared across all blocks in this checkpoint. */ protected contractsDB: PublicContractsDB; + /** Returns the current fork. */ + public getFork(): MerkleTreeWriteOperations { + return this.fork; + } + + /** Returns the native fork ID. */ + public getForkId(): number { + return this.fork.forkId; + } + /** Replaces the fork used for subsequent block builds. */ public setFork(fork: MerkleTreeWriteOperations): void { this.fork = fork; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 2880c091831c..0ad8995087e7 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2006,7 +2006,7 @@ describe('NativeWorldState', () => { }); }); - describe('commitFork', () => { + describe('registerForkForBlock', () => { let ws: NativeWorldStateService; beforeEach(async () => { @@ -2017,147 +2017,59 @@ describe('NativeWorldState', () => { await ws.close(); }); - it('makes getCommitted() return the fork state', async () => { - const fork = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork); - - // Snapshot the fork's state before committing (fork is destroyed after commit) - const forkStateRef = await fork.getStateReference(); - - await ws.commitFork(fork); - - const committedStateRef = await ws.getCommitted().getStateReference(); - expect(committedStateRef).toEqual(forkStateRef); - }); - - it('produces same state as sync_block', async () => { - // Build block on a fork - const fork = await ws.fork(); - const { block, messages } = await mockBlock(BlockNumber(1), 2, fork); - await ws.commitFork(fork); - - // Get state after commitFork - const commitForkState = await ws.getCommitted().getStateReference(); - - // Create a fresh world state and sync the same block via handleL2BlockAndMessages - const ws2 = await NativeWorldStateService.tmp(); - await ws2.handleL2BlockAndMessages(block, messages); - const syncBlockState = await ws2.getCommitted().getStateReference(); - await ws2.close(); - - expect(commitForkState).toEqual(syncBlockState); - }); - - it('fails if tip has moved', async () => { - // Build and sync block 1 via handleL2BlockAndMessages - const setupFork = await ws.fork(); - const { block, messages } = await mockBlock(BlockNumber(1), 1, setupFork); - await ws.handleL2BlockAndMessages(block, messages); - await setupFork.close(); - - // Create a fork at block 0 (now stale) - const staleFork = await ws.fork(BlockNumber(0)); - await mockBlock(BlockNumber(1), 1, staleFork); - - // commitFork should fail because tip moved from 0 to 1 - await expect(ws.commitFork(staleFork)).rejects.toThrow(); - }); - - it('handleL2BlockAndMessages skips already committed block', async () => { + it('handleL2BlockAndMessages commits a registered fork', async () => { const fork = await ws.fork(); const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); - // Snapshot the fork's state before committing const forkStateRef = await fork.getStateReference(); - await ws.commitFork(fork); - - // getCommitted() should return the committed state - const committedStateRef = await ws.getCommitted().getStateReference(); - expect(committedStateRef).toEqual(forkStateRef); - - // Sync the same block via handleL2BlockAndMessages (should be skipped since already committed) + // Register the fork, then sync — should commit the fork instead of recalculating + ws.registerForkForBlock(block.archive.root, fork.forkId); await ws.handleL2BlockAndMessages(block, messages); - // State should still match - const lmdbStateRef = await ws.getCommitted().getStateReference(); - expect(lmdbStateRef).toEqual(forkStateRef); - }); - - it('unwindBlocks after commitFork', async () => { - // Sync blocks 1..3 - const setupFork = await ws.fork(); - for (let i = 1; i <= 3; i++) { - const { block, messages } = await mockBlock(BlockNumber(i), 1, setupFork); - await ws.handleL2BlockAndMessages(block, messages); - } - await setupFork.close(); - - // Snapshot state at block 2 before the reorg - const snapshot2StateRef = await ws.getSnapshot(BlockNumber(2)).getStateReference(); - - // Create fork at block 3, build block 4, commitFork - const fork = await ws.fork(); - await mockBlock(BlockNumber(4), 1, fork); - await ws.commitFork(fork); - - // Reorg back to block 2 - await ws.unwindBlocks(BlockNumber(2)); - - // getCommitted() should return LMDB state at block 2 const committedStateRef = await ws.getCommitted().getStateReference(); - expect(committedStateRef).toEqual(snapshot2StateRef); + expect(committedStateRef).toEqual(forkStateRef); }); it('commit then create new fork at advanced tip', async () => { - // Build and commit block 1 + // Build and commit block 1 via registerFork + handleL2BlockAndMessages const fork1 = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork1); - await ws.commitFork(fork1); + const { block: block1, messages: messages1 } = await mockBlock(BlockNumber(1), 1, fork1); + ws.registerForkForBlock(block1.archive.root, (fork1 as any).forkId); + await ws.handleL2BlockAndMessages(block1, messages1); // Create new fork at latest (should be at block 1) const fork2 = await ws.fork(); - await mockBlock(BlockNumber(2), 1, fork2); - await ws.commitFork(fork2); + const { block: block2, messages: messages2 } = await mockBlock(BlockNumber(2), 1, fork2); + ws.registerForkForBlock(block2.archive.root, (fork2 as any).forkId); + await ws.handleL2BlockAndMessages(block2, messages2); // Verify canonical is at block 2 const status = await ws.getStatusSummary(); expect(status.unfinalizedBlockNumber).toEqual(2); }); - it('fork is destroyed after commitFork', async () => { + it('falls back to SYNC_BLOCK when no fork is registered', async () => { const fork = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + await fork.close(); - // Fork should be destroyed — operations on it should fail - await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); + // No registerForkForBlock call — handleL2BlockAndMessages should use SYNC_BLOCK + await ws.handleL2BlockAndMessages(block, messages); + + const status = await ws.getStatusSummary(); + expect(status.unfinalizedBlockNumber).toEqual(1); }); - it('close() after commitFork', async () => { + it('fork is destroyed after commit via handleL2BlockAndMessages', async () => { const fork = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork); - - await ws.close(); - - await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow(); - // Recreate ws so afterEach doesn't double-close - ws = await NativeWorldStateService.tmp(); - }); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); - it('committed state survives await using dispose', async () => { - // Simulate: fork created with await using, then committed. - // asyncDispose calls close() which tolerates "Fork not found" since C++ already destroyed the fork. - { - await using fork = await ws.fork(); - await mockBlock(BlockNumber(1), 1, fork); - await ws.commitFork(fork); - } + ws.registerForkForBlock(block.archive.root, fork.forkId); + await ws.handleL2BlockAndMessages(block, messages); - // The committed state should be accessible via getCommitted() - await expect(ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).resolves.toBeDefined(); - await expect(ws.getCommitted().getStateReference()).resolves.toBeDefined(); + // Fork should be destroyed — operations on it should fail + await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); }); }); }); diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index d36f4dda1503..dc0392553fb4 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -32,7 +32,6 @@ import { type WorldStateStatusFull, type WorldStateStatusSummary, blockStateReference, - buildEmptyWorldStateStatusFull, sanitizeFullStatus, sanitizeSummary, treeStateReferenceToSnapshot, @@ -49,7 +48,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: BlockHeader | undefined; // This is read heavily and only changes when data is persisted, so we cache it private cachedStatusSummary: WorldStateStatusSummary | undefined; - private committedForkStatus: WorldStateStatusFull | undefined; + /** Maps archive root (hex) → fork ID for forks that have built blocks and are awaiting SYNC_BLOCK. */ + private registeredForks = new Map(); protected constructor( protected instance: NativeWorldState, @@ -194,14 +194,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { ); } - public async commitFork(fork: MerkleTreeWriteOperations): Promise { - const forkFacade = fork as MerkleTreesForkFacade; - this.committedForkStatus = await this.instance.call( - WorldStateMessageType.COMMIT_FORK, - { forkId: forkFacade.forkId, canonical: true as const }, - this.sanitizeAndCacheSummaryFromFull.bind(this), - this.deleteCachedSummary.bind(this), - ); + public registerForkForBlock(archiveRoot: Fr, forkId: number): void { + this.registeredForks.set(archiveRoot.toString(), forkId); } public getInitialHeader(): BlockHeader { @@ -209,17 +203,18 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { - // Skip if this block is already persisted (e.g. via COMMIT_FORK). - // Don't skip if trees are out of sync — partial commits need to be detected. - const currentStatus = await this.getStatusSummary(); - if (l2Block.number <= currentStatus.unfinalizedBlockNumber && currentStatus.treesAreSynched) { - this.log.debug( - `Skipping SYNC_BLOCK for block ${l2Block.number} — already at tip ${currentStatus.unfinalizedBlockNumber}`, + // Check if a fork already built this block (registered via registerForkForBlock). + // If so, commit the fork directly instead of recalculating via SYNC_BLOCK. + const registeredForkId = this.registeredForks.get(l2Block.archive.root.toString()); + if (registeredForkId !== undefined) { + this.registeredForks.delete(l2Block.archive.root.toString()); + this.log.debug(`Committing registered fork ${registeredForkId} for block ${l2Block.number}`); + return await this.instance.call( + WorldStateMessageType.COMMIT_FORK, + { forkId: registeredForkId, canonical: true as const }, + this.sanitizeAndCacheSummaryFromFull.bind(this), + this.deleteCachedSummary.bind(this), ); - if (this.committedForkStatus) { - return this.committedForkStatus; - } - return buildEmptyWorldStateStatusFull(); } const isFirstBlock = l2Block.indexWithinCheckpoint === 0; diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 390f5f4ffbb6..2efea17c6200 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -85,8 +85,8 @@ export class ServerWorldStateSynchronizer return this.merkleTreeDb.fork(blockNumber, opts); } - public commitFork(fork: MerkleTreeWriteOperations): Promise { - return this.merkleTreeDb.commitFork(fork); + public registerForkForBlock(archiveRoot: Fr, forkId: number): void { + this.merkleTreeDb.registerForkForBlock(archiveRoot, forkId); } public backupTo(dstPath: string, compact?: boolean): Promise, string>> { diff --git a/yarn-project/world-state/src/test/utils.ts b/yarn-project/world-state/src/test/utils.ts index 1ae038308efd..adb31ea62f97 100644 --- a/yarn-project/world-state/src/test/utils.ts +++ b/yarn-project/world-state/src/test/utils.ts @@ -118,7 +118,8 @@ export async function mockBlocks( const { block, messages } = await mockBlock(BlockNumber(blockNumber), numTxs, tempFork); blocks.push(block); messagesArray.push(messages); - await worldState.commitFork(tempFork); + worldState.registerForkForBlock(block.archive.root, tempFork.forkId); + await worldState.handleL2BlockAndMessages(block, messages); } return { blocks, messages: messagesArray }; From 82553131336d0b449b3bdbd1a92d140b9b7a45cc Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 30 Mar 2026 12:09:28 -0300 Subject: [PATCH 20/29] cleanup --- .../src/native/native_world_state.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index dc0392553fb4..96366e1ce40e 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -209,12 +209,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase { if (registeredForkId !== undefined) { this.registeredForks.delete(l2Block.archive.root.toString()); this.log.debug(`Committing registered fork ${registeredForkId} for block ${l2Block.number}`); - return await this.instance.call( - WorldStateMessageType.COMMIT_FORK, - { forkId: registeredForkId, canonical: true as const }, - this.sanitizeAndCacheSummaryFromFull.bind(this), - this.deleteCachedSummary.bind(this), - ); + try { + return await this.instance.call( + WorldStateMessageType.COMMIT_FORK, + { forkId: registeredForkId, canonical: true as const }, + this.sanitizeAndCacheSummaryFromFull.bind(this), + this.deleteCachedSummary.bind(this), + ); + } catch (err) { + this.log.warn( + `Failed to commit registered fork ${registeredForkId} for block ${l2Block.number}, falling back to SYNC_BLOCK`, + { err }, + ); + } } const isFirstBlock = l2Block.indexWithinCheckpoint === 0; @@ -347,6 +354,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { * @returns The new WorldStateStatus */ public async unwindBlocks(toBlockNumber: BlockNumber) { + // Clear any registered forks — they're invalid after a reorg. + this.registeredForks.clear(); try { const result = await this.instance.call( WorldStateMessageType.UNWIND_BLOCKS, From 0c7c298c8546e3429103b3437f58df846f295b45 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 30 Mar 2026 13:18:48 -0300 Subject: [PATCH 21/29] fix ha --- .../src/sequencer/checkpoint_proposal_job.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 14cfe0e1eb27..c2c71099832b 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -571,12 +571,12 @@ export class CheckpointProposalJob implements Traceable { // Register the fork so SYNC_BLOCK can commit it instead of recalculating. this.worldState.registerForkForBlock(block.archive.root, checkpointBuilder.getForkId()); - await this.syncProposedBlockToArchiver(block); - // Force the block stream to pick up the block and commit the fork via SYNC_BLOCK. - await this.worldState.syncImmediate(); - // If this is the last block, exit the loop so we can build the checkpoint and start collecting attestations. + // If this is the last block, sync to archiver and exit the loop + // so we can build the checkpoint and start collecting attestations. if (timingInfo.isLastBlock) { + await this.syncProposedBlockToArchiver(block); + await this.worldState.syncImmediate(blockNumber); this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, { slot: this.targetSlot, blockNumber, @@ -592,11 +592,17 @@ export class CheckpointProposalJob implements Traceable { // a HA error we don't pollute our archiver with a block that won't make it to the chain. const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions); + // Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal. + await this.syncProposedBlockToArchiver(block); + // Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it. proposal && (await this.p2pClient.broadcastProposal(proposal)); + // Wait for LMDB to reach this block before creating the next fork. + // In HA mode, another peer's proposal may arrive via gossip and be committed instead of ours. + await this.worldState.syncImmediate(blockNumber); + // Create a new fork at the advanced tip for the next block. - // syncImmediate committed the previous fork via SYNC_BLOCK, so LMDB is up to date. const newFork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); checkpointBuilder.setFork(newFork); From 6cbfdad46894c1ca9778e52d044dfe9bab8d5234 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Mon, 30 Mar 2026 15:21:02 -0300 Subject: [PATCH 22/29] do not wait if skipPushProposedBlocksToArchiver --- .../src/sequencer/checkpoint_proposal_job.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index c2c71099832b..681b57786a35 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -572,11 +572,10 @@ export class CheckpointProposalJob implements Traceable { // Register the fork so SYNC_BLOCK can commit it instead of recalculating. this.worldState.registerForkForBlock(block.archive.root, checkpointBuilder.getForkId()); - // If this is the last block, sync to archiver and exit the loop - // so we can build the checkpoint and start collecting attestations. + // If this is the last block, exit the loop so we can build the checkpoint and start collecting attestations. + // The block will be synced to LMDB when the block stream picks it up from the archiver. if (timingInfo.isLastBlock) { await this.syncProposedBlockToArchiver(block); - await this.worldState.syncImmediate(blockNumber); this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, { slot: this.targetSlot, blockNumber, @@ -600,11 +599,13 @@ export class CheckpointProposalJob implements Traceable { // Wait for LMDB to reach this block before creating the next fork. // In HA mode, another peer's proposal may arrive via gossip and be committed instead of ours. - await this.worldState.syncImmediate(blockNumber); - - // Create a new fork at the advanced tip for the next block. - const newFork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); - checkpointBuilder.setFork(newFork); + // If the block was not pushed to the archiver (e.g. skipPushProposedBlocksToArchiver), skip the + // sync and reuse the current fork for the next block. + if (!this.config.skipPushProposedBlocksToArchiver) { + await this.worldState.syncImmediate(blockNumber); + const newFork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); + checkpointBuilder.setFork(newFork); + } // Wait until the next block's start time await this.waitUntilNextSubslot(timingInfo.deadline); From 8724c642073eac147f1eb0de565d537f3457afbf Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Tue, 31 Mar 2026 20:56:19 -0300 Subject: [PATCH 23/29] fix commit_fork --- .../content_addressed_append_only_tree.hpp | 9 +++ .../barretenberg/world_state/world_state.cpp | 12 ++++ .../world_state/world_state.test.cpp | 56 +++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 6c02efd2a940..465ae0828bbc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -77,6 +77,15 @@ template class ContentAddressedAppendOn void clear_initialized_from_block() { store_->clear_initialized_from_block(); } + void sync_pruning_meta(const TreeMeta& canonical) + { + TreeMeta meta; + store_->get_meta(meta); + meta.oldestHistoricBlock = canonical.oldestHistoricBlock; + meta.finalizedBlockHeight = canonical.finalizedBlockHeight; + store_->put_meta(meta); + } + /** * @brief Adds a single value to the end of the tree * @param value The value to be added diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 893f090bc36f..869b5cb4a9e0 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -263,11 +263,23 @@ WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) // Rollback canonical to clear any uncommitted state rollback(); + // Save pruning-related meta from canonical before the fork overwrites it. + // The fork's cached meta may have stale oldestHistoricBlock/finalizedBlockHeight + // from when it was created, so commit_block would overwrite LMDB with stale values. + std::array canonicalMeta; + get_all_tree_info(WorldStateRevision::committed(), canonicalMeta); + // Clear fork flags so commit_block() is allowed on fork stores for (auto& [id, tree] : fork->_trees) { std::visit([](auto&& wrapper) { wrapper.tree->clear_initialized_from_block(); }, tree); } + // Sync the fork's cached meta with canonical pruning state before committing. + // The fork's cached meta may have stale oldestHistoricBlock/finalizedBlockHeight. + for (auto& entry : fork->_trees) { + std::visit([&](auto&& wrapper) { wrapper.tree->sync_pruning_meta(canonicalMeta[entry.first]); }, entry.second); + } + // Commit fork trees to LMDB WorldStateStatusFull status; auto [success, message] = commit(fork, status); diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 8a6bc58f20f5..15bdd036fb17 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -1053,6 +1053,62 @@ TEST_F(WorldStateTest, CommitForkDestroysFork) EXPECT_THROW(ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 99 }, fork_id), std::runtime_error); } +TEST_F(WorldStateTest, CommitForkDoesNotRollBackOldestHistoricBlock) +{ + // Reproduces: fork is created, then blocks are pruned on canonical BEFORE commit_fork. + // Without the fix, commit_fork overwrites oldestHistoricBlock with the stale fork value, + // causing subsequent remove_historical_blocks to fail with "Failed to read block data". + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + + // Sync blocks 1..4 via sync_block + for (uint32_t i = 1; i <= 4; i++) { + auto tmp = ws.create_fork(i - 1); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(i * 10) }, tmp); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(i * 10 + 1) }, tmp); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 128 + i } }, 0, tmp); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 128 + i, i } }, 0, tmp); + auto sr = ws.get_state_reference(WorldStateRevision{ .forkId = tmp, .includeUncommitted = true }); + ws.delete_fork(tmp); + ws.sync_block(sr, + fr(i), + { fr(i * 10) }, + { fr(i * 10 + 1) }, + { NullifierLeafValue(128 + i) }, + { { PublicDataLeafValue(128 + i, i) } }); + } + + // Finalize block 2 so we can prune + ws.set_finalized_blocks(2); + + // Create fork at block 4 BEFORE pruning — fork captures oldestHistoricBlock = 1 + auto fork_id = ws.create_fork(4); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { fr(50) }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { fr(51) }, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 133 } }, 0, fork_id); + ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 133, 5 } }, 0, fork_id); + auto fork_sr = ws.get_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }); + ws.update_archive(fork_sr, { 5 }, fork_id); + + // Prune blocks 1..2 AFTER fork was created — canonical advances oldestHistoricBlock to 2 + ws.remove_historical_blocks(2); + auto info_after_prune = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + EXPECT_EQ(info_after_prune.meta.oldestHistoricBlock, 2); + + // commit_fork — must NOT roll back oldestHistoricBlock from 2 to 1 + ws.commit_fork(fork_id); + + auto info_after_commit = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + EXPECT_EQ(info_after_commit.meta.unfinalizedBlockHeight, 5); + EXPECT_EQ(info_after_commit.meta.oldestHistoricBlock, 2); + + // Finalize block 4 and prune — should NOT throw "Failed to read block data" + ws.set_finalized_blocks(4); + EXPECT_NO_THROW(ws.remove_historical_blocks(4)); + + auto info_final = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); + EXPECT_EQ(info_final.meta.oldestHistoricBlock, 4); +} + TEST_F(WorldStateTest, GetBlockForIndex) { WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); From dbf6279da12424b62fe9ada5c854233fd4a4e128 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 08:42:53 -0300 Subject: [PATCH 24/29] Fix close fork --- .../src/sequencer/checkpoint_proposal_job.ts | 15 ++++++++++++--- .../src/test/mock_checkpoint_builder.ts | 4 ++-- .../stdlib/src/interfaces/block-builder.ts | 4 ++-- .../validator-client/src/checkpoint_builder.ts | 5 +++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 681b57786a35..51792ae6b81f 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -264,6 +264,8 @@ export class CheckpointProposalJob implements Traceable { }; }) private async proposeCheckpoint(): Promise { + let fork: Awaited> | undefined; + let checkpointBuilder: CheckpointBuilder | undefined; try { // Get operator configured coinbase and fee recipient for this attestor const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress); @@ -321,10 +323,10 @@ export class CheckpointProposalJob implements Traceable { // Create a forked world state for the checkpoint builder. // The fork is registered for each built block so SYNC_BLOCK can commit it. // After each block, a new fork is created at the advanced tip. - const fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); + fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 }); // Create checkpoint builder for the entire slot - const checkpointBuilder = await this.checkpointsBuilder.startCheckpoint( + checkpointBuilder = await this.checkpointsBuilder.startCheckpoint( this.checkpointNumber, checkpointGlobalVariables, feeAssetPriceModifier, @@ -488,6 +490,13 @@ export class CheckpointProposalJob implements Traceable { this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err); return undefined; + } finally { + // Close forks to release native resources. Already-committed forks silently handle "Fork not found". + const currentFork = checkpointBuilder?.getFork(); + if (currentFork && currentFork !== fork) { + await currentFork.close(); + } + await fork?.close(); } } @@ -604,7 +613,7 @@ export class CheckpointProposalJob implements Traceable { if (!this.config.skipPushProposedBlocksToArchiver) { await this.worldState.syncImmediate(blockNumber); const newFork = await this.worldState.fork(undefined, { closeDelayMs: 12_000 }); - checkpointBuilder.setFork(newFork); + await checkpointBuilder.setFork(newFork); } // Wait until the next block's start time diff --git a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts index bf635de4d87c..84f54cbc204b 100644 --- a/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts +++ b/yarn-project/sequencer-client/src/test/mock_checkpoint_builder.ts @@ -67,14 +67,14 @@ export class MockCheckpointBuilder implements ICheckpointBlockBuilder { } getFork(): MerkleTreeWriteOperations { - return {} as MerkleTreeWriteOperations; + return { close: () => Promise.resolve() } as unknown as MerkleTreeWriteOperations; } getForkId(): number { return 0; } - setFork(_fork: MerkleTreeWriteOperations): void { + async setFork(_fork: MerkleTreeWriteOperations): Promise { // No-op in mock — the mock doesn't use the fork directly } diff --git a/yarn-project/stdlib/src/interfaces/block-builder.ts b/yarn-project/stdlib/src/interfaces/block-builder.ts index 108581d76cea..0bfd6aa66487 100644 --- a/yarn-project/stdlib/src/interfaces/block-builder.ts +++ b/yarn-project/stdlib/src/interfaces/block-builder.ts @@ -142,8 +142,8 @@ export interface ICheckpointBlockBuilder { /** Returns the native fork ID of the current fork. */ getForkId(): number; - /** Replaces the fork used for subsequent block builds. */ - setFork(fork: MerkleTreeWriteOperations): void; + /** Replaces the fork used for subsequent block builds, closing the previous one. */ + setFork(fork: MerkleTreeWriteOperations): Promise; } /** Interface for creating checkpoint builders. */ diff --git a/yarn-project/validator-client/src/checkpoint_builder.ts b/yarn-project/validator-client/src/checkpoint_builder.ts index b93b88d605e2..d46f4df13edd 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.ts @@ -60,8 +60,9 @@ export class CheckpointBuilder implements ICheckpointBlockBuilder { return this.fork.forkId; } - /** Replaces the fork used for subsequent block builds. */ - public setFork(fork: MerkleTreeWriteOperations): void { + /** Replaces the fork used for subsequent block builds, closing the previous one. */ + public async setFork(fork: MerkleTreeWriteOperations): Promise { + await this.fork.close(); this.fork = fork; this.checkpointBuilder.db = fork; } From 8a19f7f46a4a3fccec21936cd5f20b5d2d58d19e Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 08:53:38 -0300 Subject: [PATCH 25/29] Keep track of only one fork --- .../src/native/native_world_state.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/yarn-project/world-state/src/native/native_world_state.ts b/yarn-project/world-state/src/native/native_world_state.ts index 96366e1ce40e..654b2ca13ba2 100644 --- a/yarn-project/world-state/src/native/native_world_state.ts +++ b/yarn-project/world-state/src/native/native_world_state.ts @@ -48,8 +48,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase { protected initialHeader: BlockHeader | undefined; // This is read heavily and only changes when data is persisted, so we cache it private cachedStatusSummary: WorldStateStatusSummary | undefined; - /** Maps archive root (hex) → fork ID for forks that have built blocks and are awaiting SYNC_BLOCK. */ - private registeredForks = new Map(); + /** The single registered fork awaiting SYNC_BLOCK. Only one fork is active at a time. */ + private registeredFork: { archiveRoot: string; forkId: number } | undefined; protected constructor( protected instance: NativeWorldState, @@ -195,7 +195,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { } public registerForkForBlock(archiveRoot: Fr, forkId: number): void { - this.registeredForks.set(archiveRoot.toString(), forkId); + this.registeredFork = { archiveRoot: archiveRoot.toString(), forkId }; } public getInitialHeader(): BlockHeader { @@ -205,20 +205,20 @@ export class NativeWorldStateService implements MerkleTreeDatabase { public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise { // Check if a fork already built this block (registered via registerForkForBlock). // If so, commit the fork directly instead of recalculating via SYNC_BLOCK. - const registeredForkId = this.registeredForks.get(l2Block.archive.root.toString()); - if (registeredForkId !== undefined) { - this.registeredForks.delete(l2Block.archive.root.toString()); - this.log.debug(`Committing registered fork ${registeredForkId} for block ${l2Block.number}`); + const registered = this.registeredFork; + if (registered && registered.archiveRoot === l2Block.archive.root.toString()) { + this.registeredFork = undefined; + this.log.debug(`Committing registered fork ${registered.forkId} for block ${l2Block.number}`); try { return await this.instance.call( WorldStateMessageType.COMMIT_FORK, - { forkId: registeredForkId, canonical: true as const }, + { forkId: registered.forkId, canonical: true as const }, this.sanitizeAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this), ); } catch (err) { this.log.warn( - `Failed to commit registered fork ${registeredForkId} for block ${l2Block.number}, falling back to SYNC_BLOCK`, + `Failed to commit registered fork ${registered.forkId} for block ${l2Block.number}, falling back to SYNC_BLOCK`, { err }, ); } @@ -355,7 +355,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase { */ public async unwindBlocks(toBlockNumber: BlockNumber) { // Clear any registered forks — they're invalid after a reorg. - this.registeredForks.clear(); + this.registeredFork = undefined; try { const result = await this.instance.call( WorldStateMessageType.UNWIND_BLOCKS, From 16891b3cb880f85e0c8f52a1d748cf12bbdf4e64 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 09:24:46 -0300 Subject: [PATCH 26/29] validator using commit_fork --- .../validator-client/src/proposal_handler.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index af81e14fc8a7..17911b7bcbd9 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -18,7 +18,12 @@ import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdl import { validateCheckpoint } from '@aztec/stdlib/checkpoint'; import { getEpochAtSlot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { Gas } from '@aztec/stdlib/gas'; -import type { ITxProvider, ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; +import type { + ITxProvider, + MerkleTreeWriteOperations, + ValidatorClientFullConfig, + WorldStateSynchronizer, +} from '@aztec/stdlib/interfaces/server'; import { type L1ToL2MessageSource, accumulateCheckpointOutHashes, @@ -341,6 +346,11 @@ export class ProposalHandler { .filter(c => c.checkpointNumber < checkpointNumber) .map(c => c.checkpointOutHash); + // Fork before the block to be built + const parentBlockNumber = BlockNumber(blockNumber - 1); + await this.worldState.syncImmediate(parentBlockNumber); + await using fork = await this.worldState.fork(parentBlockNumber); + // Try re-executing the transactions in the proposal if needed let reexecutionResult; try { @@ -352,6 +362,7 @@ export class ProposalHandler { txs, l1ToL2Messages, previousCheckpointOutHashes, + fork, ); } catch (error) { this.log.error(`Error reexecuting txs while processing block proposal`, error, proposalInfo); @@ -359,9 +370,11 @@ export class ProposalHandler { return { isValid: false, blockNumber, reason, reexecutionResult }; } - // If we succeeded, push this block into the archiver (unless disabled) + // If we succeeded, push this block into the archiver and commit the fork to LMDB if (reexecutionResult?.block && this.config.skipPushProposedBlocksToArchiver === false) { + this.worldState.registerForkForBlock(reexecutionResult.block.archive.root, fork.forkId); await this.blockSource.addBlock(reexecutionResult.block); + await this.worldState.syncImmediate(blockNumber); } this.log.info( @@ -595,6 +608,7 @@ export class ProposalHandler { txs: Tx[], l1ToL2Messages: Fr[], previousCheckpointOutHashes: Fr[], + fork: MerkleTreeWriteOperations, ): Promise { const { blockHeader, txHashes } = proposal; @@ -613,11 +627,6 @@ export class ProposalHandler { const allBlocksInSlot = await this.blockSource.getBlocksForSlot(slot); const priorBlocks = allBlocksInSlot.filter(b => b.number < blockNumber && b.header.getSlot() === slot); - // Fork before the block to be built - const parentBlockNumber = BlockNumber(blockNumber - 1); - await this.worldState.syncImmediate(parentBlockNumber); - await using fork = await this.worldState.fork(parentBlockNumber); - // Verify the fork's archive root matches the proposal's expected last archive. // If they don't match, our world state synced to a different chain and reexecution would fail. const forkArchiveRoot = new Fr((await fork.getTreeInfo(MerkleTreeId.ARCHIVE)).root); From 707e0f454dd93e2126badc057282869844b50afb Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 09:56:11 -0300 Subject: [PATCH 27/29] add few test --- .../src/validator.integration.test.ts | 27 +++++++++++++- yarn-project/world-state/src/native/index.ts | 1 + .../src/native/native_world_state.test.ts | 37 ++++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/yarn-project/validator-client/src/validator.integration.test.ts b/yarn-project/validator-client/src/validator.integration.test.ts index 6b93351cf590..dd0efba88cc5 100644 --- a/yarn-project/validator-client/src/validator.integration.test.ts +++ b/yarn-project/validator-client/src/validator.integration.test.ts @@ -31,7 +31,7 @@ import { mockTx } from '@aztec/stdlib/testing'; import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees'; import { BlockHeader, type CheckpointGlobalVariables, Tx } from '@aztec/stdlib/tx'; import { ServerWorldStateSynchronizer } from '@aztec/world-state'; -import { NativeWorldStateService } from '@aztec/world-state/native'; +import { NativeWorldStateService, WorldStateMessageType } from '@aztec/world-state/native'; import { getGenesisValues } from '@aztec/world-state/testing'; import { describe, expect, it, jest } from '@jest/globals'; @@ -379,6 +379,31 @@ describe('ValidatorClient Integration', () => { }); describe('happy path', () => { + it('uses COMMIT_FORK instead of SYNC_BLOCK when validating blocks', async () => { + const blockCount = 5; + const { blocks } = await buildCheckpoint( + CheckpointNumber(1), + slotNumber, + emptyL1ToL2Messages, + emptyPreviousCheckpointOutHashes, + BlockNumber(1), + blockCount, + () => buildTxs(2), + ); + + // Spy on the attestor's native world state to track message types sent to C++ + const instance = (attestor.worldStateDb as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + await attestorValidateBlocks(blocks); + + const messageTypes = callSpy.mock.calls.map(call => call[0] as WorldStateMessageType); + expect(messageTypes.filter(t => t === WorldStateMessageType.COMMIT_FORK)).toHaveLength(blockCount); + expect(messageTypes.filter(t => t === WorldStateMessageType.SYNC_BLOCK)).toHaveLength(0); + + callSpy.mockRestore(); + }); + it('validates multiple blocks and attests to checkpoint', async () => { const { blocks, proposal } = await buildCheckpoint( CheckpointNumber(1), diff --git a/yarn-project/world-state/src/native/index.ts b/yarn-project/world-state/src/native/index.ts index 133319956c9d..775f3fb7858f 100644 --- a/yarn-project/world-state/src/native/index.ts +++ b/yarn-project/world-state/src/native/index.ts @@ -1,2 +1,3 @@ export * from './native_world_state.js'; export * from './fork_checkpoint.js'; +export { WorldStateMessageType } from './message.js'; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 0ad8995087e7..719ab85a3534 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -32,7 +32,7 @@ import { join } from 'path'; import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js'; import { assertSameState, compareChains, mockBlock, mockEmptyBlock } from '../test/utils.js'; import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE } from '../world-state-db/merkle_tree_db.js'; -import type { WorldStateStatusSummary } from './message.js'; +import { WorldStateMessageType, type WorldStateStatusSummary } from './message.js'; import { NativeWorldStateService, WORLD_STATE_DB_VERSION, WORLD_STATE_DIR } from './native_world_state.js'; jest.setTimeout(60_000); @@ -2071,5 +2071,40 @@ describe('NativeWorldState', () => { // Fork should be destroyed — operations on it should fail await expect(fork.getTreeInfo(MerkleTreeId.NULLIFIER_TREE)).rejects.toThrow('Fork not found'); }); + + it('uses COMMIT_FORK and not SYNC_BLOCK when fork is registered', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + + // Spy on the native instance to track which message types are sent + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + ws.registerForkForBlock(block.archive.root, fork.forkId); + await ws.handleL2BlockAndMessages(block, messages); + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes).toContain(WorldStateMessageType.COMMIT_FORK); + expect(messageTypes).not.toContain(WorldStateMessageType.SYNC_BLOCK); + + callSpy.mockRestore(); + }); + + it('uses SYNC_BLOCK and not COMMIT_FORK when no fork is registered', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + await fork.close(); + + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + await ws.handleL2BlockAndMessages(block, messages); + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes).toContain(WorldStateMessageType.SYNC_BLOCK); + expect(messageTypes).not.toContain(WorldStateMessageType.COMMIT_FORK); + + callSpy.mockRestore(); + }); }); }); From bfdfa1e812e8c9bc5670c4914dfcb8e898adb245 Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 10:07:49 -0300 Subject: [PATCH 28/29] more tests --- .../light/lightweight_checkpoint_builder.ts | 7 +- .../src/checkpoint_builder.ts | 2 +- .../src/native/native_world_state.test.ts | 150 ++++++++++++++++++ 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts index 278952365a66..cdb0a47dc5e5 100644 --- a/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts +++ b/yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts @@ -42,13 +42,18 @@ export class LightweightCheckpointBuilder { private blocks: L2Block[] = []; private blobFields: Fr[] = []; + /** Replaces the database used for subsequent block builds. */ + setDb(db: MerkleTreeWriteOperations): void { + this.db = db; + } + constructor( public readonly checkpointNumber: CheckpointNumber, public readonly constants: CheckpointGlobalVariables, public feeAssetPriceModifier: bigint, public readonly l1ToL2Messages: Fr[], private readonly previousCheckpointOutHashes: Fr[], - public db: MerkleTreeWriteOperations, + private db: MerkleTreeWriteOperations, bindings?: LoggerBindings, ) { this.logger = createLogger('checkpoint-builder', { diff --git a/yarn-project/validator-client/src/checkpoint_builder.ts b/yarn-project/validator-client/src/checkpoint_builder.ts index d46f4df13edd..386a17d625b7 100644 --- a/yarn-project/validator-client/src/checkpoint_builder.ts +++ b/yarn-project/validator-client/src/checkpoint_builder.ts @@ -64,7 +64,7 @@ export class CheckpointBuilder implements ICheckpointBlockBuilder { public async setFork(fork: MerkleTreeWriteOperations): Promise { await this.fork.close(); this.fork = fork; - this.checkpointBuilder.db = fork; + this.checkpointBuilder.setDb(fork); } constructor( diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 719ab85a3534..72da0361ed2d 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -2106,5 +2106,155 @@ describe('NativeWorldState', () => { callSpy.mockRestore(); }); + + it('unwind correctly reverses state committed via commit_fork', async () => { + // Commit 4 blocks via COMMIT_FORK + const treeInfosAfterBlock: Awaited>[] = []; + + for (let i = 1; i <= 4; i++) { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(i), 1, fork); + ws.registerForkForBlock(block.archive.root, fork.forkId); + await ws.handleL2BlockAndMessages(block, messages); + treeInfosAfterBlock.push(await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE)); + } + + expect((await ws.getStatusSummary()).unfinalizedBlockNumber).toBe(4); + + // Unwind back to block 2 + const unwindStatus = await ws.unwindBlocks(BlockNumber(2)); + expect(unwindStatus.summary.unfinalizedBlockNumber).toBe(2); + + // State matches what it was after block 2 + const treeInfoAfterUnwind = await ws.getCommitted().getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + expect(treeInfoAfterUnwind).toEqual(treeInfosAfterBlock[1]); + + // Can build and commit new blocks on top of the unwound state + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(3), 1, fork); + ws.registerForkForBlock(block.archive.root, fork.forkId); + await ws.handleL2BlockAndMessages(block, messages); + expect((await ws.getStatusSummary()).unfinalizedBlockNumber).toBe(3); + }); + + it('falls back to SYNC_BLOCK when COMMIT_FORK fails (fork deleted before commit)', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, fork); + + ws.registerForkForBlock(block.archive.root, fork.forkId); + await fork.close(); + + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + await ws.handleL2BlockAndMessages(block, messages); + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes).toContain(WorldStateMessageType.COMMIT_FORK); + expect(messageTypes).toContain(WorldStateMessageType.SYNC_BLOCK); + + const status = await ws.getStatusSummary(); + expect(status.unfinalizedBlockNumber).toEqual(1); + + callSpy.mockRestore(); + }); + + it('registered fork is not used after unwindBlocks', async () => { + // Commit block 1 + const fork1 = await ws.fork(); + const { block: block1, messages: messages1 } = await mockBlock(BlockNumber(1), 1, fork1); + ws.registerForkForBlock(block1.archive.root, fork1.forkId); + await ws.handleL2BlockAndMessages(block1, messages1); + + // Register fork for block 2 but don't sync it + const fork2 = await ws.fork(); + const { block: block2 } = await mockBlock(BlockNumber(2), 1, fork2); + ws.registerForkForBlock(block2.archive.root, fork2.forkId); + + // Unwind to genesis + await ws.unwindBlocks(BlockNumber(0)); + + // Build a new block 1 with different content + const fork3 = await ws.fork(); + const { block: newBlock1, messages: newMessages1 } = await mockBlock(BlockNumber(1), 2, fork3); + await fork3.close(); + + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + await ws.handleL2BlockAndMessages(newBlock1, newMessages1); + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes).toContain(WorldStateMessageType.SYNC_BLOCK); + expect(messageTypes).not.toContain(WorldStateMessageType.COMMIT_FORK); + expect((await ws.getStatusSummary()).unfinalizedBlockNumber).toEqual(1); + + callSpy.mockRestore(); + }); + + it('commits an empty block via COMMIT_FORK', async () => { + const fork = await ws.fork(); + const { block, messages } = await mockEmptyBlock(BlockNumber(1), fork); + + ws.registerForkForBlock(block.archive.root, fork.forkId); + + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + await ws.handleL2BlockAndMessages(block, messages); + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes).toContain(WorldStateMessageType.COMMIT_FORK); + expect(messageTypes).not.toContain(WorldStateMessageType.SYNC_BLOCK); + expect((await ws.getStatusSummary()).unfinalizedBlockNumber).toEqual(1); + + callSpy.mockRestore(); + }); + + it('COMMIT_FORK produces the same state as SYNC_BLOCK', async () => { + // Instance A: commit via COMMIT_FORK + const wsA = await NativeWorldStateService.tmp(); + const forkA = await wsA.fork(); + const { block, messages } = await mockBlock(BlockNumber(1), 1, forkA); + wsA.registerForkForBlock(block.archive.root, forkA.forkId); + await wsA.handleL2BlockAndMessages(block, messages); + + // Instance B: commit via SYNC_BLOCK (no fork registration) + const wsB = await NativeWorldStateService.tmp(); + await wsB.handleL2BlockAndMessages(block, messages); + + // State references must be identical + const stateRefA = await wsA.getCommitted().getStateReference(); + const stateRefB = await wsB.getCommitted().getStateReference(); + expect(stateRefA).toEqual(stateRefB); + + const archiveA = await wsA.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE); + const archiveB = await wsB.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE); + expect(archiveA).toEqual(archiveB); + + await wsA.close(); + await wsB.close(); + }); + + it('commits 5 sequential blocks via COMMIT_FORK (proposer flow)', async () => { + const blockCount = 5; + const instance = (ws as any).instance; + const callSpy = jest.spyOn(instance, 'call'); + + for (let i = 0; i < blockCount; i++) { + const fork = await ws.fork(); + const { block, messages } = await mockBlock(BlockNumber(i + 1), 1, fork); + ws.registerForkForBlock(block.archive.root, fork.forkId); + await ws.handleL2BlockAndMessages(block, messages); + } + + const messageTypes = callSpy.mock.calls.map(call => call[0]); + expect(messageTypes.filter(t => t === WorldStateMessageType.CREATE_FORK)).toHaveLength(blockCount); + expect(messageTypes.filter(t => t === WorldStateMessageType.COMMIT_FORK)).toHaveLength(blockCount); + expect(messageTypes.filter(t => t === WorldStateMessageType.SYNC_BLOCK)).toHaveLength(0); + expect((await ws.getStatusSummary()).unfinalizedBlockNumber).toEqual(blockCount); + + callSpy.mockRestore(); + }); }); }); From e16baca2e7ebef0b788e53ecb9964bce02e2afec Mon Sep 17 00:00:00 2001 From: Nikita Meshcheriakov Date: Wed, 1 Apr 2026 10:50:39 -0300 Subject: [PATCH 29/29] fix race condition --- .../barretenberg/world_state/world_state.cpp | 33 +++++++++++-------- .../barretenberg/world_state/world_state.hpp | 1 + 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 869b5cb4a9e0..c7e0ec4a04fb 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -192,6 +192,19 @@ Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const } return it->second; } + +Fork::SharedPtr WorldState::retrieve_and_remove_fork(const uint64_t& forkId) +{ + std::unique_lock lock(mtx); + auto it = _forks.find(forkId); + if (it == _forks.end()) { + throw std::runtime_error("Fork not found"); + } + Fork::SharedPtr fork = it->second; + _forks.erase(it); + return fork; +} + uint64_t WorldState::create_fork(const std::optional& blockNumber) { block_number_t blockNumberForFork = 0; @@ -234,13 +247,9 @@ void WorldState::delete_fork(const uint64_t& forkId) if (forkId == 0) { throw std::runtime_error("Unable to delete canonical fork"); } - // Retrieving the shared pointer here means we throw if the fork is not available, it also means we are not under a - // lock when we destroy the object - Fork::SharedPtr fork = retrieve_fork(forkId); - { - std::unique_lock lock(mtx); - _forks.erase(forkId); - } + // Atomically retrieve and remove so no concurrent caller can obtain a reference. + // The local shared_ptr ensures the fork is destroyed outside the lock. + Fork::SharedPtr fork = retrieve_and_remove_fork(forkId); } WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) @@ -250,7 +259,9 @@ WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) } validate_trees_are_equally_synched(); - Fork::SharedPtr fork = retrieve_fork(forkId); + // Atomically retrieve and remove so no concurrent caller can obtain a reference. + // The local shared_ptr keeps the fork alive for the duration of this method. + Fork::SharedPtr fork = retrieve_and_remove_fork(forkId); // Validate tip hasn't moved since fork was created auto archiveMeta = get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE); @@ -290,12 +301,6 @@ WorldStateStatusFull WorldState::commit_fork(const uint64_t& forkId) // Rollback canonical so it re-reads the updated LMDB state rollback(); - // Destroy the fork - { - std::unique_lock lock(mtx); - _forks.erase(forkId); - } - populate_status_summary(status); return status; } diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 4492fb8d53d2..b3bd9a5bbe7c 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -312,6 +312,7 @@ class WorldState { uint64_t maxReaders); Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; + Fork::SharedPtr retrieve_and_remove_fork(const uint64_t& forkId); Fork::SharedPtr create_new_fork(const block_number_t& blockNumber); void remove_forks_for_block(const block_number_t& blockNumber);