From 27c0e294e3cd94bc4d77f2344dc88684aa2e2da1 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 3 Jul 2026 06:24:27 -0300 Subject: [PATCH] test(e2e): extract p2p gossip scenario skeleton and add category README --- yarn-project/end-to-end/src/p2p/README.md | 78 +++ .../p2p/fee_asset_price_oracle_gossip.test.ts | 264 ++++------ .../end-to-end/src/p2p/gossip_network.test.ts | 475 ++++++++++-------- .../src/p2p/gossip_network_no_cheat.test.ts | 294 ----------- .../src/p2p/late_prover_tx_collection.test.ts | 13 +- .../end-to-end/src/p2p/p2p_network.ts | 24 + .../src/p2p/preferred_gossip_network.test.ts | 76 +-- .../end-to-end/src/p2p/rediscovery.test.ts | 13 +- .../src/p2p/reqresp/reqresp.test.ts | 8 +- .../end-to-end/src/p2p/reqresp/utils.ts | 17 +- yarn-project/end-to-end/src/p2p/shared.ts | 231 +++++++++ 11 files changed, 725 insertions(+), 768 deletions(-) create mode 100644 yarn-project/end-to-end/src/p2p/README.md delete mode 100644 yarn-project/end-to-end/src/p2p/gossip_network_no_cheat.test.ts diff --git a/yarn-project/end-to-end/src/p2p/README.md b/yarn-project/end-to-end/src/p2p/README.md new file mode 100644 index 000000000000..a4228d8f4523 --- /dev/null +++ b/yarn-project/end-to-end/src/p2p/README.md @@ -0,0 +1,78 @@ +# `p2p` e2e test category + +P2P tests run a network of validator nodes over **real libp2p** — a real bootstrap node, discv5 +discovery, the GossipSub mesh, the req/resp protocol, and real peer authentication and transport. This +is the category for any test whose subject is the networking layer itself: peer discovery and +rediscovery, gossip mesh formation and propagation, req/resp fetching, preferred-peer / supernode +topologies, and peer auth gating. + +Tests whose subject — proposals, attestations, checkpointing, pruning/recovery, offense detection, +sentinel observability — is faithfully reproduced by the in-memory `MockGossipSubNetwork` bus belong in +the `multi-node` category instead, which is far cheaper to run. Only reach for `p2p` when the behavior +under test genuinely cannot be reproduced without real networking. + +## Entrypoint + +`P2PNetworkTest` (`p2p_network.ts`) is the env-builder for this category. Unlike `MultiNodeTestContext`, +it does **not** extend `SingleNodeTestContext`; it is a parallel, self-contained builder that calls the +root `setup()` directly, then spawns real-libp2p nodes via the factories in +`../fixtures/setup_p2p_test.ts`. Its distinguishing traits: + +- A real bootstrap node (`addBootstrapNode`) with a fixed private key, so nodes discover each other via + discv5 and data directories can be reused across a restart. +- Two validator-registration paths: `applyBaseSetup()` registers the committee post-genesis via the + on-chain `MultiAdder` cheat (validators are added at `aztecTargetCommitteeSize: 0`, then activated by + warping past the validator-set lag), while tests that exercise the real onboarding flow call + `addBootstrapNode()` and register each validator through the CLI `addL1Validator` path instead. +- `waitForP2PMeshConnectivity(nodes, …)` — waits for peer connections and for the GossipSub mesh to + form on the requested topics; callers that need a proposal to reach the whole committee within a slot + raise `minMeshPeerCount` for a full mesh. +- Data-directory management: `setup()` creates one root temp dir; `dataDirFor(label)` hands out a nested + per-role path (`createNodes` appends `-`, single-node factories use it verbatim); `teardown()` + removes the whole tree in one recursive delete. Test files never `mkdtemp`/`rmSync` themselves. +- One named timing preset: `SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES` (ethSlot 4s, aztecSlot 12s, + `aztecProofSubmissionEpochs: 640` so the chain effectively never prunes), used by every file with + per-file slot/epoch overrides. `WAIT_FOR_TX_TIMEOUT` derives from the env slot duration. + +## Shared skeleton and helpers + +`shared.ts` holds the common scenario skeleton and the reusable building blocks, so test bodies read +declaratively rather than re-deriving the same ~40 lines of node-creation / mesh-wait / signer-check +boilerplate: + +- `runGossipScenario(opts)` — the shared bootstrap→createNodes→mesh→account→(submit)→verify skeleton. + Each varying part is a callback/flag: validator registration (`beforeCreateNodes`), extra prover / + monitor nodes (`createExtraNodes`), pre-submit waits (`beforeSubmit`), mesh parameters (`mesh`), + whether/how many txs to submit (`txsPerNode`, `submitSequentially`), which published checkpoint to + read signers from (`checkpointSource`), and scenario-specific verification (`afterVerify`). +- `verifyAttestationSigners(t, nodes, checkpoint)` — recovers the attestation signers from a published + checkpoint and asserts each belongs to the validator set; returns the signers for extra assertions. +- `getPublishedCheckpointForTx(node, txHash)` / `waitForFirstPublishedCheckpoint(t, nodes)` — resolve + the published checkpoint to read signers from, either from a mined tx's block or by polling for the + first one indexed by the archiver. +- `waitForNodesToSync(t, nodes)` — wait for every node to catch up to the initial node's tip. +- `maybeCheckQosAlerts(logger)` — run the QoS Grafana alert check when `CHECK_ALERTS=true`, else a no-op. +- `submitTransactions` / `submitComplexTxsTo` / `prepareTransactions` — build a throwaway wallet on a + node and submit (or pre-prove) N txs through it. + +Node factories live in `../fixtures/setup_p2p_test.ts` (`createNodes`, `createNode`, +`createNonValidatorNode`, `createProverNode`, `createValidatorConfig`, `generatePrivateKeys`), shared +with `P2PNetworkTest`. + +## Organizing principle + +Each file is one real-libp2p scenario the mock-gossip bus cannot reproduce, with a single top-level +`describe` named `e2e_p2p_*`. Files are plain `*.test.ts` (no `.parallel` suffix): each is one CI +container running the whole file. Because the nodes bind **fixed UDP/TCP ports**, two p2p test files +must never run at the same time — run them one at a time locally. + +## Files + +| File | Scenario | +|---|---| +| `gossip_network.test.ts` | The baseline gossip test, two describes on the shared skeleton: `cheat-registered validators` (4 validators + a p2p-only prover + a re-execution monitor; txs mine from every node, attestation signers match the committee, and the prover produces a proven block by collecting txs over p2p) and `on-chain-registered validators (no cheats)` (validators registered via the real `addL1Validator` CLI path with a ZkPassport mock proof, then the same propagation/signer checks). | +| `fee_asset_price_oracle_gossip.test.ts` | A fee-asset price set on a mock L1 `StateView` contract gossips through the validator network and the rollup's on-chain price converges on it across two adjustments. Runs on the shared skeleton with `txsPerNode: 0` — no txs are submitted; empty blocks carry the price. | +| `late_prover_tx_collection.test.ts` | A prover that joins after a block was already mined learns the block from its archiver (via L1 sync) but is missing the txs, and must fetch them from peers over the req/resp `BLOCK_TXS` path — the same path `ProverNode.gatherTxs` takes when preparing an epoch proof. | +| `preferred_gossip_network.test.ts` | A preferred-node (supernode) topology: preferred nodes accept only validator connections, a no-discovery validator reaches the network exclusively through them, and gossip monitors assert traffic flows only through the expected peers. Verifies exact per-role peer counts, tx mining, and attestation signers. | +| `rediscovery.test.ts` | Nodes form a mesh, then the bootstrap node and all validators are stopped and restarted from their data directories with no bootstrap ENR — they rejoin purely from the discv5 peer tables they persisted to disk. | +| `reqresp/reqresp.test.ts` | Tx gossip is disabled on the upcoming proposer nodes, so they must request the missing tx data over req/resp before they can attest; also asserts a multiple-blocks-per-slot checkpoint is produced with correctly ordered blocks. The scenario body lives in `reqresp/utils.ts`. | diff --git a/yarn-project/end-to-end/src/p2p/fee_asset_price_oracle_gossip.test.ts b/yarn-project/end-to-end/src/p2p/fee_asset_price_oracle_gossip.test.ts index a543b15893a4..21c010085ac3 100644 --- a/yarn-project/end-to-end/src/p2p/fee_asset_price_oracle_gossip.test.ts +++ b/yarn-project/end-to-end/src/p2p/fee_asset_price_oracle_gossip.test.ts @@ -1,49 +1,29 @@ -import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { RollupContract, STATE_VIEW_ADDRESS } from '@aztec/ethereum/contracts'; -import { CheckpointNumber } from '@aztec/foundation/branded-types'; -import { Signature } from '@aztec/foundation/eth-signature'; import { retryUntil } from '@aztec/foundation/retry'; -import type { SequencerClient } from '@aztec/sequencer-client'; import { tryStop } from '@aztec/stdlib/interfaces/server'; -import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; import { mnemonicToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; import { MNEMONIC, shouldCollectMetrics } from '../fixtures/fixtures.js'; -import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js'; -import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js'; +import { ATTESTER_PRIVATE_KEYS_START_INDEX, createProverNode } from '../fixtures/setup_p2p_test.js'; import { MockStateView, diffInBps } from '../shared/mock_state_view.js'; import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES } from './p2p_network.js'; +import { maybeCheckQosAlerts, runGossipScenario, waitForNodesToSync } from './shared.js'; -const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; - -// Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds +// Don't set NUM_VALIDATORS higher than 9 because each node uses a different L1 publisher account and anvil seeds. const NUM_VALIDATORS = 4; const BOOT_NODE_UDP_PORT = 4500; -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gossip-')); - jest.setTimeout(1000 * 60 * 10); -const qosAlerts: AlertConfig[] = [ - { - alert: 'SequencerTimeToCollectAttestations', - expr: 'aztec_sequencer_time_to_collect_attestations > 3500', - labels: { severity: 'error' }, - for: '10m', - annotations: {}, - }, -]; - // Tests that the fee-asset price oracle value set on a mock L1 StateView contract gossips through the -// real libp2p validator network and converges on the rollup's on-chain price. Uses P2PNetworkTest with +// real libp2p validator network and converges on the rollup's on-chain price. Runs on the shared gossip +// skeleton (runGossipScenario) with txsPerNode:0 — no txs are submitted; instead the oracle price is +// adjusted twice and the on-chain price is asserted to converge. Uses P2PNetworkTest with // SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES (ethSlot=4s, aztecSlot=12s, epoch=4, proofSubEpochs=640) plus a // real prover node. CHECK_ALERTS env var gates optional Grafana alert validation. describe('e2e_p2p_network', () => { @@ -80,172 +60,106 @@ describe('e2e_p2p_network', () => { await tryStop(proverNode); await t.stopNodes(nodes); await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } }); afterAll(async () => { - if (CHECK_ALERTS) { - const checker = new GrafanaClient(t.logger); - await checker.runAlertCheck(qosAlerts); - } + await maybeCheckQosAlerts(t.logger); }); // Deploys a MockStateView L1 contract, sets an initial oracle price, then starts 4 validator nodes // and a prover. Adjusts the oracle price twice and uses retryUntil to confirm the rollup's on-chain // price converges to each target within the gossip propagation window. it('should rollup txs from all peers', async () => { - // create the bootstrap node for the network - if (!t.bootstrapNodeEnr) { - throw new Error('Bootstrap node ENR is not available'); - } - const rollup = RollupContract.getFromL1ContractsValues(t.ctx.deployL1ContractsValues); const account = mnemonicToAccount(MNEMONIC, { addressIndex: 999 }); const walletClient = createExtendedL1Client(t.ctx.aztecNodeConfig.l1RpcUrls, account, foundry); - await t.ctx.ethCheatCodes.setBalance(account.address, 100n * 10n ** 18n); - - const mockStateView = await MockStateView.deploy(t.ctx.ethCheatCodes, walletClient, STATE_VIEW_ADDRESS); - - // The initial oracle price (default value) is 1e7 - await mockStateView.setEthPerFeeAsset(10n ** 7n); - - await t.ctx.ethCheatCodes.mineEmptyBlock(); - await t.ctx.ethCheatCodes.mine(10); - await t.ctx.ethCheatCodes.mineEmptyBlock(); - - t.logger.info('Creating validator nodes'); - nodes = await createNodes( - t.ctx.aztecNodeConfig, - t.ctx.dateProvider!, - t.bootstrapNodeEnr, - NUM_VALIDATORS, - BOOT_NODE_UDP_PORT, - t.genesis, - DATA_DIR, - // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true - shouldCollectMetrics(), - ); - - t.logger.warn(`Creating prover node`); - ({ proverNode } = await createProverNode( - { ...t.ctx.aztecNodeConfig, minTxsPerBlock: 0 }, - BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, - t.bootstrapNodeEnr, - ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, - { dateProvider: t.ctx.dateProvider! }, - t.genesis, - `${DATA_DIR}-prover`, - shouldCollectMetrics(), - )); - - // Wait for the validators to discover each other and form a gossip mesh before proceeding. - await t.waitForP2PMeshConnectivity(nodes); - - // We need to `createNodes` before we setup account, because - // those nodes actually form the committee, and so we cannot build - // blocks without them (since targetCommitteeSize is set to the number of nodes) - await t.setupAccount(); - - // Wait until the other nodes sync to the block from which we sent the tx - const targetBlock = await t.ctx.aztecNode.getBlockNumber(); - t.logger.warn(`Waiting for all nodes to sync to block number ${targetBlock}`); - await retryUntil( - async () => { - const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber())); - const checkpointNumber = (await t.monitor.run()).checkpointNumber; - t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`); - return blockNumbers.every(bn => bn >= targetBlock); - }, - `nodes to sync to block number ${targetBlock}`, - 30, - 0.5, - ); - - for (const node of nodes) { - await node.setConfig({ minTxsPerBlock: 0 }); - } - - const targetOraclePrice = (BigInt(10n ** 7n) * 1025n) / 1000n; - await mockStateView.setEthPerFeeAsset(targetOraclePrice); - t.logger.info(`Set uniswap price to ${targetOraclePrice}`); - - // Get initial on-chain price - const initialOnChainPrice = await rollup.getEthPerFeeAsset(); - t.logger.info(`Initial on-chain price: ${initialOnChainPrice}, target oracle price: ${targetOraclePrice}`); - - // Gather signers from attestations downloaded from L1. setupAccount() no longer sends a tx, - // so when the test reaches here no checkpoint may have been published yet; wait for the - // archiver to index the first published checkpoint before reading attestations. - const dataStore = (nodes[0] as AztecNodeService).getBlockSource() as Archiver; - t.logger.warn('Waiting for first checkpoint to be published and indexed by the archiver'); - const publishedCheckpoint = await retryUntil( - async () => { - const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber())); - const checkpointNumber = (await t.monitor.run()).checkpointNumber; - t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`); - const [checkpoint] = await dataStore.getCheckpoints({ from: CheckpointNumber(1), limit: 1 }); - return checkpoint; - }, - 'published checkpoint to be indexed', - 120, - 1, - ); - const signatureContext = { - chainId: t.ctx.aztecNodeConfig.l1ChainId, - rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, - }; - const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext); - const attestations = publishedCheckpoint.attestations - .filter(a => !a.signature.isEmpty()) - .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty())); - const signers = await Promise.all(attestations.map(att => att.getSender()!.toString())); - t.logger.info(`Attestation signers`, { signers }); - - // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right - const validatorAddresses = nodes.flatMap(node => - ((node as AztecNodeService).getSequencer() as SequencerClient).validatorAddresses?.map(a => a.toString()), - ); - t.logger.info(`Validator addresses`, { addresses: validatorAddresses }); - for (const signer of signers) { - expect(validatorAddresses).toContain(signer); - } - - await retryUntil( - async () => { - const currentPrice = await rollup.getEthPerFeeAsset(); - t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice}`); - return diffInBps(currentPrice, targetOraclePrice) == 0n; + let mockStateView!: MockStateView; + let initialOnChainPrice!: bigint; + let targetOraclePrice!: bigint; + + nodes = await runGossipScenario({ + t, + numValidators: NUM_VALIDATORS, + bootNodePort: BOOT_NODE_UDP_PORT, + txsPerNode: 0, + checkpointSource: 'first-published', + beforeCreateNodes: async () => { + await t.ctx.ethCheatCodes.setBalance(account.address, 100n * 10n ** 18n); + + mockStateView = await MockStateView.deploy(t.ctx.ethCheatCodes, walletClient, STATE_VIEW_ADDRESS); + + // The initial oracle price (default value) is 1e7 + await mockStateView.setEthPerFeeAsset(10n ** 7n); + + await t.ctx.ethCheatCodes.mineEmptyBlock(); + await t.ctx.ethCheatCodes.mine(10); + await t.ctx.ethCheatCodes.mineEmptyBlock(); }, - 'price convergence toward oracle', - 120, // timeout in seconds - 5, // check interval in seconds - ); - - const priceAfterFirstAlignment = await rollup.getEthPerFeeAsset(); - const targetOraclePrice2 = (BigInt(priceAfterFirstAlignment) * 995n) / 1000n; - await mockStateView.setEthPerFeeAsset(targetOraclePrice2); - t.logger.info(`Set uniswap price to ${targetOraclePrice}`); - - await retryUntil( - async () => { - const currentPrice = await rollup.getEthPerFeeAsset(); - t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice2}`); - return diffInBps(currentPrice, targetOraclePrice2) == 0n; + createExtraNodes: async () => { + t.logger.warn(`Creating prover node`); + ({ proverNode } = await createProverNode( + { ...t.ctx.aztecNodeConfig, minTxsPerBlock: 0 }, + BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, + t.bootstrapNodeEnr, + ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, + { dateProvider: t.ctx.dateProvider! }, + t.genesis, + t.dataDirFor('prover'), + shouldCollectMetrics(), + )); }, - 'price convergence toward oracle', - 120, // timeout in seconds - 5, // check interval in seconds - ); + beforeSubmit: async nodes => { + await waitForNodesToSync(t, nodes); + + for (const node of nodes) { + await node.setConfig({ minTxsPerBlock: 0 }); + } - const finalPrice = await rollup.getEthPerFeeAsset(); - t.logger.info(`Final on-chain price: ${finalPrice}`); + targetOraclePrice = (BigInt(10n ** 7n) * 1025n) / 1000n; + await mockStateView.setEthPerFeeAsset(targetOraclePrice); + t.logger.info(`Set uniswap price to ${targetOraclePrice}`); - // Verify the price moved toward the oracle price - expect(finalPrice).toBeGreaterThan(initialOnChainPrice); - expect(diffInBps(finalPrice, targetOraclePrice2)).toBe(0n); + // Get initial on-chain price + initialOnChainPrice = await rollup.getEthPerFeeAsset(); + t.logger.info(`Initial on-chain price: ${initialOnChainPrice}, target oracle price: ${targetOraclePrice}`); + }, + afterVerify: async () => { + await retryUntil( + async () => { + const currentPrice = await rollup.getEthPerFeeAsset(); + t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice}`); + return diffInBps(currentPrice, targetOraclePrice) == 0n; + }, + 'price convergence toward oracle', + 120, // timeout in seconds + 5, // check interval in seconds + ); + + const priceAfterFirstAlignment = await rollup.getEthPerFeeAsset(); + const targetOraclePrice2 = (BigInt(priceAfterFirstAlignment) * 995n) / 1000n; + await mockStateView.setEthPerFeeAsset(targetOraclePrice2); + t.logger.info(`Set uniswap price to ${targetOraclePrice}`); + + await retryUntil( + async () => { + const currentPrice = await rollup.getEthPerFeeAsset(); + t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice2}`); + return diffInBps(currentPrice, targetOraclePrice2) == 0n; + }, + 'price convergence toward oracle', + 120, // timeout in seconds + 5, // check interval in seconds + ); + + const finalPrice = await rollup.getEthPerFeeAsset(); + t.logger.info(`Final on-chain price: ${finalPrice}`); + + // Verify the price moved toward the oracle price + expect(finalPrice).toBeGreaterThan(initialOnChainPrice); + expect(diffInBps(finalPrice, targetOraclePrice2)).toBe(0n); + }, + }); }); }); diff --git a/yarn-project/end-to-end/src/p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/p2p/gossip_network.test.ts index 1edd171f69eb..a1df69dce8cd 100644 --- a/yarn-project/end-to-end/src/p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/p2p/gossip_network.test.ts @@ -1,33 +1,30 @@ -import type { Archiver } from '@aztec/archiver'; import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; -import { TxHash } from '@aztec/aztec.js/tx'; -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { Signature } from '@aztec/foundation/eth-signature'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import { addL1Validator } from '@aztec/cli/l1/validators'; +import { RollupContract } from '@aztec/ethereum/contracts'; +import { EpochNumber } from '@aztec/foundation/branded-types'; import { retryUntil } from '@aztec/foundation/retry'; -import type { SequencerClient } from '@aztec/sequencer-client'; +import { sleep } from '@aztec/foundation/sleep'; +import { MockZKPassportVerifierAbi } from '@aztec/l1-artifacts/MockZKPassportVerifierAbi'; +import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi'; import { tryStop } from '@aztec/stdlib/interfaces/server'; -import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; +import { TopicType } from '@aztec/stdlib/p2p'; +import { ZkPassportProofParams } from '@aztec/stdlib/zkpassport'; import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; +import { getContract } from 'viem'; import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { ATTESTER_PRIVATE_KEYS_START_INDEX, - createNodes, createNonValidatorNode, createProverNode, } from '../fixtures/setup_p2p_test.js'; -import { waitForTxs } from '../fixtures/wait_helpers.js'; -import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js'; -import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; -import { submitTransactions } from './shared.js'; +import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES } from './p2p_network.js'; +import { maybeCheckQosAlerts, runGossipScenario, waitForNodesToSync } from './shared.js'; -const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; - -// Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds +// Don't set NUM_VALIDATORS higher than 9 because each node uses a different L1 publisher account and anvil seeds. const NUM_VALIDATORS = 4; const NUM_TXS_PER_NODE = 2; const BOOT_NODE_UDP_PORT = process.env.BOOT_NODE_UDP_PORT ? parseInt(process.env.BOOT_NODE_UDP_PORT) : 4500; @@ -35,195 +32,277 @@ const AZTEC_SLOT_DURATION = 24; const AZTEC_EPOCH_DURATION = 4; const BLOCK_DURATION_MS = 10_000; -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gossip-')); - jest.setTimeout(1000 * 60 * 10); -const qosAlerts: AlertConfig[] = [ - { - alert: 'SequencerTimeToCollectAttestations', - expr: 'aztec_sequencer_time_to_collect_attestations > 3500', - labels: { severity: 'error' }, - for: '10m', - annotations: {}, - }, -]; - -// Tests end-to-end gossip propagation with 4 validators, a fake prover node, and a non-validator -// monitoring node (alwaysReexecuteBlockProposals:true). Uses P2PNetworkTest with real libp2p, -// SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES (ethSlot=4s, aztecSlot=24s, epoch=4, proofSubEpochs=640), -// inboxLag=2. Asserts txs are mined from all nodes, attestation signers match the validator set, -// and the prover node produces a proven block by collecting txs from p2p. +// End-to-end gossip propagation over real libp2p with 4 validators, using the shared +// bootstrap->createNodes->mesh->account->submit->verify skeleton (runGossipScenario). Both describes use +// P2PNetworkTest with SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES (ethSlot=4s, proofSubEpochs=640), aztecSlot=24s, +// blockDurationMs=10000, inboxLag=2, and differ only in how the validator set is registered. describe('e2e_p2p_network', () => { - let t: P2PNetworkTest; - let nodes: AztecNodeService[]; - let proverAztecNode: AztecNodeService; - let monitoringNode: AztecNodeService; - - beforeEach(async () => { - t = await P2PNetworkTest.create({ - testName: 'e2e_p2p_network', - numberOfNodes: 0, - numberOfValidators: NUM_VALIDATORS, - basePort: BOOT_NODE_UDP_PORT, - metricsPort: shouldCollectMetrics(), - startProverNode: false, // we'll start our own using p2p - initialConfig: { - ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, - aztecSlotDuration: AZTEC_SLOT_DURATION, - aztecEpochDuration: AZTEC_EPOCH_DURATION, - blockDurationMs: BLOCK_DURATION_MS, - slashingRoundSizeInEpochs: 2, - slashingQuorum: 5, - listenAddress: '127.0.0.1', - inboxLag: 2, - }, + // Registers validators via applyBaseSetup()'s MultiAdder cheat shortcut, then stands up 4 validators + + // a fake prover node (p2p-only tx collection) + a non-validator monitoring node + // (alwaysReexecuteBlockProposals:true). Asserts txs mine from every node, attestation signers match the + // validator set, and the prover eventually produces a proven block by collecting txs from p2p. + describe('cheat-registered validators', () => { + let t: P2PNetworkTest; + let nodes: AztecNodeService[]; + let proverAztecNode: AztecNodeService; + let monitoringNode: AztecNodeService; + + beforeEach(async () => { + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_network', + numberOfNodes: 0, + numberOfValidators: NUM_VALIDATORS, + basePort: BOOT_NODE_UDP_PORT, + metricsPort: shouldCollectMetrics(), + startProverNode: false, // we'll start our own using p2p + initialConfig: { + ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, + aztecSlotDuration: AZTEC_SLOT_DURATION, + aztecEpochDuration: AZTEC_EPOCH_DURATION, + blockDurationMs: BLOCK_DURATION_MS, + slashingRoundSizeInEpochs: 2, + slashingQuorum: 5, + listenAddress: '127.0.0.1', + inboxLag: 2, + }, + }); + + await t.setup(); + await t.applyBaseSetup(); }); - await t.setup(); - await t.applyBaseSetup(); - }); + afterEach(async () => { + await tryStop(proverAztecNode); + await tryStop(monitoringNode); + await t.stopNodes(nodes); + await t.teardown(); + }); - afterEach(async () => { - await tryStop(proverAztecNode); - await tryStop(monitoringNode); - await t.stopNodes(nodes); - await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } - }); + afterAll(async () => { + await maybeCheckQosAlerts(t.logger); + }); + + it('should rollup txs from all peers', async () => { + nodes = await runGossipScenario({ + t, + numValidators: NUM_VALIDATORS, + bootNodePort: BOOT_NODE_UDP_PORT, + txsPerNode: NUM_TXS_PER_NODE, + createExtraNodes: async () => { + // A prover node that uses p2p only (not rpc) to gather txs, to test prover tx collection. + t.logger.warn(`Creating prover node`); + ({ proverNode: proverAztecNode } = await createProverNode( + t.ctx.aztecNodeConfig, + BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, + t.bootstrapNodeEnr, + ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, + { dateProvider: t.ctx.dateProvider }, + t.genesis, + t.dataDirFor('prover'), + shouldCollectMetrics(), + )); - afterAll(async () => { - if (CHECK_ALERTS) { - const checker = new GrafanaClient(t.logger); - await checker.runAlertCheck(qosAlerts); - } + t.logger.warn(`Creating non validator node`); + const monitoringNodeConfig: AztecNodeConfig = { + ...t.ctx.aztecNodeConfig, + alwaysReexecuteBlockProposals: true, + }; + monitoringNode = await createNonValidatorNode( + monitoringNodeConfig, + t.ctx.dateProvider, + BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 2, + t.bootstrapNodeEnr, + t.genesis, + t.dataDirFor('monitor'), + shouldCollectMetrics(), + ); + }, + beforeSubmit: nodes => waitForNodesToSync(t, nodes), + afterVerify: async nodes => { + // Ensure prover node did its job and collected txs from p2p + await retryUntil( + async () => { + const provenBlock = await nodes[0].getBlockNumber('proven'); + return provenBlock > 0; + }, + 'proven block', + SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES.aztecProofSubmissionEpochs * + AZTEC_EPOCH_DURATION * + AZTEC_SLOT_DURATION, + ); + }, + }); + }); }); - // Stands up 4 validators + 1 prover + 1 re-execution monitor, submits 2 txs per node, and waits - // for all txs to mine. Checks attestation signers match the validator set and confirms the prover - // eventually produces a proven block (collecting txs from p2p rather than RPC). - it('should rollup txs from all peers', async () => { - // create the bootstrap node for the network - if (!t.bootstrapNodeEnr) { - throw new Error('Bootstrap node ENR is not available'); - } - - // create our network of nodes and submit txs into each of them - // the number of txs per node and the number of txs per rollup - // should be set so that the only way for rollups to be built - // is if the txs are successfully gossiped around the nodes. - const txsSentViaDifferentNodes: TxHash[][] = []; - t.logger.info('Creating validator nodes'); - nodes = await createNodes( - t.ctx.aztecNodeConfig, - t.ctx.dateProvider, - t.bootstrapNodeEnr, - NUM_VALIDATORS, - BOOT_NODE_UDP_PORT, - t.genesis, - DATA_DIR, - // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true - shouldCollectMetrics(), - ); - - // create a prover node that uses p2p only (not rpc) to gather txs to test prover tx collection - t.logger.warn(`Creating prover node`); - ({ proverNode: proverAztecNode } = await createProverNode( - t.ctx.aztecNodeConfig, - BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, - t.bootstrapNodeEnr, - ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, - { dateProvider: t.ctx.dateProvider }, - t.genesis, - `${DATA_DIR}-prover`, - shouldCollectMetrics(), - )); - - t.logger.warn(`Creating non validator node`); - const monitoringNodeConfig: AztecNodeConfig = { ...t.ctx.aztecNodeConfig, alwaysReexecuteBlockProposals: true }; - monitoringNode = await createNonValidatorNode( - monitoringNodeConfig, - t.ctx.dateProvider, - BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 2, - t.bootstrapNodeEnr, - t.genesis, - `${DATA_DIR}-monitor`, - shouldCollectMetrics(), - ); - - t.logger.info('Waiting for nodes to connect'); - await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS); - - // We need to `createNodes` before we setup account, because - // those nodes actually form the committee, and so we cannot build - // blocks without them (since targetCommitteeSize is set to the number of nodes) - await t.setupAccount(); - - // Wait until the other nodes sync to the block from which we sent the tx - const targetBlock = await t.ctx.aztecNode.getBlockNumber(); - t.logger.warn(`Waiting for all nodes to sync to block number ${targetBlock}`); - await retryUntil( - async () => { - const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber())); - const checkpointNumber = (await t.monitor.run()).checkpointNumber; - t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`); - return blockNumbers.every(bn => bn >= targetBlock); - }, - `nodes to sync to block number ${targetBlock}`, - 30, - 0.5, - ); - - t.logger.info('Submitting transactions'); - // Each submitTransactions call builds its own wallet/PXE, so submissions are independent and can run - // concurrently. Promise.all preserves node order, keeping txsSentViaDifferentNodes[i] aligned with nodes[i]. - const submitted = await Promise.all( - nodes.map(node => submitTransactions(t.logger, node, NUM_TXS_PER_NODE, t.fundedAccount)), - ); - txsSentViaDifferentNodes.push(...submitted); - - t.logger.info('Waiting for transactions to be mined'); - // now ensure that all txs were successfully mined - const allTxHashes = txsSentViaDifferentNodes.flat(); - await waitForTxs(nodes[0], allTxHashes, { timeout: WAIT_FOR_TX_TIMEOUT }); - t.logger.info('All transactions mined'); - - // Gather signers from attestations downloaded from L1 - const receipt = await nodes[0].getTxReceipt(txsSentViaDifferentNodes[0][0]); - const blockNumber = receipt.blockNumber!; - const dataStore = (nodes[0] as AztecNodeService).getBlockSource() as Archiver; - const blockData = await dataStore.getBlockData({ number: BlockNumber(blockNumber) }); - const [publishedCheckpoint] = await dataStore.getCheckpoints({ from: blockData!.checkpointNumber, limit: 1 }); - const signatureContext = { - chainId: t.ctx.aztecNodeConfig.l1ChainId, - rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, - }; - const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext); - const attestations = publishedCheckpoint.attestations - .filter(a => !a.signature.isEmpty()) - .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty())); - const signers = await Promise.all(attestations.map(att => att.getSender()!.toString())); - t.logger.info(`Attestation signers`, { signers }); - - // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right - const validatorAddresses = nodes.flatMap(node => - ((node as AztecNodeService).getSequencer() as SequencerClient).validatorAddresses?.map(a => a.toString()), - ); - t.logger.info(`Validator addresses`, { addresses: validatorAddresses }); - for (const signer of signers) { - expect(validatorAddresses).toContain(signer); - } - - // Ensure prover node did its job and collected txs from p2p - await retryUntil( - async () => { - const provenBlock = await nodes[0].getBlockNumber('proven'); - return provenBlock > 0; - }, - 'proven block', - SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES.aztecProofSubmissionEpochs * AZTEC_EPOCH_DURATION * AZTEC_SLOT_DURATION, - ); + // Registers validators via the real addL1Validator CLI path (with a ZkPassport mock proof) instead of + // the MultiAdder cheat shortcut, then submits txs to each node. Asserts the registration took effect + // on-chain, all txs mine, and attestation signers match the registered validator set. + describe('on-chain-registered validators (no cheats)', () => { + let t: P2PNetworkTest; + let nodes: AztecNodeService[]; + + beforeEach(async () => { + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_network', + numberOfNodes: 0, + numberOfValidators: NUM_VALIDATORS, + basePort: BOOT_NODE_UDP_PORT, + metricsPort: shouldCollectMetrics(), + initialConfig: { + ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, + aztecSlotDuration: AZTEC_SLOT_DURATION, + blockDurationMs: BLOCK_DURATION_MS, + listenAddress: '127.0.0.1', + // Allow empty blocks so the first checkpoint can be published before any txs are submitted. + // Without this, no blocks are built until txs arrive, and a failed checkpoint during tx + // submission causes block pruning that invalidates tx references. + minTxsPerBlock: 0, + inboxLag: 2, + }, + }); + + await t.setup(); + await t.addBootstrapNode(); + }); + + afterEach(async () => { + await t.stopNodes(nodes); + await t.teardown(); + }); + + afterAll(async () => { + await maybeCheckQosAlerts(t.logger); + }); + + it('should rollup txs from all peers (and add the validators without cheating)', async () => { + nodes = await runGossipScenario({ + t, + numValidators: NUM_VALIDATORS, + bootNodePort: BOOT_NODE_UDP_PORT, + txsPerNode: NUM_TXS_PER_NODE, + submitSequentially: true, + // A full mesh (N-1 peers per node) on the proposal/checkpoint topics is required: with + // skipInitialSequencer the first blocks are built by this committee, and the first checkpoint must + // reach quorum (all 4 validators) to land on L1. If those meshes are only partly formed, some + // committee members miss the first proposal, the checkpoint stalls at 2/3, and every later slot + // rebuilds a competing un-checkpointed block 1 that peers reject as `block_number_already_exists` + // — a permanent 2/3 deadlock. + mesh: { + expectedNodeCount: NUM_VALIDATORS, + timeoutSeconds: 60, + checkIntervalSeconds: 0.5, + topics: [TopicType.block_proposal, TopicType.checkpoint_proposal, TopicType.checkpoint_attestation], + minMeshPeerCount: NUM_VALIDATORS - 1, + }, + beforeCreateNodes: async () => { + expect(t.ctx.deployL1ContractsValues.l1ContractAddresses.stakingAssetHandlerAddress).toBeDefined(); + + const { validators } = t.getValidators(); + + const rollupWrapper = RollupContract.getFromL1ContractsValues(t.ctx.deployL1ContractsValues); + + const rollup = getContract({ + address: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), + abi: RollupAbi, + client: t.ctx.deployL1ContractsValues.l1Client, + }); + + const zkPassportVerifier = getContract({ + address: t.ctx.deployL1ContractsValues.l1ContractAddresses.zkPassportVerifierAddress!.toString(), + abi: MockZKPassportVerifierAbi, + client: t.ctx.deployL1ContractsValues.l1Client, + }); + + expect((await rollupWrapper.getAttesters()).length).toBe(0); + + // Use the base account as the withdrawer for all validators in this test + const withdrawerAddress = EthAddress.fromString(t.baseAccount.address); + + // Add the validators to the rollup using the same function as the CLI + for (let i = 0; i < validators.length; i++) { + const validator = validators[i]; + const mockPassportProof = ZkPassportProofParams.random().toBuffer(); + await addL1Validator({ + rpcUrls: t.ctx.aztecNodeConfig.l1RpcUrls, + chainId: t.ctx.aztecNodeConfig.l1ChainId, + privateKey: t.baseAccountPrivateKey, + mnemonic: undefined, + attesterAddress: EthAddress.fromString(validator.attester.toString()), + withdrawerAddress, + stakingAssetHandlerAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.stakingAssetHandlerAddress!, + proofParams: mockPassportProof, + blsSecretKey: Fr.random().toBigInt(), + log: t.logger.info, + debugLogger: t.logger, + }); + + // mock nullifiers - increment the id in the mock zk passport verifier + t.logger.info('Incrementing unique identifier in mock zk passport verifier'); + await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ + hash: await zkPassportVerifier.write.incrementUniqueIdentifier(), + }); + } + + await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ + hash: await rollup.write.flushEntryQueue(), + }); + + const attestersImmedatelyAfterAdding = await rollupWrapper.getAttesters(); + expect(attestersImmedatelyAfterAdding.length).toBe(validators.length); + + // Check that the validators are added correctly + for (const validator of validators) { + const info = await rollupWrapper.getAttesterView(validator.attester.toString()); + expect(info.config.withdrawer.toChecksumString()).toBe(withdrawerAddress.toChecksumString()); + } + + // Wait for the validators to be added to the rollup + const timestamp = await t.ctx.cheatCodes.rollup.advanceToEpoch( + EpochNumber(t.ctx.aztecNodeConfig.lagInEpochsForValidatorSet + 1), + ); + + // Changes have now taken effect + const attesters = await rollupWrapper.getAttesters(); + expect(attesters.length).toBe(validators.length); + expect(attesters.length).toBe(NUM_VALIDATORS); + + // Send and await a tx to make sure we mine a block for the warp to correctly progress. + await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ + hash: await t.ctx.deployL1ContractsValues.l1Client.sendTransaction({ + to: t.baseAccount.address, + value: 1n, + account: t.baseAccount, + }), + }); + + // Set the system time in the node, only after we have warped the time and waited for a block + // Time is only set in the NEXT block + t.ctx.dateProvider.setTime(Number(timestamp) * 1000); + }, + beforeSubmit: async nodes => { + // Wait for the first checkpoint to be published to L1 before submitting transactions. + // With skipInitialSequencer, no blocks exist from setup, so the first blocks are built by the + // validator committee. If we submit txs before a checkpoint lands on L1, a failed checkpoint + // publish can prune locally-proposed blocks, causing txs to reference pruned block headers. + t.logger.info('Waiting for first checkpoint to be published'); + await retryUntil( + async () => (await nodes[0].getBlockNumber('checkpointed')) > 0, + 'first checkpoint published', + 120, + ); + t.logger.info('First checkpoint published'); + + // Wait for the next L1 block so that all nodes' getCurrentMinFees() caches are + // refreshed after the first L2 checkpoint is published. Without this, some wallets + // may estimate fees based on pre-checkpoint values (very low due to fee decay), + // while receiving nodes already see the post-checkpoint fees (much higher). + const ethereumSlotDuration = t.ctx.aztecNodeConfig.ethereumSlotDuration ?? 4; + await sleep((ethereumSlotDuration + 1) * 1000); + }, + }); + }); }); }); diff --git a/yarn-project/end-to-end/src/p2p/gossip_network_no_cheat.test.ts b/yarn-project/end-to-end/src/p2p/gossip_network_no_cheat.test.ts deleted file mode 100644 index f78bcccfa651..000000000000 --- a/yarn-project/end-to-end/src/p2p/gossip_network_no_cheat.test.ts +++ /dev/null @@ -1,294 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; -import { waitForTx } from '@aztec/aztec.js/node'; -import { TxHash } from '@aztec/aztec.js/tx'; -import { addL1Validator } from '@aztec/cli/l1/validators'; -import { RollupContract } from '@aztec/ethereum/contracts'; -import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; -import { Signature } from '@aztec/foundation/eth-signature'; -import { retryUntil } from '@aztec/foundation/retry'; -import { sleep } from '@aztec/foundation/sleep'; -import { MockZKPassportVerifierAbi } from '@aztec/l1-artifacts/MockZKPassportVerifierAbi'; -import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi'; -import type { SequencerClient } from '@aztec/sequencer-client'; -import { CheckpointAttestation, ConsensusPayload, TopicType } from '@aztec/stdlib/p2p'; -import { ZkPassportProofParams } from '@aztec/stdlib/zkpassport'; - -import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import { getContract } from 'viem'; - -import { shouldCollectMetrics } from '../fixtures/fixtures.js'; -import { createNodes } from '../fixtures/setup_p2p_test.js'; -import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js'; -import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; -import { submitTransactions } from './shared.js'; - -const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; - -// Don't set this to a higher value than 9 because each node will use a different L1 publisher account and anvil seeds -const NUM_VALIDATORS = 4; -const NUM_TXS_PER_NODE = 2; -const BOOT_NODE_UDP_PORT = 4500; -const BLOCK_DURATION_MS = 10_000; - -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gossip-')); - -jest.setTimeout(1000 * 60 * 10); - -const qosAlerts: AlertConfig[] = [ - { - alert: 'SequencerTimeToCollectAttestations', - expr: 'aztec_sequencer_time_to_collect_attestations > 3500', - labels: { severity: 'error' }, - for: '10m', - annotations: {}, - }, -]; - -// Tests gossip propagation using the real CLI validator-registration path (addL1Validator + ZkPassport mock) -// instead of the MultiAdder cheat shortcut used by applyBaseSetup. Uses P2PNetworkTest with -// SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES (ethSlot=4s, aztecSlot=24s, proofSubEpochs=640) and real libp2p. -// Distinct from gossip_network.test.ts specifically because of the validator-registration flow. -describe('e2e_p2p_network', () => { - let t: P2PNetworkTest; - let nodes: AztecNodeService[]; - - beforeEach(async () => { - t = await P2PNetworkTest.create({ - testName: 'e2e_p2p_network', - numberOfNodes: 0, - numberOfValidators: NUM_VALIDATORS, - basePort: BOOT_NODE_UDP_PORT, - metricsPort: shouldCollectMetrics(), - initialConfig: { - ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, - aztecSlotDuration: 24, - blockDurationMs: BLOCK_DURATION_MS, - listenAddress: '127.0.0.1', - // Allow empty blocks so the first checkpoint can be published before any txs are submitted. - // Without this, no blocks are built until txs arrive, and a failed checkpoint during tx - // submission causes block pruning that invalidates tx references. - minTxsPerBlock: 0, - inboxLag: 2, - }, - }); - - await t.setup(); - await t.addBootstrapNode(); - }); - - afterEach(async () => { - await t.stopNodes(nodes); - await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } - }); - - afterAll(async () => { - if (CHECK_ALERTS) { - const checker = new GrafanaClient(t.logger); - await checker.runAlertCheck(qosAlerts); - } - }); - - // Registers validators via the real addL1Validator CLI path (with ZkPassport mock proof), submits - // transactions to each node, and asserts all txs are mined and that attestation signers match - // the registered validator set. Validates the non-cheat registration flow end-to-end. - it('should rollup txs from all peers (and add the validators without cheating)', async () => { - // create the bootstrap node for the network - if (!t.bootstrapNodeEnr) { - throw new Error('Bootstrap node ENR is not available'); - } - - expect(t.ctx.deployL1ContractsValues.l1ContractAddresses.stakingAssetHandlerAddress).toBeDefined(); - - const { validators } = t.getValidators(); - - const rollupWrapper = RollupContract.getFromL1ContractsValues(t.ctx.deployL1ContractsValues); - - const rollup = getContract({ - address: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), - abi: RollupAbi, - client: t.ctx.deployL1ContractsValues.l1Client, - }); - - const zkPassportVerifier = getContract({ - address: t.ctx.deployL1ContractsValues.l1ContractAddresses.zkPassportVerifierAddress!.toString(), - abi: MockZKPassportVerifierAbi, - client: t.ctx.deployL1ContractsValues.l1Client, - }); - - expect((await rollupWrapper.getAttesters()).length).toBe(0); - - // Use the base account as the withdrawer for all validators in this test - const withdrawerAddress = EthAddress.fromString(t.baseAccount.address); - - // Add the validators to the rollup using the same function as the CLI - for (let i = 0; i < validators.length; i++) { - const validator = validators[i]; - const mockPassportProof = ZkPassportProofParams.random().toBuffer(); - await addL1Validator({ - rpcUrls: t.ctx.aztecNodeConfig.l1RpcUrls, - chainId: t.ctx.aztecNodeConfig.l1ChainId, - privateKey: t.baseAccountPrivateKey, - mnemonic: undefined, - attesterAddress: EthAddress.fromString(validator.attester.toString()), - withdrawerAddress, - stakingAssetHandlerAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.stakingAssetHandlerAddress!, - proofParams: mockPassportProof, - blsSecretKey: Fr.random().toBigInt(), - log: t.logger.info, - debugLogger: t.logger, - }); - - // mock nullifiers - increment the id in the mock zk passport verifier - t.logger.info('Incrementing unique identifier in mock zk passport verifier'); - await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ - hash: await zkPassportVerifier.write.incrementUniqueIdentifier(), - }); - } - - await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ - hash: await rollup.write.flushEntryQueue(), - }); - - const attestersImmedatelyAfterAdding = await rollupWrapper.getAttesters(); - expect(attestersImmedatelyAfterAdding.length).toBe(validators.length); - - // Check that the validators are added correctly - for (const validator of validators) { - const info = await rollupWrapper.getAttesterView(validator.attester.toString()); - expect(info.config.withdrawer.toChecksumString()).toBe(withdrawerAddress.toChecksumString()); - } - - // Wait for the validators to be added to the rollup - const timestamp = await t.ctx.cheatCodes.rollup.advanceToEpoch( - EpochNumber(t.ctx.aztecNodeConfig.lagInEpochsForValidatorSet + 1), - ); - - // Changes have now taken effect - const attesters = await rollupWrapper.getAttesters(); - expect(attesters.length).toBe(validators.length); - expect(attesters.length).toBe(NUM_VALIDATORS); - - // Send and await a tx to make sure we mine a block for the warp to correctly progress. - await t.ctx.deployL1ContractsValues.l1Client.waitForTransactionReceipt({ - hash: await t.ctx.deployL1ContractsValues.l1Client.sendTransaction({ - to: t.baseAccount.address, - value: 1n, - account: t.baseAccount, - }), - }); - - // Set the system time in the node, only after we have warped the time and waited for a block - // Time is only set in the NEXT block - t.ctx.dateProvider.setTime(Number(timestamp) * 1000); - - // create our network of nodes and submit txs into each of them - // the number of txs per node and the number of txs per rollup - // should be set so that the only way for rollups to be built - // is if the txs are successfully gossiped around the nodes. - const txsSentViaDifferentNodes: TxHash[][] = []; - t.logger.info('Creating nodes'); - nodes = await createNodes( - t.ctx.aztecNodeConfig, - t.ctx.dateProvider, - t.bootstrapNodeEnr, - NUM_VALIDATORS, - BOOT_NODE_UDP_PORT, - t.genesis, - DATA_DIR, - // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true - shouldCollectMetrics(), - ); - - // Wait for the gossipsub mesh to fully form before the committee starts producing. With - // skipInitialSequencer, the first blocks are built by this committee, and the first checkpoint - // must reach quorum (all 4 validators) to land on L1. If the proposal/checkpoint meshes are only - // partly formed, some committee members miss the first proposal, the first checkpoint stalls at - // 2/3, and every later slot rebuilds a competing un-checkpointed block 1 that peers reject as - // `block_number_already_exists` — a permanent 2/3 deadlock. Require a full mesh (N-1 peers per - // node) on the proposal/checkpoint topics so the first proposal reaches the whole committee. - await t.waitForP2PMeshConnectivity( - nodes, - NUM_VALIDATORS, - 60, - 0.5, - [TopicType.block_proposal, TopicType.checkpoint_proposal, TopicType.checkpoint_attestation], - NUM_VALIDATORS - 1, - ); - - // Wait for the first checkpoint to be published to L1 before submitting transactions. - // With skipInitialSequencer, no blocks exist from setup, so the first blocks are built by the - // validator committee. If we submit txs before a checkpoint lands on L1, a failed checkpoint - // publish can prune locally-proposed blocks, causing txs to reference pruned block headers. - t.logger.info('Waiting for first checkpoint to be published'); - await retryUntil( - async () => (await nodes[0].getBlockNumber('checkpointed')) > 0, - 'first checkpoint published', - 120, - ); - t.logger.info('First checkpoint published'); - - // We need to `createNodes` before we setup account, because - // those nodes actually form the committee, and so we cannot build - // blocks without them (since targetCommitteeSize is set to the number of nodes) - await t.setupAccount(); - - // Wait for the next L1 block so that all nodes' getCurrentMinFees() caches are - // refreshed after the first L2 checkpoint is published. Without this, some wallets - // may estimate fees based on pre-checkpoint values (very low due to fee decay), - // while receiving nodes already see the post-checkpoint fees (much higher). - const ethereumSlotDuration = t.ctx.aztecNodeConfig.ethereumSlotDuration ?? 4; - await sleep((ethereumSlotDuration + 1) * 1000); - - t.logger.info('Submitting transactions'); - for (const node of nodes) { - const txs = await submitTransactions(t.logger, node, NUM_TXS_PER_NODE, t.fundedAccount); - txsSentViaDifferentNodes.push(txs); - } - - t.logger.info('Waiting for transactions to be mined'); - // now ensure that all txs were successfully mined - await Promise.all( - txsSentViaDifferentNodes.flatMap((txs, i) => - txs.map((txHash, j) => { - t.logger.info(`Waiting for tx ${i}-${j}: ${txHash.toString()} to be mined`); - return waitForTx(nodes[0], txHash, { timeout: WAIT_FOR_TX_TIMEOUT }); - }), - ), - ); - t.logger.info('All transactions mined'); - - // Gather signers from attestations downloaded from L1 - const blockNumber = await nodes[0].getTxReceipt(txsSentViaDifferentNodes[0][0]).then(r => r.blockNumber!); - const dataStore = (nodes[0] as AztecNodeService).getBlockSource() as Archiver; - const blockData = await dataStore.getBlockData({ number: BlockNumber(blockNumber) }); - const [publishedCheckpoint] = await dataStore.getCheckpoints({ from: blockData!.checkpointNumber, limit: 1 }); - const signatureContext = { - chainId: t.ctx.aztecNodeConfig.l1ChainId, - rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, - }; - const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext); - const attestations = publishedCheckpoint.attestations - .filter(a => !a.signature.isEmpty()) - .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty())); - const signers = await Promise.all(attestations.map(att => att.getSender()!.toString())); - t.logger.info(`Attestation signers`, { signers }); - - // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right - const validatorAddresses = nodes.flatMap(node => - ((node as AztecNodeService).getSequencer() as SequencerClient).validatorAddresses?.map(v => v.toString()), - ); - t.logger.info(`Validator addresses`, { addresses: validatorAddresses }); - for (const signer of signers) { - expect(validatorAddresses).toContain(signer); - } - }); -}); diff --git a/yarn-project/end-to-end/src/p2p/late_prover_tx_collection.test.ts b/yarn-project/end-to-end/src/p2p/late_prover_tx_collection.test.ts index 3735a18cf930..0c46f4693c49 100644 --- a/yarn-project/end-to-end/src/p2p/late_prover_tx_collection.test.ts +++ b/yarn-project/end-to-end/src/p2p/late_prover_tx_collection.test.ts @@ -6,9 +6,6 @@ import { tryStop } from '@aztec/stdlib/interfaces/server'; import type { Tx } from '@aztec/stdlib/tx'; import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js'; @@ -27,8 +24,6 @@ const BOOT_NODE_UDP_PORT = process.env.BOOT_NODE_UDP_PORT ? parseInt(process.env const AZTEC_SLOT_DURATION = 12; const AZTEC_EPOCH_DURATION = 4; -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'late-prover-')); - jest.setTimeout(1000 * 60 * 10); // Tests the reqresp BLOCK_TXS path for a prover that joins after a block has already been mined. The @@ -69,10 +64,6 @@ describe('e2e_p2p_late_prover_tx_collection', () => { await tryStop(proverNode); await t.stopNodes(nodes); await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } - fs.rmSync(`${DATA_DIR}-late-prover`, { recursive: true, force: true, maxRetries: 3 }); }); // Mines a block with 2 txs via 4 validators, then starts a prover node late (after gossip has already @@ -92,7 +83,7 @@ describe('e2e_p2p_late_prover_tx_collection', () => { NUM_VALIDATORS, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('validator'), shouldCollectMetrics(), ); await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS); @@ -128,7 +119,7 @@ describe('e2e_p2p_late_prover_tx_collection', () => { ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, { dateProvider: t.ctx.dateProvider }, t.genesis, - `${DATA_DIR}-late-prover`, + t.dataDirFor('late-prover'), shouldCollectMetrics(), )); diff --git a/yarn-project/end-to-end/src/p2p/p2p_network.ts b/yarn-project/end-to-end/src/p2p/p2p_network.ts index e8d95d08901e..045ca4529360 100644 --- a/yarn-project/end-to-end/src/p2p/p2p_network.ts +++ b/yarn-project/end-to-end/src/p2p/p2p_network.ts @@ -25,7 +25,10 @@ import type { GenesisData } from '@aztec/stdlib/world-state'; import { ZkPassportProofParams } from '@aztec/stdlib/zkpassport'; import { getGenesisValues } from '@aztec/world-state/testing'; +import fs from 'fs'; import getPort from 'get-port'; +import os from 'os'; +import path from 'path'; import { type GetContractReturnType, getAddress, getContract } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; @@ -84,6 +87,9 @@ export class P2PNetworkTest { public bootstrapNode?: BootstrapNode; + // Root temp directory holding every node's data dir. Created in setup(), removed in teardown(). + public dataDir!: string; + // Store setup options for use in setup() private setupOptions: SetupOptions; private deployL1ContractsArgs: any; @@ -194,6 +200,19 @@ export class P2PNetworkTest { return this.hardcodedAccountData; } + /** + * Returns a per-role data directory nested under this test's root data dir. The multi-node factories + * (`createNodes`) append `-` to this base for each node they spawn; single-node factories + * (`createProverNode`, `createNonValidatorNode`, `createNode`) use it verbatim. Everything lives under + * `this.dataDir`, so a single recursive remove in teardown() cleans up all of it. + */ + dataDirFor(label: string): string { + if (!this.dataDir) { + throw new Error('Call setup to initialize the data directory.'); + } + return path.join(this.dataDir, label); + } + async addBootstrapNode() { this.logger.info('Adding bootstrap node'); const telemetry = await getEndToEndTestTelemetryClient(this.metricsPort); @@ -356,6 +375,8 @@ export class P2PNetworkTest { async setup() { this.logger.info('Setting up subsystems from fresh'); + this.dataDir = fs.mkdtempSync(path.join(os.tmpdir(), `${this.testName}-`)); + // Pre-compute hardcoded account data so it gets funded in genesis. const contract = new SchnorrHardcodedKeyAccountContract(); const secret = Fr.random(); @@ -488,6 +509,9 @@ export class P2PNetworkTest { await this.monitor.stop(); await tryStop(this.bootstrapNode, this.logger); await teardown(this.context); + if (this.dataDir) { + fs.rmSync(this.dataDir, { recursive: true, force: true, maxRetries: 3 }); + } } async getContracts(): Promise<{ diff --git a/yarn-project/end-to-end/src/p2p/preferred_gossip_network.test.ts b/yarn-project/end-to-end/src/p2p/preferred_gossip_network.test.ts index a3da5c55ff03..b2242e916abb 100644 --- a/yarn-project/end-to-end/src/p2p/preferred_gossip_network.test.ts +++ b/yarn-project/end-to-end/src/p2p/preferred_gossip_network.test.ts @@ -1,27 +1,21 @@ -import type { Archiver } from '@aztec/archiver'; import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; import { waitForTx } from '@aztec/aztec.js/node'; import { TxHash } from '@aztec/aztec.js/tx'; -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { Signature } from '@aztec/foundation/eth-signature'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { ENR, type P2PClient, type P2PService, type PeerId } from '@aztec/p2p'; -import type { SequencerClient } from '@aztec/sequencer-client'; -import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { createNodes } from '../fixtures/setup_p2p_test.js'; -import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js'; import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; -import { submitTransactions } from './shared.js'; - -const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; +import { + getPublishedCheckpointForTx, + maybeCheckQosAlerts, + submitTransactions, + verifyAttestationSigners, +} from './shared.js'; /** * This test builds a network using preferred nodes (supernodes) @@ -48,20 +42,8 @@ const NUM_TXS_PER_NODE = 2; const BOOT_NODE_UDP_PORT = 4500; const BLOCK_DURATION_MS = 10_000; -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gossip-')); - jest.setTimeout(1000 * 180 * 10); -const qosAlerts: AlertConfig[] = [ - { - alert: 'SequencerTimeToCollectAttestations', - expr: 'aztec_sequencer_time_to_collect_attestations > 3500', - labels: { severity: 'error' }, - for: '10m', - annotations: {}, - }, -]; - // Tests the preferred-node (supernode) topology: preferred nodes only accept validator connections; // a no-discovery validator connects exclusively through preferred nodes; gossip monitors assert that // traffic flows only through expected peers. Verifies txs mine and attestation signers match validators. @@ -165,16 +147,10 @@ describe('e2e_p2p_preferred_network', () => { afterEach(async () => { await t.stopNodes([t.ctx.aztecNodeService].concat(nodes).concat(validators).concat(preferredNodes)); await t.teardown(); - for (let i = 0; i < NUM_NODES + NUM_VALIDATORS + NUM_PREFERRED_NODES; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } }); afterAll(async () => { - if (CHECK_ALERTS) { - const checker = new GrafanaClient(t.logger); - await checker.runAlertCheck(qosAlerts); - } + await maybeCheckQosAlerts(t.logger); }); // Creates a 7-node topology (2 regular + 2 preferred + 2 validators + 1 no-discovery validator), @@ -210,7 +186,7 @@ describe('e2e_p2p_preferred_network', () => { NUM_PREFERRED_NODES, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('node'), // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true shouldCollectMetrics(), indexOffset, @@ -244,7 +220,7 @@ describe('e2e_p2p_preferred_network', () => { NUM_NODES, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('node'), // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true shouldCollectMetrics(), indexOffset, @@ -267,7 +243,7 @@ describe('e2e_p2p_preferred_network', () => { NUM_VALIDATORS - 1, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('node'), // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true shouldCollectMetrics(), indexOffset, @@ -291,7 +267,7 @@ describe('e2e_p2p_preferred_network', () => { 1, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('node'), // To collect metrics - run in aztec-packages `docker compose --profile metrics up` and set COLLECT_METRICS=true shouldCollectMetrics(), indexOffset, @@ -372,7 +348,7 @@ describe('e2e_p2p_preferred_network', () => { t.logger.info('Waiting for transactions to be mined'); // now ensure that all txs were successfully mined - const receipts = await Promise.all( + await Promise.all( txsSentViaDifferentNodes.flatMap((txs, i) => txs.map((txHash, j) => { t.logger.info(`Waiting for tx ${i}-${j}: ${txHash.toString()} to be mined`); @@ -382,31 +358,9 @@ describe('e2e_p2p_preferred_network', () => { ); t.logger.info('All transactions mined'); - // Gather signers from attestations downloaded from L1 - const blockNumber = receipts[0].blockNumber!; - const dataStore = (nodes[0] as AztecNodeService).getBlockSource() as Archiver; - const blockData = await dataStore.getBlockData({ number: BlockNumber(blockNumber) }); - const [publishedCheckpoint] = await dataStore.getCheckpoints({ from: blockData!.checkpointNumber, limit: 1 }); - const signatureContext = { - chainId: t.ctx.aztecNodeConfig.l1ChainId, - rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, - }; - const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext); - const attestations = publishedCheckpoint.attestations - .filter(a => !a.signature.isEmpty()) - .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty())); - const signers = await Promise.all(attestations.map(att => att.getSender()!.toString())); - t.logger.info(`Attestation signers`, { signers }); - + // Gather signers from attestations downloaded from L1 and check they belong to the validator set. + const publishedCheckpoint = await getPublishedCheckpointForTx(nodes[0], txsSentViaDifferentNodes[0][0]); + const signers = await verifyAttestationSigners(t, validators, publishedCheckpoint); expect(signers.length).toEqual(validators.length); - - // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right - const validatorAddresses = validators.flatMap(node => - ((node as AztecNodeService).getSequencer() as SequencerClient).validatorAddresses?.map(a => a.toString()), - ); - t.logger.info(`Validator addresses`, { addresses: validatorAddresses }); - for (const signer of signers) { - expect(validatorAddresses).toContain(signer); - } }); }); diff --git a/yarn-project/end-to-end/src/p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/p2p/rediscovery.test.ts index 3b526dd479d0..7eed53674dd9 100644 --- a/yarn-project/end-to-end/src/p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/p2p/rediscovery.test.ts @@ -2,10 +2,6 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import { waitForTx } from '@aztec/aztec.js/node'; import { TxHash } from '@aztec/aztec.js/tx'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; - import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { createNode, createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; @@ -17,8 +13,6 @@ const NUM_TXS_PER_NODE = 2; const BOOT_NODE_UDP_PORT = 4500; const BLOCK_DURATION_MS = 10_000; -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'rediscovery-')); - // Tests that nodes can rediscover each other from their stored peer tables after a full restart, // without a bootstrap node. Uses P2PNetworkTest real libp2p, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES // (ethSlot=4s, aztecSlot=24s, proofSubEpochs=640), 4 validators, inboxLag=2. @@ -50,9 +44,6 @@ describe('e2e_p2p_rediscovery', () => { t.logger.info('Stopping nodes and cleaning up data directories'); await t.stopNodes(nodes); await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } }); // Forms an initial 4-node mesh, stops the bootstrap node, then restarts each validator from its data @@ -67,7 +58,7 @@ describe('e2e_p2p_rediscovery', () => { NUM_VALIDATORS, BOOT_NODE_UDP_PORT, t.genesis, - DATA_DIR, + t.dataDirFor('validator'), // To collect metrics - run in aztec-packages `docker compose --profile metrics up` shouldCollectMetrics(), ); @@ -102,7 +93,7 @@ describe('e2e_p2p_rediscovery', () => { undefined, i, t.genesis, - `${DATA_DIR}-${i}`, + `${t.dataDirFor('validator')}-${i}`, ); t.logger.info(`Node ${i} restarted`); newNodes.push(newNode); diff --git a/yarn-project/end-to-end/src/p2p/reqresp/reqresp.test.ts b/yarn-project/end-to-end/src/p2p/reqresp/reqresp.test.ts index 010ec516f263..48dfcfa092b1 100644 --- a/yarn-project/end-to-end/src/p2p/reqresp/reqresp.test.ts +++ b/yarn-project/end-to-end/src/p2p/reqresp/reqresp.test.ts @@ -3,9 +3,7 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import { jest } from '@jest/globals'; import type { P2PNetworkTest } from '../p2p_network.js'; -import { cleanupReqrespTest, createReqrespDataDir, createReqrespTest, runReqrespTxTest } from './utils.js'; - -const DATA_DIR = createReqrespDataDir(); +import { cleanupReqrespTest, createReqrespTest, runReqrespTxTest } from './utils.js'; // Under pipelining a 36s aztec slot plus build-slot/target-slot round trip + L1 // publish exceeds the default 5 min jest test timeout. Allow 15 min. @@ -24,7 +22,7 @@ describe('e2e_p2p_reqresp_tx', () => { }); afterEach(async () => { - await cleanupReqrespTest({ t, nodes, dataDir: DATA_DIR }); + await cleanupReqrespTest({ t, nodes }); }); it('should produce an attestation by requesting tx data over the p2p network', async () => { @@ -42,6 +40,6 @@ describe('e2e_p2p_reqresp_tx', () => { * * Delegates to runReqrespTxTest in utils.ts; see that helper for the full flow. */ - nodes = await runReqrespTxTest({ t, dataDir: DATA_DIR }); + nodes = await runReqrespTxTest({ t }); }); }); diff --git a/yarn-project/end-to-end/src/p2p/reqresp/utils.ts b/yarn-project/end-to-end/src/p2p/reqresp/utils.ts index 19e06a4fc14d..f26d6f183a11 100644 --- a/yarn-project/end-to-end/src/p2p/reqresp/utils.ts +++ b/yarn-project/end-to-end/src/p2p/reqresp/utils.ts @@ -9,9 +9,6 @@ import { timesAsync } from '@aztec/foundation/collection'; import { retryUntil } from '@aztec/foundation/retry'; import { expect, jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; import { getBootNodeUdpPort, shouldCollectMetrics } from '../../fixtures/fixtures.js'; import { createNodes } from '../../fixtures/setup_p2p_test.js'; @@ -23,8 +20,6 @@ export const NUM_VALIDATORS = 6; export const NUM_TXS_PER_NODE = 4; export const BOOT_NODE_UDP_PORT = getBootNodeUdpPort(); -export const createReqrespDataDir = () => fs.mkdtempSync(path.join(os.tmpdir(), 'reqresp-')); - type ReqrespOptions = { disableStatusHandshake?: boolean; }; @@ -58,25 +53,21 @@ export async function createReqrespTest(options: ReqrespOptions = {}): Promise

BOOT_NODE_UDP_PORT + 1 + nodeIndex; export async function runReqrespTxTest(params: { t: P2PNetworkTest; - dataDir: string; disableStatusHandshake?: boolean; }): Promise { - const { t, dataDir, disableStatusHandshake = false } = params; + const { t, disableStatusHandshake = false } = params; if (!t.bootstrapNodeEnr) { throw new Error('Bootstrap node ENR is not available'); @@ -94,7 +85,7 @@ export async function runReqrespTxTest(params: { NUM_VALIDATORS, BOOT_NODE_UDP_PORT, t.genesis, - dataDir, + t.dataDirFor('validator'), shouldCollectMetrics(), ); diff --git a/yarn-project/end-to-end/src/p2p/shared.ts b/yarn-project/end-to-end/src/p2p/shared.ts index 9f0415817753..fe17f29d6c3b 100644 --- a/yarn-project/end-to-end/src/p2p/shared.ts +++ b/yarn-project/end-to-end/src/p2p/shared.ts @@ -1,19 +1,35 @@ import type { InitialAccountData } from '@aztec/accounts/testing'; +import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import type { AztecAddress } from '@aztec/aztec.js/addresses'; import { NO_WAIT, getContractInstanceFromInstantiationParams } from '@aztec/aztec.js/contracts'; import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { TxHash } from '@aztec/aztec.js/tx'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; +import { Signature } from '@aztec/foundation/eth-signature'; +import { retryUntil } from '@aztec/foundation/retry'; import type { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { TestContract, TestContractArtifact } from '@aztec/noir-test-contracts.js/Test'; import { getPXEConfig, getPXEConfig as getRpcConfig } from '@aztec/pxe/server'; +import type { SequencerClient } from '@aztec/sequencer-client'; +import { CheckpointAttestation, ConsensusPayload, type TopicType } from '@aztec/stdlib/p2p'; +import { expect } from '@jest/globals'; + +import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { SchnorrHardcodedKeyAccountContract } from '../fixtures/schnorr_hardcoded_account_contract.js'; +import { createNodes } from '../fixtures/setup_p2p_test.js'; +import { waitForTxs } from '../fixtures/wait_helpers.js'; +import { type AlertConfig, GrafanaClient } from '../quality_of_service/grafana_client.js'; import { submitTxsTo } from '../shared/submit-transactions.js'; import { TestWallet } from '../test-wallet/test_wallet.js'; import { type ProvenTx, proveInteraction } from '../test-wallet/utils.js'; +import { type P2PNetworkTest, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; + +/** One published checkpoint as returned by the archiver, including its attestations. */ +type PublishedCheckpoint = Awaited>[number]; // submits a set of transactions to the provided Private eXecution Environment (PXE) export const submitComplexTxsTo = async ( @@ -96,3 +112,218 @@ export async function prepareTransactions( return tx; }); } + +const CHECK_ALERTS = process.env.CHECK_ALERTS === 'true'; + +const qosAlerts: AlertConfig[] = [ + { + alert: 'SequencerTimeToCollectAttestations', + expr: 'aztec_sequencer_time_to_collect_attestations > 3500', + labels: { severity: 'error' }, + for: '10m', + annotations: {}, + }, +]; + +/** Runs the shared p2p QoS Grafana alert check when CHECK_ALERTS=true; a no-op otherwise. */ +export async function maybeCheckQosAlerts(logger: Logger): Promise { + if (CHECK_ALERTS) { + const checker = new GrafanaClient(logger); + await checker.runAlertCheck(qosAlerts); + } +} + +/** Waits until every node's synced block number reaches the initial node's current tip. */ +export async function waitForNodesToSync(t: P2PNetworkTest, nodes: AztecNodeService[]): Promise { + const targetBlock = await t.ctx.aztecNode.getBlockNumber(); + t.logger.warn(`Waiting for all nodes to sync to block number ${targetBlock}`); + await retryUntil( + async () => { + const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber())); + const checkpointNumber = (await t.monitor.run()).checkpointNumber; + t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`); + return blockNumbers.every(bn => bn >= targetBlock); + }, + `nodes to sync to block number ${targetBlock}`, + 30, + 0.5, + ); +} + +/** Reads the published checkpoint containing the block that mined the given tx. */ +export async function getPublishedCheckpointForTx( + node: AztecNodeService, + txHash: TxHash, +): Promise { + const receipt = await node.getTxReceipt(txHash); + const blockNumber = receipt.blockNumber!; + const dataStore = node.getBlockSource() as Archiver; + const blockData = await dataStore.getBlockData({ number: BlockNumber(blockNumber) }); + const [publishedCheckpoint] = await dataStore.getCheckpoints({ from: blockData!.checkpointNumber, limit: 1 }); + return publishedCheckpoint; +} + +/** Polls until the archiver has indexed the first published checkpoint, then returns it. */ +export async function waitForFirstPublishedCheckpoint( + t: P2PNetworkTest, + nodes: AztecNodeService[], + timeoutSeconds = 120, +): Promise { + const dataStore = nodes[0].getBlockSource() as Archiver; + t.logger.warn('Waiting for first checkpoint to be published and indexed by the archiver'); + return await retryUntil( + async () => { + const blockNumbers = await Promise.all(nodes.map(node => node.getBlockNumber())); + const checkpointNumber = (await t.monitor.run()).checkpointNumber; + t.logger.info(`Current block numbers ${blockNumbers} (checkpoint number on L1 is ${checkpointNumber})`); + const [checkpoint] = await dataStore.getCheckpoints({ from: CheckpointNumber(1), limit: 1 }); + return checkpoint; + }, + 'published checkpoint to be indexed', + timeoutSeconds, + 1, + ); +} + +/** + * Recovers the attestation signers from a published checkpoint and asserts each one belongs to the + * validator set formed by the given nodes. Returns the recovered signer addresses so callers can add + * scenario-specific assertions (e.g. an exact signer count). + */ +export async function verifyAttestationSigners( + t: P2PNetworkTest, + nodes: AztecNodeService[], + publishedCheckpoint: PublishedCheckpoint, +): Promise { + const signatureContext = { + chainId: t.ctx.aztecNodeConfig.l1ChainId, + rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, + }; + const payload = ConsensusPayload.fromCheckpoint(publishedCheckpoint.checkpoint, signatureContext); + const attestations = publishedCheckpoint.attestations + .filter(a => !a.signature.isEmpty()) + .map(a => new CheckpointAttestation(payload, a.signature, Signature.empty())); + const signers = await Promise.all(attestations.map(att => att.getSender()!.toString())); + t.logger.info(`Attestation signers`, { signers }); + + // Check that the signers found are part of the proposer nodes to ensure the archiver fetched them right + const validatorAddresses = nodes.flatMap(node => + (node.getSequencer() as SequencerClient).validatorAddresses?.map(a => a.toString()), + ); + t.logger.info(`Validator addresses`, { addresses: validatorAddresses }); + for (const signer of signers) { + expect(validatorAddresses).toContain(signer); + } + return signers; +} + +/** Options that parameterize {@link runGossipScenario}. */ +export interface GossipScenarioOptions { + /** The initialized P2P network test (setup() + validator registration already done). */ + t: P2PNetworkTest; + /** Number of validator nodes forming the committee. */ + numValidators: number; + /** Base UDP port for the validator nodes. */ + bootNodePort: number; + /** Transactions submitted per validator node; 0 skips tx submission entirely (e.g. the oracle test). */ + txsPerNode: number; + /** Submit tx batches sequentially instead of concurrently. */ + submitSequentially?: boolean; + /** Overrides for the P2P mesh-connectivity wait; unset fields fall back to waitForP2PMeshConnectivity defaults. */ + mesh?: { + expectedNodeCount?: number; + timeoutSeconds?: number; + checkIntervalSeconds?: number; + topics?: TopicType[]; + minMeshPeerCount?: number; + }; + /** Which published checkpoint the attestation signers are read from (defaults to the first tx's block). */ + checkpointSource?: 'first-tx' | 'first-published'; + /** Runs after validator registration but before the validator nodes are created. */ + beforeCreateNodes?: () => Promise; + /** Creates extra non-validator nodes (prover/monitor) once the validator nodes exist. */ + createExtraNodes?: (nodes: AztecNodeService[]) => Promise; + /** Runs after the account is registered but before txs are submitted (sync waits, checkpoint waits, sleeps). */ + beforeSubmit?: (nodes: AztecNodeService[]) => Promise; + /** Scenario-specific verification run after attestation-signer verification (proven block, price convergence). */ + afterVerify?: (nodes: AztecNodeService[]) => Promise; +} + +/** + * Shared skeleton for the p2p gossip tests: create the validator nodes and any extra nodes, wait for + * the mesh to form, register the account, optionally submit and mine txs, then verify the attestation + * signers of a published checkpoint. Each varying part (validator registration, extra nodes, pre-submit + * waits, scenario-specific verification) is supplied via the callbacks in {@link GossipScenarioOptions}. + * Returns the validator nodes so the caller can track them for teardown. + */ +export async function runGossipScenario(opts: GossipScenarioOptions): Promise { + const { t, numValidators, bootNodePort, txsPerNode } = opts; + + if (!t.bootstrapNodeEnr) { + throw new Error('Bootstrap node ENR is not available'); + } + + await opts.beforeCreateNodes?.(); + + t.logger.info('Creating validator nodes'); + const nodes = await createNodes( + t.ctx.aztecNodeConfig, + t.ctx.dateProvider, + t.bootstrapNodeEnr, + numValidators, + bootNodePort, + t.genesis, + t.dataDirFor('validator'), + shouldCollectMetrics(), + ); + + await opts.createExtraNodes?.(nodes); + + t.logger.info('Waiting for nodes to connect'); + await t.waitForP2PMeshConnectivity( + nodes, + opts.mesh?.expectedNodeCount ?? numValidators, + opts.mesh?.timeoutSeconds, + opts.mesh?.checkIntervalSeconds, + opts.mesh?.topics, + opts.mesh?.minMeshPeerCount, + ); + + // We need to create the nodes before we setup the account, because those nodes form the committee + // and blocks cannot be built without them (targetCommitteeSize is set to the number of nodes). + await t.setupAccount(); + + await opts.beforeSubmit?.(nodes); + + let firstTxHash: TxHash | undefined; + if (txsPerNode > 0) { + t.logger.info('Submitting transactions'); + const submitOne = (node: AztecNodeService) => submitTransactions(t.logger, node, txsPerNode, t.fundedAccount); + // Each submitTransactions call builds its own wallet/PXE, so submissions are independent. When run + // concurrently, Promise.all preserves node order so submitted[i] stays aligned with nodes[i]. + const submitted: TxHash[][] = []; + if (opts.submitSequentially) { + for (const node of nodes) { + submitted.push(await submitOne(node)); + } + } else { + submitted.push(...(await Promise.all(nodes.map(submitOne)))); + } + firstTxHash = submitted[0][0]; + + t.logger.info('Waiting for transactions to be mined'); + await waitForTxs(nodes[0], submitted.flat(), { timeout: WAIT_FOR_TX_TIMEOUT }); + t.logger.info('All transactions mined'); + } + + const publishedCheckpoint = + opts.checkpointSource === 'first-published' + ? await waitForFirstPublishedCheckpoint(t, nodes) + : await getPublishedCheckpointForTx(nodes[0], firstTxHash!); + + await verifyAttestationSigners(t, nodes, publishedCheckpoint); + + await opts.afterVerify?.(nodes); + + return nodes; +}