diff --git a/Cargo.toml b/Cargo.toml index f9ba495..df4feb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/mempool", "crates/rpc", "crates/metrics", + "crates/sync", ] resolver = "2" @@ -148,9 +149,11 @@ informalsystems-malachitebft-metrics = { version = "0.5", default-features = fal # when sync requests fail, enabling retries instead of silent timeouts # - sync: Adds SkipToHeight effect to allow nodes to skip ahead when peers # have pruned old blocks and cannot serve the required heights +# - app: Updated to use new sync config fields (tip_first_sync, tip_first_buffer) [patch.crates-io] informalsystems-malachitebft-network = { path = "vendor/malachitebft-network" } informalsystems-malachitebft-sync = { path = "vendor/malachitebft-sync" } +informalsystems-malachitebft-app = { path = "vendor/malachitebft-app" } [profile.release] opt-level = 3 diff --git a/crates/execution/src/engine.rs b/crates/execution/src/engine.rs index 6be748a..993ba95 100644 --- a/crates/execution/src/engine.rs +++ b/crates/execution/src/engine.rs @@ -329,6 +329,14 @@ impl ExecutionEngine

{ self.database.provider() } + /// Get the current block number. + /// + /// Returns the block number of the most recently executed block, or 0 if + /// no blocks have been executed yet. + pub fn current_block_number(&self) -> u64 { + self.current_block + } + /// Process all transactions in a block. /// /// Returns a tuple containing receipts, cumulative gas used, logs, and total fees. diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 336418b..7019a2a 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -21,6 +21,7 @@ cipherbft-consensus = { path = "../consensus", features = ["malachite"] } cipherbft-rpc = { path = "../rpc" } cipherbft-mempool = { path = "../mempool" } cipherbft-metrics = { path = "../metrics" } +cipherbft-sync = { path = "../sync" } # Ethereum primitives (for genesis types in tests and transaction parsing) alloy-primitives = { version = "1", features = ["serde"] } diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index ff35df3..b43e0ca 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -77,6 +77,38 @@ pub const DEFAULT_RPC_WS_PORT: u16 = 8546; /// Default metrics port (Prometheus standard) pub const DEFAULT_METRICS_PORT: u16 = 9100; +/// Default snap sync enabled setting +pub const DEFAULT_SNAP_SYNC_ENABLED: bool = true; + +/// Default minimum peers required to start sync +pub const DEFAULT_MIN_SYNC_PEERS: usize = 3; + +/// Default sync request timeout in seconds +pub const DEFAULT_SYNC_TIMEOUT_SECS: u64 = 30; + +/// Default block gap threshold to trigger snap sync (vs block-by-block) +pub const DEFAULT_SNAP_SYNC_THRESHOLD: u64 = 1024; + +/// Serde default function for snap_sync_enabled +fn default_snap_sync_enabled() -> bool { + DEFAULT_SNAP_SYNC_ENABLED +} + +/// Serde default function for min_sync_peers +fn default_min_sync_peers() -> usize { + DEFAULT_MIN_SYNC_PEERS +} + +/// Serde default function for sync_timeout +fn default_sync_timeout() -> u64 { + DEFAULT_SYNC_TIMEOUT_SECS +} + +/// Serde default function for snap_sync_threshold +fn default_snap_sync_threshold() -> u64 { + DEFAULT_SNAP_SYNC_THRESHOLD +} + /// Serde default function for rpc_http_port fn default_rpc_http_port() -> u16 { DEFAULT_RPC_HTTP_PORT @@ -109,6 +141,52 @@ pub struct PeerConfig { pub worker_addrs: Vec, } +/// Sync configuration for state synchronization +/// +/// Controls how the node performs snap sync when joining the network +/// or recovering from being behind. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncConfig { + /// Enable snap sync for fast bootstrap. + /// + /// When enabled, nodes that are significantly behind will use snap sync + /// to download state directly rather than replaying all blocks. + #[serde(default = "default_snap_sync_enabled")] + pub snap_sync_enabled: bool, + + /// Minimum peers required to start sync. + /// + /// The node will wait until it has at least this many peers with + /// consistent state before beginning snap sync. + #[serde(default = "default_min_sync_peers")] + pub min_sync_peers: usize, + + /// Sync request timeout in seconds. + /// + /// Maximum time to wait for a response to a sync request before + /// retrying with a different peer. + #[serde(default = "default_sync_timeout")] + pub sync_timeout_secs: u64, + + /// Block gap threshold to trigger snap sync (vs block-by-block). + /// + /// If the node is behind by more than this many blocks, it will + /// use snap sync. Otherwise, it will sync block-by-block. + #[serde(default = "default_snap_sync_threshold")] + pub snap_sync_threshold: u64, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + snap_sync_enabled: DEFAULT_SNAP_SYNC_ENABLED, + min_sync_peers: DEFAULT_MIN_SYNC_PEERS, + sync_timeout_secs: DEFAULT_SYNC_TIMEOUT_SECS, + snap_sync_threshold: DEFAULT_SNAP_SYNC_THRESHOLD, + } + } +} + /// Node configuration /// /// Keys are loaded from the keyring backend specified by `keyring_backend`. @@ -199,6 +277,13 @@ pub struct NodeConfig { /// Port for Prometheus metrics endpoint (default: 9100) #[serde(default = "default_metrics_port")] pub metrics_port: u16, + + /// Sync configuration for state synchronization. + /// + /// Controls snap sync behavior for fast bootstrap when joining + /// the network or recovering from being behind. + #[serde(default)] + pub sync: SyncConfig, } /// Test configuration with keypairs for local testing @@ -253,6 +338,7 @@ impl NodeConfig { rpc_http_port: DEFAULT_RPC_HTTP_PORT + (index as u16), rpc_ws_port: DEFAULT_RPC_WS_PORT + (index as u16), metrics_port: DEFAULT_METRICS_PORT + (index as u16), + sync: SyncConfig::default(), }; LocalTestConfig { diff --git a/crates/node/src/execution_bridge.rs b/crates/node/src/execution_bridge.rs index f6d8cca..57cb54e 100644 --- a/crates/node/src/execution_bridge.rs +++ b/crates/node/src/execution_bridge.rs @@ -631,6 +631,86 @@ impl ExecutionBridge { let execution = self.execution.read().await; Arc::clone(execution.staking_precompile()) } + + /// Get the last executed block hash. + /// + /// Returns the hash of the most recently executed block, or B256::ZERO + /// if no blocks have been executed yet. + pub fn last_block_hash(&self) -> B256 { + self.last_block_hash + .read() + .map(|guard| *guard) + .unwrap_or(B256::ZERO) + } + + /// Get the current block number. + /// + /// Returns the block number of the most recently executed block. + /// This requires acquiring the execution lock. + pub async fn current_block_number(&self) -> u64 { + let execution = self.execution.read().await; + execution.current_block_number() + } + + /// Execute a block directly from BlockInput (for sync replay). + /// + /// This method is used during sync to replay blocks that have been + /// downloaded from peers. It bypasses the Cut conversion since the + /// transactions are already flattened. + /// + /// # Arguments + /// + /// * `input` - Block input containing ordered transactions + /// + /// # Returns + /// + /// Returns `BlockExecutionResult` containing execution result with properly + /// computed block hash and parent hash. + pub async fn execute_block_input( + &self, + input: BlockInput, + ) -> anyhow::Result { + let block_number = input.block_number; + let timestamp = input.timestamp; + let parent_hash = input.parent_hash; + let transactions = input.transactions.clone(); + + let mut execution = self.execution.write().await; + + let result = execution + .execute_block(input) + .map_err(|e| anyhow::anyhow!("Block execution failed: {}", e))?; + + // Compute and store the new block hash for the next block's parent_hash + let new_block_hash = compute_block_hash( + block_number, + timestamp, + parent_hash, + result.state_root, + result.transactions_root, + result.receipts_root, + ); + + // Update the last block hash for the next execution + if let Ok(mut guard) = self.last_block_hash.write() { + *guard = new_block_hash; + } + + debug!( + height = block_number, + block_hash = %new_block_hash, + parent_hash = %parent_hash, + "Sync block hash updated" + ); + + Ok(BlockExecutionResult { + execution_result: result, + block_hash: new_block_hash, + parent_hash, + timestamp, + executed_transactions: transactions, + }) + } } /// Compute a deterministic block hash from block components. diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 0b13430..9c6c6f1 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -15,15 +15,20 @@ pub mod network; pub mod network_api; pub mod node; pub mod supervisor; +pub mod sync_executor; +pub mod sync_network; +pub mod sync_runner; +pub mod sync_server; pub mod util; pub mod worker_pool_adapter; pub use client_config::ClientConfig; pub use config::{ - generate_keypair, generate_local_configs, LocalTestConfig, NodeConfig, PeerConfig, + generate_keypair, generate_local_configs, LocalTestConfig, NodeConfig, PeerConfig, SyncConfig, CIPHERD_GENESIS_PATH_ENV, CIPHERD_HOME_ENV, DEFAULT_GENESIS_FILENAME, DEFAULT_HOME_DIR, DEFAULT_KEYRING_BACKEND, DEFAULT_KEYS_DIR, DEFAULT_KEY_NAME, DEFAULT_METRICS_PORT, - DEFAULT_RPC_HTTP_PORT, DEFAULT_RPC_WS_PORT, + DEFAULT_MIN_SYNC_PEERS, DEFAULT_RPC_HTTP_PORT, DEFAULT_RPC_WS_PORT, DEFAULT_SNAP_SYNC_ENABLED, + DEFAULT_SNAP_SYNC_THRESHOLD, DEFAULT_SYNC_TIMEOUT_SECS, }; pub use execution_bridge::ExecutionBridge; pub use execution_sync::{ExecutionSyncConfig, ExecutionSyncTracker, SyncAction}; @@ -36,4 +41,8 @@ pub use mempool_state::ExecutionStateValidator; pub use network_api::{NodeNetworkApi, TcpNetworkApi}; pub use node::Node; pub use supervisor::{NodeSupervisor, ShutdownError}; +pub use sync_executor::ExecutionBridgeSyncExecutor; +pub use sync_network::{create_sync_adapter, wire_sync_to_network, SyncNetworkAdapter}; +pub use sync_runner::{create_sync_manager, run_snap_sync, should_snap_sync, SyncResult}; +pub use sync_server::SnapSyncServer; pub use worker_pool_adapter::WorkerPoolAdapter; diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 2989611..e6a82f3 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -560,6 +560,7 @@ fn cmd_init( rpc_http_port: cipherd::DEFAULT_RPC_HTTP_PORT, rpc_ws_port: cipherd::DEFAULT_RPC_WS_PORT, metrics_port: cipherd::DEFAULT_METRICS_PORT, + sync: cipherd::SyncConfig::default(), }; let config_path = config_dir.join("node.json"); @@ -975,6 +976,7 @@ fn cmd_testnet_init_files( rpc_http_port: cipherd::DEFAULT_RPC_HTTP_PORT + (i as u16 * 10), rpc_ws_port: cipherd::DEFAULT_RPC_WS_PORT + (i as u16 * 10), metrics_port: cipherd::DEFAULT_METRICS_PORT + (i as u16 * 10), + sync: cipherd::SyncConfig::default(), }; let config_path = config_dir.join("node.json"); diff --git a/crates/node/src/network.rs b/crates/node/src/network.rs index cc450ff..74830eb 100644 --- a/crates/node/src/network.rs +++ b/crates/node/src/network.rs @@ -38,6 +38,7 @@ use cipherbft_metrics::network::{ P2P_BYTES_RECEIVED, P2P_BYTES_SENT, P2P_CONNECTION_ERRORS, P2P_MESSAGES_RECEIVED, P2P_MESSAGES_SENT, P2P_PEERS_CONNECTED, P2P_PEERS_INBOUND, P2P_PEERS_OUTBOUND, }; +use cipherbft_sync::{OutgoingSnapMessage, SnapSyncMessage, SyncNetworkSender}; use cipherbft_types::ValidatorId; use std::collections::HashMap; use std::net::SocketAddr; @@ -58,6 +59,19 @@ pub enum NetworkMessage { Dcl(Box), /// Worker message (for Worker) Worker(WorkerMessage), + /// Snap sync message (for state synchronization) + SnapSync(SnapSyncMessage), +} + +impl NetworkMessage { + /// Get message type name for logging + pub fn type_name(&self) -> &'static str { + match self { + NetworkMessage::Dcl(msg) => msg.type_name(), + NetworkMessage::Worker(msg) => msg.type_name(), + NetworkMessage::SnapSync(msg) => msg.message_type(), + } + } } /// TCP-based Primary network implementation @@ -71,6 +85,9 @@ pub struct TcpPrimaryNetwork { peer_addrs: HashMap, /// Incoming message channel incoming_tx: mpsc::Sender<(ValidatorId, DclMessage)>, + /// Snap sync network sender for forwarding sync messages + /// Uses RwLock for interior mutability (can be set after Arc creation) + snap_sync_sender: RwLock>, /// Whether the network listener is active listening: AtomicBool, } @@ -101,10 +118,19 @@ impl TcpPrimaryNetwork { peers: Arc::new(RwLock::new(HashMap::new())), peer_addrs, incoming_tx, + snap_sync_sender: RwLock::new(None), listening: AtomicBool::new(false), } } + /// Set the snap sync sender for forwarding incoming sync messages + /// + /// This method uses interior mutability via RwLock, so it can be called + /// even after the network is wrapped in Arc. + pub async fn set_snap_sync_sender(&self, sender: SyncNetworkSender) { + *self.snap_sync_sender.write().await = Some(sender); + } + /// Returns whether the network listener is active. pub fn is_listening(&self) -> bool { self.listening.load(Ordering::Acquire) @@ -245,8 +271,20 @@ impl TcpPrimaryNetwork { break; } } - Ok(_) => { - warn!("Received non-DCL message on Primary connection"); + Ok(NetworkMessage::SnapSync(snap_msg)) => { + debug!("Received snap sync message: {}", snap_msg.message_type()); + // Forward to sync manager via channel + let sender_guard = self.snap_sync_sender.read().await; + if let Some(ref sender) = *sender_guard { + // For incoming connections, we don't have a validator ID yet + // Use "unknown" as peer ID - the sync manager can track peers by socket + sender + .forward_message("unknown".to_string(), snap_msg) + .await; + } + } + Ok(NetworkMessage::Worker(_)) => { + warn!("Received Worker message on Primary connection"); } Err(e) => { error!("Failed to deserialize message: {}", e); @@ -295,8 +333,13 @@ impl TcpPrimaryNetwork { // This is critical: without this, responses (e.g., attestations) sent back on // this connection would never be read, causing attestation timeouts. let incoming_tx = self.incoming_tx.clone(); + let snap_sender = self.snap_sync_sender.read().await.clone(); + let peer_id = format!("{:?}", validator_id); tokio::spawn(async move { - if let Err(e) = Self::handle_outgoing_connection_reader(reader, incoming_tx).await { + if let Err(e) = + Self::handle_outgoing_connection_reader(reader, incoming_tx, peer_id, snap_sender) + .await + { debug!("Outgoing connection reader ended: {}", e); } // Connection ended - decrement metrics @@ -316,6 +359,8 @@ impl TcpPrimaryNetwork { async fn handle_outgoing_connection_reader( mut reader: tokio::io::ReadHalf, incoming_tx: mpsc::Sender<(ValidatorId, DclMessage)>, + peer_id: String, + snap_sync_sender: Option, ) -> Result<()> { let mut buf = BytesMut::with_capacity(4096); @@ -358,8 +403,18 @@ impl TcpPrimaryNetwork { break; } } - Ok(_) => { - warn!("Received non-DCL message on outgoing connection"); + Ok(NetworkMessage::SnapSync(snap_msg)) => { + debug!( + "Received snap sync message on outgoing connection: {}", + snap_msg.message_type() + ); + // Forward to sync manager via channel + if let Some(ref sender) = snap_sync_sender { + sender.forward_message(peer_id.clone(), snap_msg).await; + } + } + Ok(NetworkMessage::Worker(_)) => { + warn!("Received Worker message on outgoing connection"); } Err(e) => { error!( @@ -424,6 +479,7 @@ impl TcpPrimaryNetwork { _ => "other", }, NetworkMessage::Worker(_) => "worker", + NetworkMessage::SnapSync(_) => "snap_sync", }; // Serialize message AFTER releasing peers lock @@ -454,6 +510,70 @@ impl TcpPrimaryNetwork { } } } + + /// Send a snap sync message to a specific peer by validator ID + pub async fn send_snap_sync(&self, validator_id: ValidatorId, message: SnapSyncMessage) { + let msg = NetworkMessage::SnapSync(message); + if let Err(e) = self.send_to(validator_id, &msg).await { + warn!("Failed to send snap sync to {:?}: {}", validator_id, e); + } + } + + /// Broadcast a snap sync message to all peers + pub async fn broadcast_snap_sync(&self, message: SnapSyncMessage) { + let msg = NetworkMessage::SnapSync(message); + self.broadcast(&msg).await; + } + + /// Start processing outgoing snap sync messages from the adapter + /// + /// This spawns a background task that reads from the outgoing message channel + /// and sends messages via the TCP network. + pub fn start_snap_sync_sender( + self: Arc, + mut outgoing_rx: mpsc::Receiver, + ) { + tokio::spawn(async move { + while let Some(outgoing) = outgoing_rx.recv().await { + match outgoing.target_peer { + Some(peer_id) => { + // Parse peer_id string to extract validator ID + // Format is typically "ValidatorId(0x...)" from Debug + if let Some(vid) = Self::parse_peer_id(&peer_id) { + self.send_snap_sync(vid, outgoing.message).await; + } else { + warn!("Could not parse peer ID: {}", peer_id); + } + } + None => { + // Broadcast to all peers + self.broadcast_snap_sync(outgoing.message).await; + } + } + } + info!("Snap sync sender task ended"); + }); + } + + /// Parse a peer ID string back to ValidatorId + /// + /// Handles both raw hex strings and "ValidatorId(0x...)" format + pub fn parse_peer_id(peer_id: &str) -> Option { + // Try to extract hex from "ValidatorId(0x...)" format + let hex_str = if let Some(start) = peer_id.find("0x") { + let end = peer_id[start..].find(')').unwrap_or(peer_id.len() - start); + &peer_id[start + 2..start + end] + } else if let Some(stripped) = peer_id.strip_prefix("0x") { + stripped + } else { + peer_id + }; + + // Trim any trailing characters + let hex_str = hex_str.trim_end_matches(|c: char| !c.is_ascii_hexdigit()); + + parse_validator_id(hex_str).ok() + } } #[async_trait] diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 6c6c4ff..68e344b 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -20,6 +20,7 @@ use crate::execution_bridge::{BlockExecutionResult, ExecutionBridge}; use crate::execution_sync::{ExecutionSyncConfig, ExecutionSyncTracker, SyncAction}; use crate::network::{TcpPrimaryNetwork, TcpWorkerNetwork}; use crate::supervisor::NodeSupervisor; +use crate::sync_network::{create_sync_adapter, wire_sync_to_network}; use alloy_primitives::{Address, B256}; use anyhow::{Context, Result}; use cipherbft_consensus::{ @@ -548,6 +549,25 @@ impl Node { // Store reference for RPC NetworkApi before moving into adapter primary_network_opt = Some(Arc::clone(&primary_network)); + // Wire up snap sync network adapter + // This enables the sync manager to communicate with peers via the TCP network + // + // TODO(snap-sync): Integrate with actual snap sync triggering logic + // Current state: + // - Network wiring is set up to forward snap sync messages + // - The sync_adapter is created but not yet connected to StateSyncManager + // - When implementing snap sync trigger (e.g., detecting node is behind), + // create StateSyncManager and use this adapter with run_snap_sync() + // - Consider storing sync_adapter in Node struct for on-demand sync + let (_sync_adapter, network_to_sync_tx, sync_to_network_rx) = create_sync_adapter(); + wire_sync_to_network( + &primary_network, + Arc::clone(&primary_network), + network_to_sync_tx, + sync_to_network_rx, + ) + .await; + // Start primary listener Arc::clone(&primary_network) .start_listener(self.config.primary_listen) @@ -1065,9 +1085,12 @@ impl Node { // Spawn Sync actor for state synchronization // Use higher parallelism to catch up faster when significantly behind + // Note: Blocks must be applied sequentially, but having more in-flight + // reduces wait time between blocks and saturates network bandwidth + // Note: tip_first_sync disabled - requires consensus to support checkpoint-based sync let sync_config = SyncConfig::new(true) - .with_parallel_requests(20) // Increased from default 5 for faster catch-up - .with_request_timeout(Duration::from_secs(30)); // Longer timeout for slower networks + .with_parallel_requests(50) // High parallelism ensures blocks ready when needed + .with_request_timeout(Duration::from_secs(5)); // Fail fast, try other peers quickly let sync = spawn_sync(ctx.clone(), network.clone(), host.clone(), sync_config).await?; // Build and spawn Consensus engine with sync support diff --git a/crates/node/src/sync_executor.rs b/crates/node/src/sync_executor.rs new file mode 100644 index 0000000..c39da9c --- /dev/null +++ b/crates/node/src/sync_executor.rs @@ -0,0 +1,145 @@ +//! SyncExecutor implementation for ExecutionBridge +//! +//! Bridges the sync crate's execution trait to the node's execution bridge. + +use crate::execution_bridge::ExecutionBridge; +use alloy_primitives::B256; +use async_trait::async_trait; +use cipherbft_execution::types::BlockInput; +use cipherbft_storage::BlockStore; +use cipherbft_sync::error::{Result, SyncError}; +use cipherbft_sync::execution::{SyncBlock, SyncExecutionResult, SyncExecutor}; +use std::sync::Arc; +use tracing::{debug, info, warn}; + +/// Wrapper that implements SyncExecutor using ExecutionBridge +/// +/// This executor bridges the sync crate's execution interface to the node's +/// execution bridge, enabling snap sync to execute downloaded blocks. +/// +/// # Type Parameters +/// +/// * `B` - The block store implementation for state root verification +pub struct ExecutionBridgeSyncExecutor { + bridge: Arc, + block_store: Arc, +} + +impl ExecutionBridgeSyncExecutor { + /// Create a new sync executor wrapping an execution bridge + /// + /// # Arguments + /// + /// * `bridge` - The execution bridge for block execution + /// * `block_store` - The block store for state root verification + pub fn new(bridge: Arc, block_store: Arc) -> Self { + Self { + bridge, + block_store, + } + } +} + +#[async_trait] +impl SyncExecutor for ExecutionBridgeSyncExecutor { + async fn execute_block(&self, block: SyncBlock) -> Result { + debug!( + block_number = block.block_number, + txs = block.transactions.len(), + "Executing sync block" + ); + + // Convert SyncBlock to BlockInput + let block_input = BlockInput { + block_number: block.block_number, + timestamp: block.timestamp, + transactions: block.transactions.clone(), + parent_hash: block.parent_hash, + gas_limit: block.gas_limit, + base_fee_per_gas: block.base_fee_per_gas, + beneficiary: block.beneficiary, + }; + + // Execute through the bridge + let result = self + .bridge + .execute_block_input(block_input) + .await + .map_err(|e| SyncError::Storage(format!("execution failed: {}", e)))?; + + info!( + block_number = block.block_number, + gas_used = result.execution_result.gas_used, + "Sync block executed" + ); + + Ok(SyncExecutionResult { + block_number: block.block_number, + state_root: if result.execution_result.state_root != B256::ZERO { + Some(result.execution_result.state_root) + } else { + None + }, + block_hash: result.block_hash, + gas_used: result.execution_result.gas_used, + transaction_count: block.transactions.len(), + }) + } + + async fn last_block_hash(&self) -> B256 { + self.bridge.last_block_hash() + } + + async fn last_block_number(&self) -> u64 { + self.bridge.current_block_number().await + } + + async fn verify_state_root(&self, height: u64, expected: B256) -> Result { + debug!(height, %expected, "Verifying state root"); + + // Get the block at the given height from storage + let block = self + .block_store + .get_block_by_number(height) + .await + .map_err(|e| SyncError::Storage(format!("failed to get block {}: {}", height, e)))?; + + match block { + Some(block) => { + let stored_state_root = B256::from(block.state_root); + if stored_state_root == expected { + debug!( + height, + %expected, + "State root verification passed" + ); + Ok(true) + } else { + warn!( + height, + %expected, + %stored_state_root, + "State root mismatch" + ); + Ok(false) + } + } + None => { + // Block not found - this is expected during sync before blocks are stored + // Return true to allow sync to proceed, actual verification happens + // when we reconstruct state + debug!( + height, + "Block not found for state root verification (expected during sync)" + ); + Ok(true) + } + } + } +} + +#[cfg(test)] +mod tests { + // Tests would require mocking ExecutionBridge and BlockStore which is complex + // Integration tests are more appropriate for this module +} diff --git a/crates/node/src/sync_network.rs b/crates/node/src/sync_network.rs new file mode 100644 index 0000000..13c90ca --- /dev/null +++ b/crates/node/src/sync_network.rs @@ -0,0 +1,331 @@ +//! Network adapter for snap sync protocol +//! +//! This module provides the bridge between the node's network layer and the +//! [`StateSyncManager`](cipherbft_sync::StateSyncManager). It handles the routing +//! of snap sync messages to and from peers. +//! +//! # Architecture +//! +//! ```text +//! +-----------------+ +-------------------+ +----------------+ +//! | Network Layer | <-> | SyncNetworkAdapter| <-> | StateSyncManager | +//! +-----------------+ +-------------------+ +----------------+ +//! ``` +//! +//! The adapter uses channels to decouple the network I/O from the sync logic, +//! allowing both to operate concurrently. +//! +//! # Integration with TcpPrimaryNetwork +//! +//! The [`wire_sync_to_network`] function connects the sync adapter to the +//! TCP network layer: +//! +//! ```text +//! Incoming: TcpPrimaryNetwork -> SyncNetworkSender -> channels -> SyncNetworkAdapter +//! Outgoing: SyncNetworkAdapter -> channels -> start_snap_sync_sender task -> TcpPrimaryNetwork +//! ``` + +use crate::network::TcpPrimaryNetwork; +use cipherbft_sync::protocol::SnapSyncMessage; +use cipherbft_sync::SyncNetworkSender; +use std::sync::Arc; +use tokio::sync::mpsc; + +/// Snap sync network adapter +/// +/// Bridges between the node's network layer and StateSyncManager. +/// Uses async channels for non-blocking message passing. +pub struct SyncNetworkAdapter { + /// Outbound message sender (to network layer) + outbound_tx: mpsc::Sender<(String, SnapSyncMessage)>, + /// Inbound message receiver (from network layer) + inbound_rx: mpsc::Receiver<(String, SnapSyncMessage)>, +} + +impl SyncNetworkAdapter { + /// Create a new adapter with provided channels + /// + /// # Arguments + /// + /// * `outbound_tx` - Channel sender for outgoing messages to the network + /// * `inbound_rx` - Channel receiver for incoming messages from the network + pub fn new( + outbound_tx: mpsc::Sender<(String, SnapSyncMessage)>, + inbound_rx: mpsc::Receiver<(String, SnapSyncMessage)>, + ) -> Self { + Self { + outbound_tx, + inbound_rx, + } + } + + /// Send a message to a specific peer + /// + /// # Arguments + /// + /// * `peer_id` - The peer's identifier + /// * `message` - The snap sync message to send + /// + /// # Returns + /// + /// Returns `Ok(())` if the message was queued successfully, or an error + /// if the channel is closed. + pub async fn send(&self, peer_id: &str, message: SnapSyncMessage) -> Result<(), String> { + self.outbound_tx + .send((peer_id.to_string(), message)) + .await + .map_err(|e| format!("failed to send snap sync message: {}", e)) + } + + /// Receive the next inbound message + /// + /// # Returns + /// + /// Returns `Some((peer_id, message))` if a message is available, + /// or `None` if the channel has been closed. + pub async fn recv(&mut self) -> Option<(String, SnapSyncMessage)> { + self.inbound_rx.recv().await + } + + /// Try to receive a message without blocking + /// + /// # Returns + /// + /// Returns `Some((peer_id, message))` if a message is immediately available, + /// `None` if no message is available (does not block). + pub fn try_recv(&mut self) -> Option<(String, SnapSyncMessage)> { + self.inbound_rx.try_recv().ok() + } + + /// Broadcast a message to multiple peers + /// + /// # Arguments + /// + /// * `peer_ids` - List of peer identifiers to send to + /// * `message` - The snap sync message to broadcast + /// + /// # Returns + /// + /// Returns the number of peers the message was successfully queued for. + pub async fn broadcast(&self, peer_ids: &[String], message: &SnapSyncMessage) -> usize { + let mut success_count = 0; + for peer_id in peer_ids { + if self.send(peer_id, message.clone()).await.is_ok() { + success_count += 1; + } + } + success_count + } +} + +/// Channel buffer size for sync messages +const SYNC_CHANNEL_SIZE: usize = 1000; + +/// Create sync network adapter with channels +/// +/// Returns a tuple containing: +/// - The adapter for use by the sync manager +/// - Sender for the network layer to forward incoming messages to sync +/// - Receiver for the network layer to get outgoing messages from sync +/// +/// # Example +/// +/// ```ignore +/// let (adapter, network_to_sync_tx, sync_to_network_rx) = create_sync_adapter(); +/// +/// // Network layer uses: +/// // network_to_sync_tx.send((peer_id, message)) - forward incoming +/// // sync_to_network_rx.recv() - get outgoing +/// +/// // Sync manager uses: +/// // adapter.recv() - get incoming +/// // adapter.send(peer_id, message) - send outgoing +/// ``` +#[allow(clippy::type_complexity)] +pub fn create_sync_adapter() -> ( + SyncNetworkAdapter, + mpsc::Sender<(String, SnapSyncMessage)>, // for network to send to sync + mpsc::Receiver<(String, SnapSyncMessage)>, // for network to receive from sync +) { + // Channel for messages from network to sync (inbound to sync) + let (inbound_tx, inbound_rx) = mpsc::channel(SYNC_CHANNEL_SIZE); + // Channel for messages from sync to network (outbound from sync) + let (outbound_tx, outbound_rx) = mpsc::channel(SYNC_CHANNEL_SIZE); + + let adapter = SyncNetworkAdapter::new(outbound_tx, inbound_rx); + (adapter, inbound_tx, outbound_rx) +} + +/// Wire the sync adapter to the TcpPrimaryNetwork +/// +/// This function sets up the bidirectional message routing between the sync +/// adapter and the TCP network layer: +/// +/// - **Incoming**: Messages received by `TcpPrimaryNetwork` are forwarded to the +/// sync adapter via `SyncNetworkSender` +/// - **Outgoing**: Messages sent by the sync adapter are routed through the +/// `start_snap_sync_sender` task to `TcpPrimaryNetwork` +/// +/// # Arguments +/// +/// * `network` - Mutable reference to the TCP primary network (for setting sender) +/// * `network_arc` - Arc reference to the same network (for starting sender task) +/// * `network_to_sync_tx` - Sender from `create_sync_adapter` for incoming messages +/// * `sync_to_network_rx` - Receiver from `create_sync_adapter` for outgoing messages +/// +/// # Example +/// +/// ```ignore +/// // Create the sync adapter +/// let (adapter, network_to_sync_tx, sync_to_network_rx) = create_sync_adapter(); +/// +/// // Wire it to the network +/// wire_sync_to_network( +/// &mut network, +/// network.clone(), // Arc +/// network_to_sync_tx, +/// sync_to_network_rx, +/// ); +/// +/// // Now use the adapter with the sync manager +/// ``` +pub async fn wire_sync_to_network( + network: &TcpPrimaryNetwork, + network_arc: Arc, + network_to_sync_tx: mpsc::Sender<(String, SnapSyncMessage)>, + sync_to_network_rx: mpsc::Receiver<(String, SnapSyncMessage)>, +) { + // Create a SyncNetworkSender that forwards to our channel-based adapter + // This wraps the (String, SnapSyncMessage) channel in the SyncNetworkSender interface + let sync_sender = create_bridge_sender(network_to_sync_tx); + network.set_snap_sync_sender(sync_sender).await; + + // Start the outgoing message processor task + // This reads from sync_to_network_rx and sends via TcpPrimaryNetwork + start_outgoing_processor(network_arc, sync_to_network_rx); +} + +/// Create a SyncNetworkSender that bridges to the local channel +/// +/// The sync crate's SyncNetworkSender expects IncomingSnapMessage, but our local +/// adapter uses (String, SnapSyncMessage) tuples. This creates a bridge sender. +fn create_bridge_sender(tx: mpsc::Sender<(String, SnapSyncMessage)>) -> SyncNetworkSender { + // Create a channel that the SyncNetworkSender will use + let (bridge_tx, mut bridge_rx) = + mpsc::channel::(SYNC_CHANNEL_SIZE); + + // Spawn a task that forwards from bridge to local adapter format + tokio::spawn(async move { + while let Some(incoming) = bridge_rx.recv().await { + if tx.send((incoming.peer_id, incoming.message)).await.is_err() { + break; + } + } + }); + + SyncNetworkSender::new(bridge_tx) +} + +/// Start the outgoing message processor +/// +/// Reads messages from the sync adapter's outgoing channel and sends them +/// via the TcpPrimaryNetwork. +fn start_outgoing_processor( + network: Arc, + mut rx: mpsc::Receiver<(String, SnapSyncMessage)>, +) { + tokio::spawn(async move { + while let Some((peer_id, message)) = rx.recv().await { + if peer_id == "*" || peer_id.is_empty() { + // Broadcast to all peers + network.broadcast_snap_sync(message).await; + } else { + // Parse peer_id and send to specific peer + // The peer_id is in format "ValidatorId(0x...)" from Debug + if let Some(vid) = TcpPrimaryNetwork::parse_peer_id(&peer_id) { + network.send_snap_sync(vid, message).await; + } else { + tracing::warn!( + "Could not parse peer ID for outgoing sync message: {}", + peer_id + ); + } + } + } + tracing::info!("Outgoing sync message processor ended"); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use cipherbft_sync::protocol::{SnapshotInfo, StatusResponse}; + + #[tokio::test] + async fn test_adapter_send_recv() { + let (adapter, _inbound_tx, mut outbound_rx) = create_sync_adapter(); + + // Send a message through the adapter + let msg = SnapSyncMessage::GetStatus; + adapter.send("peer1", msg).await.unwrap(); + + // Verify it comes out on the network side + let (peer_id, received) = outbound_rx.recv().await.unwrap(); + assert_eq!(peer_id, "peer1"); + assert!(matches!(received, SnapSyncMessage::GetStatus)); + } + + #[tokio::test] + async fn test_adapter_recv_from_network() { + let (mut adapter, inbound_tx, _outbound_rx) = create_sync_adapter(); + + // Simulate network sending a message + let status = StatusResponse { + tip_height: 100, + tip_hash: B256::ZERO, + snapshots: vec![ + SnapshotInfo { + height: 90, + state_root: B256::repeat_byte(0x01), + block_hash: B256::repeat_byte(0x02), + }, + SnapshotInfo { + height: 80, + state_root: B256::repeat_byte(0x03), + block_hash: B256::repeat_byte(0x04), + }, + ], + }; + inbound_tx + .send(("peer2".to_string(), SnapSyncMessage::Status(status))) + .await + .unwrap(); + + // Verify adapter receives it + let (peer_id, received) = adapter.recv().await.unwrap(); + assert_eq!(peer_id, "peer2"); + assert!(matches!(received, SnapSyncMessage::Status(_))); + } + + #[tokio::test] + async fn test_broadcast() { + let (adapter, _inbound_tx, mut outbound_rx) = create_sync_adapter(); + + let peers = vec![ + "peer1".to_string(), + "peer2".to_string(), + "peer3".to_string(), + ]; + let msg = SnapSyncMessage::GetStatus; + + let count = adapter.broadcast(&peers, &msg).await; + assert_eq!(count, 3); + + // Verify all messages were sent + for _ in 0..3 { + let (_, received) = outbound_rx.recv().await.unwrap(); + assert!(matches!(received, SnapSyncMessage::GetStatus)); + } + } +} diff --git a/crates/node/src/sync_runner.rs b/crates/node/src/sync_runner.rs new file mode 100644 index 0000000..b2b68fd --- /dev/null +++ b/crates/node/src/sync_runner.rs @@ -0,0 +1,1224 @@ +//! Snap sync runner for node startup +//! +//! This module provides the entry point for running snap sync during node +//! initialization. It coordinates the sync process and integrates with the +//! node's lifecycle. +//! +//! # Sync Decision Logic +//! +//! The node decides whether to run snap sync based on: +//! 1. Is snap sync enabled in config? +//! 2. How far behind is the node compared to the network tip? +//! 3. Is the gap larger than the configured threshold? +//! +//! If snap sync is needed, the node will: +//! 1. Discover peers and find a common snapshot +//! 2. Download account state from the snapshot +//! 3. Download contract storage +//! 4. Verify the state root +//! 5. Sync remaining blocks to reach the tip + +use crate::config::SyncConfig as NodeSyncConfig; +use crate::sync_network::SyncNetworkAdapter; +use cipherbft_sync::protocol::{ + AccountRangeResponse, BlockRangeRequest, BlockRangeResponse, SnapSyncMessage, + StorageRangeResponse, +}; +use cipherbft_sync::snap::accounts::PendingRange; +use cipherbft_sync::snap::storage::PendingStorageRange; +use cipherbft_sync::{ + StateSyncManager, SyncBlock, SyncConfig as ManagerSyncConfig, SyncError, SyncExecutor, +}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tracing::{debug, error, info, warn}; + +/// Maximum time to wait for discovery phase +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); + +/// Interval between discovery polls +const DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// Maximum time to wait for a single sync request (used for reference) +#[allow(dead_code)] +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// Maximum concurrent account range requests +const MAX_CONCURRENT_ACCOUNT_REQUESTS: usize = 8; + +/// Maximum concurrent storage range requests +const MAX_CONCURRENT_STORAGE_REQUESTS: usize = 4; + +/// Maximum concurrent block range requests +const MAX_CONCURRENT_BLOCK_REQUESTS: usize = 4; + +/// Result of snap sync attempt +#[derive(Debug, Clone, PartialEq)] +pub enum SyncResult { + /// Sync completed successfully + Completed { + /// Height reached after sync + final_height: u64, + /// Duration of sync process + duration: Duration, + }, + /// Node was already synced, no action needed + AlreadySynced { + /// Current local height + local_tip: u64, + }, + /// Sync was skipped (disabled or below threshold) + Skipped { + /// Reason for skipping + reason: String, + }, + /// Sync failed + Failed { + /// Error description + error: String, + }, +} + +/// Tracks in-flight requests for timeout handling +#[derive(Debug)] +struct InFlightRequest { + /// The range/request data + data: T, + /// Peer handling this request + peer_id: String, + /// When the request was sent + sent_at: Instant, +} + +impl InFlightRequest { + fn new(data: T, peer_id: String) -> Self { + Self { + data, + peer_id, + sent_at: Instant::now(), + } + } + + fn is_timed_out(&self, timeout: Duration) -> bool { + self.sent_at.elapsed() > timeout + } +} + +/// Run snap sync to completion +/// +/// This function orchestrates the snap sync process, returning when sync +/// is complete or when an error occurs. +/// +/// # Arguments +/// +/// * `manager` - The state sync manager +/// * `network` - Network adapter for peer communication +/// * `local_tip` - Current local blockchain height +/// * `network_tip` - Network's reported tip height +/// * `config` - Sync configuration from node config +/// +/// # Returns +/// +/// Returns `SyncResult` indicating the outcome of the sync attempt. +/// +/// # Example +/// +/// ```ignore +/// let result = run_snap_sync( +/// &mut manager, +/// &mut network, +/// local_tip, +/// network_tip, +/// &config, +/// ).await; +/// +/// match result { +/// SyncResult::Completed { final_height, duration } => { +/// info!("Sync completed at height {} in {:?}", final_height, duration); +/// } +/// SyncResult::AlreadySynced { local_tip } => { +/// info!("Already synced at height {}", local_tip); +/// } +/// SyncResult::Skipped { reason } => { +/// info!("Sync skipped: {}", reason); +/// } +/// SyncResult::Failed { error } => { +/// error!("Sync failed: {}", error); +/// } +/// } +/// ``` +pub async fn run_snap_sync( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + local_tip: u64, + network_tip: u64, + config: &NodeSyncConfig, +) -> SyncResult { + // Check if snap sync is enabled + if !config.snap_sync_enabled { + return SyncResult::Skipped { + reason: "snap sync is disabled in configuration".to_string(), + }; + } + + // Check if snap sync is needed based on block gap + let gap = network_tip.saturating_sub(local_tip); + if gap < config.snap_sync_threshold { + info!( + local_tip, + network_tip, + gap, + threshold = config.snap_sync_threshold, + "Node is close to tip, skipping snap sync" + ); + return SyncResult::Skipped { + reason: format!( + "block gap ({}) is below threshold ({})", + gap, config.snap_sync_threshold + ), + }; + } + + info!( + local_tip, + network_tip, + gap, + threshold = config.snap_sync_threshold, + "Starting snap sync - node is significantly behind" + ); + + let start_time = Instant::now(); + + // Start discovery phase + if let Err(e) = manager.start_discovery() { + return SyncResult::Failed { + error: format!("failed to start discovery: {}", e), + }; + } + + // Phase 1: Discovery - find peers and select snapshot + let snapshot = match run_discovery_phase(manager, network, config).await { + Ok(snapshot) => snapshot, + Err(e) => { + return SyncResult::Failed { + error: format!("discovery failed: {}", e), + }; + } + }; + + info!( + height = snapshot.block_number, + state_root = %snapshot.state_root, + "Selected snapshot for sync" + ); + + // Start snap sync with the selected snapshot + if let Err(e) = manager.start_snap_sync(snapshot.clone()) { + return SyncResult::Failed { + error: format!("failed to start snap sync: {}", e), + }; + } + + // Phase 2: Account sync + if let Err(e) = run_account_sync_phase(manager, network, config).await { + return SyncResult::Failed { + error: format!("account sync failed: {}", e), + }; + } + + // Transition to storage sync + if let Err(e) = manager.start_storage_sync() { + return SyncResult::Failed { + error: format!("failed to start storage sync: {}", e), + }; + } + + // Phase 3: Storage sync + if let Err(e) = run_storage_sync_phase(manager, network, config).await { + return SyncResult::Failed { + error: format!("storage sync failed: {}", e), + }; + } + + // Verify state root + if let Err(e) = manager.verify_state_root() { + return SyncResult::Failed { + error: format!("state root verification failed: {}", e), + }; + } + + // Start block sync + if let Err(e) = manager.start_block_sync() { + return SyncResult::Failed { + error: format!("failed to start block sync: {}", e), + }; + } + + // Check if we're already at tip (no blocks to sync) + if manager.is_complete() { + let duration = start_time.elapsed(); + info!( + final_height = snapshot.block_number, + duration_secs = duration.as_secs(), + "Snap sync completed (no blocks to sync)" + ); + return SyncResult::Completed { + final_height: snapshot.block_number, + duration, + }; + } + + // Phase 4: Block sync - download and execute remaining blocks + // Note: This phase requires a SyncExecutor which should be passed in + // For now, we just download the blocks; execution is handled separately + if let Err(e) = run_block_download_phase(manager, network, config).await { + return SyncResult::Failed { + error: format!("block sync failed: {}", e), + }; + } + + // Mark sync as complete + if let Err(e) = manager.complete_sync() { + return SyncResult::Failed { + error: format!("failed to complete sync: {}", e), + }; + } + + let duration = start_time.elapsed(); + let final_height = manager + .block_syncer_mut() + .map(|s| s.stats().executed_up_to) + .unwrap_or(snapshot.block_number); + + info!( + final_height, + duration_secs = duration.as_secs(), + "Snap sync completed successfully" + ); + + SyncResult::Completed { + final_height, + duration, + } +} + +/// Run snap sync with a provided executor for block execution +/// +/// This is the full-featured version that includes block execution. +pub async fn run_snap_sync_with_executor( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + executor: Arc, + local_tip: u64, + network_tip: u64, + config: &NodeSyncConfig, +) -> SyncResult { + // Run the main sync phases (discovery, accounts, storage) + let result = run_snap_sync(manager, network, local_tip, network_tip, config).await; + + // If we completed with blocks to execute, run the execution phase + if let SyncResult::Completed { + final_height, + duration, + } = &result + { + // Check if block syncer exists and has blocks + let needs_execution = manager.block_syncer_mut().is_some_and(|s| !s.is_complete()); + + if needs_execution { + if let Err(e) = run_block_execution_phase(manager, executor).await { + return SyncResult::Failed { + error: format!("block execution failed: {}", e), + }; + } + + // Update final height after execution + let executed_height = manager + .block_syncer_mut() + .map(|s| s.stats().executed_up_to) + .unwrap_or(*final_height); + + return SyncResult::Completed { + final_height: executed_height, + duration: *duration, + }; + } + + return SyncResult::Completed { + final_height: *final_height, + duration: *duration, + }; + } + + result +} + +/// Check if snap sync is needed for the current node state +/// +/// This is a quick check that can be used before initializing the full +/// sync machinery. +/// +/// # Arguments +/// +/// * `local_tip` - Current local blockchain height +/// * `network_tip` - Network's reported tip height +/// * `config` - Sync configuration from node config +/// +/// # Returns +/// +/// `true` if snap sync should be initiated, `false` otherwise +pub fn should_snap_sync(local_tip: u64, network_tip: u64, config: &NodeSyncConfig) -> bool { + if !config.snap_sync_enabled { + debug!("Snap sync disabled in configuration"); + return false; + } + + let gap = network_tip.saturating_sub(local_tip); + let needs_sync = gap >= config.snap_sync_threshold; + + if needs_sync { + info!( + local_tip, + network_tip, + gap, + threshold = config.snap_sync_threshold, + "Snap sync recommended" + ); + } else { + debug!( + local_tip, + network_tip, + gap, + threshold = config.snap_sync_threshold, + "Block-by-block sync sufficient" + ); + } + + needs_sync +} + +/// Create a StateSyncManager with configuration derived from NodeSyncConfig +/// +/// # Arguments +/// +/// * `config` - Node sync configuration +/// +/// # Returns +/// +/// A configured `StateSyncManager` ready for sync operations +pub fn create_sync_manager(config: &NodeSyncConfig) -> StateSyncManager { + let manager_config = ManagerSyncConfig { + min_peers: config.min_sync_peers, + max_retries: 3, + request_timeout: Duration::from_secs(config.sync_timeout_secs), + discovery_timeout: Duration::from_secs(30), + }; + + StateSyncManager::new(manager_config) +} + +// ============================================================================= +// Phase Implementations +// ============================================================================= + +use cipherbft_sync::snapshot::StateSnapshot; + +/// Run the discovery phase to find peers and select a snapshot +async fn run_discovery_phase( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + config: &NodeSyncConfig, +) -> Result { + info!("Starting discovery phase"); + + let discovery_start = Instant::now(); + + // Broadcast GetStatus to all peers + // Using "*" as peer_id triggers broadcast in the network adapter + if let Err(e) = network.send("*", SnapSyncMessage::GetStatus).await { + warn!("Failed to broadcast GetStatus: {}", e); + } + + // Poll for responses and check for snapshot agreement + loop { + // Check for timeout + if discovery_start.elapsed() > DISCOVERY_TIMEOUT { + return Err(SyncError::InsufficientPeers { + needed: config.min_sync_peers as u32, + available: manager.peers().peer_count() as u32, + }); + } + + // Process incoming messages + while let Some((peer_id, message)) = network.try_recv() { + match message { + SnapSyncMessage::Status(status) => { + debug!( + peer = %peer_id, + tip_height = status.tip_height, + snapshots = ?status.snapshots, + "Received status from peer" + ); + + // Add peer if not already known + manager.add_peer(peer_id.clone()); + manager.handle_status(&peer_id, status); + } + other => { + debug!( + peer = %peer_id, + msg_type = other.message_type(), + "Ignoring non-status message during discovery" + ); + } + } + } + + // Check if we have enough peers and snapshot agreement + match manager.try_complete_discovery()? { + Some(snapshot) => { + info!( + height = snapshot.block_number, + peers = manager.peers().peer_count(), + "Discovery complete, snapshot selected" + ); + return Ok(snapshot); + } + None => { + // Not enough peers or agreement yet, wait and retry + tokio::time::sleep(DISCOVERY_POLL_INTERVAL).await; + } + } + } +} + +/// Run the account sync phase +async fn run_account_sync_phase( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + config: &NodeSyncConfig, +) -> Result<(), SyncError> { + info!("Starting account sync phase"); + + let max_retries = 3u32; + let timeout = Duration::from_secs(config.sync_timeout_secs); + + // Track in-flight requests: request_id -> (range, peer_id, sent_time) + let mut in_flight: HashMap> = HashMap::new(); + let mut next_request_id: u64 = 0; + + loop { + // Check if account sync is complete + if manager.is_account_sync_complete() && in_flight.is_empty() { + info!("Account sync phase complete"); + return Ok(()); + } + + // Check for timed out requests + let timed_out: Vec = in_flight + .iter() + .filter(|(_, req)| req.is_timed_out(timeout)) + .map(|(id, _)| *id) + .collect(); + + for request_id in timed_out { + if let Some(req) = in_flight.remove(&request_id) { + warn!( + request_id, + peer = %req.peer_id, + "Account range request timed out" + ); + manager.peers_mut().request_failed(&req.peer_id); + + // Re-queue the range for retry + if let Some(account_syncer) = manager.account_syncer_mut() { + account_syncer.handle_failure(req.data, max_retries); + } + } + } + + // Send new requests if we have capacity + while in_flight.len() < MAX_CONCURRENT_ACCOUNT_REQUESTS { + // Get next range to request + let range = match manager.account_syncer_mut().and_then(|s| s.next_range()) { + Some(r) => r, + None => break, // No more ranges to request + }; + + // Select best peer for this request + let peer = match manager.peers().select_peer(None) { + Some(p) => p.peer_id.clone(), + None => { + // No peers available, re-queue the range + if let Some(account_syncer) = manager.account_syncer_mut() { + account_syncer.handle_failure(range, max_retries); + } + break; + } + }; + + // Create and send the request + let request = manager + .account_syncer_mut() + .map(|s| s.create_request(&range)) + .ok_or_else(|| SyncError::InvalidState("no account syncer".into()))?; + + let msg = SnapSyncMessage::GetAccountRange(request); + + if let Err(e) = network.send(&peer, msg).await { + warn!(peer = %peer, error = %e, "Failed to send account range request"); + manager.peers_mut().request_failed(&peer); + if let Some(account_syncer) = manager.account_syncer_mut() { + account_syncer.handle_failure(range, max_retries); + } + continue; + } + + // Track the in-flight request + manager.peers_mut().request_started(&peer); + in_flight.insert(next_request_id, InFlightRequest::new(range, peer)); + next_request_id += 1; + + debug!( + request_id = next_request_id - 1, + "Sent account range request" + ); + } + + // Process incoming responses + while let Some((peer_id, message)) = network.try_recv() { + match message { + SnapSyncMessage::AccountRange(response) => { + // Use the echoed request_id for direct lookup + let request_id = response.request_id; + + if let Some(req) = in_flight.remove(&request_id) { + // Verify response came from expected peer + if req.peer_id != peer_id { + warn!( + request_id, + expected = %req.peer_id, + actual = %peer_id, + "Account response from unexpected peer, ignoring" + ); + // Re-insert - might still come from correct peer + in_flight.insert(request_id, req); + continue; + } + + let latency = req.sent_at.elapsed(); + let bytes = estimate_account_response_size(&response); + + // Process the response + let process_result = manager + .account_syncer_mut() + .map(|s| s.process_response(req.data.clone(), response)); + + match process_result { + Some(Ok(())) => { + manager + .peers_mut() + .request_completed(&peer_id, latency, bytes); + debug!( + peer = %peer_id, + latency_ms = latency.as_millis(), + "Account range response processed" + ); + } + Some(Err(e)) => { + warn!( + peer = %peer_id, + error = %e, + "Failed to process account range response" + ); + manager.handle_peer_error(&peer_id, &e); + if let Some(account_syncer) = manager.account_syncer_mut() { + account_syncer.handle_failure(req.data, max_retries); + } + } + None => { + warn!("No account syncer available"); + } + } + } + } + SnapSyncMessage::Status(status) => { + // Update peer status even during account sync + manager.handle_status(&peer_id, status); + } + _ => { + debug!( + peer = %peer_id, + msg_type = message.message_type(), + "Ignoring unexpected message during account sync" + ); + } + } + } + + // Small delay to prevent busy-spinning + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// Run the storage sync phase +async fn run_storage_sync_phase( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + config: &NodeSyncConfig, +) -> Result<(), SyncError> { + info!("Starting storage sync phase"); + + let max_retries = 3u32; + let timeout = Duration::from_secs(config.sync_timeout_secs); + + // Track in-flight requests + let mut in_flight: HashMap> = HashMap::new(); + let mut next_request_id: u64 = 0; + + loop { + // Check if storage sync is complete + if manager.is_storage_sync_complete() && in_flight.is_empty() { + info!("Storage sync phase complete"); + return Ok(()); + } + + // Check for timed out requests + let timed_out: Vec = in_flight + .iter() + .filter(|(_, req)| req.is_timed_out(timeout)) + .map(|(id, _)| *id) + .collect(); + + for request_id in timed_out { + if let Some(req) = in_flight.remove(&request_id) { + warn!( + request_id, + peer = %req.peer_id, + account = %req.data.account, + "Storage range request timed out" + ); + manager.peers_mut().request_failed(&req.peer_id); + + if let Some(storage_syncer) = manager.storage_syncer_mut() { + storage_syncer.handle_failure(req.data, max_retries); + } + } + } + + // Send new requests if we have capacity + while in_flight.len() < MAX_CONCURRENT_STORAGE_REQUESTS { + let range = match manager.storage_syncer_mut().and_then(|s| s.next_range()) { + Some(r) => r, + None => break, + }; + + let peer = match manager.peers().select_peer(None) { + Some(p) => p.peer_id.clone(), + None => { + if let Some(storage_syncer) = manager.storage_syncer_mut() { + storage_syncer.handle_failure(range, max_retries); + } + break; + } + }; + + let request = manager + .storage_syncer_mut() + .map(|s| s.create_request(&range)) + .ok_or_else(|| SyncError::InvalidState("no storage syncer".into()))?; + + let msg = SnapSyncMessage::GetStorageRange(request); + + if let Err(e) = network.send(&peer, msg).await { + warn!(peer = %peer, error = %e, "Failed to send storage range request"); + manager.peers_mut().request_failed(&peer); + if let Some(storage_syncer) = manager.storage_syncer_mut() { + storage_syncer.handle_failure(range, max_retries); + } + continue; + } + + manager.peers_mut().request_started(&peer); + in_flight.insert(next_request_id, InFlightRequest::new(range, peer)); + next_request_id += 1; + + debug!( + request_id = next_request_id - 1, + "Sent storage range request" + ); + } + + // Process incoming responses + while let Some((peer_id, message)) = network.try_recv() { + match message { + SnapSyncMessage::StorageRange(response) => { + // Use the echoed request_id for direct lookup + let request_id = response.request_id; + + if let Some(req) = in_flight.remove(&request_id) { + // Verify response came from expected peer + if req.peer_id != peer_id { + warn!( + request_id, + expected = %req.peer_id, + actual = %peer_id, + "Storage response from unexpected peer, ignoring" + ); + // Re-insert - might still come from correct peer + in_flight.insert(request_id, req); + continue; + } + + let latency = req.sent_at.elapsed(); + let bytes = estimate_storage_response_size(&response); + + let process_result = manager + .storage_syncer_mut() + .map(|s| s.process_response(req.data.clone(), response)); + + match process_result { + Some(Ok(())) => { + manager + .peers_mut() + .request_completed(&peer_id, latency, bytes); + debug!( + peer = %peer_id, + latency_ms = latency.as_millis(), + "Storage range response processed" + ); + } + Some(Err(e)) => { + warn!( + peer = %peer_id, + error = %e, + "Failed to process storage range response" + ); + manager.handle_peer_error(&peer_id, &e); + if let Some(storage_syncer) = manager.storage_syncer_mut() { + storage_syncer.handle_failure(req.data, max_retries); + } + } + None => { + warn!("No storage syncer available"); + } + } + } + } + SnapSyncMessage::Status(status) => { + manager.handle_status(&peer_id, status); + } + _ => { + debug!( + peer = %peer_id, + msg_type = message.message_type(), + "Ignoring unexpected message during storage sync" + ); + } + } + } + + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// Pending block range for tracking in-flight block requests +#[derive(Clone, Debug)] +struct PendingBlockRange { + start: u64, + count: u32, + retries: u32, +} + +/// Run the block download phase +async fn run_block_download_phase( + manager: &mut StateSyncManager, + network: &mut SyncNetworkAdapter, + config: &NodeSyncConfig, +) -> Result<(), SyncError> { + info!("Starting block download phase"); + + let max_retries = 3u32; + let timeout = Duration::from_secs(config.sync_timeout_secs); + + // Track in-flight block requests + let mut in_flight: HashMap> = HashMap::new(); + let mut next_request_id: u64 = 0; + + loop { + // Check if block download is complete + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + // Check if we have all blocks downloaded (pending_ranges empty and no in-flight) + let stats = block_syncer.stats(); + if stats.pending_ranges == 0 && in_flight.is_empty() { + info!( + downloaded = stats.total_downloaded, + "Block download phase complete" + ); + return Ok(()); + } + + // Check for timed out requests + let timed_out: Vec = in_flight + .iter() + .filter(|(_, req)| req.is_timed_out(timeout)) + .map(|(id, _)| *id) + .collect(); + + for request_id in timed_out { + if let Some(req) = in_flight.remove(&request_id) { + warn!( + request_id, + peer = %req.peer_id, + start = req.data.start, + "Block range request timed out" + ); + manager.peers_mut().request_failed(&req.peer_id); + + // Re-queue the block range + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + let pending = cipherbft_sync::blocks::PendingBlockRange { + start: req.data.start, + count: req.data.count, + retries: req.data.retries, + }; + block_syncer.handle_download_failure(pending, max_retries); + } + } + + // Send new requests if we have capacity + while in_flight.len() < MAX_CONCURRENT_BLOCK_REQUESTS { + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + let range = match block_syncer.next_range() { + Some(r) => r, + None => break, + }; + + let peer = match manager.peers().select_peer(None) { + Some(p) => p.peer_id.clone(), + None => { + // Re-queue the range + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + block_syncer.handle_download_failure(range, max_retries); + break; + } + }; + + let request_id = next_request_id; + next_request_id += 1; + + let request = BlockRangeRequest { + request_id, + start_height: range.start, + count: range.count, + }; + + let msg = SnapSyncMessage::GetBlocks(request); + + if let Err(e) = network.send(&peer, msg).await { + warn!(peer = %peer, error = %e, "Failed to send block range request"); + manager.peers_mut().request_failed(&peer); + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + block_syncer.handle_download_failure(range, max_retries); + continue; + } + + manager.peers_mut().request_started(&peer); + + let pending = PendingBlockRange { + start: range.start, + count: range.count, + retries: range.retries, + }; + in_flight.insert(request_id, InFlightRequest::new(pending, peer)); + + debug!( + request_id = next_request_id - 1, + start = range.start, + count = range.count, + "Sent block range request" + ); + } + + // Process incoming responses + while let Some((peer_id, message)) = network.try_recv() { + match message { + SnapSyncMessage::Blocks(response) => { + // Use the echoed request_id for direct lookup + let request_id = response.request_id; + + if let Some(req) = in_flight.remove(&request_id) { + // Verify response came from expected peer + if req.peer_id != peer_id { + warn!( + request_id, + expected = %req.peer_id, + actual = %peer_id, + "Response from unexpected peer, ignoring" + ); + // Re-insert the request - it may still come from the right peer + in_flight.insert(request_id, req); + continue; + } + + let latency = req.sent_at.elapsed(); + let bytes = estimate_block_response_size(&response); + + let pending = cipherbft_sync::blocks::PendingBlockRange { + start: req.data.start, + count: req.data.count, + retries: req.data.retries, + }; + + let process_result = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))? + .process_response(pending, response); + + match process_result { + Ok(()) => { + manager + .peers_mut() + .request_completed(&peer_id, latency, bytes); + debug!( + peer = %peer_id, + latency_ms = latency.as_millis(), + "Block range response processed" + ); + } + Err(e) => { + warn!( + peer = %peer_id, + error = %e, + "Failed to process block range response" + ); + manager.handle_peer_error(&peer_id, &e); + let pending = cipherbft_sync::blocks::PendingBlockRange { + start: req.data.start, + count: req.data.count, + retries: req.data.retries, + }; + if let Some(block_syncer) = manager.block_syncer_mut() { + block_syncer.handle_download_failure(pending, max_retries); + } + } + } + } + } + SnapSyncMessage::Status(status) => { + manager.handle_status(&peer_id, status); + } + _ => { + debug!( + peer = %peer_id, + msg_type = message.message_type(), + "Ignoring unexpected message during block download" + ); + } + } + } + + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// Run the block execution phase (requires a SyncExecutor) +async fn run_block_execution_phase( + manager: &mut StateSyncManager, + executor: Arc, +) -> Result<(), SyncError> { + info!("Starting block execution phase"); + + loop { + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + // Check if execution is complete + if block_syncer.is_complete() { + let stats = block_syncer.stats(); + info!( + executed = stats.total_executed, + target = stats.target_height, + "Block execution phase complete" + ); + return Ok(()); + } + + // Get next block to execute + let downloaded_block = match block_syncer.next_executable_block() { + Some(b) => b, + None => { + // No blocks ready yet, wait + tokio::time::sleep(Duration::from_millis(50)).await; + continue; + } + }; + + // Deserialize and execute the block + let sync_block = SyncBlock::from_bytes(&downloaded_block.data) + .map_err(|e| SyncError::Storage(format!("failed to deserialize block: {}", e)))?; + + debug!( + height = sync_block.block_number, + txs = sync_block.transactions.len(), + "Executing block" + ); + + match executor.execute_block(sync_block).await { + Ok(result) => { + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + block_syncer.block_executed(result.block_number, result.state_root); + + debug!( + height = result.block_number, + gas_used = result.gas_used, + txs = result.transaction_count, + "Block executed successfully" + ); + } + Err(e) => { + error!( + height = downloaded_block.height, + error = %e, + "Block execution failed" + ); + + let block_syncer = manager + .block_syncer_mut() + .ok_or_else(|| SyncError::InvalidState("no block syncer".into()))?; + + block_syncer.handle_execution_failure(downloaded_block.height); + + // For now, fail on execution error + // In production, you might want to retry or recover + return Err(e); + } + } + } +} + +// ============================================================================= +// Helper Functions +// ============================================================================= + +/// Estimate the size of an account range response in bytes +fn estimate_account_response_size(response: &AccountRangeResponse) -> u64 { + // ~100 bytes per account + proof size + let accounts_size = response.accounts.len() as u64 * 100; + let proof_size: u64 = response.proof.iter().map(|p| p.len() as u64).sum(); + accounts_size + proof_size +} + +/// Estimate the size of a storage range response in bytes +fn estimate_storage_response_size(response: &StorageRangeResponse) -> u64 { + // 64 bytes per slot (key + value) + proof size + let slots_size = response.slots.len() as u64 * 64; + let proof_size: u64 = response.proof.iter().map(|p| p.len() as u64).sum(); + slots_size + proof_size +} + +/// Estimate the size of a block range response in bytes +fn estimate_block_response_size(response: &BlockRangeResponse) -> u64 { + response.blocks.iter().map(|b| b.len() as u64).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use cipherbft_sync::SyncPhase; + + fn test_config() -> NodeSyncConfig { + NodeSyncConfig { + snap_sync_enabled: true, + min_sync_peers: 3, + sync_timeout_secs: 30, + snap_sync_threshold: 1024, + } + } + + #[test] + fn test_should_snap_sync_enabled() { + let config = test_config(); + + // Below threshold - should not sync + assert!(!should_snap_sync(9000, 9500, &config)); + + // At threshold - should sync + assert!(should_snap_sync(9000, 10024, &config)); + + // Above threshold - should sync + assert!(should_snap_sync(1000, 10000, &config)); + } + + #[test] + fn test_should_snap_sync_disabled() { + let mut config = test_config(); + config.snap_sync_enabled = false; + + // Even with large gap, should not sync if disabled + assert!(!should_snap_sync(0, 100000, &config)); + } + + #[test] + fn test_should_snap_sync_already_synced() { + let config = test_config(); + + // Local ahead of network (shouldn't happen but handle gracefully) + assert!(!should_snap_sync(10000, 9000, &config)); + + // Exactly at tip + assert!(!should_snap_sync(10000, 10000, &config)); + } + + #[test] + fn test_create_sync_manager() { + let config = test_config(); + let manager = create_sync_manager(&config); + + // Manager should start in discovery phase + assert!(matches!(manager.phase(), SyncPhase::Discovery)); + assert!(!manager.is_complete()); + } + + #[tokio::test] + async fn test_run_snap_sync_disabled() { + let mut config = test_config(); + config.snap_sync_enabled = false; + + let mut manager = create_sync_manager(&config); + let (mut adapter, _tx, _rx) = crate::sync_network::create_sync_adapter(); + + let result = run_snap_sync(&mut manager, &mut adapter, 0, 100000, &config).await; + + assert!(matches!(result, SyncResult::Skipped { .. })); + } + + #[tokio::test] + async fn test_run_snap_sync_below_threshold() { + let config = test_config(); + let mut manager = create_sync_manager(&config); + let (mut adapter, _tx, _rx) = crate::sync_network::create_sync_adapter(); + + let result = run_snap_sync(&mut manager, &mut adapter, 9900, 10000, &config).await; + + assert!(matches!(result, SyncResult::Skipped { .. })); + } +} diff --git a/crates/node/src/sync_server.rs b/crates/node/src/sync_server.rs new file mode 100644 index 0000000..1cd74f9 --- /dev/null +++ b/crates/node/src/sync_server.rs @@ -0,0 +1,685 @@ +//! Snap Sync Server - Handles incoming P2P snap sync requests +//! +//! This module implements server-side handlers for the snap sync protocol. +//! When peers request state data (accounts, storage, blocks), this server +//! responds with data from local storage. +//! +//! # Protocol +//! +//! Uses `/cipherbft/snap/1.0.0` request-response protocol. +//! +//! # Handlers +//! +//! - `GetStatus` -> `Status`: Returns local tip and available snapshots +//! - `GetAccountRange` -> `AccountRange`: Returns accounts in address range +//! - `GetStorageRange` -> `StorageRange`: Returns storage slots for an account +//! - `GetBlocks` -> `Blocks`: Returns blocks in height range + +use alloy_primitives::{Address, Bytes, B256, U256}; +use cipherbft_storage::{Block, BlockStore, EvmStore, SyncStore}; +use cipherbft_sync::protocol::{ + AccountData, AccountRangeRequest, AccountRangeResponse, BlockRangeRequest, BlockRangeResponse, + SnapSyncMessage, SnapshotInfo, StatusResponse, StorageRangeRequest, StorageRangeResponse, + MAX_ACCOUNTS_PER_RESPONSE, MAX_BLOCKS_PER_RESPONSE, MAX_STORAGE_PER_RESPONSE, +}; +use std::sync::Arc; +use tracing::{debug, warn}; + +/// Snap sync server that handles incoming requests from peers. +/// +/// This server queries local storage to serve state data to syncing peers. +/// It holds references to the various storage backends needed to respond +/// to different request types. +pub struct SnapSyncServer +where + B: BlockStore, + E: EvmStore, + S: SyncStore, +{ + /// Block storage for serving block data + block_store: Arc, + /// EVM state storage for accounts and storage slots + evm_store: Arc, + /// Sync storage for snapshot metadata + sync_store: Arc, +} + +impl SnapSyncServer +where + B: BlockStore, + E: EvmStore, + S: SyncStore, +{ + /// Create a new snap sync server. + /// + /// # Arguments + /// + /// * `block_store` - Storage for blocks + /// * `evm_store` - Storage for EVM accounts and storage + /// * `sync_store` - Storage for sync snapshots + pub fn new(block_store: Arc, evm_store: Arc, sync_store: Arc) -> Self { + Self { + block_store, + evm_store, + sync_store, + } + } + + /// Handle an incoming snap sync request and return the response. + /// + /// This is the main entry point for processing incoming messages. + /// It dispatches to the appropriate handler based on message type. + /// + /// # Arguments + /// + /// * `request` - The incoming snap sync message + /// + /// # Returns + /// + /// The response message, or None if the request doesn't need a response + /// (e.g., if it's already a response message). + pub async fn handle_request(&self, request: SnapSyncMessage) -> Option { + match request { + SnapSyncMessage::GetStatus => Some(self.handle_get_status().await), + SnapSyncMessage::GetAccountRange(req) => Some(self.handle_get_account_range(req).await), + SnapSyncMessage::GetStorageRange(req) => Some(self.handle_get_storage_range(req).await), + SnapSyncMessage::GetBlocks(req) => Some(self.handle_get_blocks(req).await), + // Response messages don't need handling - they're for the client side + SnapSyncMessage::Status(_) + | SnapSyncMessage::AccountRange(_) + | SnapSyncMessage::StorageRange(_) + | SnapSyncMessage::Blocks(_) => None, + } + } + + /// Handle GetStatus request. + /// + /// Returns the local blockchain tip (height and hash) along with + /// a list of available snapshot heights for sync. + async fn handle_get_status(&self) -> SnapSyncMessage { + debug!("Handling GetStatus request"); + + // Get latest block number and hash + let (tip_height, tip_hash) = match self.block_store.get_latest_block_number().await { + Ok(Some(height)) => match self.block_store.get_block_by_number(height).await { + Ok(Some(block)) => (height, B256::from(block.hash)), + Ok(None) => { + warn!("Latest block {} not found in storage", height); + (0, B256::ZERO) + } + Err(e) => { + warn!("Failed to get latest block: {}", e); + (0, B256::ZERO) + } + }, + Ok(None) => { + debug!("No blocks in storage, returning genesis state"); + (0, B256::ZERO) + } + Err(e) => { + warn!("Failed to get latest block number: {}", e); + (0, B256::ZERO) + } + }; + + // Get available snapshot heights and convert to SnapshotInfo + let snapshot_heights = match self.sync_store.list_snapshot_heights().await { + Ok(heights) => heights, + Err(e) => { + warn!("Failed to list snapshot heights: {}", e); + Vec::new() + } + }; + + // Build full SnapshotInfo for each height + let mut snapshots = Vec::with_capacity(snapshot_heights.len()); + for height in snapshot_heights { + // Get snapshot details from sync store + if let Ok(Some(stored_snapshot)) = self.sync_store.get_snapshot(height).await { + snapshots.push(SnapshotInfo { + height: stored_snapshot.block_number, + state_root: B256::from(stored_snapshot.state_root), + block_hash: B256::from(stored_snapshot.block_hash), + }); + } + } + + debug!( + tip_height, + ?tip_hash, + snapshot_count = snapshots.len(), + "Returning status" + ); + + SnapSyncMessage::Status(StatusResponse { + tip_height, + tip_hash, + snapshots, + }) + } + + /// Handle GetAccountRange request. + /// + /// Queries accounts from local storage within the specified address range. + /// Returns accounts along with merkle proofs (currently stubbed). + async fn handle_get_account_range(&self, req: AccountRangeRequest) -> SnapSyncMessage { + debug!( + snapshot_height = req.snapshot_height, + start = ?req.start_address, + limit = ?req.limit_address, + max_accounts = req.max_accounts, + "Handling GetAccountRange request" + ); + + // Cap the number of accounts to return + let max_accounts = req.max_accounts.min(MAX_ACCOUNTS_PER_RESPONSE) as usize; + + // Get all accounts from storage and filter by range + // Note: This is a simplified implementation. A production version would + // use cursor-based iteration with seek to the start address. + let accounts_result = self.evm_store.get_all_accounts(); + + let (accounts, more) = match accounts_result { + Ok(all_accounts) => { + let start_bytes: [u8; 20] = req.start_address.into(); + let limit_bytes: [u8; 20] = req.limit_address.into(); + + // Filter accounts within the range + let mut filtered: Vec = all_accounts + .into_iter() + .filter(|(addr, _)| *addr >= start_bytes && *addr < limit_bytes) + .take(max_accounts + 1) // Take one extra to check if there's more + .map(|(addr, account)| AccountData { + address: Address::from(addr), + nonce: account.nonce, + balance: U256::from_be_bytes(account.balance), + code_hash: B256::from(account.code_hash), + storage_root: B256::from(account.storage_root), + }) + .collect(); + + // Check if there are more accounts + let more = filtered.len() > max_accounts; + if more { + filtered.pop(); // Remove the extra account + } + + (filtered, more) + } + Err(e) => { + warn!("Failed to get accounts: {}", e); + (Vec::new(), false) + } + }; + + debug!( + accounts_returned = accounts.len(), + more, "Returning account range" + ); + + // TODO: Generate actual merkle proofs using alloy-trie + // For now, return empty proof - client should verify against state root + let proof = Vec::new(); + + SnapSyncMessage::AccountRange(AccountRangeResponse { + request_id: req.request_id, + accounts, + proof, + more, + }) + } + + /// Handle GetStorageRange request. + /// + /// Queries storage slots for a specific account within the slot range. + /// Returns slots along with merkle proofs (currently stubbed). + async fn handle_get_storage_range(&self, req: StorageRangeRequest) -> SnapSyncMessage { + debug!( + snapshot_height = req.snapshot_height, + account = ?req.account, + start_slot = ?req.start_slot, + limit_slot = ?req.limit_slot, + max_slots = req.max_slots, + "Handling GetStorageRange request" + ); + + // Cap the number of slots to return + let max_slots = req.max_slots.min(MAX_STORAGE_PER_RESPONSE) as usize; + + // Convert account address to bytes + let account_bytes: [u8; 20] = req.account.into(); + + // Get all storage for this account and filter by range + let storage_result = self.evm_store.get_all_storage(&account_bytes); + + let (slots, more) = match storage_result { + Ok(all_storage) => { + let start_bytes: [u8; 32] = req.start_slot.into(); + let limit_bytes: [u8; 32] = req.limit_slot.into(); + + // Filter slots within the range + let mut filtered: Vec<(B256, B256)> = all_storage + .into_iter() + .filter(|(slot, _)| *slot >= start_bytes && *slot < limit_bytes) + .take(max_slots + 1) // Take one extra to check if there's more + .map(|(slot, value)| (B256::from(slot), B256::from(value))) + .collect(); + + // Check if there are more slots + let more = filtered.len() > max_slots; + if more { + filtered.pop(); // Remove the extra slot + } + + (filtered, more) + } + Err(e) => { + warn!("Failed to get storage for account {:?}: {}", req.account, e); + (Vec::new(), false) + } + }; + + debug!( + slots_returned = slots.len(), + more, "Returning storage range" + ); + + // TODO: Generate actual merkle proofs using alloy-trie + let proof = Vec::new(); + + SnapSyncMessage::StorageRange(StorageRangeResponse { + request_id: req.request_id, + slots, + proof, + more, + }) + } + + /// Handle GetBlocks request. + /// + /// Fetches blocks from storage in the specified height range. + /// Returns serialized blocks. + async fn handle_get_blocks(&self, req: BlockRangeRequest) -> SnapSyncMessage { + debug!( + start_height = req.start_height, + count = req.count, + "Handling GetBlocks request" + ); + + // Cap the number of blocks to return + let count = req.count.min(MAX_BLOCKS_PER_RESPONSE) as u64; + + let mut blocks = Vec::new(); + + for height in req.start_height..(req.start_height + count) { + match self.block_store.get_block_by_number(height).await { + Ok(Some(block)) => { + // Serialize block to bytes + match serialize_block(&block) { + Ok(bytes) => blocks.push(bytes), + Err(e) => { + warn!("Failed to serialize block {}: {}", height, e); + break; + } + } + } + Ok(None) => { + // No more blocks available + debug!("Block {} not found, stopping", height); + break; + } + Err(e) => { + warn!("Failed to get block {}: {}", height, e); + break; + } + } + } + + debug!(blocks_returned = blocks.len(), "Returning blocks"); + + SnapSyncMessage::Blocks(BlockRangeResponse { + request_id: req.request_id, + blocks, + }) + } +} + +/// Serialize a block to bytes for network transmission. +/// +/// Uses bincode for efficient serialization. +fn serialize_block(block: &Block) -> Result { + let bytes = bincode::serialize(block)?; + Ok(Bytes::from(bytes)) +} + +/// Deserialize a block from bytes received over the network. +/// +/// Used by the client side to reconstruct blocks from responses. +#[allow(dead_code)] +pub fn deserialize_block(bytes: &[u8]) -> Result { + bincode::deserialize(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use cipherbft_storage::{ + EvmAccount, EvmStoreResult, InMemorySyncStore, StoredSyncSnapshot, SyncStore, + }; + use std::collections::HashMap; + use std::sync::RwLock; + + /// In-memory block store for testing + struct TestBlockStore { + blocks: RwLock>, + } + + impl TestBlockStore { + fn new() -> Self { + Self { + blocks: RwLock::new(HashMap::new()), + } + } + + fn add_block(&self, block: Block) { + self.blocks.write().unwrap().insert(block.number, block); + } + } + + #[async_trait::async_trait] + impl BlockStore for TestBlockStore { + async fn put_block(&self, block: &Block) -> cipherbft_storage::BlockStoreResult<()> { + self.blocks + .write() + .unwrap() + .insert(block.number, block.clone()); + Ok(()) + } + + async fn get_block_by_number( + &self, + number: u64, + ) -> cipherbft_storage::BlockStoreResult> { + Ok(self.blocks.read().unwrap().get(&number).cloned()) + } + + async fn get_block_by_hash( + &self, + hash: &[u8; 32], + ) -> cipherbft_storage::BlockStoreResult> { + Ok(self + .blocks + .read() + .unwrap() + .values() + .find(|b| &b.hash == hash) + .cloned()) + } + + async fn get_block_number_by_hash( + &self, + hash: &[u8; 32], + ) -> cipherbft_storage::BlockStoreResult> { + Ok(self + .blocks + .read() + .unwrap() + .values() + .find(|b| &b.hash == hash) + .map(|b| b.number)) + } + + async fn get_latest_block_number( + &self, + ) -> cipherbft_storage::BlockStoreResult> { + Ok(self.blocks.read().unwrap().keys().max().copied()) + } + + async fn get_earliest_block_number( + &self, + ) -> cipherbft_storage::BlockStoreResult> { + Ok(self.blocks.read().unwrap().keys().min().copied()) + } + + async fn has_block(&self, number: u64) -> cipherbft_storage::BlockStoreResult { + Ok(self.blocks.read().unwrap().contains_key(&number)) + } + + async fn delete_block(&self, number: u64) -> cipherbft_storage::BlockStoreResult<()> { + self.blocks.write().unwrap().remove(&number); + Ok(()) + } + } + + /// In-memory EVM store for testing + #[allow(clippy::type_complexity)] + struct TestEvmStore { + accounts: RwLock>, + storage: RwLock>, + } + + impl TestEvmStore { + fn new() -> Self { + Self { + accounts: RwLock::new(HashMap::new()), + storage: RwLock::new(HashMap::new()), + } + } + + #[allow(dead_code)] + fn add_account(&self, address: [u8; 20], account: EvmAccount) { + self.accounts.write().unwrap().insert(address, account); + } + + #[allow(dead_code)] + fn add_storage(&self, address: [u8; 20], slot: [u8; 32], value: [u8; 32]) { + self.storage.write().unwrap().insert((address, slot), value); + } + } + + impl EvmStore for TestEvmStore { + fn get_account(&self, address: &[u8; 20]) -> EvmStoreResult> { + Ok(self.accounts.read().unwrap().get(address).cloned()) + } + + fn set_account(&self, address: &[u8; 20], account: EvmAccount) -> EvmStoreResult<()> { + self.accounts.write().unwrap().insert(*address, account); + Ok(()) + } + + fn delete_account(&self, address: &[u8; 20]) -> EvmStoreResult<()> { + self.accounts.write().unwrap().remove(address); + Ok(()) + } + + fn get_code( + &self, + _code_hash: &[u8; 32], + ) -> EvmStoreResult> { + Ok(None) + } + + fn set_code( + &self, + _code_hash: &[u8; 32], + _bytecode: cipherbft_storage::EvmBytecode, + ) -> EvmStoreResult<()> { + Ok(()) + } + + fn get_storage(&self, address: &[u8; 20], slot: &[u8; 32]) -> EvmStoreResult<[u8; 32]> { + Ok(self + .storage + .read() + .unwrap() + .get(&(*address, *slot)) + .copied() + .unwrap_or([0u8; 32])) + } + + fn set_storage( + &self, + address: &[u8; 20], + slot: &[u8; 32], + value: [u8; 32], + ) -> EvmStoreResult<()> { + self.storage + .write() + .unwrap() + .insert((*address, *slot), value); + Ok(()) + } + + fn get_block_hash(&self, _number: u64) -> EvmStoreResult> { + Ok(None) + } + + fn set_block_hash(&self, _number: u64, _hash: [u8; 32]) -> EvmStoreResult<()> { + Ok(()) + } + + fn get_all_accounts(&self) -> EvmStoreResult> { + let accounts = self.accounts.read().unwrap(); + let mut result: Vec<_> = accounts.iter().map(|(k, v)| (*k, v.clone())).collect(); + result.sort_by_key(|(k, _)| *k); + Ok(result) + } + + fn get_all_storage(&self, address: &[u8; 20]) -> EvmStoreResult> { + let storage = self.storage.read().unwrap(); + let mut result: Vec<_> = storage + .iter() + .filter(|((addr, _), _)| addr == address) + .map(|((_, slot), value)| (*slot, *value)) + .collect(); + result.sort_by_key(|(k, _)| *k); + Ok(result) + } + + fn get_current_block(&self) -> EvmStoreResult> { + Ok(None) + } + + fn set_current_block(&self, _block_number: u64) -> EvmStoreResult<()> { + Ok(()) + } + } + + fn make_test_block(number: u64) -> Block { + let mut hash = [0u8; 32]; + hash[0] = (number & 0xff) as u8; + hash[1] = ((number >> 8) & 0xff) as u8; + + Block { + hash, + number, + parent_hash: [number.saturating_sub(1) as u8; 32], + ommers_hash: [0u8; 32], + beneficiary: [1u8; 20], + state_root: [2u8; 32], + transactions_root: [3u8; 32], + receipts_root: [4u8; 32], + logs_bloom: vec![0u8; 256], + difficulty: [0u8; 32], + gas_limit: 30_000_000, + gas_used: 21_000, + timestamp: 1700000000 + number, + extra_data: vec![], + mix_hash: [5u8; 32], + nonce: [0u8; 8], + base_fee_per_gas: Some(1_000_000_000), + transaction_hashes: vec![], + transaction_count: 0, + total_difficulty: [0u8; 32], + size: 500, + } + } + + #[tokio::test] + async fn test_handle_get_status() { + let block_store = Arc::new(TestBlockStore::new()); + let evm_store = Arc::new(TestEvmStore::new()); + let sync_store = Arc::new(InMemorySyncStore::new()); + + // Add some blocks + block_store.add_block(make_test_block(1)); + block_store.add_block(make_test_block(2)); + block_store.add_block(make_test_block(3)); + + // Add a snapshot + sync_store + .put_snapshot(StoredSyncSnapshot::new( + 10000, [0xab; 32], [0xcd; 32], 123456, + )) + .await + .unwrap(); + + let server = SnapSyncServer::new(block_store, evm_store, sync_store); + + let response = server.handle_request(SnapSyncMessage::GetStatus).await; + + match response { + Some(SnapSyncMessage::Status(status)) => { + assert_eq!(status.tip_height, 3); + // Verify we got one snapshot with correct data + assert_eq!(status.snapshots.len(), 1); + assert_eq!(status.snapshots[0].height, 10000); + assert_eq!(status.snapshots[0].state_root, B256::from([0xcd; 32])); + assert_eq!(status.snapshots[0].block_hash, B256::from([0xab; 32])); + } + _ => panic!("Expected Status response"), + } + } + + #[tokio::test] + async fn test_handle_get_blocks() { + let block_store = Arc::new(TestBlockStore::new()); + let evm_store = Arc::new(TestEvmStore::new()); + let sync_store = Arc::new(InMemorySyncStore::new()); + + // Add blocks + for i in 1..=10 { + block_store.add_block(make_test_block(i)); + } + + let server = SnapSyncServer::new(block_store, evm_store, sync_store); + + let request = SnapSyncMessage::GetBlocks(BlockRangeRequest { + request_id: 42, + start_height: 5, + count: 3, + }); + + let response = server.handle_request(request).await; + + match response { + Some(SnapSyncMessage::Blocks(blocks_response)) => { + assert_eq!(blocks_response.blocks.len(), 3); + // Verify we can deserialize the blocks + for (i, block_bytes) in blocks_response.blocks.iter().enumerate() { + let block = deserialize_block(block_bytes).unwrap(); + assert_eq!(block.number, 5 + i as u64); + } + } + _ => panic!("Expected Blocks response"), + } + } + + #[tokio::test] + async fn test_response_messages_return_none() { + let block_store = Arc::new(TestBlockStore::new()); + let evm_store = Arc::new(TestEvmStore::new()); + let sync_store = Arc::new(InMemorySyncStore::new()); + + let server = SnapSyncServer::new(block_store, evm_store, sync_store); + + // Response messages should return None (no response needed) + let status_response = SnapSyncMessage::Status(StatusResponse { + tip_height: 100, + tip_hash: B256::ZERO, + snapshots: vec![], + }); + + assert!(server.handle_request(status_response).await.is_none()); + } +} diff --git a/crates/storage/src/blocks.rs b/crates/storage/src/blocks.rs index a7961cb..615838a 100644 --- a/crates/storage/src/blocks.rs +++ b/crates/storage/src/blocks.rs @@ -16,6 +16,7 @@ //! ``` use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use crate::error::StorageError; @@ -25,7 +26,7 @@ pub type BlockStoreResult = Result; /// Stored block data structure. /// /// Contains all the information returned by `eth_getBlockByNumber`. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Block { /// Block hash (32 bytes) pub hash: [u8; 32], diff --git a/crates/storage/src/evm.rs b/crates/storage/src/evm.rs index 54d0c39..2a2b5fe 100644 --- a/crates/storage/src/evm.rs +++ b/crates/storage/src/evm.rs @@ -145,6 +145,27 @@ pub trait EvmStore: Send + Sync { /// * `hash` - 32-byte block hash fn set_block_hash(&self, number: u64, hash: [u8; 32]) -> EvmStoreResult<()>; + /// Get all accounts from the database. + /// + /// Returns all accounts stored in the database, ordered by address. + /// This is useful for state sync, debugging, and migration purposes. + /// + /// # Returns + /// A vector of (address, account) tuples ordered by address. + fn get_all_accounts(&self) -> EvmStoreResult>; + + /// Get all storage slots for a specific address. + /// + /// Returns all storage slots stored for the given address, ordered by slot. + /// This is useful for state sync and contract inspection. + /// + /// # Arguments + /// * `address` - The 20-byte Ethereum address to get storage for + /// + /// # Returns + /// A vector of (slot, value) tuples ordered by slot. + fn get_all_storage(&self, address: &[u8; 20]) -> EvmStoreResult>; + /// Get the current block number (last executed block). /// /// This method retrieves the persisted block number for execution engine recovery. diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index ad1f676..a7fa196 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -63,6 +63,7 @@ pub mod persistent_state; pub mod pruning; pub mod receipts; pub mod staking; +pub mod sync; pub mod tables; pub mod transactions; pub mod wal; @@ -85,6 +86,10 @@ pub use persistent_state::{ pub use pruning::{PruningConfig, PruningHandle, PruningTask}; pub use receipts::{Log, Receipt, ReceiptStore, ReceiptStoreResult}; pub use staking::{StakingStore, StakingStoreResult, StoredValidator}; +pub use sync::{ + InMemorySyncStore, StoredAccountProgress, StoredBlockProgress, StoredStorageProgress, + StoredSyncPhase, StoredSyncProgress, StoredSyncSnapshot, SyncStore, SYNC_SNAPSHOT_INTERVAL, +}; pub use transactions::{Transaction, TransactionStore, TransactionStoreResult}; pub use wal::{Wal, WalEntry}; @@ -92,5 +97,5 @@ pub use wal::{Wal, WalEntry}; #[cfg(feature = "mdbx")] pub use mdbx::{ Database, DatabaseConfig, MdbxBatchStore, MdbxBlockStore, MdbxDclStore, MdbxEvmStore, - MdbxLogStore, MdbxReceiptStore, MdbxStakingStore, MdbxTransactionStore, MdbxWal, + MdbxLogStore, MdbxReceiptStore, MdbxStakingStore, MdbxSyncStore, MdbxTransactionStore, MdbxWal, }; diff --git a/crates/storage/src/mdbx/evm.rs b/crates/storage/src/mdbx/evm.rs index d247f40..e56aa17 100644 --- a/crates/storage/src/mdbx/evm.rs +++ b/crates/storage/src/mdbx/evm.rs @@ -343,6 +343,16 @@ impl EvmStore for MdbxEvmStore { Ok(()) } + fn get_all_accounts(&self) -> EvmStoreResult> { + // Delegate to the inherent method + MdbxEvmStore::get_all_accounts(self) + } + + fn get_all_storage(&self, address: &[u8; 20]) -> EvmStoreResult> { + // Delegate to the inherent method + MdbxEvmStore::get_all_storage(self, address) + } + fn get_current_block(&self) -> EvmStoreResult> { let start = Instant::now(); let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; diff --git a/crates/storage/src/mdbx/mod.rs b/crates/storage/src/mdbx/mod.rs index 0360d89..2229b5d 100644 --- a/crates/storage/src/mdbx/mod.rs +++ b/crates/storage/src/mdbx/mod.rs @@ -29,6 +29,7 @@ mod persistent_state; mod provider; mod receipts; mod staking; +mod sync; mod tables; mod transactions; mod wal; @@ -42,6 +43,7 @@ pub use persistent_state::{MdbxPersistentStateBuilder, MdbxStatePersistence}; pub use provider::{MdbxDclStore, MdbxDclStoreTx}; pub use receipts::MdbxReceiptStore; pub use staking::MdbxStakingStore; +pub use sync::MdbxSyncStore; pub use tables::{ // EVM table types AddressKey, @@ -82,12 +84,15 @@ pub use tables::{ StakingValidators, StorageSlotKey, StoredAccount, + // Sync table types + StoredAccountProgress, // Consensus value types StoredAggregatedAttestation, StoredBatch, StoredBatchDigest, // Block value types StoredBlock, + StoredBlockProgress, StoredBloom, StoredBytecode, StoredCar, @@ -100,7 +105,11 @@ pub use tables::{ StoredProposal, StoredReceipt, StoredStakingMetadata, + StoredStorageProgress, StoredStorageValue, + StoredSyncPhase, + StoredSyncProgressState, + StoredSyncSnapshot, // Transaction table types StoredTransaction, StoredValidator, @@ -109,6 +118,9 @@ pub use tables::{ StoredVote, StoredVotes, StoredWalEntry, + SyncProgress, + SyncProgressKey, + SyncSnapshots, Tables, Transactions, TransactionsByBlock, diff --git a/crates/storage/src/mdbx/sync.rs b/crates/storage/src/mdbx/sync.rs new file mode 100644 index 0000000..7cc1ee0 --- /dev/null +++ b/crates/storage/src/mdbx/sync.rs @@ -0,0 +1,438 @@ +//! MDBX-based implementation of sync storage. +//! +//! This module provides the [`MdbxSyncStore`] implementation of [`SyncStore`] trait +//! using MDBX as the backing storage engine. + +use std::sync::Arc; + +use async_trait::async_trait; +use reth_db::Database; +use reth_db_api::cursor::DbCursorRO; +use reth_db_api::transaction::{DbTx, DbTxMut}; + +use super::database::DatabaseEnv; +use super::tables::{ + BlockNumberKey, StoredAccountProgress, StoredBlockProgress, StoredStorageProgress, + StoredSyncPhase, StoredSyncProgressState, StoredSyncSnapshot, SyncProgress, SyncProgressKey, + SyncSnapshots, +}; +use crate::error::{Result, StorageError}; +use crate::sync::{ + StoredSyncProgress as SyncStoredSyncProgress, StoredSyncSnapshot as SyncStoredSyncSnapshot, + SyncStore, +}; + +/// Helper to convert database errors to storage errors. +fn db_err(e: impl std::fmt::Display) -> StorageError { + StorageError::Database(e.to_string()) +} + +/// MDBX-based sync storage implementation. +/// +/// This implementation uses reth-db (MDBX) for persistent storage of sync state. +pub struct MdbxSyncStore { + db: Arc, +} + +impl MdbxSyncStore { + /// Create a new MDBX sync store. + /// + /// # Arguments + /// * `db` - Shared reference to the MDBX database environment + pub fn new(db: Arc) -> Self { + Self { db } + } + + /// Convert from sync module's StoredSyncSnapshot to MDBX's StoredSyncSnapshot + fn to_mdbx_snapshot(snapshot: &SyncStoredSyncSnapshot) -> StoredSyncSnapshot { + StoredSyncSnapshot { + block_number: snapshot.block_number, + block_hash: snapshot.block_hash, + state_root: snapshot.state_root, + timestamp: snapshot.timestamp, + } + } + + /// Convert from MDBX's StoredSyncSnapshot to sync module's StoredSyncSnapshot + fn from_mdbx_snapshot(snapshot: &StoredSyncSnapshot) -> SyncStoredSyncSnapshot { + SyncStoredSyncSnapshot { + block_number: snapshot.block_number, + block_hash: snapshot.block_hash, + state_root: snapshot.state_root, + timestamp: snapshot.timestamp, + } + } + + /// Convert from sync module's StoredSyncProgress to MDBX's StoredSyncProgressState + fn to_mdbx_progress(progress: &SyncStoredSyncProgress) -> StoredSyncProgressState { + use crate::sync::StoredSyncPhase as SyncPhase; + + let phase = match &progress.phase { + SyncPhase::Discovery => StoredSyncPhase::Discovery, + SyncPhase::SnapSyncAccounts => StoredSyncPhase::SnapSyncAccounts, + SyncPhase::SnapSyncStorage => StoredSyncPhase::SnapSyncStorage, + SyncPhase::SnapSyncVerification => StoredSyncPhase::SnapSyncVerification, + SyncPhase::BlockSync => StoredSyncPhase::BlockSync, + SyncPhase::Complete => StoredSyncPhase::Complete, + }; + + let target_snapshot = progress + .target_snapshot + .as_ref() + .map(Self::to_mdbx_snapshot); + + let account_progress = StoredAccountProgress { + completed_up_to: progress.account_progress.completed_up_to, + accounts_needing_storage: progress.account_progress.accounts_needing_storage.clone(), + total_accounts: progress.account_progress.total_accounts, + total_bytes: progress.account_progress.total_bytes, + }; + + let storage_progress: Vec<([u8; 20], StoredStorageProgress)> = progress + .storage_progress + .iter() + .map(|(addr, prog)| { + ( + *addr, + StoredStorageProgress { + completed_up_to: prog.completed_up_to, + total_slots: prog.total_slots, + }, + ) + }) + .collect(); + + let block_progress = StoredBlockProgress { + start_height: progress.block_progress.start_height, + executed_up_to: progress.block_progress.executed_up_to, + target_height: progress.block_progress.target_height, + }; + + StoredSyncProgressState { + phase, + target_snapshot, + account_progress, + storage_progress, + block_progress, + } + } + + /// Convert from MDBX's StoredSyncProgressState to sync module's StoredSyncProgress + fn from_mdbx_progress(progress: &StoredSyncProgressState) -> SyncStoredSyncProgress { + use crate::sync::{ + StoredAccountProgress as SyncAccountProgress, StoredBlockProgress as SyncBlockProgress, + StoredStorageProgress as SyncStorageProgress, StoredSyncPhase as SyncPhase, + }; + + let phase = match &progress.phase { + StoredSyncPhase::Discovery => SyncPhase::Discovery, + StoredSyncPhase::SnapSyncAccounts => SyncPhase::SnapSyncAccounts, + StoredSyncPhase::SnapSyncStorage => SyncPhase::SnapSyncStorage, + StoredSyncPhase::SnapSyncVerification => SyncPhase::SnapSyncVerification, + StoredSyncPhase::BlockSync => SyncPhase::BlockSync, + StoredSyncPhase::Complete => SyncPhase::Complete, + }; + + let target_snapshot = progress + .target_snapshot + .as_ref() + .map(Self::from_mdbx_snapshot); + + let account_progress = SyncAccountProgress { + completed_up_to: progress.account_progress.completed_up_to, + accounts_needing_storage: progress.account_progress.accounts_needing_storage.clone(), + total_accounts: progress.account_progress.total_accounts, + total_bytes: progress.account_progress.total_bytes, + }; + + let storage_progress: Vec<([u8; 20], SyncStorageProgress)> = progress + .storage_progress + .iter() + .map(|(addr, prog)| { + ( + *addr, + SyncStorageProgress { + completed_up_to: prog.completed_up_to, + total_slots: prog.total_slots, + }, + ) + }) + .collect(); + + let block_progress = SyncBlockProgress { + start_height: progress.block_progress.start_height, + executed_up_to: progress.block_progress.executed_up_to, + target_height: progress.block_progress.target_height, + }; + + SyncStoredSyncProgress { + phase, + target_snapshot, + account_progress, + storage_progress, + block_progress, + } + } +} + +#[async_trait] +impl SyncStore for MdbxSyncStore { + async fn put_snapshot(&self, snapshot: SyncStoredSyncSnapshot) -> Result<()> { + let tx = self.db.tx_mut().map_err(|e| db_err(e.to_string()))?; + + let key = BlockNumberKey::new(snapshot.block_number); + let stored = Self::to_mdbx_snapshot(&snapshot); + + tx.put::(key, stored.into()) + .map_err(|e| db_err(e.to_string()))?; + + tx.commit().map_err(|e| db_err(e.to_string()))?; + + Ok(()) + } + + async fn get_snapshot(&self, block_number: u64) -> Result> { + let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; + + let key = BlockNumberKey::new(block_number); + let result = tx + .get::(key) + .map_err(|e| db_err(e.to_string()))?; + + Ok(result.map(|stored| Self::from_mdbx_snapshot(&stored.0))) + } + + async fn get_latest_snapshot(&self) -> Result> { + let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; + + let mut cursor = tx + .cursor_read::() + .map_err(|e| db_err(e.to_string()))?; + + let last_entry = cursor.last().map_err(|e| db_err(e.to_string()))?; + + Ok(last_entry.map(|(_, stored)| Self::from_mdbx_snapshot(&stored.0))) + } + + async fn get_snapshot_at_or_before( + &self, + block_number: u64, + ) -> Result> { + let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; + + let mut cursor = tx + .cursor_read::() + .map_err(|e| db_err(e.to_string()))?; + + // Seek to the key and work backwards if needed + let key = BlockNumberKey::new(block_number); + let entry = cursor.seek(key).map_err(|e| db_err(e.to_string()))?; + + match entry { + Some((found_key, stored)) if found_key.0 == block_number => { + Ok(Some(Self::from_mdbx_snapshot(&stored.0))) + } + Some((found_key, _)) if found_key.0 > block_number => { + // We went past, go back one + let prev = cursor.prev().map_err(|e| db_err(e.to_string()))?; + Ok(prev.map(|(_, stored)| Self::from_mdbx_snapshot(&stored.0))) + } + Some((_, stored)) => Ok(Some(Self::from_mdbx_snapshot(&stored.0))), + None => { + // Seek went past end, get last entry + let last = cursor.last().map_err(|e| db_err(e.to_string()))?; + match last { + Some((k, stored)) if k.0 <= block_number => { + Ok(Some(Self::from_mdbx_snapshot(&stored.0))) + } + _ => Ok(None), + } + } + } + } + + async fn delete_snapshot(&self, block_number: u64) -> Result<()> { + let tx = self.db.tx_mut().map_err(|e| db_err(e.to_string()))?; + + let key = BlockNumberKey::new(block_number); + tx.delete::(key, None) + .map_err(|e| db_err(e.to_string()))?; + + tx.commit().map_err(|e| db_err(e.to_string()))?; + + Ok(()) + } + + async fn list_snapshot_heights(&self) -> Result> { + let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; + + let mut cursor = tx + .cursor_read::() + .map_err(|e| db_err(e.to_string()))?; + + let mut heights = Vec::new(); + let mut entry = cursor.first().map_err(|e| db_err(e.to_string()))?; + + while let Some((key, _)) = entry { + heights.push(key.0); + entry = cursor.next().map_err(|e| db_err(e.to_string()))?; + } + + Ok(heights) + } + + async fn get_progress(&self) -> Result> { + let tx = self.db.tx().map_err(|e| db_err(e.to_string()))?; + + let result = tx + .get::(SyncProgressKey) + .map_err(|e| db_err(e.to_string()))?; + + Ok(result.map(|stored| Self::from_mdbx_progress(&stored.0))) + } + + async fn put_progress(&self, progress: SyncStoredSyncProgress) -> Result<()> { + let tx = self.db.tx_mut().map_err(|e| db_err(e.to_string()))?; + + let stored = Self::to_mdbx_progress(&progress); + tx.put::(SyncProgressKey, stored.into()) + .map_err(|e| db_err(e.to_string()))?; + + tx.commit().map_err(|e| db_err(e.to_string()))?; + + Ok(()) + } + + async fn delete_progress(&self) -> Result<()> { + let tx = self.db.tx_mut().map_err(|e| db_err(e.to_string()))?; + + tx.delete::(SyncProgressKey, None) + .map_err(|e| db_err(e.to_string()))?; + + tx.commit().map_err(|e| db_err(e.to_string()))?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mdbx::Database; + use crate::sync::{StoredSyncPhase, StoredSyncProgress, StoredSyncSnapshot}; + + fn create_test_db() -> (Arc, tempfile::TempDir) { + let (db, temp_dir) = Database::open_temp().unwrap(); + (Arc::clone(db.env()), temp_dir) + } + + #[tokio::test] + async fn test_snapshot_operations() { + let (db, _temp_dir) = create_test_db(); + let store = MdbxSyncStore::new(db); + + // Create and store snapshots + let snapshot1 = StoredSyncSnapshot::new(10000, [0xab; 32], [0xcd; 32], 1234567890); + let snapshot2 = StoredSyncSnapshot::new(20000, [0xef; 32], [0x12; 32], 1234567900); + + store.put_snapshot(snapshot1.clone()).await.unwrap(); + store.put_snapshot(snapshot2.clone()).await.unwrap(); + + // Get by block number + let retrieved = store.get_snapshot(10000).await.unwrap().unwrap(); + assert_eq!(retrieved.block_number, snapshot1.block_number); + assert_eq!(retrieved.block_hash, snapshot1.block_hash); + + // Get latest + let latest = store.get_latest_snapshot().await.unwrap().unwrap(); + assert_eq!(latest.block_number, snapshot2.block_number); + + // Get at or before + let at_or_before = store + .get_snapshot_at_or_before(15000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_or_before.block_number, snapshot1.block_number); + + // List heights + let heights = store.list_snapshot_heights().await.unwrap(); + assert_eq!(heights, vec![10000, 20000]); + + // Delete + store.delete_snapshot(10000).await.unwrap(); + assert!(store.get_snapshot(10000).await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_progress_operations() { + let (db, _temp_dir) = create_test_db(); + let store = MdbxSyncStore::new(db); + + // Initially no progress + assert!(store.get_progress().await.unwrap().is_none()); + + // Store progress + let mut progress = StoredSyncProgress::new(); + progress.phase = StoredSyncPhase::SnapSyncAccounts; + progress.account_progress.total_accounts = 5000; + + store.put_progress(progress.clone()).await.unwrap(); + + // Get progress + let retrieved = store.get_progress().await.unwrap().unwrap(); + assert_eq!(retrieved.phase, StoredSyncPhase::SnapSyncAccounts); + assert_eq!(retrieved.account_progress.total_accounts, 5000); + + // Delete progress + store.delete_progress().await.unwrap(); + assert!(store.get_progress().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_snapshot_at_or_before() { + let (db, _temp_dir) = create_test_db(); + let store = MdbxSyncStore::new(db); + + // Add snapshots at 10k, 20k, 30k + for height in [10000u64, 20000, 30000] { + let snapshot = StoredSyncSnapshot::new(height, [height as u8; 32], [0; 32], height); + store.put_snapshot(snapshot).await.unwrap(); + } + + // Query at exact boundaries + let at_10k = store + .get_snapshot_at_or_before(10000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_10k.block_number, 10000); + + // Query between snapshots + let at_15k = store + .get_snapshot_at_or_before(15000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_15k.block_number, 10000); + + let at_25k = store + .get_snapshot_at_or_before(25000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_25k.block_number, 20000); + + // Query past all snapshots + let at_40k = store + .get_snapshot_at_or_before(40000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_40k.block_number, 30000); + + // Query before first snapshot + let at_5k = store.get_snapshot_at_or_before(5000).await.unwrap(); + assert!(at_5k.is_none()); + } +} diff --git a/crates/storage/src/mdbx/tables.rs b/crates/storage/src/mdbx/tables.rs index 740e69a..f53fda7 100644 --- a/crates/storage/src/mdbx/tables.rs +++ b/crates/storage/src/mdbx/tables.rs @@ -1422,6 +1422,151 @@ impl Decompress for UnitKey { } } +// ============================================================================= +// Sync Tables (for snap sync and state sync) +// ============================================================================= + +/// Key for SyncProgress table (singleton, always "current") +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, Default, PartialOrd, Ord, Serialize, Deserialize, +)] +pub struct SyncProgressKey; + +impl Encode for SyncProgressKey { + type Encoded = [u8; 7]; // "current" + + fn encode(self) -> Self::Encoded { + *b"current" + } +} + +impl Decode for SyncProgressKey { + fn decode(_value: &[u8]) -> Result { + Ok(Self) + } +} + +impl Compress for SyncProgressKey { + type Compressed = Vec; + + fn compress(self) -> Self::Compressed { + b"current".to_vec() + } + + fn compress_to_buf>(&self, buf: &mut B) { + buf.put_slice(b"current"); + } +} + +impl Decompress for SyncProgressKey { + fn decompress(_value: &[u8]) -> Result { + Ok(Self) + } +} + +/// Stored sync snapshot for MDBX persistence +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StoredSyncSnapshot { + /// Block height this snapshot represents + pub block_number: u64, + /// Block hash at this height (32 bytes) + pub block_hash: [u8; 32], + /// State root (MPT root of all accounts, 32 bytes) + pub state_root: [u8; 32], + /// Unix timestamp when snapshot was created + pub timestamp: u64, +} + +/// Stored sync phase for MDBX persistence +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub enum StoredSyncPhase { + /// Finding peers and selecting snapshot + #[default] + Discovery, + /// Downloading accounts + SnapSyncAccounts, + /// Downloading storage for accounts + SnapSyncStorage, + /// Final state root verification + SnapSyncVerification, + /// Downloading and executing blocks + BlockSync, + /// Sync complete + Complete, +} + +/// Stored account progress for sync resumability +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StoredAccountProgress { + /// Last completed address (exclusive upper bound, 20 bytes) + pub completed_up_to: Option<[u8; 20]>, + /// Addresses that need storage downloaded + pub accounts_needing_storage: Vec<[u8; 20]>, + /// Total accounts downloaded + pub total_accounts: u64, + /// Total bytes downloaded + pub total_bytes: u64, +} + +/// Stored storage slot progress per account +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StoredStorageProgress { + /// Last completed slot (exclusive upper bound, 32 bytes) + pub completed_up_to: Option<[u8; 32]>, + /// Total slots downloaded + pub total_slots: u64, +} + +/// Stored block sync progress +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StoredBlockProgress { + /// First block needed (snapshot height + 1) + pub start_height: u64, + /// Last block successfully executed + pub executed_up_to: u64, + /// Target height to sync to + pub target_height: u64, +} + +/// Stored sync progress state for MDBX persistence +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StoredSyncProgressState { + /// Current sync phase + pub phase: StoredSyncPhase, + /// Target snapshot being synced to + pub target_snapshot: Option, + /// Account download progress + pub account_progress: StoredAccountProgress, + /// Storage download progress per account (address -> progress) + pub storage_progress: Vec<([u8; 20], StoredStorageProgress)>, + /// Block sync progress + pub block_progress: StoredBlockProgress, +} + +/// SyncSnapshots table: BlockNumber -> StoredSyncSnapshot +/// Stores state snapshots at regular intervals (every 10,000 blocks) +#[derive(Debug, Clone, Copy, Default)] +pub struct SyncSnapshots; + +impl Table for SyncSnapshots { + const NAME: &'static str = "SyncSnapshots"; + const DUPSORT: bool = false; + type Key = BlockNumberKey; + type Value = BincodeValue; +} + +/// SyncProgress table: SyncProgressKey -> StoredSyncProgressState +/// Stores current sync progress for resumability (single row) +#[derive(Debug, Clone, Copy, Default)] +pub struct SyncProgress; + +impl Table for SyncProgress { + const NAME: &'static str = "SyncProgress"; + const DUPSORT: bool = false; + type Key = SyncProgressKey; + type Value = BincodeValue; +} + // ============================================================================= // TableInfo and TableSet implementation for CipherBFT custom tables // ============================================================================= @@ -1467,6 +1612,9 @@ pub enum CipherBftTable { // Persistent state tables StateSnapshots, StateDeltas, + // Sync tables + SyncSnapshots, + SyncProgress, } impl CipherBftTable { @@ -1510,6 +1658,9 @@ impl CipherBftTable { // Persistent state tables Self::StateSnapshots, Self::StateDeltas, + // Sync tables + Self::SyncSnapshots, + Self::SyncProgress, ]; } @@ -1546,6 +1697,8 @@ impl TableInfo for CipherBftTable { Self::BlockBlooms => BlockBlooms::NAME, Self::StateSnapshots => StateSnapshots::NAME, Self::StateDeltas => StateDeltas::NAME, + Self::SyncSnapshots => SyncSnapshots::NAME, + Self::SyncProgress => SyncProgress::NAME, } } @@ -1581,6 +1734,8 @@ impl TableInfo for CipherBftTable { Self::BlockBlooms => BlockBlooms::DUPSORT, Self::StateSnapshots => StateSnapshots::DUPSORT, Self::StateDeltas => StateDeltas::DUPSORT, + Self::SyncSnapshots => SyncSnapshots::DUPSORT, + Self::SyncProgress => SyncProgress::DUPSORT, } } } @@ -1629,6 +1784,9 @@ impl Tables { // Persistent state tables StateSnapshots::NAME, StateDeltas::NAME, + // Sync tables + SyncSnapshots::NAME, + SyncProgress::NAME, ]; } diff --git a/crates/storage/src/sync.rs b/crates/storage/src/sync.rs new file mode 100644 index 0000000..4c5c176 --- /dev/null +++ b/crates/storage/src/sync.rs @@ -0,0 +1,343 @@ +//! Sync storage for snap sync snapshots and progress tracking +//! +//! This module provides storage for state synchronization, enabling: +//! - Persistence of snapshots at regular intervals (every 10,000 blocks) +//! - Progress tracking for resumable sync operations +//! +//! # Tables +//! +//! | Table | Key | Value | Description | +//! |-------|-----|-------|-------------| +//! | SyncSnapshots | BlockNumber | StoredSyncSnapshot | State snapshots at interval blocks | +//! | SyncProgress | "current" | StoredSyncProgress | Current sync progress (single row) | + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use crate::error::Result; + +/// Interval between sync snapshots (in blocks) +pub const SYNC_SNAPSHOT_INTERVAL: u64 = 10_000; + +/// Stored sync snapshot metadata +/// +/// Represents a point-in-time state snapshot that can be used as a +/// starting point for state synchronization. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StoredSyncSnapshot { + /// Block height this snapshot represents + pub block_number: u64, + /// Block hash at this height (32 bytes) + pub block_hash: [u8; 32], + /// State root (MPT root of all accounts, 32 bytes) + pub state_root: [u8; 32], + /// Unix timestamp when snapshot was created + pub timestamp: u64, +} + +impl StoredSyncSnapshot { + /// Create a new stored sync snapshot + pub fn new( + block_number: u64, + block_hash: [u8; 32], + state_root: [u8; 32], + timestamp: u64, + ) -> Self { + Self { + block_number, + block_hash, + state_root, + timestamp, + } + } +} + +/// Sync phase enumeration for persistence +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub enum StoredSyncPhase { + /// Finding peers and selecting snapshot + #[default] + Discovery, + /// Downloading accounts + SnapSyncAccounts, + /// Downloading storage for accounts + SnapSyncStorage, + /// Final state root verification + SnapSyncVerification, + /// Downloading and executing blocks + BlockSync, + /// Sync complete + Complete, +} + +/// Stored account progress for sync resumability +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StoredAccountProgress { + /// Last completed address (exclusive upper bound, 20 bytes) + pub completed_up_to: Option<[u8; 20]>, + /// Addresses that need storage downloaded + pub accounts_needing_storage: Vec<[u8; 20]>, + /// Total accounts downloaded + pub total_accounts: u64, + /// Total bytes downloaded + pub total_bytes: u64, +} + +/// Stored storage slot progress per account +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StoredStorageProgress { + /// Last completed slot (exclusive upper bound, 32 bytes) + pub completed_up_to: Option<[u8; 32]>, + /// Total slots downloaded + pub total_slots: u64, +} + +/// Stored block sync progress +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StoredBlockProgress { + /// First block needed (snapshot height + 1) + pub start_height: u64, + /// Last block successfully executed + pub executed_up_to: u64, + /// Target height to sync to + pub target_height: u64, +} + +/// Stored sync progress state +/// +/// Contains all the information needed to resume a sync operation +/// after a restart. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StoredSyncProgress { + /// Current sync phase + pub phase: StoredSyncPhase, + /// Target snapshot being synced to + pub target_snapshot: Option, + /// Account download progress + pub account_progress: StoredAccountProgress, + /// Storage download progress per account (address -> progress) + pub storage_progress: Vec<([u8; 20], StoredStorageProgress)>, + /// Block sync progress + pub block_progress: StoredBlockProgress, +} + +impl StoredSyncProgress { + /// Create a new empty sync progress state + pub fn new() -> Self { + Self::default() + } + + /// Reset progress for fresh sync + pub fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Trait for sync storage operations +/// +/// Provides CRUD operations for sync snapshots and progress tracking. +#[async_trait] +pub trait SyncStore: Send + Sync { + // ======================================================================== + // Snapshot Operations + // ======================================================================== + + /// Store a sync snapshot at a specific block number + async fn put_snapshot(&self, snapshot: StoredSyncSnapshot) -> Result<()>; + + /// Get a snapshot by block number + async fn get_snapshot(&self, block_number: u64) -> Result>; + + /// Get the latest snapshot + async fn get_latest_snapshot(&self) -> Result>; + + /// Get snapshot at or before a specific block number + async fn get_snapshot_at_or_before( + &self, + block_number: u64, + ) -> Result>; + + /// Delete a snapshot by block number + async fn delete_snapshot(&self, block_number: u64) -> Result<()>; + + /// List all snapshot block numbers + async fn list_snapshot_heights(&self) -> Result>; + + // ======================================================================== + // Progress Operations + // ======================================================================== + + /// Get current sync progress (there's only one, keyed by "current") + async fn get_progress(&self) -> Result>; + + /// Store sync progress (overwrites existing) + async fn put_progress(&self, progress: StoredSyncProgress) -> Result<()>; + + /// Delete sync progress (used after successful sync completion) + async fn delete_progress(&self) -> Result<()>; +} + +/// In-memory implementation of SyncStore for testing +pub struct InMemorySyncStore { + snapshots: parking_lot::RwLock>, + progress: parking_lot::RwLock>, +} + +impl InMemorySyncStore { + /// Create a new in-memory sync store + pub fn new() -> Self { + Self { + snapshots: parking_lot::RwLock::new(std::collections::BTreeMap::new()), + progress: parking_lot::RwLock::new(None), + } + } +} + +impl Default for InMemorySyncStore { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl SyncStore for InMemorySyncStore { + async fn put_snapshot(&self, snapshot: StoredSyncSnapshot) -> Result<()> { + let block_number = snapshot.block_number; + self.snapshots.write().insert(block_number, snapshot); + Ok(()) + } + + async fn get_snapshot(&self, block_number: u64) -> Result> { + Ok(self.snapshots.read().get(&block_number).cloned()) + } + + async fn get_latest_snapshot(&self) -> Result> { + Ok(self.snapshots.read().values().last().cloned()) + } + + async fn get_snapshot_at_or_before( + &self, + block_number: u64, + ) -> Result> { + Ok(self + .snapshots + .read() + .range(..=block_number) + .next_back() + .map(|(_, s)| s.clone())) + } + + async fn delete_snapshot(&self, block_number: u64) -> Result<()> { + self.snapshots.write().remove(&block_number); + Ok(()) + } + + async fn list_snapshot_heights(&self) -> Result> { + Ok(self.snapshots.read().keys().copied().collect()) + } + + async fn get_progress(&self) -> Result> { + Ok(self.progress.read().clone()) + } + + async fn put_progress(&self, progress: StoredSyncProgress) -> Result<()> { + *self.progress.write() = Some(progress); + Ok(()) + } + + async fn delete_progress(&self) -> Result<()> { + *self.progress.write() = None; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_snapshot_operations() { + let store = InMemorySyncStore::new(); + + // Create and store snapshots + let snapshot1 = StoredSyncSnapshot::new(10000, [0xab; 32], [0xcd; 32], 1234567890); + let snapshot2 = StoredSyncSnapshot::new(20000, [0xef; 32], [0x12; 32], 1234567900); + + store.put_snapshot(snapshot1.clone()).await.unwrap(); + store.put_snapshot(snapshot2.clone()).await.unwrap(); + + // Get by block number + let retrieved = store.get_snapshot(10000).await.unwrap().unwrap(); + assert_eq!(retrieved, snapshot1); + + // Get latest + let latest = store.get_latest_snapshot().await.unwrap().unwrap(); + assert_eq!(latest, snapshot2); + + // Get at or before + let at_or_before = store + .get_snapshot_at_or_before(15000) + .await + .unwrap() + .unwrap(); + assert_eq!(at_or_before, snapshot1); + + // List heights + let heights = store.list_snapshot_heights().await.unwrap(); + assert_eq!(heights, vec![10000, 20000]); + + // Delete + store.delete_snapshot(10000).await.unwrap(); + assert!(store.get_snapshot(10000).await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_progress_operations() { + let store = InMemorySyncStore::new(); + + // Initially no progress + assert!(store.get_progress().await.unwrap().is_none()); + + // Store progress + let mut progress = StoredSyncProgress::new(); + progress.phase = StoredSyncPhase::SnapSyncAccounts; + progress.account_progress.total_accounts = 5000; + + store.put_progress(progress.clone()).await.unwrap(); + + // Get progress + let retrieved = store.get_progress().await.unwrap().unwrap(); + assert_eq!(retrieved.phase, StoredSyncPhase::SnapSyncAccounts); + assert_eq!(retrieved.account_progress.total_accounts, 5000); + + // Delete progress + store.delete_progress().await.unwrap(); + assert!(store.get_progress().await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_progress_serialization() { + let mut progress = StoredSyncProgress::new(); + progress.phase = StoredSyncPhase::SnapSyncStorage; + progress.target_snapshot = Some(StoredSyncSnapshot::new( + 10000, [0xab; 32], [0xcd; 32], 1234567890, + )); + progress.account_progress.completed_up_to = Some([0x80; 20]); + progress.account_progress.total_accounts = 1000; + progress.storage_progress.push(( + [0x01; 20], + StoredStorageProgress { + completed_up_to: Some([0xff; 32]), + total_slots: 500, + }, + )); + + // Serialize and deserialize + let encoded = bincode::serialize(&progress).unwrap(); + let decoded: StoredSyncProgress = bincode::deserialize(&encoded).unwrap(); + + assert_eq!(decoded.phase, StoredSyncPhase::SnapSyncStorage); + assert_eq!(decoded.account_progress.total_accounts, 1000); + assert_eq!(decoded.storage_progress.len(), 1); + } +} diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml new file mode 100644 index 0000000..4f359f4 --- /dev/null +++ b/crates/sync/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "cipherbft-sync" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +description = "State synchronization for CipherBFT nodes" + +[dependencies] +# Internal +cipherbft-types = { path = "../types" } +cipherbft-crypto = { path = "../crypto" } +cipherbft-storage = { path = "../storage" } +cipherbft-execution = { path = "../execution" } + +# Async +tokio = { workspace = true, features = ["sync", "time"] } +async-trait = { workspace = true } +futures = { workspace = true } + +# Serialization +serde = { workspace = true, features = ["derive"] } +bincode = { workspace = true } +bytes = { workspace = true } + +# Ethereum types +alloy-primitives = { workspace = true } +alloy-rlp = { workspace = true } +alloy-trie = "0.9" + +# Error handling +thiserror = { workspace = true } + +# Logging +tracing = { workspace = true } + +# Metrics +prometheus = { workspace = true } +once_cell = "1.19" + +# Collections +lru = "0.12" + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util"] } +tempfile = { workspace = true } +rand = { workspace = true } + +[lints.rust] +unsafe_code = "deny" diff --git a/crates/sync/src/blocks.rs b/crates/sync/src/blocks.rs new file mode 100644 index 0000000..2fd97a1 --- /dev/null +++ b/crates/sync/src/blocks.rs @@ -0,0 +1,336 @@ +//! Block synchronization after snap sync + +#![allow(dead_code)] // Module is foundational, will be used by sync orchestrator + +use crate::error::{Result, SyncError}; +use crate::protocol::{BlockRangeRequest, BlockRangeResponse}; +use alloy_primitives::{Bytes, B256}; +use std::collections::VecDeque; + +/// Block batch size for parallel requests +pub const BLOCK_BATCH_SIZE: u32 = 64; + +/// State root verification interval (must match execution layer) +pub const STATE_ROOT_INTERVAL: u64 = 100; + +/// Pending block range to download +#[derive(Clone, Debug)] +pub struct PendingBlockRange { + /// Start height (inclusive) + pub start: u64, + /// Number of blocks to fetch + pub count: u32, + /// Retry count + pub retries: u32, +} + +/// Downloaded block awaiting execution +#[derive(Clone, Debug)] +pub struct DownloadedBlock { + /// Block height + pub height: u64, + /// Serialized block data + pub data: Bytes, +} + +/// Block syncer state +pub struct BlockSyncer { + /// Start height (snapshot + 1) + start_height: u64, + /// Target height to sync to + target_height: u64, + /// Last successfully executed block + executed_up_to: u64, + /// Pending ranges to download + pending_ranges: Vec, + /// Downloaded blocks awaiting execution (sorted by height) + downloaded: VecDeque, + /// Expected state roots at checkpoints (height -> root) + checkpoint_roots: Vec<(u64, B256)>, + /// Total blocks downloaded + total_downloaded: u64, + /// Total blocks executed + total_executed: u64, + /// Next request ID for correlation + next_request_id: u64, +} + +impl BlockSyncer { + /// Create a new block syncer + pub fn new(start_height: u64, target_height: u64) -> Self { + // Create pending ranges covering start to target + let mut pending_ranges = Vec::new(); + let mut current = start_height; + + while current <= target_height { + let count = std::cmp::min(BLOCK_BATCH_SIZE, (target_height - current + 1) as u32); + pending_ranges.push(PendingBlockRange { + start: current, + count, + retries: 0, + }); + current += count as u64; + } + + // Reverse so we pop from end (lowest heights first) + pending_ranges.reverse(); + + Self { + start_height, + target_height, + executed_up_to: start_height.saturating_sub(1), + pending_ranges, + downloaded: VecDeque::new(), + checkpoint_roots: Vec::new(), + total_downloaded: 0, + total_executed: 0, + next_request_id: 1, + } + } + + /// Resume from existing progress + pub fn resume(start_height: u64, executed_up_to: u64, target_height: u64) -> Self { + let mut syncer = Self::new(executed_up_to + 1, target_height); + syncer.start_height = start_height; + syncer.executed_up_to = executed_up_to; + syncer + } + + /// Check if sync is complete + pub fn is_complete(&self) -> bool { + self.executed_up_to >= self.target_height + } + + /// Check if we have blocks ready to execute + pub fn has_executable_blocks(&self) -> bool { + self.downloaded + .front() + .is_some_and(|b| b.height == self.executed_up_to + 1) + } + + /// Get next range to request + pub fn next_range(&mut self) -> Option { + self.pending_ranges.pop() + } + + /// Create request for a range with unique request ID + pub fn create_request(&mut self, range: &PendingBlockRange) -> BlockRangeRequest { + let request_id = self.next_request_id; + self.next_request_id += 1; + BlockRangeRequest { + request_id, + start_height: range.start, + count: range.count, + } + } + + /// Process downloaded blocks + pub fn process_response( + &mut self, + range: PendingBlockRange, + response: BlockRangeResponse, + ) -> Result<()> { + if response.blocks.is_empty() { + return Err(SyncError::malformed("unknown", "empty block response")); + } + + // Add blocks to download queue + for (i, block_data) in response.blocks.into_iter().enumerate() { + let height = range.start + i as u64; + self.downloaded.push_back(DownloadedBlock { + height, + data: block_data, + }); + self.total_downloaded += 1; + } + + // Sort by height (insertion sort since mostly sorted) + self.sort_downloaded(); + + Ok(()) + } + + /// Get next block to execute (if available in sequence) + pub fn next_executable_block(&mut self) -> Option { + if self.has_executable_blocks() { + self.downloaded.pop_front() + } else { + None + } + } + + /// Mark block as successfully executed + pub fn block_executed(&mut self, height: u64, state_root: Option) { + self.executed_up_to = height; + self.total_executed += 1; + + // Store checkpoint state root + if let Some(root) = state_root { + if height.is_multiple_of(STATE_ROOT_INTERVAL) { + self.checkpoint_roots.push((height, root)); + } + } + } + + /// Handle failed download + pub fn handle_download_failure(&mut self, range: PendingBlockRange, max_retries: u32) { + if range.retries < max_retries { + self.pending_ranges.push(PendingBlockRange { + retries: range.retries + 1, + ..range + }); + } else if range.count > 1 { + // Split the range + let mid = range.count / 2; + self.pending_ranges.push(PendingBlockRange { + start: range.start, + count: mid, + retries: 0, + }); + self.pending_ranges.push(PendingBlockRange { + start: range.start + mid as u64, + count: range.count - mid, + retries: 0, + }); + } + // If single block fails too many times, we have a problem + } + + /// Handle execution failure + pub fn handle_execution_failure(&mut self, height: u64) { + // Re-download the failed block + self.pending_ranges.push(PendingBlockRange { + start: height, + count: 1, + retries: 0, + }); + + // Remove any downloaded blocks at or after this height + self.downloaded.retain(|b| b.height < height); + } + + /// Get sync progress as percentage + pub fn progress(&self) -> f64 { + let total = self.target_height - self.start_height + 1; + if total == 0 { + return 100.0; + } + let done = self.executed_up_to.saturating_sub(self.start_height) + 1; + (done as f64 / total as f64) * 100.0 + } + + /// Get sync statistics + pub fn stats(&self) -> BlockSyncStats { + BlockSyncStats { + start_height: self.start_height, + target_height: self.target_height, + executed_up_to: self.executed_up_to, + total_downloaded: self.total_downloaded, + total_executed: self.total_executed, + pending_ranges: self.pending_ranges.len() as u64, + downloaded_pending: self.downloaded.len() as u64, + } + } + + /// Sort downloaded blocks by height + fn sort_downloaded(&mut self) { + // Convert to vec, sort, convert back + let mut blocks: Vec<_> = self.downloaded.drain(..).collect(); + blocks.sort_by_key(|b| b.height); + self.downloaded = blocks.into_iter().collect(); + } +} + +/// Block sync statistics +#[derive(Clone, Debug, Default)] +pub struct BlockSyncStats { + /// Starting block height + pub start_height: u64, + /// Target block height + pub target_height: u64, + /// Last executed block + pub executed_up_to: u64, + /// Total blocks downloaded + pub total_downloaded: u64, + /// Total blocks executed + pub total_executed: u64, + /// Pending download ranges + pub pending_ranges: u64, + /// Downloaded blocks awaiting execution + pub downloaded_pending: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_block_syncer_creation() { + let syncer = BlockSyncer::new(10001, 10100); + + assert!(!syncer.is_complete()); + assert_eq!(syncer.start_height, 10001); + assert_eq!(syncer.target_height, 10100); + } + + #[test] + fn test_pending_ranges_creation() { + let syncer = BlockSyncer::new(1, 200); + + // Should create multiple ranges of BLOCK_BATCH_SIZE + let stats = syncer.stats(); + assert!(stats.pending_ranges > 0); + } + + #[test] + fn test_block_execution_flow() { + let mut syncer = BlockSyncer::new(1, 10); + + // Simulate downloading blocks + let range = syncer.next_range().unwrap(); + let response = BlockRangeResponse { + request_id: 1, + blocks: (1..=10).map(|_| Bytes::from(vec![0u8; 100])).collect(), + }; + + syncer.process_response(range, response).unwrap(); + + // Should have executable blocks + assert!(syncer.has_executable_blocks()); + + // Execute blocks + while let Some(block) = syncer.next_executable_block() { + syncer.block_executed(block.height, None); + } + + assert!(syncer.is_complete()); + } + + #[test] + fn test_progress_calculation() { + let mut syncer = BlockSyncer::new(1, 100); + + // Initial progress is 0% (executed_up_to = 0, which is before start_height = 1) + // done = 0.saturating_sub(1) + 1 = 0 + 1 = 1, but actually we haven't executed block 1 yet + // The progress formula counts from start, so initial is 1/100 = 1% + assert!(syncer.progress() < 2.0); + + syncer.executed_up_to = 50; + assert!((syncer.progress() - 50.0).abs() < 1.0); + + syncer.executed_up_to = 100; + assert!((syncer.progress() - 100.0).abs() < 0.1); + } + + #[test] + fn test_resume() { + let syncer = BlockSyncer::resume(10001, 10050, 10100); + + assert_eq!(syncer.start_height, 10001); + assert_eq!(syncer.executed_up_to, 10050); + assert_eq!(syncer.target_height, 10100); + + // Progress should reflect resumed state + assert!(syncer.progress() > 40.0); + } +} diff --git a/crates/sync/src/error.rs b/crates/sync/src/error.rs new file mode 100644 index 0000000..c082e58 --- /dev/null +++ b/crates/sync/src/error.rs @@ -0,0 +1,240 @@ +//! Synchronization error types + +use alloy_primitives::B256; +use std::time::Duration; +use thiserror::Error; + +/// Result type alias for sync operations +pub type Result = std::result::Result; + +/// Sync error categories +#[derive(Debug, Error)] +pub enum SyncError { + // === Network Errors === + /// Peer disconnected during sync + #[error("peer {0} disconnected")] + PeerDisconnected(String), + + /// Request timed out + #[error("request timed out after {0:?}")] + Timeout(Duration), + + /// No peers available for sync + #[error("insufficient peers: need {needed}, have {available}")] + InsufficientPeers { + /// Number of peers needed + needed: u32, + /// Number of peers available + available: u32, + }, + + // === Verification Errors === + /// Invalid merkle proof from peer + #[error("invalid proof from peer {peer}: {reason}")] + InvalidProof { + /// Peer that sent the invalid proof + peer: String, + /// Reason for invalidity + reason: String, + }, + + /// State root mismatch + #[error("state root mismatch: expected {expected}, got {actual}")] + StateRootMismatch { + /// Expected state root + expected: B256, + /// Actual state root received + actual: B256, + }, + + /// Malformed response from peer + #[error("malformed response from peer {peer}: {reason}")] + MalformedResponse { + /// Peer that sent the malformed response + peer: String, + /// Reason for considering the response malformed + reason: String, + }, + + // === State Errors === + /// No valid snapshot found + #[error("no valid snapshot found at or before height {0}")] + NoValidSnapshot(u64), + + /// Invalid sync state transition + #[error("invalid state transition: {0}")] + InvalidState(String), + + /// Snapshot height mismatch + #[error("snapshot height mismatch: requested {requested}, got {actual}")] + SnapshotHeightMismatch { + /// Requested snapshot height + requested: u64, + /// Actual snapshot height received + actual: u64, + }, + + // === Storage Errors === + /// Storage operation failed + #[error("storage error: {0}")] + Storage(String), + + /// Sync progress corrupted + #[error("sync progress corrupted: {0}")] + ProgressCorrupted(String), + + // === Configuration Errors === + /// Invalid configuration + #[error("invalid configuration: {0}")] + Config(String), +} + +impl SyncError { + /// Create an invalid proof error + pub fn invalid_proof(peer: impl Into, reason: impl Into) -> Self { + Self::InvalidProof { + peer: peer.into(), + reason: reason.into(), + } + } + + /// Create a malformed response error + pub fn malformed(peer: impl Into, reason: impl Into) -> Self { + Self::MalformedResponse { + peer: peer.into(), + reason: reason.into(), + } + } + + /// Check if this error indicates peer misbehavior (should ban peer) + pub fn is_peer_misbehavior(&self) -> bool { + matches!( + self, + Self::InvalidProof { .. } + | Self::MalformedResponse { .. } + | Self::StateRootMismatch { .. } + ) + } + + /// Check if this error is retriable with different peer + pub fn is_retriable(&self) -> bool { + matches!( + self, + Self::PeerDisconnected(_) | Self::Timeout(_) | Self::MalformedResponse { .. } + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_peer_disconnected_display() { + let err = SyncError::PeerDisconnected("peer123".to_string()); + assert_eq!(err.to_string(), "peer peer123 disconnected"); + } + + #[test] + fn test_timeout_display() { + let err = SyncError::Timeout(Duration::from_secs(5)); + assert!(err.to_string().contains("5")); + } + + #[test] + fn test_insufficient_peers_display() { + let err = SyncError::InsufficientPeers { + needed: 3, + available: 1, + }; + let msg = err.to_string(); + assert!(msg.contains("3")); + assert!(msg.contains("1")); + } + + #[test] + fn test_invalid_proof_constructor() { + let err = SyncError::invalid_proof("peer1", "bad merkle path"); + match err { + SyncError::InvalidProof { peer, reason } => { + assert_eq!(peer, "peer1"); + assert_eq!(reason, "bad merkle path"); + } + _ => panic!("expected InvalidProof variant"), + } + } + + #[test] + fn test_malformed_constructor() { + let err = SyncError::malformed("peer2", "missing field"); + match err { + SyncError::MalformedResponse { peer, reason } => { + assert_eq!(peer, "peer2"); + assert_eq!(reason, "missing field"); + } + _ => panic!("expected MalformedResponse variant"), + } + } + + #[test] + fn test_state_root_mismatch_display() { + let expected = B256::from([1u8; 32]); + let actual = B256::from([2u8; 32]); + let err = SyncError::StateRootMismatch { expected, actual }; + let msg = err.to_string(); + assert!(msg.contains("state root mismatch")); + } + + #[test] + fn test_is_peer_misbehavior() { + // Should be true for misbehavior errors + assert!(SyncError::invalid_proof("p", "r").is_peer_misbehavior()); + assert!(SyncError::malformed("p", "r").is_peer_misbehavior()); + assert!(SyncError::StateRootMismatch { + expected: B256::ZERO, + actual: B256::ZERO, + } + .is_peer_misbehavior()); + + // Should be false for other errors + assert!(!SyncError::PeerDisconnected("p".to_string()).is_peer_misbehavior()); + assert!(!SyncError::Timeout(Duration::from_secs(1)).is_peer_misbehavior()); + assert!(!SyncError::NoValidSnapshot(100).is_peer_misbehavior()); + assert!(!SyncError::Storage("err".to_string()).is_peer_misbehavior()); + } + + #[test] + fn test_is_retriable() { + // Should be retriable + assert!(SyncError::PeerDisconnected("p".to_string()).is_retriable()); + assert!(SyncError::Timeout(Duration::from_secs(1)).is_retriable()); + assert!(SyncError::malformed("p", "r").is_retriable()); + + // Should not be retriable + assert!(!SyncError::invalid_proof("p", "r").is_retriable()); + assert!(!SyncError::StateRootMismatch { + expected: B256::ZERO, + actual: B256::ZERO, + } + .is_retriable()); + assert!(!SyncError::NoValidSnapshot(100).is_retriable()); + assert!(!SyncError::Config("bad".to_string()).is_retriable()); + } + + #[test] + fn test_no_valid_snapshot_display() { + let err = SyncError::NoValidSnapshot(12345); + assert!(err.to_string().contains("12345")); + } + + #[test] + fn test_snapshot_height_mismatch_display() { + let err = SyncError::SnapshotHeightMismatch { + requested: 100, + actual: 50, + }; + let msg = err.to_string(); + assert!(msg.contains("100")); + assert!(msg.contains("50")); + } +} diff --git a/crates/sync/src/execution.rs b/crates/sync/src/execution.rs new file mode 100644 index 0000000..a7ba85a --- /dev/null +++ b/crates/sync/src/execution.rs @@ -0,0 +1,222 @@ +//! Block execution integration for sync +//! +//! Provides the bridge between downloaded blocks and the execution engine. + +#![allow(dead_code)] // Will be used by node integration + +use crate::error::{Result, SyncError}; +use alloy_primitives::{Address, Bytes, B256}; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +/// Block data ready for execution +/// +/// This is the format blocks are stored in for sync purposes, +/// containing all data needed to replay the block. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncBlock { + /// Block height + pub block_number: u64, + /// Block timestamp (unix seconds) + pub timestamp: u64, + /// Parent block hash + pub parent_hash: B256, + /// Transactions (RLP-encoded) + pub transactions: Vec, + /// Block gas limit + pub gas_limit: u64, + /// Base fee per gas (EIP-1559) + pub base_fee_per_gas: Option, + /// Block beneficiary (proposer address for rewards) + pub beneficiary: Address, +} + +impl SyncBlock { + /// Create a new sync block + pub fn new( + block_number: u64, + timestamp: u64, + parent_hash: B256, + transactions: Vec, + gas_limit: u64, + base_fee_per_gas: Option, + beneficiary: Address, + ) -> Self { + Self { + block_number, + timestamp, + parent_hash, + transactions, + gas_limit, + base_fee_per_gas, + beneficiary, + } + } + + /// Serialize to bytes for network transfer + pub fn to_bytes(&self) -> Result { + bincode::serialize(self) + .map(Bytes::from) + .map_err(|e| SyncError::Storage(format!("serialization error: {}", e))) + } + + /// Deserialize from bytes + pub fn from_bytes(data: &[u8]) -> Result { + bincode::deserialize(data) + .map_err(|e| SyncError::Storage(format!("deserialization error: {}", e))) + } +} + +/// Result of executing a sync block +#[derive(Clone, Debug)] +pub struct SyncExecutionResult { + /// Block number that was executed + pub block_number: u64, + /// State root after execution (may be None if not at checkpoint) + pub state_root: Option, + /// Block hash after execution + pub block_hash: B256, + /// Gas used in this block + pub gas_used: u64, + /// Number of transactions executed + pub transaction_count: usize, +} + +/// Trait for executing blocks during sync +/// +/// Implemented by the node to provide execution capability to the sync manager. +#[async_trait] +pub trait SyncExecutor: Send + Sync { + /// Execute a block and return the result + /// + /// The executor should: + /// 1. Validate the block can be executed (parent hash matches) + /// 2. Execute all transactions + /// 3. Return state root if at checkpoint interval + /// 4. Store the block and receipts + async fn execute_block(&self, block: SyncBlock) -> Result; + + /// Get the last executed block hash + async fn last_block_hash(&self) -> B256; + + /// Get the last executed block number + async fn last_block_number(&self) -> u64; + + /// Verify the state root at a checkpoint height + async fn verify_state_root(&self, height: u64, expected: B256) -> Result; +} + +/// A no-op executor for testing +#[derive(Clone, Debug, Default)] +pub struct MockSyncExecutor { + last_hash: B256, + last_number: u64, +} + +impl MockSyncExecutor { + /// Create a new mock executor + pub fn new() -> Self { + Self::default() + } + + /// Create with initial state + pub fn with_state(last_number: u64, last_hash: B256) -> Self { + Self { + last_hash, + last_number, + } + } +} + +#[async_trait] +impl SyncExecutor for MockSyncExecutor { + async fn execute_block(&self, block: SyncBlock) -> Result { + // Mock execution - just return success + Ok(SyncExecutionResult { + block_number: block.block_number, + state_root: if block.block_number.is_multiple_of(100) { + Some(B256::repeat_byte(0xab)) + } else { + None + }, + block_hash: B256::repeat_byte(0xcd), + gas_used: 21000 * block.transactions.len() as u64, + transaction_count: block.transactions.len(), + }) + } + + async fn last_block_hash(&self) -> B256 { + self.last_hash + } + + async fn last_block_number(&self) -> u64 { + self.last_number + } + + async fn verify_state_root(&self, _height: u64, _expected: B256) -> Result { + Ok(true) + } +} + +/// State root checkpoint interval (matches execution layer) +pub const STATE_ROOT_CHECKPOINT_INTERVAL: u64 = 100; + +/// Check if a block height is a state root checkpoint +pub fn is_state_root_checkpoint(height: u64) -> bool { + height > 0 && height.is_multiple_of(STATE_ROOT_CHECKPOINT_INTERVAL) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sync_block_serialization() { + let block = SyncBlock::new( + 100, + 1234567890, + B256::repeat_byte(0x01), + vec![Bytes::from(vec![0x02, 0x03])], + 30_000_000, + Some(1_000_000_000), + Address::repeat_byte(0x04), + ); + + let bytes = block.to_bytes().unwrap(); + let decoded = SyncBlock::from_bytes(&bytes).unwrap(); + + assert_eq!(decoded.block_number, 100); + assert_eq!(decoded.timestamp, 1234567890); + assert_eq!(decoded.transactions.len(), 1); + } + + #[test] + fn test_checkpoint_detection() { + assert!(!is_state_root_checkpoint(0)); + assert!(!is_state_root_checkpoint(50)); + assert!(is_state_root_checkpoint(100)); + assert!(!is_state_root_checkpoint(150)); + assert!(is_state_root_checkpoint(200)); + } + + #[tokio::test] + async fn test_mock_executor() { + let executor = MockSyncExecutor::new(); + + let block = SyncBlock::new( + 100, + 0, + B256::ZERO, + vec![Bytes::from(vec![0x01])], + 30_000_000, + None, + Address::ZERO, + ); + + let result = executor.execute_block(block).await.unwrap(); + + assert_eq!(result.block_number, 100); + assert!(result.state_root.is_some()); // 100 is a checkpoint + assert_eq!(result.transaction_count, 1); + } +} diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs new file mode 100644 index 0000000..6736186 --- /dev/null +++ b/crates/sync/src/lib.rs @@ -0,0 +1,34 @@ +//! State Synchronization for CipherBFT +//! +//! Provides snap sync-based state synchronization for nodes joining the network +//! or recovering from being behind. + +#![deny(unsafe_code)] +#![warn(missing_docs)] + +pub mod error; +pub mod execution; +pub mod metrics; +pub mod network; +pub mod progress; +pub mod protocol; +pub mod snapshot; + +pub mod blocks; +mod manager; +pub mod peers; +pub mod snap; + +pub use error::{Result, SyncError}; +pub use execution::{MockSyncExecutor, SyncBlock, SyncExecutionResult, SyncExecutor}; +pub use manager::{StateSyncManager, SyncConfig}; +pub use network::{ + IncomingSnapMessage, OutgoingSnapMessage, SyncNetworkAdapter, SyncNetworkSender, + SNAP_CHANNEL_CAPACITY, +}; +pub use progress::{ + AccountProgress, BlockProgress, ProgressTracker, SnapSubPhase, StorageProgress, SyncPhase, + SyncProgressState, +}; +pub use protocol::{SnapSyncMessage, SnapshotInfo}; +pub use snapshot::{SnapshotAgreement, StateSnapshot, SNAPSHOT_INTERVAL}; diff --git a/crates/sync/src/manager.rs b/crates/sync/src/manager.rs new file mode 100644 index 0000000..0d1a207 --- /dev/null +++ b/crates/sync/src/manager.rs @@ -0,0 +1,464 @@ +//! State sync manager - main orchestrator + +#![allow(dead_code)] // Will be used by node integration + +use crate::blocks::BlockSyncer; +use crate::error::{Result, SyncError}; +use crate::peers::PeerManager; +use crate::progress::{ProgressTracker, SnapSubPhase, SyncPhase}; +use crate::protocol::StatusResponse; +use crate::snap::accounts::AccountRangeSyncer; +use crate::snap::storage::StorageRangeSyncer; +use crate::snapshot::{SnapshotAgreement, StateSnapshot}; +use alloy_primitives::B256; +use std::collections::HashMap; +use std::time::{Duration, Instant}; +use tracing::{info, warn}; + +/// Minimum peers required to start sync +pub const MIN_PEERS_FOR_SYNC: usize = 3; + +/// Discovery timeout +pub const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); + +/// Sync configuration +#[derive(Clone, Debug)] +pub struct SyncConfig { + /// Minimum peers required to start sync + pub min_peers: usize, + /// Maximum retries per request + pub max_retries: u32, + /// Request timeout + pub request_timeout: Duration, + /// Discovery timeout + pub discovery_timeout: Duration, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + min_peers: MIN_PEERS_FOR_SYNC, + max_retries: 3, + request_timeout: Duration::from_secs(10), + discovery_timeout: DISCOVERY_TIMEOUT, + } + } +} + +/// Main state sync manager +pub struct StateSyncManager { + /// Configuration + config: SyncConfig, + /// Peer manager + peers: PeerManager, + /// Progress tracker + progress: ProgressTracker, + /// Target snapshot (set after discovery) + target_snapshot: Option, + /// Account range syncer (active during snap sync) + account_syncer: Option, + /// Storage range syncer (active during snap sync) + storage_syncer: Option, + /// Block syncer (active during block sync) + block_syncer: Option, + /// Discovery start time + discovery_started: Option, + /// Current network tip height + network_tip: u64, +} + +impl StateSyncManager { + /// Create a new sync manager + pub fn new(config: SyncConfig) -> Self { + Self { + config, + peers: PeerManager::new(), + progress: ProgressTracker::new(), + target_snapshot: None, + account_syncer: None, + storage_syncer: None, + block_syncer: None, + discovery_started: None, + network_tip: 0, + } + } + + /// Create with default config + pub fn with_defaults() -> Self { + Self::new(SyncConfig::default()) + } + + /// Get current sync phase + pub fn phase(&self) -> &SyncPhase { + &self.progress.state().phase + } + + /// Check if sync is complete + pub fn is_complete(&self) -> bool { + matches!(self.progress.state().phase, SyncPhase::Complete) + } + + /// Get sync progress percentage + pub fn progress_percent(&self) -> f64 { + self.progress.state().overall_progress() + } + + /// Add a peer + pub fn add_peer(&mut self, peer_id: String) { + self.peers.add_peer(peer_id); + } + + /// Remove a peer + pub fn remove_peer(&mut self, peer_id: &str) { + self.peers.remove_peer(peer_id); + } + + /// Handle status response from peer + pub fn handle_status(&mut self, peer_id: &str, status: StatusResponse) { + self.peers.update_status(peer_id, status.clone()); + + // Update network tip + if status.tip_height > self.network_tip { + self.network_tip = status.tip_height; + } + } + + // === Phase: Discovery === + + /// Start discovery phase + pub fn start_discovery(&mut self) -> Result<()> { + info!("Starting sync discovery phase"); + self.progress.set_phase(SyncPhase::Discovery)?; + self.discovery_started = Some(Instant::now()); + Ok(()) + } + + /// Check if discovery is complete (enough peers with snapshot agreement) + pub fn try_complete_discovery(&mut self) -> Result> { + let peer_count = self.peers.peer_count(); + + if peer_count < self.config.min_peers { + // Check timeout + if let Some(started) = self.discovery_started { + if started.elapsed() > self.config.discovery_timeout { + return Err(SyncError::InsufficientPeers { + needed: self.config.min_peers as u32, + available: peer_count as u32, + }); + } + } + return Ok(None); + } + + // Find best snapshot with agreement + if let Some(agreement) = self.find_snapshot_agreement() { + if agreement.has_quorum(self.config.min_peers) { + info!( + height = agreement.snapshot.block_number, + peers = agreement.peer_count, + "Found snapshot with peer agreement" + ); + return Ok(Some(agreement.snapshot)); + } + } + + Ok(None) + } + + /// Find snapshot with best peer agreement + /// + /// Requires agreement on both height AND state root for security. + /// Two peers reporting the same height but different state roots + /// indicates a chain fork or malicious peer. + fn find_snapshot_agreement(&self) -> Option { + // Collect all snapshots from peers, keyed by (height, state_root) + // This ensures we only count agreement when peers agree on BOTH values + let mut snapshot_votes: HashMap<(u64, B256, B256), Vec> = HashMap::new(); + + for (peer_id, peer) in self.peers.iter() { + if let Some(status) = &peer.status { + for snapshot_info in &status.snapshots { + // Key by (height, state_root, block_hash) for full agreement + let key = ( + snapshot_info.height, + snapshot_info.state_root, + snapshot_info.block_hash, + ); + snapshot_votes.entry(key).or_default().push(peer_id.clone()); + } + } + } + + // Find highest snapshot with most votes + // Only consider snapshots where peers agree on height AND state root + snapshot_votes + .into_iter() + .filter(|(_, peers)| peers.len() >= self.config.min_peers) + .max_by_key(|((height, _, _), peers)| (*height, peers.len())) + .map(|((height, state_root, block_hash), peers)| { + SnapshotAgreement { + snapshot: StateSnapshot::new( + height, block_hash, state_root, + 0, // Timestamp not included in snapshot info - could be added later + ), + peer_count: peers.len(), + peers, + } + }) + } + + // === Phase: Snap Sync === + + /// Start snap sync phase with selected snapshot + pub fn start_snap_sync(&mut self, snapshot: StateSnapshot) -> Result<()> { + info!( + height = snapshot.block_number, + state_root = %snapshot.state_root, + "Starting snap sync" + ); + + self.target_snapshot = Some(snapshot.clone()); + self.account_syncer = Some(AccountRangeSyncer::new(snapshot)); + self.progress + .set_phase(SyncPhase::SnapSync(SnapSubPhase::Accounts))?; + + Ok(()) + } + + /// Check if account sync is complete + pub fn is_account_sync_complete(&self) -> bool { + self.account_syncer.as_ref().is_none_or(|s| s.is_complete()) + } + + /// Transition to storage sync + pub fn start_storage_sync(&mut self) -> Result<()> { + let snapshot = self + .target_snapshot + .clone() + .ok_or_else(|| SyncError::InvalidState("no target snapshot".into()))?; + + let accounts_with_storage: Vec<_> = self + .account_syncer + .as_ref() + .map(|s| s.accounts_needing_storage().to_vec()) + .unwrap_or_default(); + + info!( + accounts = accounts_with_storage.len(), + "Starting storage sync for accounts" + ); + + // Create storage syncer with accounts that need storage + let storage_accounts: Vec<_> = accounts_with_storage + .into_iter() + .map(|addr| (addr, B256::ZERO)) // TODO: track actual storage roots + .collect(); + + self.storage_syncer = Some(StorageRangeSyncer::new(snapshot, storage_accounts)); + self.progress + .set_phase(SyncPhase::SnapSync(SnapSubPhase::Storage))?; + + Ok(()) + } + + /// Check if storage sync is complete + pub fn is_storage_sync_complete(&self) -> bool { + self.storage_syncer.as_ref().is_none_or(|s| s.is_complete()) + } + + /// Verify final state root + pub fn verify_state_root(&self) -> Result<()> { + // TODO: Compute actual state root from downloaded state and compare + // For now, assume verification passes + info!("State root verification passed"); + Ok(()) + } + + // === Phase: Block Sync === + + /// Start block sync phase + pub fn start_block_sync(&mut self) -> Result<()> { + let snapshot = self + .target_snapshot + .as_ref() + .ok_or_else(|| SyncError::InvalidState("no target snapshot".into()))?; + + let start_height = snapshot.block_number + 1; + let target_height = self.network_tip; + + if start_height > target_height { + info!("Already at tip, no blocks to sync"); + self.progress.set_phase(SyncPhase::Complete)?; + return Ok(()); + } + + info!( + start = start_height, + target = target_height, + blocks = target_height - start_height + 1, + "Starting block sync" + ); + + self.block_syncer = Some(BlockSyncer::new(start_height, target_height)); + self.progress.set_phase(SyncPhase::BlockSync)?; + + Ok(()) + } + + /// Check if block sync is complete + pub fn is_block_sync_complete(&self) -> bool { + self.block_syncer.as_ref().is_none_or(|s| s.is_complete()) + } + + /// Mark sync as complete + pub fn complete_sync(&mut self) -> Result<()> { + info!("Sync complete!"); + self.progress.set_phase(SyncPhase::Complete)?; + + // Clean up syncers + self.account_syncer = None; + self.storage_syncer = None; + self.block_syncer = None; + + Ok(()) + } + + // === Error Handling === + + /// Handle sync error from a peer + pub fn handle_peer_error(&mut self, peer_id: &str, error: &SyncError) { + warn!(peer = peer_id, error = %error, "Peer error during sync"); + self.peers.handle_misbehavior(peer_id, error); + } + + // === Getters for syncers === + + /// Get mutable reference to account syncer + pub fn account_syncer_mut(&mut self) -> Option<&mut AccountRangeSyncer> { + self.account_syncer.as_mut() + } + + /// Get mutable reference to storage syncer + pub fn storage_syncer_mut(&mut self) -> Option<&mut StorageRangeSyncer> { + self.storage_syncer.as_mut() + } + + /// Get mutable reference to block syncer + pub fn block_syncer_mut(&mut self) -> Option<&mut BlockSyncer> { + self.block_syncer.as_mut() + } + + /// Get peer manager + pub fn peers(&self) -> &PeerManager { + &self.peers + } + + /// Get mutable peer manager + pub fn peers_mut(&mut self) -> &mut PeerManager { + &mut self.peers + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_manager_creation() { + let manager = StateSyncManager::with_defaults(); + assert!(matches!(manager.phase(), SyncPhase::Discovery)); + assert!(!manager.is_complete()); + } + + #[test] + fn test_phase_transitions() { + let mut manager = StateSyncManager::with_defaults(); + + // Start discovery + manager.start_discovery().unwrap(); + assert!(matches!(manager.phase(), SyncPhase::Discovery)); + + // Start snap sync + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::repeat_byte(0xab), 12345); + manager.start_snap_sync(snapshot).unwrap(); + assert!(matches!( + manager.phase(), + SyncPhase::SnapSync(SnapSubPhase::Accounts) + )); + + // Transition to storage + manager.start_storage_sync().unwrap(); + assert!(matches!( + manager.phase(), + SyncPhase::SnapSync(SnapSubPhase::Storage) + )); + } + + #[test] + fn test_peer_management() { + use crate::protocol::SnapshotInfo; + + let mut manager = StateSyncManager::with_defaults(); + + manager.add_peer("peer1".to_string()); + manager.add_peer("peer2".to_string()); + + let status = StatusResponse { + tip_height: 100000, + tip_hash: B256::ZERO, + snapshots: vec![ + SnapshotInfo { + height: 90000, + state_root: B256::repeat_byte(0xab), + block_hash: B256::repeat_byte(0xcd), + }, + SnapshotInfo { + height: 80000, + state_root: B256::repeat_byte(0xef), + block_hash: B256::repeat_byte(0x12), + }, + ], + }; + + manager.handle_status("peer1", status); + assert_eq!(manager.network_tip, 100000); + } + + #[test] + fn test_block_sync_start() { + let mut manager = StateSyncManager::with_defaults(); + manager.network_tip = 10100; + + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::ZERO, 0); + manager.start_snap_sync(snapshot).unwrap(); + manager.start_storage_sync().unwrap(); + manager.start_block_sync().unwrap(); + + assert!(matches!(manager.phase(), SyncPhase::BlockSync)); + assert!(manager.block_syncer.is_some()); + } + + #[test] + fn test_already_at_tip() { + let mut manager = StateSyncManager::with_defaults(); + manager.network_tip = 10000; // Same as snapshot + + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::ZERO, 0); + manager.target_snapshot = Some(snapshot); + manager.start_block_sync().unwrap(); + + // Should go directly to complete + assert!(matches!(manager.phase(), SyncPhase::Complete)); + } + + #[test] + fn test_sync_completion() { + let mut manager = StateSyncManager::with_defaults(); + manager.complete_sync().unwrap(); + + assert!(manager.is_complete()); + assert!(manager.account_syncer.is_none()); + assert!(manager.storage_syncer.is_none()); + assert!(manager.block_syncer.is_none()); + } +} diff --git a/crates/sync/src/metrics.rs b/crates/sync/src/metrics.rs new file mode 100644 index 0000000..8b427b1 --- /dev/null +++ b/crates/sync/src/metrics.rs @@ -0,0 +1,184 @@ +//! Sync layer metrics + +use once_cell::sync::Lazy; +use prometheus::{ + register_counter_vec, register_gauge, register_histogram, CounterVec, Gauge, Histogram, +}; + +/// Current sync phase (0=Discovery, 1=SnapSync, 2=BlockSync, 3=Complete) +pub static SYNC_PHASE: Lazy = Lazy::new(|| { + register_gauge!( + "sync_phase", + "Current sync phase (0=Discovery, 1=SnapSync, 2=BlockSync, 3=Complete)" + ) + .expect("Failed to register sync_phase metric") +}); + +/// Overall sync progress percentage (0-100) +pub static SYNC_PROGRESS_PERCENT: Lazy = Lazy::new(|| { + register_gauge!("sync_progress_percent", "Overall sync progress percentage") + .expect("Failed to register sync_progress_percent metric") +}); + +/// Target snapshot height +pub static SYNC_TARGET_HEIGHT: Lazy = Lazy::new(|| { + register_gauge!("sync_target_height", "Target snapshot height for sync") + .expect("Failed to register sync_target_height metric") +}); + +/// Current synced height +pub static SYNC_CURRENT_HEIGHT: Lazy = Lazy::new(|| { + register_gauge!("sync_current_height", "Current synced block height") + .expect("Failed to register sync_current_height metric") +}); + +/// Number of connected sync peers +pub static SYNC_PEER_COUNT: Lazy = Lazy::new(|| { + register_gauge!("sync_peer_count", "Number of connected sync peers") + .expect("Failed to register sync_peer_count metric") +}); + +/// Accounts downloaded during snap sync +pub static SYNC_ACCOUNTS_DOWNLOADED: Lazy = Lazy::new(|| { + register_gauge!("sync_accounts_downloaded", "Number of accounts downloaded") + .expect("Failed to register sync_accounts_downloaded metric") +}); + +/// Storage slots downloaded during snap sync +pub static SYNC_STORAGE_SLOTS_DOWNLOADED: Lazy = Lazy::new(|| { + register_gauge!( + "sync_storage_slots_downloaded", + "Number of storage slots downloaded" + ) + .expect("Failed to register sync_storage_slots_downloaded metric") +}); + +/// Bytes downloaded during sync +pub static SYNC_BYTES_DOWNLOADED: Lazy = Lazy::new(|| { + register_gauge!( + "sync_bytes_downloaded", + "Total bytes downloaded during sync" + ) + .expect("Failed to register sync_bytes_downloaded metric") +}); + +/// Blocks executed during block sync +pub static SYNC_BLOCKS_EXECUTED: Lazy = Lazy::new(|| { + register_gauge!( + "sync_blocks_executed", + "Number of blocks executed during sync" + ) + .expect("Failed to register sync_blocks_executed metric") +}); + +/// Sync request latency histogram +pub static SYNC_REQUEST_LATENCY: Lazy = Lazy::new(|| { + register_histogram!( + "sync_request_latency_seconds", + "Sync request latency in seconds", + vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] + ) + .expect("Failed to register sync_request_latency metric") +}); + +/// Sync requests by type and status +pub static SYNC_REQUESTS: Lazy = Lazy::new(|| { + register_counter_vec!( + "sync_requests_total", + "Total sync requests by type and status", + &["type", "status"] + ) + .expect("Failed to register sync_requests metric") +}); + +/// Sync errors by type +pub static SYNC_ERRORS: Lazy = Lazy::new(|| { + register_counter_vec!("sync_errors_total", "Total sync errors by type", &["type"]) + .expect("Failed to register sync_errors metric") +}); + +/// Update sync phase metric +pub fn set_sync_phase(phase: u8) { + SYNC_PHASE.set(phase as f64); +} + +/// Update sync progress +pub fn set_sync_progress(percent: f64) { + SYNC_PROGRESS_PERCENT.set(percent); +} + +/// Update target height +pub fn set_target_height(height: u64) { + SYNC_TARGET_HEIGHT.set(height as f64); +} + +/// Update current height +pub fn set_current_height(height: u64) { + SYNC_CURRENT_HEIGHT.set(height as f64); +} + +/// Update peer count +pub fn set_peer_count(count: usize) { + SYNC_PEER_COUNT.set(count as f64); +} + +/// Update accounts downloaded +pub fn set_accounts_downloaded(count: u64) { + SYNC_ACCOUNTS_DOWNLOADED.set(count as f64); +} + +/// Update storage slots downloaded +pub fn set_storage_slots_downloaded(count: u64) { + SYNC_STORAGE_SLOTS_DOWNLOADED.set(count as f64); +} + +/// Update bytes downloaded +pub fn set_bytes_downloaded(bytes: u64) { + SYNC_BYTES_DOWNLOADED.set(bytes as f64); +} + +/// Update blocks executed +pub fn set_blocks_executed(count: u64) { + SYNC_BLOCKS_EXECUTED.set(count as f64); +} + +/// Record a sync request +pub fn record_request(request_type: &str, success: bool) { + let status = if success { "success" } else { "failure" }; + SYNC_REQUESTS + .with_label_values(&[request_type, status]) + .inc(); +} + +/// Record request latency +pub fn record_latency(seconds: f64) { + SYNC_REQUEST_LATENCY.observe(seconds); +} + +/// Record a sync error +pub fn record_error(error_type: &str) { + SYNC_ERRORS.with_label_values(&[error_type]).inc(); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_phase_metric() { + set_sync_phase(1); + assert_eq!(SYNC_PHASE.get(), 1.0); + } + + #[test] + fn test_progress_metric() { + set_sync_progress(50.0); + assert_eq!(SYNC_PROGRESS_PERCENT.get(), 50.0); + } + + #[test] + fn test_request_counter() { + record_request("account_range", true); + // Counter incremented + } +} diff --git a/crates/sync/src/network.rs b/crates/sync/src/network.rs new file mode 100644 index 0000000..6b5ccd2 --- /dev/null +++ b/crates/sync/src/network.rs @@ -0,0 +1,193 @@ +//! Network adapter for snap sync +//! +//! Provides a channel-based interface between the network layer and the sync manager. +//! This decouples the sync logic from the network implementation. + +use crate::protocol::SnapSyncMessage; +use tokio::sync::mpsc; +use tracing::warn; + +/// Message with peer context for incoming messages +#[derive(Debug, Clone)] +pub struct IncomingSnapMessage { + /// Peer that sent the message + pub peer_id: String, + /// The snap sync message + pub message: SnapSyncMessage, +} + +/// Message with target peer for outgoing messages +#[derive(Debug, Clone)] +pub struct OutgoingSnapMessage { + /// Target peer (None for broadcast) + pub target_peer: Option, + /// The snap sync message to send + pub message: SnapSyncMessage, +} + +/// Channel capacity for snap sync messages +pub const SNAP_CHANNEL_CAPACITY: usize = 256; + +/// Network adapter connecting sync manager to network layer +/// +/// The sync manager uses this to receive messages from peers +/// and send messages to peers without knowing about the network +/// implementation details. +pub struct SyncNetworkAdapter { + /// Receiver for incoming snap sync messages from network + incoming_rx: mpsc::Receiver, + /// Sender for outgoing snap sync messages to network + outgoing_tx: mpsc::Sender, +} + +impl SyncNetworkAdapter { + /// Create a new adapter with connected channel ends + /// + /// Returns the adapter and the channel ends that should be given to the network layer. + #[allow(clippy::type_complexity)] + pub fn new() -> ( + Self, + mpsc::Sender, + mpsc::Receiver, + ) { + let (incoming_tx, incoming_rx) = mpsc::channel(SNAP_CHANNEL_CAPACITY); + let (outgoing_tx, outgoing_rx) = mpsc::channel(SNAP_CHANNEL_CAPACITY); + + let adapter = Self { + incoming_rx, + outgoing_tx, + }; + + (adapter, incoming_tx, outgoing_rx) + } + + /// Receive the next incoming message (async) + pub async fn recv(&mut self) -> Option { + self.incoming_rx.recv().await + } + + /// Try to receive a message without blocking + pub fn try_recv(&mut self) -> Option { + self.incoming_rx.try_recv().ok() + } + + /// Send a message to a specific peer + pub async fn send(&self, peer_id: String, message: SnapSyncMessage) { + let outgoing = OutgoingSnapMessage { + target_peer: Some(peer_id), + message, + }; + if self.outgoing_tx.send(outgoing).await.is_err() { + warn!("Failed to send snap sync message - channel closed"); + } + } + + /// Broadcast a message to all peers + pub async fn broadcast(&self, message: SnapSyncMessage) { + let outgoing = OutgoingSnapMessage { + target_peer: None, + message, + }; + if self.outgoing_tx.send(outgoing).await.is_err() { + warn!("Failed to broadcast snap sync message - channel closed"); + } + } + + /// Request status from all peers + pub async fn request_status_from_all(&self) { + self.broadcast(SnapSyncMessage::GetStatus).await; + } +} + +/// Handle for the network layer to send incoming messages to sync +#[derive(Clone)] +pub struct SyncNetworkSender { + tx: mpsc::Sender, +} + +impl SyncNetworkSender { + /// Create from the sender channel + pub fn new(tx: mpsc::Sender) -> Self { + Self { tx } + } + + /// Forward a received snap sync message from a peer + pub async fn forward_message(&self, peer_id: String, message: SnapSyncMessage) { + let incoming = IncomingSnapMessage { peer_id, message }; + if self.tx.send(incoming).await.is_err() { + warn!("Failed to forward snap sync message - sync receiver dropped"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::StatusResponse; + use alloy_primitives::B256; + + #[tokio::test] + async fn test_adapter_creation() { + let (adapter, _incoming_tx, _outgoing_rx) = SyncNetworkAdapter::new(); + // Adapter should be created successfully + drop(adapter); + } + + #[tokio::test] + async fn test_send_and_receive() { + let (mut adapter, incoming_tx, mut outgoing_rx) = SyncNetworkAdapter::new(); + + // Send incoming message + let msg = IncomingSnapMessage { + peer_id: "peer1".to_string(), + message: SnapSyncMessage::GetStatus, + }; + incoming_tx.send(msg).await.unwrap(); + + // Receive it through adapter + let received = adapter.recv().await.unwrap(); + assert_eq!(received.peer_id, "peer1"); + assert!(matches!(received.message, SnapSyncMessage::GetStatus)); + + // Send outgoing message + adapter + .send( + "peer2".to_string(), + SnapSyncMessage::Status(StatusResponse { + tip_height: 100, + tip_hash: B256::ZERO, + snapshots: vec![], + }), + ) + .await; + + // Receive outgoing message + let outgoing = outgoing_rx.recv().await.unwrap(); + assert_eq!(outgoing.target_peer, Some("peer2".to_string())); + assert!(matches!(outgoing.message, SnapSyncMessage::Status(_))); + } + + #[tokio::test] + async fn test_broadcast() { + let (adapter, _incoming_tx, mut outgoing_rx) = SyncNetworkAdapter::new(); + + adapter.broadcast(SnapSyncMessage::GetStatus).await; + + let outgoing = outgoing_rx.recv().await.unwrap(); + assert_eq!(outgoing.target_peer, None); // Broadcast has no specific target + assert!(matches!(outgoing.message, SnapSyncMessage::GetStatus)); + } + + #[tokio::test] + async fn test_network_sender() { + let (mut adapter, incoming_tx, _outgoing_rx) = SyncNetworkAdapter::new(); + + let sender = SyncNetworkSender::new(incoming_tx); + sender + .forward_message("peer1".to_string(), SnapSyncMessage::GetStatus) + .await; + + let received = adapter.recv().await.unwrap(); + assert_eq!(received.peer_id, "peer1"); + } +} diff --git a/crates/sync/src/peers.rs b/crates/sync/src/peers.rs new file mode 100644 index 0000000..354c162 --- /dev/null +++ b/crates/sync/src/peers.rs @@ -0,0 +1,353 @@ +//! Peer tracking and selection for sync + +#![allow(dead_code)] // Module is used by other sync components (WIP) + +use crate::error::SyncError; +use crate::protocol::StatusResponse; +use std::collections::HashMap; +use std::time::{Duration, Instant}; + +/// Maximum concurrent requests per peer +pub const MAX_REQUESTS_PER_PEER: u32 = 4; + +/// Total maximum concurrent requests +pub const MAX_TOTAL_REQUESTS: u32 = 16; + +/// Default request timeout +pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// Ban duration for misbehaving peers +pub const BAN_DURATION: Duration = Duration::from_secs(3600); + +/// Peer performance metrics +#[derive(Clone, Debug)] +pub struct PeerMetrics { + /// Exponential moving average of response latency + pub avg_latency_ms: f64, + /// Throughput in bytes/sec + pub throughput_bps: f64, + /// Request success rate (0.0 - 1.0) + pub success_rate: f64, + /// Total completed requests + pub total_requests: u64, + /// Total failed requests + pub failed_requests: u64, +} + +impl Default for PeerMetrics { + fn default() -> Self { + Self { + avg_latency_ms: 100.0, // Assume 100ms initially + throughput_bps: 0.0, + success_rate: 1.0, // Assume good until proven otherwise + total_requests: 0, + failed_requests: 0, + } + } +} + +impl PeerMetrics { + /// Update metrics after successful request + pub fn record_success(&mut self, latency: Duration, bytes: u64) { + let latency_ms = latency.as_secs_f64() * 1000.0; + + // Exponential moving average (alpha = 0.3) + self.avg_latency_ms = 0.7 * self.avg_latency_ms + 0.3 * latency_ms; + + if latency.as_secs_f64() > 0.0 { + let bps = bytes as f64 / latency.as_secs_f64(); + self.throughput_bps = 0.7 * self.throughput_bps + 0.3 * bps; + } + + self.total_requests += 1; + self.success_rate = 1.0 - (self.failed_requests as f64 / self.total_requests as f64); + } + + /// Update metrics after failed request + pub fn record_failure(&mut self) { + self.total_requests += 1; + self.failed_requests += 1; + self.success_rate = 1.0 - (self.failed_requests as f64 / self.total_requests as f64); + + // Penalize latency on failure + self.avg_latency_ms += 500.0; + } +} + +/// Tracked peer state +#[derive(Clone, Debug)] +pub struct PeerState { + /// Peer identifier + pub peer_id: String, + /// Peer's reported status + pub status: Option, + /// Performance metrics + pub metrics: PeerMetrics, + /// Current in-flight requests + pub pending_requests: u32, + /// Last successful interaction + pub last_seen: Instant, + /// Ban expiry (if banned) + pub banned_until: Option, +} + +impl PeerState { + /// Create a new peer state + pub fn new(peer_id: String) -> Self { + Self { + peer_id, + status: None, + metrics: PeerMetrics::default(), + pending_requests: 0, + last_seen: Instant::now(), + banned_until: None, + } + } + + /// Calculate peer score (higher is better) + pub fn score(&self) -> f64 { + if self.is_banned() { + return f64::NEG_INFINITY; + } + + let latency_score = 1000.0 / (self.metrics.avg_latency_ms + 10.0); + let throughput_score = self.metrics.throughput_bps / 1_000_000.0; // MB/s + let reliability_score = self.metrics.success_rate * 10.0; + let capacity_score = (MAX_REQUESTS_PER_PEER - self.pending_requests) as f64; + + latency_score + throughput_score + reliability_score + capacity_score + } + + /// Check if peer is banned + pub fn is_banned(&self) -> bool { + self.banned_until.is_some_and(|t| Instant::now() < t) + } + + /// Check if peer can accept more requests + pub fn can_accept_request(&self) -> bool { + !self.is_banned() && self.pending_requests < MAX_REQUESTS_PER_PEER + } + + /// Check if peer has the snapshot we need + pub fn has_snapshot(&self, height: u64) -> bool { + self.status + .as_ref() + .is_some_and(|s| s.snapshots.iter().any(|snap| snap.height == height)) + } +} + +/// Peer manager for sync +pub struct PeerManager { + peers: HashMap, + total_pending: u32, +} + +impl PeerManager { + /// Create a new peer manager + pub fn new() -> Self { + Self { + peers: HashMap::new(), + total_pending: 0, + } + } + + /// Add or update a peer + pub fn add_peer(&mut self, peer_id: String) { + self.peers + .entry(peer_id.clone()) + .or_insert_with(|| PeerState::new(peer_id)); + } + + /// Update peer status + pub fn update_status(&mut self, peer_id: &str, status: StatusResponse) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.status = Some(status); + peer.last_seen = Instant::now(); + } + } + + /// Remove a peer + pub fn remove_peer(&mut self, peer_id: &str) { + if let Some(peer) = self.peers.remove(peer_id) { + self.total_pending = self.total_pending.saturating_sub(peer.pending_requests); + } + } + + /// Get number of available peers + pub fn peer_count(&self) -> usize { + self.peers.values().filter(|p| !p.is_banned()).count() + } + + /// Get peers that have a specific snapshot + pub fn peers_with_snapshot(&self, height: u64) -> Vec<&PeerState> { + self.peers + .values() + .filter(|p| !p.is_banned() && p.has_snapshot(height)) + .collect() + } + + /// Select best peer for a request + pub fn select_peer(&self, snapshot_height: Option) -> Option<&PeerState> { + if self.total_pending >= MAX_TOTAL_REQUESTS { + return None; + } + + self.peers + .values() + .filter(|p| p.can_accept_request()) + .filter(|p| snapshot_height.is_none_or(|h| p.has_snapshot(h))) + .max_by(|a, b| a.score().partial_cmp(&b.score()).unwrap()) + } + + /// Mark request started for peer + pub fn request_started(&mut self, peer_id: &str) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.pending_requests += 1; + self.total_pending += 1; + } + } + + /// Mark request completed for peer + pub fn request_completed(&mut self, peer_id: &str, latency: Duration, bytes: u64) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.pending_requests = peer.pending_requests.saturating_sub(1); + peer.metrics.record_success(latency, bytes); + peer.last_seen = Instant::now(); + } + self.total_pending = self.total_pending.saturating_sub(1); + } + + /// Mark request failed for peer + pub fn request_failed(&mut self, peer_id: &str) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.pending_requests = peer.pending_requests.saturating_sub(1); + peer.metrics.record_failure(); + } + self.total_pending = self.total_pending.saturating_sub(1); + } + + /// Ban a misbehaving peer + pub fn ban_peer(&mut self, peer_id: &str, duration: Duration) { + if let Some(peer) = self.peers.get_mut(peer_id) { + peer.banned_until = Some(Instant::now() + duration); + // Return pending requests + self.total_pending = self.total_pending.saturating_sub(peer.pending_requests); + peer.pending_requests = 0; + } + } + + /// Handle peer misbehavior + pub fn handle_misbehavior(&mut self, peer_id: &str, error: &SyncError) { + if error.is_peer_misbehavior() { + self.ban_peer(peer_id, BAN_DURATION); + } else if error.is_retriable() { + self.request_failed(peer_id); + } + } +} + +impl Default for PeerManager { + fn default() -> Self { + Self::new() + } +} + +impl PeerManager { + /// Iterate over all peers + pub fn iter(&self) -> impl Iterator { + self.peers.iter() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::SnapshotInfo; + use alloy_primitives::B256; + + fn make_status(height: u64, snapshot_heights: Vec) -> StatusResponse { + StatusResponse { + tip_height: height, + tip_hash: B256::ZERO, + snapshots: snapshot_heights + .into_iter() + .map(|h| SnapshotInfo { + height: h, + state_root: B256::repeat_byte((h % 256) as u8), + block_hash: B256::repeat_byte(((h / 256) % 256) as u8), + }) + .collect(), + } + } + + #[test] + fn test_peer_scoring() { + let mut peer = PeerState::new("peer1".to_string()); + + // Default score + let initial_score = peer.score(); + + // Record some success + peer.metrics + .record_success(Duration::from_millis(50), 10000); + let better_score = peer.score(); + + assert!(better_score > initial_score); + } + + #[test] + fn test_peer_selection() { + let mut manager = PeerManager::new(); + + manager.add_peer("peer1".to_string()); + manager.add_peer("peer2".to_string()); + + manager.update_status("peer1", make_status(100000, vec![90000, 80000])); + manager.update_status("peer2", make_status(100000, vec![90000])); + + // Both have snapshot 90000 + let peer = manager.select_peer(Some(90000)); + assert!(peer.is_some()); + + // Only peer1 has snapshot 80000 + let peer = manager.select_peer(Some(80000)); + assert_eq!(peer.map(|p| p.peer_id.as_str()), Some("peer1")); + } + + #[test] + fn test_peer_banning() { + let mut manager = PeerManager::new(); + manager.add_peer("bad_peer".to_string()); + + assert_eq!(manager.peer_count(), 1); + + manager.ban_peer("bad_peer", Duration::from_secs(60)); + + assert_eq!(manager.peer_count(), 0); + assert!(manager.select_peer(None).is_none()); + } + + #[test] + fn test_request_tracking() { + let mut manager = PeerManager::new(); + manager.add_peer("peer1".to_string()); + + manager.request_started("peer1"); + assert_eq!(manager.total_pending, 1); + + manager.request_completed("peer1", Duration::from_millis(100), 5000); + assert_eq!(manager.total_pending, 0); + } + + #[test] + fn test_misbehavior_handling() { + let mut manager = PeerManager::new(); + manager.add_peer("peer1".to_string()); + + // Invalid proof should ban + let error = SyncError::invalid_proof("peer1", "bad proof"); + manager.handle_misbehavior("peer1", &error); + + assert_eq!(manager.peer_count(), 0); + } +} diff --git a/crates/sync/src/progress.rs b/crates/sync/src/progress.rs new file mode 100644 index 0000000..c5c6b6d --- /dev/null +++ b/crates/sync/src/progress.rs @@ -0,0 +1,591 @@ +//! Sync progress tracking for resumability + +use crate::snapshot::StateSnapshot; +use alloy_primitives::{Address, B256}; +use cipherbft_storage::{ + StoredAccountProgress, StoredBlockProgress, StoredStorageProgress, StoredSyncPhase, + StoredSyncProgress, StoredSyncSnapshot, SyncStore, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Current sync phase +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum SyncPhase { + /// Finding peers and selecting snapshot + Discovery, + /// Downloading account/storage state + SnapSync(SnapSubPhase), + /// Downloading and executing blocks + BlockSync, + /// Sync complete + Complete, +} + +/// Sub-phases within snap sync +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum SnapSubPhase { + /// Downloading accounts + Accounts, + /// Downloading storage for accounts + Storage, + /// Final state root verification + Verification, +} + +/// Progress state persisted to disk +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncProgressState { + /// Current sync phase + pub phase: SyncPhase, + /// Target snapshot being synced to + pub target_snapshot: Option, + /// Account download progress + pub account_progress: AccountProgress, + /// Storage download progress per account + pub storage_progress: BTreeMap, + /// Block sync progress + pub block_progress: BlockProgress, +} + +/// Account range download progress +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AccountProgress { + /// Last completed address (exclusive upper bound) + pub completed_up_to: Option

, + /// Addresses that need storage downloaded + pub accounts_needing_storage: Vec
, + /// Total accounts downloaded + pub total_accounts: u64, + /// Total bytes downloaded + pub total_bytes: u64, +} + +/// Storage slot download progress +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct StorageProgress { + /// Last completed slot (exclusive upper bound) + pub completed_up_to: Option, + /// Total slots downloaded + pub total_slots: u64, +} + +/// Block sync progress +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct BlockProgress { + /// First block needed (snapshot height + 1) + pub start_height: u64, + /// Last block successfully executed + pub executed_up_to: u64, + /// Target height to sync to + pub target_height: u64, +} + +impl Default for SyncProgressState { + fn default() -> Self { + Self { + phase: SyncPhase::Discovery, + target_snapshot: None, + account_progress: AccountProgress::default(), + storage_progress: BTreeMap::new(), + block_progress: BlockProgress::default(), + } + } +} + +impl SyncProgressState { + /// Create a new progress state + pub fn new() -> Self { + Self::default() + } + + /// Reset progress for fresh sync + pub fn reset(&mut self) { + *self = Self::default(); + } + + /// Calculate overall sync progress as percentage + pub fn overall_progress(&self) -> f64 { + match &self.phase { + SyncPhase::Discovery => 0.0, + SyncPhase::SnapSync(sub) => { + let base = match sub { + SnapSubPhase::Accounts => 0.0, + SnapSubPhase::Storage => 33.0, + SnapSubPhase::Verification => 66.0, + }; + base + self.snap_phase_progress() * 33.0 / 100.0 + } + SyncPhase::BlockSync => 66.0 + self.block_sync_progress() * 34.0 / 100.0, + SyncPhase::Complete => 100.0, + } + } + + fn snap_phase_progress(&self) -> f64 { + // Estimate based on address space coverage + if let Some(addr) = &self.account_progress.completed_up_to { + let first_byte = addr.0[0] as f64; + (first_byte / 255.0) * 100.0 + } else { + 0.0 + } + } + + fn block_sync_progress(&self) -> f64 { + let bp = &self.block_progress; + if bp.target_height <= bp.start_height { + return 100.0; + } + let total = bp.target_height - bp.start_height; + let done = bp.executed_up_to.saturating_sub(bp.start_height); + (done as f64 / total as f64) * 100.0 + } +} + +// ============================================================================ +// Conversion between in-memory and storage types +// ============================================================================ + +impl From<&SyncPhase> for StoredSyncPhase { + fn from(phase: &SyncPhase) -> Self { + match phase { + SyncPhase::Discovery => StoredSyncPhase::Discovery, + SyncPhase::SnapSync(SnapSubPhase::Accounts) => StoredSyncPhase::SnapSyncAccounts, + SyncPhase::SnapSync(SnapSubPhase::Storage) => StoredSyncPhase::SnapSyncStorage, + SyncPhase::SnapSync(SnapSubPhase::Verification) => { + StoredSyncPhase::SnapSyncVerification + } + SyncPhase::BlockSync => StoredSyncPhase::BlockSync, + SyncPhase::Complete => StoredSyncPhase::Complete, + } + } +} + +impl From<&StoredSyncPhase> for SyncPhase { + fn from(phase: &StoredSyncPhase) -> Self { + match phase { + StoredSyncPhase::Discovery => SyncPhase::Discovery, + StoredSyncPhase::SnapSyncAccounts => SyncPhase::SnapSync(SnapSubPhase::Accounts), + StoredSyncPhase::SnapSyncStorage => SyncPhase::SnapSync(SnapSubPhase::Storage), + StoredSyncPhase::SnapSyncVerification => { + SyncPhase::SnapSync(SnapSubPhase::Verification) + } + StoredSyncPhase::BlockSync => SyncPhase::BlockSync, + StoredSyncPhase::Complete => SyncPhase::Complete, + } + } +} + +impl From<&SyncProgressState> for StoredSyncProgress { + fn from(state: &SyncProgressState) -> Self { + StoredSyncProgress { + phase: (&state.phase).into(), + target_snapshot: state.target_snapshot.as_ref().map(|s| StoredSyncSnapshot { + block_number: s.block_number, + block_hash: s.block_hash.0, + state_root: s.state_root.0, + timestamp: s.timestamp, + }), + account_progress: StoredAccountProgress { + completed_up_to: state.account_progress.completed_up_to.map(|a| a.0 .0), + accounts_needing_storage: state + .account_progress + .accounts_needing_storage + .iter() + .map(|a| a.0 .0) + .collect(), + total_accounts: state.account_progress.total_accounts, + total_bytes: state.account_progress.total_bytes, + }, + storage_progress: state + .storage_progress + .iter() + .map(|(addr, progress)| { + ( + addr.0 .0, + StoredStorageProgress { + completed_up_to: progress.completed_up_to.map(|b| b.0), + total_slots: progress.total_slots, + }, + ) + }) + .collect(), + block_progress: StoredBlockProgress { + start_height: state.block_progress.start_height, + executed_up_to: state.block_progress.executed_up_to, + target_height: state.block_progress.target_height, + }, + } + } +} + +impl From<&StoredSyncProgress> for SyncProgressState { + fn from(stored: &StoredSyncProgress) -> Self { + SyncProgressState { + phase: (&stored.phase).into(), + target_snapshot: stored.target_snapshot.as_ref().map(|s| { + StateSnapshot::new( + s.block_number, + B256::from(s.block_hash), + B256::from(s.state_root), + s.timestamp, + ) + }), + account_progress: AccountProgress { + completed_up_to: stored.account_progress.completed_up_to.map(Address::from), + accounts_needing_storage: stored + .account_progress + .accounts_needing_storage + .iter() + .map(|a| Address::from(*a)) + .collect(), + total_accounts: stored.account_progress.total_accounts, + total_bytes: stored.account_progress.total_bytes, + }, + storage_progress: stored + .storage_progress + .iter() + .map(|(addr, progress)| { + ( + Address::from(*addr), + StorageProgress { + completed_up_to: progress.completed_up_to.map(B256::from), + total_slots: progress.total_slots, + }, + ) + }) + .collect(), + block_progress: BlockProgress { + start_height: stored.block_progress.start_height, + executed_up_to: stored.block_progress.executed_up_to, + target_height: stored.block_progress.target_height, + }, + } + } +} + +/// Progress tracker with persistence +pub struct ProgressTracker { + state: SyncProgressState, +} + +impl ProgressTracker { + /// Create a new progress tracker + pub fn new() -> Self { + Self { + state: SyncProgressState::new(), + } + } + + /// Load progress from storage (or create fresh) + /// + /// This synchronous version creates a fresh tracker. + /// Use `load_from_store` for async loading from storage. + pub fn load_or_create() -> Self { + Self::new() + } + + /// Load progress from a sync store asynchronously + /// + /// If progress exists in storage, loads and returns it. + /// Otherwise, returns a fresh tracker. + pub async fn load_from_store(store: &S) -> crate::Result { + match store.get_progress().await { + Ok(Some(stored)) => { + tracing::info!("Loaded sync progress from storage"); + Ok(Self { + state: (&stored).into(), + }) + } + Ok(None) => { + tracing::debug!("No existing sync progress, starting fresh"); + Ok(Self::new()) + } + Err(e) => { + tracing::warn!("Failed to load sync progress: {}, starting fresh", e); + Ok(Self::new()) + } + } + } + + /// Get current progress state + pub fn state(&self) -> &SyncProgressState { + &self.state + } + + /// Get mutable progress state + pub fn state_mut(&mut self) -> &mut SyncProgressState { + &mut self.state + } + + /// Persist current progress to storage (no-op without store) + /// + /// This synchronous version is a no-op for backwards compatibility. + /// Use `persist_to_store` for actual persistence. + pub fn persist(&self) -> crate::Result<()> { + Ok(()) + } + + /// Persist current progress to a sync store asynchronously + pub async fn persist_to_store(&self, store: &S) -> crate::Result<()> { + let stored: StoredSyncProgress = (&self.state).into(); + store + .put_progress(stored) + .await + .map_err(|e| crate::SyncError::Storage(format!("failed to persist progress: {}", e)))?; + tracing::trace!("Persisted sync progress"); + Ok(()) + } + + /// Clear progress from storage after successful sync completion + pub async fn clear_from_store(&self, store: &S) -> crate::Result<()> { + store + .delete_progress() + .await + .map_err(|e| crate::SyncError::Storage(format!("failed to clear progress: {}", e)))?; + tracing::info!("Cleared sync progress from storage"); + Ok(()) + } + + /// Update phase and persist + pub fn set_phase(&mut self, phase: SyncPhase) -> crate::Result<()> { + self.state.phase = phase; + self.persist() + } + + /// Update phase and persist to store + pub async fn set_phase_and_persist( + &mut self, + phase: SyncPhase, + store: &S, + ) -> crate::Result<()> { + self.state.phase = phase; + self.persist_to_store(store).await + } + + /// Mark account range as complete + pub fn complete_account_range( + &mut self, + up_to: Address, + accounts: u64, + bytes: u64, + needs_storage: Vec
, + ) -> crate::Result<()> { + self.state.account_progress.completed_up_to = Some(up_to); + self.state.account_progress.total_accounts += accounts; + self.state.account_progress.total_bytes += bytes; + self.state + .account_progress + .accounts_needing_storage + .extend(needs_storage); + self.persist() + } + + /// Mark account range as complete and persist to store + pub async fn complete_account_range_and_persist( + &mut self, + up_to: Address, + accounts: u64, + bytes: u64, + needs_storage: Vec
, + store: &S, + ) -> crate::Result<()> { + self.state.account_progress.completed_up_to = Some(up_to); + self.state.account_progress.total_accounts += accounts; + self.state.account_progress.total_bytes += bytes; + self.state + .account_progress + .accounts_needing_storage + .extend(needs_storage); + self.persist_to_store(store).await + } + + /// Mark storage range as complete for account + pub fn complete_storage_range( + &mut self, + account: Address, + up_to: Option, + slots: u64, + ) -> crate::Result<()> { + let progress = self.state.storage_progress.entry(account).or_default(); + progress.completed_up_to = up_to; + progress.total_slots += slots; + self.persist() + } + + /// Mark storage range as complete and persist to store + pub async fn complete_storage_range_and_persist( + &mut self, + account: Address, + up_to: Option, + slots: u64, + store: &S, + ) -> crate::Result<()> { + let progress = self.state.storage_progress.entry(account).or_default(); + progress.completed_up_to = up_to; + progress.total_slots += slots; + self.persist_to_store(store).await + } + + /// Mark block as executed + pub fn complete_block(&mut self, height: u64) -> crate::Result<()> { + self.state.block_progress.executed_up_to = height; + self.persist() + } + + /// Mark block as executed and persist to store + pub async fn complete_block_and_persist( + &mut self, + height: u64, + store: &S, + ) -> crate::Result<()> { + self.state.block_progress.executed_up_to = height; + self.persist_to_store(store).await + } +} + +impl Default for ProgressTracker { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_progress_phases() { + let mut state = SyncProgressState::new(); + assert_eq!(state.phase, SyncPhase::Discovery); + assert_eq!(state.overall_progress(), 0.0); + + state.phase = SyncPhase::Complete; + assert_eq!(state.overall_progress(), 100.0); + } + + #[test] + fn test_block_progress_calculation() { + let mut state = SyncProgressState::new(); + state.phase = SyncPhase::BlockSync; + state.block_progress = BlockProgress { + start_height: 10000, + executed_up_to: 15000, + target_height: 20000, + }; + + // 50% of block sync = 66% + (50% * 34%) = 83% + let progress = state.overall_progress(); + assert!(progress > 82.0 && progress < 84.0); + } + + #[test] + fn test_progress_tracker() { + let mut tracker = ProgressTracker::new(); + + tracker + .set_phase(SyncPhase::SnapSync(SnapSubPhase::Accounts)) + .unwrap(); + assert_eq!( + tracker.state().phase, + SyncPhase::SnapSync(SnapSubPhase::Accounts) + ); + + tracker + .complete_account_range( + Address::repeat_byte(0x80), + 1000, + 50000, + vec![Address::repeat_byte(0x01)], + ) + .unwrap(); + + assert_eq!(tracker.state().account_progress.total_accounts, 1000); + assert_eq!( + tracker + .state() + .account_progress + .accounts_needing_storage + .len(), + 1 + ); + } + + #[test] + fn test_progress_serialization() { + let mut state = SyncProgressState::new(); + state.phase = SyncPhase::SnapSync(SnapSubPhase::Storage); + state.account_progress.total_accounts = 5000; + + let encoded = bincode::serialize(&state).unwrap(); + let decoded: SyncProgressState = bincode::deserialize(&encoded).unwrap(); + + assert_eq!(decoded.phase, SyncPhase::SnapSync(SnapSubPhase::Storage)); + assert_eq!(decoded.account_progress.total_accounts, 5000); + } + + #[test] + fn test_phase_conversion_roundtrip() { + let phases = vec![ + SyncPhase::Discovery, + SyncPhase::SnapSync(SnapSubPhase::Accounts), + SyncPhase::SnapSync(SnapSubPhase::Storage), + SyncPhase::SnapSync(SnapSubPhase::Verification), + SyncPhase::BlockSync, + SyncPhase::Complete, + ]; + + for phase in phases { + let stored: StoredSyncPhase = (&phase).into(); + let back: SyncPhase = (&stored).into(); + assert_eq!(phase, back); + } + } + + #[test] + fn test_progress_state_conversion_roundtrip() { + let mut state = SyncProgressState::new(); + state.phase = SyncPhase::SnapSync(SnapSubPhase::Storage); + state.target_snapshot = Some(StateSnapshot::new( + 10000, + B256::repeat_byte(0xab), + B256::repeat_byte(0xcd), + 123456, + )); + state.account_progress.completed_up_to = Some(Address::repeat_byte(0x42)); + state.account_progress.total_accounts = 5000; + state.account_progress.accounts_needing_storage = vec![Address::repeat_byte(0x01)]; + state.storage_progress.insert( + Address::repeat_byte(0x01), + StorageProgress { + completed_up_to: Some(B256::repeat_byte(0xff)), + total_slots: 100, + }, + ); + state.block_progress.start_height = 10000; + state.block_progress.executed_up_to = 10050; + state.block_progress.target_height = 10100; + + let stored: StoredSyncProgress = (&state).into(); + let back: SyncProgressState = (&stored).into(); + + assert_eq!(state.phase, back.phase); + assert_eq!( + state.target_snapshot.as_ref().map(|s| s.block_number), + back.target_snapshot.as_ref().map(|s| s.block_number) + ); + assert_eq!( + state.account_progress.completed_up_to, + back.account_progress.completed_up_to + ); + assert_eq!( + state.account_progress.total_accounts, + back.account_progress.total_accounts + ); + assert_eq!(state.storage_progress.len(), back.storage_progress.len()); + assert_eq!( + state.block_progress.executed_up_to, + back.block_progress.executed_up_to + ); + } +} diff --git a/crates/sync/src/protocol.rs b/crates/sync/src/protocol.rs new file mode 100644 index 0000000..93fa868 --- /dev/null +++ b/crates/sync/src/protocol.rs @@ -0,0 +1,239 @@ +//! Snap sync P2P protocol messages +//! +//! Protocol identifier: `/cipherbft/snap/1.0.0` + +use alloy_primitives::{Address, Bytes, B256, U256}; +use serde::{Deserialize, Serialize}; + +/// Protocol identifier for snap sync +pub const SNAP_PROTOCOL_ID: &str = "/cipherbft/snap/1.0.0"; + +/// Maximum accounts per range response +pub const MAX_ACCOUNTS_PER_RESPONSE: u32 = 4096; + +/// Maximum storage slots per range response +pub const MAX_STORAGE_PER_RESPONSE: u32 = 8192; + +/// Maximum blocks per response +pub const MAX_BLOCKS_PER_RESPONSE: u32 = 128; + +/// Snap sync protocol messages +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum SnapSyncMessage { + // === Discovery === + /// Request peer's sync status + GetStatus, + + /// Response with current status + Status(StatusResponse), + + // === Account Ranges === + /// Request accounts in address range + GetAccountRange(AccountRangeRequest), + + /// Response with accounts and proof + AccountRange(AccountRangeResponse), + + // === Storage Ranges === + /// Request storage slots for account + GetStorageRange(StorageRangeRequest), + + /// Response with storage slots and proof + StorageRange(StorageRangeResponse), + + // === Blocks === + /// Request block range + GetBlocks(BlockRangeRequest), + + /// Response with blocks + Blocks(BlockRangeResponse), +} + +/// Peer status response +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StatusResponse { + /// Peer's tip block height + pub tip_height: u64, + /// Peer's tip block hash + pub tip_hash: B256, + /// Available snapshots with state roots (sorted by height descending) + /// Each entry contains (height, state_root, block_hash) + pub snapshots: Vec, +} + +/// Snapshot information advertised by a peer +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SnapshotInfo { + /// Block height of the snapshot + pub height: u64, + /// State root (MPT root) at this height + pub state_root: B256, + /// Block hash at this height + pub block_hash: B256, +} + +/// Account range request +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AccountRangeRequest { + /// Unique request identifier for response correlation + pub request_id: u64, + /// Snapshot height to sync from + pub snapshot_height: u64, + /// Expected state root at snapshot + pub state_root: B256, + /// Start of address range (inclusive) + pub start_address: Address, + /// End of address range (exclusive) + pub limit_address: Address, + /// Maximum accounts to return + pub max_accounts: u32, +} + +/// Account range response +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AccountRangeResponse { + /// Request ID echoed back for correlation + pub request_id: u64, + /// Accounts in range: (address, nonce, balance, code_hash, storage_root) + pub accounts: Vec, + /// Merkle proof nodes + pub proof: Vec, + /// True if more accounts exist after this range + pub more: bool, +} + +/// Account data for sync +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AccountData { + /// Account address + pub address: Address, + /// Account nonce + pub nonce: u64, + /// Account balance + pub balance: U256, + /// Hash of contract bytecode (or empty account hash) + pub code_hash: B256, + /// Root of storage trie + pub storage_root: B256, +} + +/// Storage range request +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StorageRangeRequest { + /// Unique request identifier for response correlation + pub request_id: u64, + /// Snapshot height + pub snapshot_height: u64, + /// Expected state root + pub state_root: B256, + /// Account to fetch storage for + pub account: Address, + /// Account's storage root (for verification) + pub storage_root: B256, + /// Start storage slot (inclusive) + pub start_slot: B256, + /// End storage slot (exclusive) + pub limit_slot: B256, + /// Maximum slots to return + pub max_slots: u32, +} + +/// Storage range response +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StorageRangeResponse { + /// Request ID echoed back for correlation + pub request_id: u64, + /// Storage slots: (key, value) + pub slots: Vec<(B256, B256)>, + /// Merkle proof nodes + pub proof: Vec, + /// True if more slots exist after this range + pub more: bool, +} + +/// Block range request +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BlockRangeRequest { + /// Unique request identifier for response correlation + pub request_id: u64, + /// Start block height (inclusive) + pub start_height: u64, + /// Number of blocks to fetch + pub count: u32, +} + +/// Block range response +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BlockRangeResponse { + /// Request ID echoed back for correlation + pub request_id: u64, + /// Serialized blocks + pub blocks: Vec, +} + +impl SnapSyncMessage { + /// Get message type name for logging + pub fn message_type(&self) -> &'static str { + match self { + Self::GetStatus => "GetStatus", + Self::Status(_) => "Status", + Self::GetAccountRange(_) => "GetAccountRange", + Self::AccountRange(_) => "AccountRange", + Self::GetStorageRange(_) => "GetStorageRange", + Self::StorageRange(_) => "StorageRange", + Self::GetBlocks(_) => "GetBlocks", + Self::Blocks(_) => "Blocks", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_message_serialization_roundtrip() { + let msg = SnapSyncMessage::GetAccountRange(AccountRangeRequest { + request_id: 1, + snapshot_height: 10000, + state_root: B256::ZERO, + start_address: Address::ZERO, + limit_address: Address::repeat_byte(0xff), + max_accounts: 1000, + }); + + let encoded = bincode::serialize(&msg).unwrap(); + let decoded: SnapSyncMessage = bincode::deserialize(&encoded).unwrap(); + + assert!(matches!(decoded, SnapSyncMessage::GetAccountRange(_))); + } + + #[test] + fn test_status_response() { + let status = StatusResponse { + tip_height: 1_000_000, + tip_hash: B256::repeat_byte(0xab), + snapshots: vec![ + SnapshotInfo { + height: 990_000, + state_root: B256::repeat_byte(0x01), + block_hash: B256::repeat_byte(0x02), + }, + SnapshotInfo { + height: 980_000, + state_root: B256::repeat_byte(0x03), + block_hash: B256::repeat_byte(0x04), + }, + SnapshotInfo { + height: 970_000, + state_root: B256::repeat_byte(0x05), + block_hash: B256::repeat_byte(0x06), + }, + ], + }; + + let encoded = bincode::serialize(&status).unwrap(); + // Status response is now larger due to state roots and block hashes + assert!(encoded.len() < 1500); + } +} diff --git a/crates/sync/src/snap/accounts.rs b/crates/sync/src/snap/accounts.rs new file mode 100644 index 0000000..4aa5092 --- /dev/null +++ b/crates/sync/src/snap/accounts.rs @@ -0,0 +1,317 @@ +//! Account range synchronization + +#![allow(dead_code)] // Module is foundational, will be used by sync orchestrator + +use crate::error::Result; +use crate::protocol::{AccountRangeRequest, AccountRangeResponse, MAX_ACCOUNTS_PER_RESPONSE}; +use crate::snap::verify::verify_account_range_proof; +use crate::snapshot::StateSnapshot; +use alloy_primitives::Address; +use alloy_trie::EMPTY_ROOT_HASH; + +/// Number of parallel address ranges to download +pub const PARALLEL_RANGES: usize = 16; + +/// Pending account range to download +#[derive(Clone, Debug)] +pub struct PendingRange { + /// Start address (inclusive) + pub start: Address, + /// End address (exclusive) + pub end: Address, + /// Number of retry attempts + pub retries: u32, +} + +impl PendingRange { + /// Create initial ranges covering the full address space + pub fn initial_ranges() -> Vec { + let mut ranges = Vec::with_capacity(PARALLEL_RANGES); + let step = u128::MAX / PARALLEL_RANGES as u128; + + for i in 0..PARALLEL_RANGES { + let start_val = step * i as u128; + let end_val = if i == PARALLEL_RANGES - 1 { + u128::MAX + } else { + step * (i + 1) as u128 + }; + + // Convert u128 to Address (use lower 16 bytes, padded) + let start = address_from_u128(start_val); + let end = address_from_u128(end_val); + + ranges.push(PendingRange { + start, + end, + retries: 0, + }); + } + ranges + } + + /// Split this range in half for retry + pub fn split(&self) -> (Self, Self) { + let mid = midpoint_address(&self.start, &self.end); + ( + PendingRange { + start: self.start, + end: mid, + retries: self.retries, + }, + PendingRange { + start: mid, + end: self.end, + retries: self.retries, + }, + ) + } +} + +/// Account range syncer +pub struct AccountRangeSyncer { + /// Target snapshot + snapshot: StateSnapshot, + /// Pending ranges to download + pending: Vec, + /// Accounts that need storage downloaded + accounts_with_storage: Vec
, + /// Total accounts downloaded + total_accounts: u64, + /// Total bytes downloaded + total_bytes: u64, + /// Next request ID for correlation + next_request_id: u64, +} + +impl AccountRangeSyncer { + /// Create a new account range syncer + pub fn new(snapshot: StateSnapshot) -> Self { + Self { + snapshot, + pending: PendingRange::initial_ranges(), + accounts_with_storage: Vec::new(), + total_accounts: 0, + total_bytes: 0, + next_request_id: 1, + } + } + + /// Resume from progress state + pub fn resume(snapshot: StateSnapshot, completed_up_to: Option
) -> Self { + let pending = if let Some(addr) = completed_up_to { + // Resume from where we left off + vec![PendingRange { + start: addr, + end: Address::repeat_byte(0xff), + retries: 0, + }] + } else { + PendingRange::initial_ranges() + }; + + Self { + snapshot, + pending, + accounts_with_storage: Vec::new(), + total_accounts: 0, + total_bytes: 0, + next_request_id: 1, + } + } + + /// Check if sync is complete + pub fn is_complete(&self) -> bool { + self.pending.is_empty() + } + + /// Get next range to request + pub fn next_range(&mut self) -> Option { + self.pending.pop() + } + + /// Create request for a range with unique request ID + pub fn create_request(&mut self, range: &PendingRange) -> AccountRangeRequest { + let request_id = self.next_request_id; + self.next_request_id += 1; + AccountRangeRequest { + request_id, + snapshot_height: self.snapshot.block_number, + state_root: self.snapshot.state_root, + start_address: range.start, + limit_address: range.end, + max_accounts: MAX_ACCOUNTS_PER_RESPONSE, + } + } + + /// Process response for a range + pub fn process_response( + &mut self, + range: PendingRange, + response: AccountRangeResponse, + ) -> Result<()> { + // Verify the proof with the actual range start address + self.verify_account_proof(&range, &response)?; + + // Track accounts with storage + for account in &response.accounts { + if account.storage_root != EMPTY_ROOT_HASH { + self.accounts_with_storage.push(account.address); + } + } + + self.total_accounts += response.accounts.len() as u64; + self.total_bytes += estimate_response_size(&response); + + // If more accounts exist, add continuation range + if response.more { + if let Some(last) = response.accounts.last() { + // Next range starts after last account + let next_start = increment_address(last.address); + self.pending.push(PendingRange { + start: next_start, + end: range.end, + retries: 0, + }); + } + } + + Ok(()) + } + + /// Handle failed request + pub fn handle_failure(&mut self, range: PendingRange, max_retries: u32) { + if range.retries < max_retries { + // Retry with incremented counter + self.pending.push(PendingRange { + retries: range.retries + 1, + ..range + }); + } else { + // Split range and retry both halves + let (left, right) = range.split(); + self.pending.push(left); + self.pending.push(right); + } + } + + /// Verify account range proof using MPT proof verification. + fn verify_account_proof( + &self, + range: &PendingRange, + response: &AccountRangeResponse, + ) -> Result<()> { + // Use the actual range start address for proper verification + verify_account_range_proof( + self.snapshot.state_root, + range.start, + &response.accounts, + &response.proof, + ) + } + + /// Get accounts that need storage downloaded + pub fn accounts_needing_storage(&self) -> &[Address] { + &self.accounts_with_storage + } + + /// Get sync statistics + pub fn stats(&self) -> (u64, u64, usize) { + (self.total_accounts, self.total_bytes, self.pending.len()) + } +} + +// Helper functions + +fn address_from_u128(val: u128) -> Address { + let bytes = val.to_be_bytes(); + let mut addr_bytes = [0u8; 20]; + // Use the lower 16 bytes, padded + addr_bytes[4..20].copy_from_slice(&bytes); + Address::from(addr_bytes) +} + +fn midpoint_address(start: &Address, end: &Address) -> Address { + // Simple midpoint calculation + let start_bytes = start.as_slice(); + let end_bytes = end.as_slice(); + let mut mid_bytes = [0u8; 20]; + + let mut carry = 0u16; + for i in (0..20).rev() { + let sum = start_bytes[i] as u16 + end_bytes[i] as u16 + carry; + mid_bytes[i] = (sum / 2) as u8; + carry = (sum % 2) << 8; + } + + Address::from(mid_bytes) +} + +fn increment_address(addr: Address) -> Address { + let mut bytes = addr.0; + for i in (0..20).rev() { + if bytes[i] < 255 { + bytes[i] += 1; + break; + } + bytes[i] = 0; + } + Address::from(bytes) +} + +fn estimate_response_size(response: &AccountRangeResponse) -> u64 { + // Rough estimate: 100 bytes per account + proof size + let accounts_size = response.accounts.len() as u64 * 100; + let proof_size: u64 = response.proof.iter().map(|p| p.len() as u64).sum(); + accounts_size + proof_size +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + + #[test] + fn test_initial_ranges() { + let ranges = PendingRange::initial_ranges(); + assert_eq!(ranges.len(), PARALLEL_RANGES); + + // First range starts at 0 + assert_eq!(ranges[0].start, Address::ZERO); + } + + #[test] + fn test_range_split() { + let range = PendingRange { + start: Address::ZERO, + end: Address::repeat_byte(0xff), + retries: 0, + }; + + let (left, right) = range.split(); + assert_eq!(left.start, range.start); + assert_eq!(right.end, range.end); + // Midpoint should be between start and end + assert!(left.end == right.start); + } + + #[test] + fn test_syncer_creation() { + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::repeat_byte(0xab), 12345); + + let syncer = AccountRangeSyncer::new(snapshot); + assert!(!syncer.is_complete()); + assert_eq!(syncer.accounts_needing_storage().len(), 0); + } + + #[test] + fn test_increment_address() { + let addr = Address::ZERO; + let next = increment_address(addr); + assert_eq!(next.0[19], 1); + + let addr = Address::repeat_byte(0xff); + let next = increment_address(addr); + assert_eq!(next, Address::ZERO); // Overflow wraps + } +} diff --git a/crates/sync/src/snap/mod.rs b/crates/sync/src/snap/mod.rs new file mode 100644 index 0000000..81e4532 --- /dev/null +++ b/crates/sync/src/snap/mod.rs @@ -0,0 +1,5 @@ +//! Snap sync components + +pub mod accounts; +pub mod storage; +pub mod verify; diff --git a/crates/sync/src/snap/storage.rs b/crates/sync/src/snap/storage.rs new file mode 100644 index 0000000..3af1f4c --- /dev/null +++ b/crates/sync/src/snap/storage.rs @@ -0,0 +1,269 @@ +//! Storage range synchronization + +#![allow(dead_code)] // Module is foundational, will be used by sync orchestrator + +use crate::error::Result; +use crate::protocol::{StorageRangeRequest, StorageRangeResponse, MAX_STORAGE_PER_RESPONSE}; +use crate::snap::verify::verify_storage_range_proof; +use crate::snapshot::StateSnapshot; +use alloy_primitives::{Address, B256}; +use std::collections::VecDeque; + +/// Pending storage range to download +#[derive(Clone, Debug)] +pub struct PendingStorageRange { + /// Account address + pub account: Address, + /// Account's storage root + pub storage_root: B256, + /// Start slot (inclusive) + pub start: B256, + /// End slot (exclusive) + pub end: B256, + /// Retry count + pub retries: u32, +} + +impl PendingStorageRange { + /// Create initial range for an account (full storage space) + pub fn new(account: Address, storage_root: B256) -> Self { + Self { + account, + storage_root, + start: B256::ZERO, + end: B256::repeat_byte(0xff), + retries: 0, + } + } +} + +/// Storage range syncer +pub struct StorageRangeSyncer { + /// Target snapshot + snapshot: StateSnapshot, + /// Accounts pending storage download + pending_accounts: VecDeque<(Address, B256)>, + /// Current account being synced + current_ranges: Vec, + /// Total slots downloaded + total_slots: u64, + /// Total bytes downloaded + total_bytes: u64, + /// Completed accounts count + completed_accounts: u64, + /// Next request ID for correlation + next_request_id: u64, +} + +impl StorageRangeSyncer { + /// Create a new storage range syncer + pub fn new(snapshot: StateSnapshot, accounts: Vec<(Address, B256)>) -> Self { + Self { + snapshot, + pending_accounts: accounts.into_iter().collect(), + current_ranges: Vec::new(), + total_slots: 0, + total_bytes: 0, + completed_accounts: 0, + next_request_id: 1, + } + } + + /// Check if sync is complete + pub fn is_complete(&self) -> bool { + self.pending_accounts.is_empty() && self.current_ranges.is_empty() + } + + /// Get next range to request + pub fn next_range(&mut self) -> Option { + // First try current ranges + if let Some(range) = self.current_ranges.pop() { + return Some(range); + } + + // Start next account + if let Some((account, storage_root)) = self.pending_accounts.pop_front() { + return Some(PendingStorageRange::new(account, storage_root)); + } + + None + } + + /// Create request for a range with unique request ID + pub fn create_request(&mut self, range: &PendingStorageRange) -> StorageRangeRequest { + let request_id = self.next_request_id; + self.next_request_id += 1; + StorageRangeRequest { + request_id, + snapshot_height: self.snapshot.block_number, + state_root: self.snapshot.state_root, + account: range.account, + storage_root: range.storage_root, + start_slot: range.start, + limit_slot: range.end, + max_slots: MAX_STORAGE_PER_RESPONSE, + } + } + + /// Process response for a range + pub fn process_response( + &mut self, + range: PendingStorageRange, + response: StorageRangeResponse, + ) -> Result<()> { + // Verify the proof + self.verify_storage_proof(&range, &response)?; + + self.total_slots += response.slots.len() as u64; + self.total_bytes += estimate_storage_response_size(&response); + + // If more slots exist, add continuation range + if response.more { + if let Some((last_key, _)) = response.slots.last() { + let next_start = increment_b256(*last_key); + self.current_ranges.push(PendingStorageRange { + account: range.account, + storage_root: range.storage_root, + start: next_start, + end: range.end, + retries: 0, + }); + } + } else { + // Account storage complete + self.completed_accounts += 1; + } + + Ok(()) + } + + /// Handle failed request + pub fn handle_failure(&mut self, range: PendingStorageRange, max_retries: u32) { + if range.retries < max_retries { + self.current_ranges.push(PendingStorageRange { + retries: range.retries + 1, + ..range + }); + } else { + // Re-queue the account to try again later + self.pending_accounts + .push_back((range.account, range.storage_root)); + } + } + + /// Verify storage range proof using MPT proof verification. + fn verify_storage_proof( + &self, + range: &PendingStorageRange, + response: &StorageRangeResponse, + ) -> Result<()> { + verify_storage_range_proof( + range.storage_root, + range.start, + &response.slots, + &response.proof, + ) + } + + /// Get sync statistics + pub fn stats(&self) -> StorageSyncStats { + StorageSyncStats { + total_slots: self.total_slots, + total_bytes: self.total_bytes, + completed_accounts: self.completed_accounts, + pending_accounts: self.pending_accounts.len() as u64, + pending_ranges: self.current_ranges.len() as u64, + } + } +} + +/// Storage sync statistics +#[derive(Clone, Debug, Default)] +pub struct StorageSyncStats { + /// Total storage slots downloaded + pub total_slots: u64, + /// Total bytes downloaded + pub total_bytes: u64, + /// Number of accounts completed + pub completed_accounts: u64, + /// Number of accounts still pending + pub pending_accounts: u64, + /// Number of ranges in progress + pub pending_ranges: u64, +} + +// Helper functions + +fn increment_b256(val: B256) -> B256 { + let mut bytes = val.0; + for i in (0..32).rev() { + if bytes[i] < 255 { + bytes[i] += 1; + return B256::from(bytes); + } + bytes[i] = 0; + } + B256::ZERO // Overflow wraps +} + +fn estimate_storage_response_size(response: &StorageRangeResponse) -> u64 { + // 64 bytes per slot (key + value) + proof size + let slots_size = response.slots.len() as u64 * 64; + let proof_size: u64 = response.proof.iter().map(|p| p.len() as u64).sum(); + slots_size + proof_size +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pending_storage_range() { + let range = PendingStorageRange::new(Address::repeat_byte(0x42), B256::repeat_byte(0xab)); + + assert_eq!(range.start, B256::ZERO); + assert_eq!(range.account, Address::repeat_byte(0x42)); + } + + #[test] + fn test_storage_syncer_creation() { + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::repeat_byte(0xab), 12345); + + let accounts = vec![ + (Address::repeat_byte(0x01), B256::repeat_byte(0xaa)), + (Address::repeat_byte(0x02), B256::repeat_byte(0xbb)), + ]; + + let syncer = StorageRangeSyncer::new(snapshot, accounts); + assert!(!syncer.is_complete()); + + let stats = syncer.stats(); + assert_eq!(stats.pending_accounts, 2); + } + + #[test] + fn test_storage_syncer_next_range() { + let snapshot = StateSnapshot::new(10000, B256::ZERO, B256::ZERO, 0); + let accounts = vec![(Address::repeat_byte(0x01), B256::repeat_byte(0xaa))]; + + let mut syncer = StorageRangeSyncer::new(snapshot, accounts); + + let range = syncer.next_range(); + assert!(range.is_some()); + assert_eq!(range.unwrap().account, Address::repeat_byte(0x01)); + + // No more ranges + assert!(syncer.next_range().is_none()); + } + + #[test] + fn test_increment_b256() { + let val = B256::ZERO; + let next = increment_b256(val); + assert_eq!(next.0[31], 1); + + let val = B256::repeat_byte(0xff); + let next = increment_b256(val); + assert_eq!(next, B256::ZERO); // Overflow + } +} diff --git a/crates/sync/src/snap/verify.rs b/crates/sync/src/snap/verify.rs new file mode 100644 index 0000000..6ea83b5 --- /dev/null +++ b/crates/sync/src/snap/verify.rs @@ -0,0 +1,351 @@ +//! Merkle proof verification for snap sync + +use crate::error::{Result, SyncError}; +use crate::protocol::AccountData; +use alloy_primitives::{keccak256, Address, Bytes, B256}; +use alloy_rlp::Encodable; +use alloy_trie::{proof::verify_proof as trie_verify_proof, Nibbles, EMPTY_ROOT_HASH}; + +/// RLP-encoded account for trie verification. +/// This matches Ethereum's account encoding in the state trie. +#[derive(alloy_rlp::RlpEncodable)] +struct RlpAccount { + nonce: u64, + balance: alloy_primitives::U256, + storage_root: B256, + code_hash: B256, +} + +impl From<&AccountData> for RlpAccount { + fn from(account: &AccountData) -> Self { + Self { + nonce: account.nonce, + balance: account.balance, + storage_root: account.storage_root, + code_hash: account.code_hash, + } + } +} + +/// Verify an account range proof against the state root. +/// +/// This verifies that: +/// 1. The proof is valid (connects to state_root) +/// 2. Each account exists at its claimed address +/// 3. The range is complete (no gaps between accounts) +pub fn verify_account_range_proof( + state_root: B256, + start_address: Address, + accounts: &[AccountData], + proof: &[Bytes], +) -> Result<()> { + if accounts.is_empty() { + // Empty range is valid if proof shows no keys in range + // SECURITY: Empty range claims REQUIRE proofs to prevent peers from + // falsely claiming "no accounts exist" when accounts do exist + return verify_empty_range(state_root, start_address, proof); + } + + // TODO: Server proof generation not yet implemented + // When proofs are empty but accounts are returned, we skip individual proof + // verification. The final state root check will catch any malicious data. + // This is acceptable because: + // 1. We DO require proofs for empty range claims (above) + // 2. The final state root verification catches any omitted/modified accounts + // 3. Malicious peers will be detected and banned when sync completes + if !proof.is_empty() { + // Verify each account exists in the trie + for account in accounts { + let key = Nibbles::unpack(keccak256(account.address)); + let rlp_account = RlpAccount::from(account); + let mut encoded = Vec::new(); + rlp_account.encode(&mut encoded); + + // Use alloy-trie's proof verification + verify_proof(state_root, &key, Some(&encoded), proof)?; + } + } + + // Verify range completeness (no missing accounts between returned ones) + // This check runs regardless of whether proofs are present + verify_range_completeness(state_root, start_address, accounts, proof)?; + + Ok(()) +} + +/// Verify a storage range proof against the account's storage root. +pub fn verify_storage_range_proof( + storage_root: B256, + start_slot: B256, + slots: &[(B256, B256)], + proof: &[Bytes], +) -> Result<()> { + if slots.is_empty() { + // SECURITY: Empty storage claims REQUIRE proofs + return verify_empty_storage_range(storage_root, start_slot, proof); + } + + // TODO: Server proof generation not yet implemented + // Skip individual slot verification when proofs are empty. + // Final state root check will catch any malicious data. + if !proof.is_empty() { + // Verify each slot exists in the storage trie + for (key, value) in slots { + let nibble_key = Nibbles::unpack(keccak256(key)); + let encoded_value = alloy_rlp::encode(value); + + verify_proof(storage_root, &nibble_key, Some(&encoded_value), proof)?; + } + } + + Ok(()) +} + +/// Verify a single proof against expected value. +fn verify_proof( + root: B256, + key: &Nibbles, + expected_value: Option<&[u8]>, + proof_nodes: &[Bytes], +) -> Result<()> { + // Convert expected value to owned Vec for alloy-trie API + let expected = expected_value.map(|v| v.to_vec()); + + // Use alloy-trie's proof verification + match trie_verify_proof(root, *key, expected, proof_nodes) { + Ok(()) => Ok(()), + Err(e) => Err(SyncError::invalid_proof( + "peer", + format!("proof verification failed: {}", e), + )), + } +} + +/// Verify that an empty range response is valid. +fn verify_empty_range(state_root: B256, start: Address, proof: &[Bytes]) -> Result<()> { + // For empty range, proof should show no keys exist at or after start + if state_root == EMPTY_ROOT_HASH { + // Empty state is valid + return Ok(()); + } + + // An empty range claim requires a non-empty proof + if proof.is_empty() { + return Err(SyncError::invalid_proof( + "peer", + "empty proof for empty range claim - proof required to demonstrate absence", + )); + } + + // Verify the proof shows absence of the key + // This is an exclusion proof - we expect the key NOT to exist + let key = Nibbles::unpack(keccak256(start)); + + // For exclusion proofs, expected_value is None + // The proof MUST verify successfully - invalid proofs are rejected + trie_verify_proof(state_root, key, None, proof).map_err(|e| { + SyncError::invalid_proof( + "peer", + format!("empty range exclusion proof invalid: {}", e), + ) + }) +} + +/// Verify that no accounts are missing between the returned ones. +/// +/// This performs several validations: +/// 1. Accounts are sorted by their keccak256 hash (trie key order) +/// 2. Accounts are within the requested range +/// 3. First account is at or after the start address +/// +/// Note: Full boundary proof verification would require walking the trie structure +/// to prove no keys exist between consecutive accounts. This is complex and +/// partially covered by the individual account proofs. Malicious peers that omit +/// accounts will be detected when the final state root doesn't match. +fn verify_range_completeness( + _state_root: B256, + start: Address, + accounts: &[AccountData], + _proof: &[Bytes], +) -> Result<()> { + if accounts.is_empty() { + return Ok(()); + } + + // Verify first account is at or after start address + let first_hash = keccak256(accounts[0].address); + let start_hash = keccak256(start); + if first_hash < start_hash { + return Err(SyncError::invalid_proof( + "peer", + format!( + "first account {} is before requested start {}", + accounts[0].address, start + ), + )); + } + + // Verify accounts are in sorted order by their trie key (keccak256 hash) + // This is required for range completeness - gaps would show up as unsorted + let mut prev_hash = first_hash; + for (i, account) in accounts.iter().enumerate().skip(1) { + let curr_hash = keccak256(account.address); + if curr_hash <= prev_hash { + return Err(SyncError::invalid_proof( + "peer", + format!( + "accounts not in sorted order at index {}: {} should be after {}", + i, + account.address, + accounts[i - 1].address + ), + )); + } + prev_hash = curr_hash; + } + + // Note: Full range completeness would verify boundary proofs showing no keys + // exist between consecutive accounts. This requires trie structure walking. + // The current implementation relies on: + // 1. Individual account proofs verifying each account exists + // 2. Sorted order verification above + // 3. Final state root verification catching any omissions + // + // A malicious peer omitting accounts would produce an incorrect final state root, + // causing sync to fail and the peer to be banned. + + Ok(()) +} + +/// Verify empty storage range. +fn verify_empty_storage_range(storage_root: B256, start: B256, proof: &[Bytes]) -> Result<()> { + if storage_root == EMPTY_ROOT_HASH { + return Ok(()); + } + + // An empty storage range claim requires a non-empty proof + if proof.is_empty() { + return Err(SyncError::invalid_proof( + "peer", + "empty proof for empty storage range claim - proof required to demonstrate absence", + )); + } + + // Verify the proof shows absence of the key + let key = Nibbles::unpack(keccak256(start)); + + // The proof MUST verify successfully - invalid proofs are rejected + trie_verify_proof(storage_root, key, None, proof).map_err(|e| { + SyncError::invalid_proof( + "peer", + format!("empty storage range exclusion proof invalid: {}", e), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::U256; + + #[test] + fn test_rlp_account_encoding() { + let account = AccountData { + address: Address::ZERO, + nonce: 1, + balance: U256::from(100), + code_hash: B256::ZERO, + storage_root: B256::ZERO, + }; + + let rlp = RlpAccount::from(&account); + let mut encoded = Vec::new(); + rlp.encode(&mut encoded); + + // RLP encoding should produce valid bytes + assert!(!encoded.is_empty()); + } + + #[test] + fn test_empty_state_verification() { + // Empty state root with empty accounts should pass + let result = verify_account_range_proof( + EMPTY_ROOT_HASH, + Address::ZERO, + &[], + &[Bytes::from(vec![0x80])], // RLP empty + ); + + assert!(result.is_ok()); + } + + #[test] + fn test_empty_storage_verification() { + let result = verify_storage_range_proof(EMPTY_ROOT_HASH, B256::ZERO, &[], &[]); + + assert!(result.is_ok()); + } + + #[test] + fn test_empty_proof_empty_range_fails() { + // Empty range claims REQUIRE proofs - this is security critical + // A peer claiming "no accounts exist" must prove it + + // Empty accounts with empty proof on non-empty state should fail + let result = verify_account_range_proof( + B256::repeat_byte(0xab), // Non-empty state root + Address::ZERO, + &[], // Empty accounts = empty range claim + &[], // Empty proof + ); + assert!(result.is_err()); + + // Empty storage with empty proof on non-empty storage root should fail + let result = verify_storage_range_proof( + B256::repeat_byte(0xab), // Non-empty storage root + B256::ZERO, + &[], // Empty slots = empty range claim + &[], // Empty proof + ); + assert!(result.is_err()); + } + + #[test] + fn test_empty_proof_with_data_allowed_temporarily() { + // TODO: When server implements proof generation, this test should expect errors + // For now, empty proofs are allowed when data IS returned + // (security relies on final state root verification) + + let account = AccountData { + address: Address::ZERO, // Use ZERO to ensure it's at/after start + nonce: 1, + balance: U256::from(100), + code_hash: B256::ZERO, + storage_root: EMPTY_ROOT_HASH, + }; + + // Non-empty accounts with empty proof - temporarily allowed + let result = verify_account_range_proof( + B256::repeat_byte(0xab), + Address::ZERO, + &[account], + &[], // Empty proof + ); + assert!( + result.is_ok(), + "Empty proof with accounts should temporarily pass" + ); + + // Non-empty storage with empty proof - temporarily allowed + let result = verify_storage_range_proof( + B256::repeat_byte(0xab), + B256::ZERO, + &[(B256::ZERO, B256::repeat_byte(0x42))], + &[], // Empty proof + ); + assert!( + result.is_ok(), + "Empty proof with slots should temporarily pass" + ); + } +} diff --git a/crates/sync/src/snapshot.rs b/crates/sync/src/snapshot.rs new file mode 100644 index 0000000..e16cc4a --- /dev/null +++ b/crates/sync/src/snapshot.rs @@ -0,0 +1,102 @@ +//! State snapshot types and management + +use alloy_primitives::B256; +use serde::{Deserialize, Serialize}; + +/// Snapshot creation interval in blocks +pub const SNAPSHOT_INTERVAL: u64 = 10_000; + +/// State snapshot metadata +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct StateSnapshot { + /// Block height this snapshot represents + pub block_number: u64, + /// Block hash at this height + pub block_hash: B256, + /// State root (MPT root of all accounts) + pub state_root: B256, + /// Unix timestamp when snapshot was created + pub timestamp: u64, +} + +impl StateSnapshot { + /// Create a new snapshot + pub fn new(block_number: u64, block_hash: B256, state_root: B256, timestamp: u64) -> Self { + Self { + block_number, + block_hash, + state_root, + timestamp, + } + } + + /// Check if this is a valid snapshot height + pub fn is_valid_snapshot_height(height: u64) -> bool { + height > 0 && height.is_multiple_of(SNAPSHOT_INTERVAL) + } + + /// Get the nearest snapshot height at or below the given height + pub fn nearest_snapshot_height(height: u64) -> u64 { + (height / SNAPSHOT_INTERVAL) * SNAPSHOT_INTERVAL + } +} + +/// Snapshot agreement from peers +#[derive(Clone, Debug)] +pub struct SnapshotAgreement { + /// The agreed snapshot + pub snapshot: StateSnapshot, + /// Number of peers agreeing on this snapshot + pub peer_count: usize, + /// Peers that provided this snapshot + pub peers: Vec, +} + +impl SnapshotAgreement { + /// Check if we have enough peers agreeing (2f+1 for f=7, so 15 of 21) + /// For sync, we use a lower threshold since we verify proofs + pub fn has_quorum(&self, min_peers: usize) -> bool { + self.peer_count >= min_peers + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_snapshot_heights() { + assert!(!StateSnapshot::is_valid_snapshot_height(0)); + assert!(!StateSnapshot::is_valid_snapshot_height(1)); + assert!(!StateSnapshot::is_valid_snapshot_height(9999)); + assert!(StateSnapshot::is_valid_snapshot_height(10000)); + assert!(!StateSnapshot::is_valid_snapshot_height(10001)); + assert!(StateSnapshot::is_valid_snapshot_height(20000)); + } + + #[test] + fn test_nearest_snapshot_height() { + assert_eq!(StateSnapshot::nearest_snapshot_height(0), 0); + assert_eq!(StateSnapshot::nearest_snapshot_height(1), 0); + assert_eq!(StateSnapshot::nearest_snapshot_height(9999), 0); + assert_eq!(StateSnapshot::nearest_snapshot_height(10000), 10000); + assert_eq!(StateSnapshot::nearest_snapshot_height(10001), 10000); + assert_eq!(StateSnapshot::nearest_snapshot_height(19999), 10000); + assert_eq!(StateSnapshot::nearest_snapshot_height(25000), 20000); + } + + #[test] + fn test_snapshot_serialization() { + let snapshot = StateSnapshot::new( + 10000, + B256::repeat_byte(0xab), + B256::repeat_byte(0xcd), + 1234567890, + ); + + let encoded = bincode::serialize(&snapshot).unwrap(); + let decoded: StateSnapshot = bincode::deserialize(&encoded).unwrap(); + + assert_eq!(snapshot, decoded); + } +} diff --git a/vendor/malachitebft-app/.cargo-ok b/vendor/malachitebft-app/.cargo-ok new file mode 100644 index 0000000..5f8b795 --- /dev/null +++ b/vendor/malachitebft-app/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/vendor/malachitebft-app/.cargo_vcs_info.json b/vendor/malachitebft-app/.cargo_vcs_info.json new file mode 100644 index 0000000..184e173 --- /dev/null +++ b/vendor/malachitebft-app/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "8a64f82061690b70e0522b71f32152295ccb0df5" + }, + "path_in_vcs": "code/crates/app" +} \ No newline at end of file diff --git a/vendor/malachitebft-app/Cargo.toml b/vendor/malachitebft-app/Cargo.toml new file mode 100644 index 0000000..f51f09a --- /dev/null +++ b/vendor/malachitebft-app/Cargo.toml @@ -0,0 +1,124 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.85" +name = "informalsystems-malachitebft-app" +version = "0.5.0" +build = false +publish = true +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "High-level interface for building applications on top of the Malachite BFT consensus engine" +readme = "README.md" +license = "Apache-2.0" +repository = "https://github.com/informalsystems/malachite" + +[package.metadata.docs.rs] +all-features = true + +[features] +borsh = ["malachitebft-core-consensus/borsh"] + +[lib] +name = "informalsystems_malachitebft_app" +path = "src/lib.rs" + +[dependencies.async-trait] +version = "0.1.88" + +[dependencies.derive-where] +version = "1.5.0" + +[dependencies.eyre] +version = "0.6" + +[dependencies.libp2p-identity] +version = "0.2.12" + +[dependencies.malachitebft-codec] +version = "0.5.0" +package = "informalsystems-malachitebft-codec" + +[dependencies.malachitebft-config] +version = "0.5.0" +package = "informalsystems-malachitebft-config" + +[dependencies.malachitebft-core-consensus] +version = "0.5.0" +package = "informalsystems-malachitebft-core-consensus" + +[dependencies.malachitebft-core-types] +version = "0.5.0" +package = "informalsystems-malachitebft-core-types" + +[dependencies.malachitebft-engine] +version = "0.5.0" +package = "informalsystems-malachitebft-engine" + +[dependencies.malachitebft-metrics] +version = "0.5.0" +package = "informalsystems-malachitebft-metrics" + +[dependencies.malachitebft-network] +version = "0.5.0" +package = "informalsystems-malachitebft-network" + +[dependencies.malachitebft-peer] +version = "0.5.0" +default-features = false +package = "informalsystems-malachitebft-peer" + +[dependencies.malachitebft-sync] +version = "0.5.0" +package = "informalsystems-malachitebft-sync" + +[dependencies.malachitebft-wal] +version = "0.5.0" +package = "informalsystems-malachitebft-wal" + +[dependencies.ractor] +version = "0.14.6" +features = [ + "async-trait", + "tokio_runtime", +] +default-features = false + +[dependencies.rand] +version = "0.8.5" +features = [ + "std_rng", + "small_rng", +] + +[dependencies.serde] +version = "1.0" +default-features = false + +[dependencies.tokio] +version = "1.46.1" + +[dependencies.tracing] +version = "0.1.41" + +[lints.clippy] +disallowed_types = "deny" +doc_overindented_list_items = "allow" + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = ["cfg(coverage_nightly)"] diff --git a/vendor/malachitebft-app/Cargo.toml.orig b/vendor/malachitebft-app/Cargo.toml.orig new file mode 100644 index 0000000..bfbdd7a --- /dev/null +++ b/vendor/malachitebft-app/Cargo.toml.orig @@ -0,0 +1,41 @@ +[package] +name = "informalsystems-malachitebft-app" +description = "High-level interface for building applications on top of the Malachite BFT consensus engine" +version.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true +readme = "../../../README.md" + +[package.metadata.docs.rs] +all-features = true + +[features] +borsh = ["malachitebft-core-consensus/borsh"] + +[dependencies] +malachitebft-codec.workspace = true +malachitebft-config.workspace = true +malachitebft-core-consensus.workspace = true +malachitebft-core-types.workspace = true +malachitebft-engine.workspace = true +malachitebft-metrics.workspace = true +malachitebft-network.workspace = true +malachitebft-peer.workspace = true +malachitebft-sync.workspace = true +malachitebft-wal.workspace = true + +async-trait = { workspace = true } +derive-where = { workspace = true } +eyre = { workspace = true } +libp2p-identity = { workspace = true } +ractor = { workspace = true } +rand = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/vendor/malachitebft-app/README.md b/vendor/malachitebft-app/README.md new file mode 100644 index 0000000..65cb5ee --- /dev/null +++ b/vendor/malachitebft-app/README.md @@ -0,0 +1,251 @@ +

+Malachite +

+ +

+ Flexible BFT Consensus Engine in Rust
+ State-of-the-art implementation of Tendermint +

+ +--- + +
+ +[![Build Status][build-image]][build-link] +[![Quint tests][quint-image]][quint-link] +[![MBT tests][mbt-test-image]][mbt-test-link] +[![Code coverage][coverage-image]][coverage-link] + +[![Apache 2.0 Licensed][license-image]][license-link] +![Rust Stable][rustc-image] +![Rust 1.82+][rustc-version] +[![Quint 0.22][quint-version]][quint-repo] + +[![Telegram Chat][tg-badge]][tg-url] + +[![Documentation][docs-main-image]][docs-main-link] + +
+ +## About + +Malachite is a Byzantine-fault tolerant (BFT) consensus engine implemented in Rust. + +Malachite `/ˈmæl.ə.kaɪt/` is pronounced as follows: __"Mala"__ (like in Malaysia) + __"kite"__ (like the flying toy). + +> [!IMPORTANT] +> Malachite is alpha software and under heavy development. +> The software is provided "as is" and has not been externally audited; use at your own risk. + +### Goals + +The goal is for Malachite to enable developers to decentralize whatever the future may bring—sequencers, social networks, Layer 1s, etc. +Therefore, Malachite addresses a particular void in the market: The lack of flexible, reliable, and high-performance distributed systems foundations, such as BFT consensus libraries. + +### Features + +#### Tendermint as a Library +Bundled with Malachite comes a state-of-the-art implementation of the Tendermint BFT consensus algorithm. +Tendermint is an [optimistically responsive][responsive] consensus algorithm, and therefore exhibits high-performance, and has found adoption in many decentralized systems through its implementation in Go as part of [CometBFT](https://github.com/cometbft/cometbft/). + +#### Design +Key [design decisions][announcement] in Malachite are heavily inspired by lessons and experiences of maintaining CometBFT throughout the years. +Malachite addresses numerous points of technical debt in the design of consensus engines, resulting in a lean, flexible, and reliable solution that performs at the highest level. + +#### Reliability and Performance +Parts of Malachite were co-designed with their formal specification and model checking, notably for the Tendermint algorithm, which improved the confidence and reliability of this core library. + +Early [experiments][announcement] with Malachite show an average finalization latency of 780 ms at a scale of 100 validators with 1MB blocks. +Depending on the setup, Malachite can clear up to 2.5 blocks per second or finalize up to 13.5 MB/s (around 50,000 transactions per second). + +We publish regular performance benchmarks on the [dashboard][website-dashboard]. + +#### Use-cases +Malachite originated as a consensus core for the Starknet L2 decentralized sequencer. +It will serve as the core consensus library in the [Madara][madara] and [Pathfinder][pathfinder] Starknet clients. +Malachite is also being used for Farcaster’s newest backend layer called [Snapchain](https://github.com/farcasterxyz/snapchain-v0/). +Thanks to its flexible design, Malachite is amenable to a broad range of environments, and a number of other teams are building and exploring in private. +Please [reach-out if interested][tg-url], we would love to speak with more teams. + +To follow-up with use-cases and more general announcements, see the [blog][website-announcements]. + +## Overview + +### Repository + +The repository is split into three areas, each covering one of the important aspects of this project: + +1. [code](./code): Contains the Rust implementation of the Tendermint consensus algorithm, split across multiple Rust crates. +2. [docs](./docs): Contains Architectural Decision Records (ADRs) and other documentation, such as the 2018 paper describing the core consensus algorithm. +3. [specs](./specs): Contains English and [Quint][quint-repo] specifications. + +### Crates and Status + +> [!NOTE] +> The actual name of each crate is prefixed with `informalsystems-malachitebft-`. +> For instance, the crate denoted by `core-consensus` below can be found on crates.io as `informalsystems-malachitebft-core-consensus`. + +#### Core consensus algorithm + +| Crate name | Crate | Docs | +|:------------------------------------------------------|:--------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------:| +| [core-consensus](code/crates/core-consensus) | [![core-consensus][core-consensus-crate-image]][core-consensus-crate-link] | [![core-consensus Docs][core-consensus-docs-image]][core-consensus-docs-link] | +| [core-driver](code/crates/core-driver) | [![core-driver][core-driver-crate-image]][core-driver-crate-link] | [![core-driver Docs][core-driver-docs-image]][core-driver-docs-link] | +| [core-state-machine](code/crates/core-state-machine) | [![core-state-machine][core-state-machine-crate-image]][core-state-machine-crate-link] | [![core-state-machine Docs][core-state-machine-docs-image]][core-state-machine-docs-link] | +| [core-types](code/crates/core-types) | [![core-types][core-types-crate-image]][core-types-crate-link] | [![core-types Docs][core-types-docs-image]][core-types-docs-link] | +| [core-votekeeper](code/crates/core-votekeeper) | [![core-votekeeper][core-votekeeper-crate-image]][core-votekeeper-crate-link] | [![core-votekeeper Docs][core-votekeeper-docs-image]][core-votekeeper-docs-link] | + +#### Consensus engine + +| Crate name | Crate | Docs | +|-------------------------------------------:|:-----------------------------------------------------------------:|:--------------------------------------------------------------------:| +| [app-channel](./code/crates/app-channel) | [![app-channel][app-channel-crate-image]][app-channel-crate-link] | [![app-channel Docs][app-channel-docs-image]][app-channel-docs-link] | +| [app](./code/crates/app) | [![app][app-crate-image]][app-crate-link] | [![app Docs][app-docs-image]][app-docs-link] | +| [codec](./code/crates/codec) | [![codec][codec-crate-image]][codec-crate-link] | [![codec Docs][codec-docs-image]][codec-docs-link] | +| [config](./code/crates/config) | [![config][config-crate-image]][config-crate-link] | [![config Docs][config-docs-image]][config-docs-link] | +| [discovery](./code/crates/discovery) | [![discovery][discovery-crate-image]][discovery-crate-link] | [![discovery Docs][discovery-docs-image]][discovery-docs-link] | +| [engine](./code/crates/engine) | [![engine][engine-crate-image]][engine-crate-link] | [![engine Docs][engine-docs-image]][engine-docs-link] | +| [metrics](./code/crates/metrics) | [![metrics][metrics-crate-image]][metrics-crate-link] | [![metrics Docs][metrics-docs-image]][metrics-docs-link] | +| [network](./code/crates/network) | [![network][network-crate-image]][network-crate-link] | [![network Docs][network-docs-image]][network-docs-link] | +| [peer](./code/crates/peer) | [![peer][peer-crate-image]][peer-crate-link] | [![peer Docs][peer-docs-image]][peer-docs-link] | +| [proto](./code/crates/proto) | [![proto][proto-crate-image]][proto-crate-link] | [![proto Docs][proto-docs-image]][proto-docs-link] | +| [sync](./code/crates/sync) | [![sync][sync-crate-image]][sync-crate-link] | [![sync Docs][sync-docs-image]][sync-docs-link] | +| [wal](./code/crates/wal) | [![wal][wal-crate-image]][wal-crate-link] | [![wal Docs][wal-docs-image]][wal-docs-link] | + + +### Building with Malachite + +As a guiding point to understand how to use Malachite, please read [ARCHITECTURE.md](ARCHITECTURE.md). + +You can also check out the [examples](./code/examples) for a more in-depth experience. + +### Contributing + +If you would like to contribute to the Malachite open-source codebase, please see [CONTRIBUTING.md](./CONTRIBUTING.md). +We invite all contributors. + +## Requirements + +- Rust v1.82+ ([rustup.rs](https://rustup.rs)) +- Quint v0.22+ ([github.com](https://github.com/informalsystems/quint)) + +## Join Us + +Malachite is developed by [Informal Systems](https://informal.systems). + +If you'd like to work full-time on challenging problems of distributed systems and decentralization, +[we're always looking for talented people to join](https://informal.systems/careers)! + + +## Acknowledgements + +Malachite would not have been possible without the kind support of the Starknet ecosystem. +We are grateful to StarkWare Industries for prompting the initial discussions of building Tendermint in Rust, to Starknet Foundation for funding and fostering a collaborative environment, and to both of these organizations plus numerous others in the ecosystem for their constructive feedback on earlier designs of Malachite. + +We are also thankful for the collaboration with Farcaster. +This led to further refinements and maturing of the Malachite codebase, and their approach to building complex systems and shipping valuable products is an inspiration for us. + +## License + +Copyright © 2024 Informal Systems Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use the files in this repository except in compliance with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +[docs-main-image]: https://img.shields.io/badge/docs-main-blue?logo=googledocs&logoColor=white +[docs-main-link]: https://informalsystems.github.io/malachite/ +[build-image]: https://github.com/informalsystems/malachite/actions/workflows/rust.yml/badge.svg +[build-link]: https://github.com/informalsystems/malachite/actions/workflows/rust.yml +[quint-image]: https://github.com/informalsystems/malachite/actions/workflows/quint.yml/badge.svg +[quint-link]: https://github.com/informalsystems/malachite/actions/workflows/quint.yml +[mbt-test-image]: https://github.com/informalsystems/malachite/actions/workflows/mbt.yml/badge.svg +[mbt-test-link]: https://github.com/informalsystems/malachite/actions/workflows/mbt.yml +[coverage-image]: https://codecov.io/gh/informalsystems/malachite/graph/badge.svg?token=B9KY7B6DJF +[coverage-link]: https://codecov.io/gh/informalsystems/malachite +[license-image]: https://img.shields.io/badge/license-Apache_2.0-blue.svg +[license-link]: https://github.com/informalsystems/hermes/blob/master/LICENSE +[rustc-image]: https://img.shields.io/badge/Rust-stable-orange.svg +[rustc-version]: https://img.shields.io/badge/Rust-1.82+-orange.svg +[quint-version]: https://img.shields.io/badge/Quint-0.22-purple.svg +[quint-repo]: https://github.com/informalsystems/quint +[tg-url]: https://t.me/MalachiteEngine +[tg-badge]: https://img.shields.io/badge/Malachite-Engine-blue.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTEyIDI0YzYuNjI3IDAgMTItNS4zNzMgMTItMTJTMTguNjI3IDAgMTIgMCAwIDUuMzczIDAgMTJzNS4zNzMgMTIgMTIgMTJaIiBmaWxsPSJ1cmwoI2EpIi8+PHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik01LjQyNSAxMS44NzFhNzk2LjQxNCA3OTYuNDE0IDAgMCAxIDYuOTk0LTMuMDE4YzMuMzI4LTEuMzg4IDQuMDI3LTEuNjI4IDQuNDc3LTEuNjM4LjEgMCAuMzIuMDIuNDcuMTQuMTIuMS4xNS4yMy4xNy4zMy4wMi4xLjA0LjMxLjAyLjQ3LS4xOCAxLjg5OC0uOTYgNi41MDQtMS4zNiA4LjYyMi0uMTcuOS0uNSAxLjE5OS0uODE5IDEuMjI5LS43LjA2LTEuMjI5LS40Ni0xLjg5OC0uOS0xLjA2LS42ODktMS42NDktMS4xMTktMi42NzgtMS43OTgtMS4xOS0uNzgtLjQyLTEuMjA5LjI2LTEuOTA4LjE4LS4xOCAzLjI0Ny0yLjk3OCAzLjMwNy0zLjIyOC4wMS0uMDMuMDEtLjE1LS4wNi0uMjEtLjA3LS4wNi0uMTctLjA0LS4yNS0uMDItLjExLjAyLTEuNzg4IDEuMTQtNS4wNTYgMy4zNDgtLjQ4LjMzLS45MDkuNDktMS4yOTkuNDgtLjQzLS4wMS0xLjI0OC0uMjQtMS44NjgtLjQ0LS43NS0uMjQtMS4zNDktLjM3LTEuMjk5LS43OS4wMy0uMjIuMzMtLjQ0Ljg5LS42NjlaIiBmaWxsPSIjZmZmIi8+PGRlZnM+PGxpbmVhckdyYWRpZW50IGlkPSJhIiB4MT0iMTEuOTkiIHkxPSIwIiB4Mj0iMTEuOTkiIHkyPSIyMy44MSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPjxzdG9wIHN0b3AtY29sb3I9IiMyQUFCRUUiLz48c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiMyMjlFRDkiLz48L2xpbmVhckdyYWRpZW50PjwvZGVmcz48L3N2Zz4K +[responsive]: https://informal.systems/blog/tendermint-responsiveness +[announcement]: https://informal.systems/blog/malachite-decentralize-whatever +[madara]: https://github.com/madara-alliance/madara +[pathfinder]: https://github.com/eqlabs/pathfinder +[website-dashboard]: https://informal.systems/malachite/dashboard +[website-announcements]: https://informal.systems/malachite/blog + +[//]: # (crates.io and docs.rs: links and badges) +[core-consensus-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-core-consensus +[core-consensus-crate-link]: https://crates.io/crates/informalsystems-malachitebft-core-consensus +[core-driver-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-core-driver +[core-driver-crate-link]: https://crates.io/crates/informalsystems-malachitebft-core-driver +[core-state-machine-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-core-state-machine +[core-state-machine-crate-link]: https://crates.io/crates/informalsystems-malachitebft-core-state-machine +[core-types-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-core-types +[core-types-crate-link]: https://crates.io/crates/informalsystems-malachitebft-core-types +[core-votekeeper-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-core-votekeeper +[core-votekeeper-crate-link]: https://crates.io/crates/informalsystems-malachitebft-core-votekeeper +[core-consensus-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-core-consensus +[core-consensus-docs-link]: https://docs.rs/informalsystems-malachitebft-core-consensus +[core-driver-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-core-driver +[core-driver-docs-link]: https://docs.rs/informalsystems-malachitebft-core-driver +[core-state-machine-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-core-state-machine +[core-state-machine-docs-link]: https://docs.rs/informalsystems-malachitebft-core-state-machine +[core-types-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-core-types +[core-types-docs-link]: https://docs.rs/informalsystems-malachitebft-core-types +[core-votekeeper-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-core-votekeeper +[core-votekeeper-docs-link]: https://docs.rs/informalsystems-malachitebft-core-votekeeper +[app-channel-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-app-channel +[app-channel-crate-link]: https://crates.io/crates/informalsystems-malachitebft-app-channel +[app-channel-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-app-channel +[app-channel-docs-link]: https://docs.rs/informalsystems-malachitebft-app-channel +[app-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-app +[app-crate-link]: https://crates.io/crates/informalsystems-malachitebft-app +[codec-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-codec +[codec-crate-link]: https://crates.io/crates/informalsystems-malachitebft-codec +[config-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-config +[config-crate-link]: https://crates.io/crates/informalsystems-malachitebft-config +[discovery-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-discovery +[discovery-crate-link]: https://crates.io/crates/informalsystems-malachitebft-discovery +[engine-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-engine +[engine-crate-link]: https://crates.io/crates/informalsystems-malachitebft-engine +[metrics-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-metrics +[metrics-crate-link]: https://crates.io/crates/informalsystems-malachitebft-metrics +[network-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-network +[network-crate-link]: https://crates.io/crates/informalsystems-malachitebft-network +[peer-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-peer +[peer-crate-link]: https://crates.io/crates/informalsystems-malachitebft-peer +[proto-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-proto +[proto-crate-link]: https://crates.io/crates/informalsystems-malachitebft-proto +[sync-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-sync +[sync-crate-link]: https://crates.io/crates/informalsystems-malachitebft-sync +[wal-crate-image]: https://img.shields.io/crates/v/informalsystems-malachitebft-wal +[wal-crate-link]: https://crates.io/crates/informalsystems-malachitebft-wal +[app-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-app +[app-docs-link]: https://docs.rs/informalsystems-malachitebft-app +[codec-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-codec +[codec-docs-link]: https://docs.rs/informalsystems-malachitebft-codec +[config-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-config +[config-docs-link]: https://docs.rs/informalsystems-malachitebft-config +[discovery-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-discovery +[discovery-docs-link]: https://docs.rs/informalsystems-malachitebft-discovery +[engine-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-engine +[engine-docs-link]: https://docs.rs/informalsystems-malachitebft-engine +[metrics-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-metrics +[metrics-docs-link]: https://docs.rs/informalsystems-malachitebft-metrics +[network-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-network +[network-docs-link]: https://docs.rs/informalsystems-malachitebft-network +[peer-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-peer +[peer-docs-link]: https://docs.rs/informalsystems-malachitebft-peer +[proto-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-proto +[proto-docs-link]: https://docs.rs/informalsystems-malachitebft-proto +[sync-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-sync +[sync-docs-link]: https://docs.rs/informalsystems-malachitebft-sync +[wal-docs-image]: https://img.shields.io/docsrs/informalsystems-malachitebft-wal +[wal-docs-link]: https://docs.rs/informalsystems-malachitebft-wal diff --git a/vendor/malachitebft-app/src/lib.rs b/vendor/malachitebft-app/src/lib.rs new file mode 100644 index 0000000..c0b5c59 --- /dev/null +++ b/vendor/malachitebft-app/src/lib.rs @@ -0,0 +1,25 @@ +// TODO: Enforce proper documentation +// #![warn( +// missing_docs, +// clippy::empty_docs, +// clippy::missing_errors_doc, +// rustdoc::broken_intra_doc_links, +// rustdoc::missing_crate_level_docs, +// rustdoc::missing_doc_code_examples +// )] + +pub mod node; +pub mod part_store; +pub mod spawn; +pub mod types; + +pub mod events { + pub use malachitebft_engine::util::events::{RxEvent, TxEvent}; +} + +pub use malachitebft_config as config; +pub use malachitebft_core_consensus as consensus; +pub use malachitebft_engine as engine; +pub use malachitebft_engine::util::streaming; +pub use malachitebft_metrics as metrics; +pub use malachitebft_wal as wal; diff --git a/vendor/malachitebft-app/src/node.rs b/vendor/malachitebft-app/src/node.rs new file mode 100644 index 0000000..4b67d2b --- /dev/null +++ b/vendor/malachitebft-app/src/node.rs @@ -0,0 +1,113 @@ +#![allow(clippy::too_many_arguments)] + +use std::path::PathBuf; + +use async_trait::async_trait; +use rand::{CryptoRng, RngCore}; +use serde::de::DeserializeOwned; +use serde::Serialize; +use tokio::task::JoinHandle; + +use malachitebft_config::{ + ConsensusConfig, DiscoveryConfig, RuntimeConfig, TransportProtocol, ValueSyncConfig, +}; +use malachitebft_core_types::SigningProvider; +use malachitebft_engine::node::NodeRef; +use malachitebft_engine::util::events::RxEvent; + +use crate::types::core::{Context, PrivateKey, PublicKey, VotingPower}; +use crate::types::Keypair; + +pub struct EngineHandle { + pub actor: NodeRef, + pub handle: JoinHandle<()>, +} + +#[async_trait] +pub trait NodeHandle +where + Self: Send + Sync + 'static, + Ctx: Context, +{ + fn subscribe(&self) -> RxEvent; + async fn kill(&self, reason: Option) -> eyre::Result<()>; +} + +pub trait NodeConfig { + fn moniker(&self) -> &str; + fn consensus(&self) -> &ConsensusConfig; + fn value_sync(&self) -> &ValueSyncConfig; +} + +#[async_trait] +pub trait Node { + type Context: Context; + type Config: NodeConfig + Serialize + DeserializeOwned; + type Genesis: Serialize + DeserializeOwned; + type PrivateKeyFile: Serialize + DeserializeOwned; + type SigningProvider: SigningProvider; + type NodeHandle: NodeHandle; + + async fn start(&self) -> eyre::Result; + + async fn run(self) -> eyre::Result<()>; + + fn get_home_dir(&self) -> PathBuf; + + fn load_config(&self) -> eyre::Result; + + fn get_address(&self, pk: &PublicKey) -> ::Address; + + fn get_public_key(&self, pk: &PrivateKey) -> PublicKey; + + fn get_keypair(&self, pk: PrivateKey) -> Keypair; + + fn load_private_key(&self, file: Self::PrivateKeyFile) -> PrivateKey; + + fn load_private_key_file(&self) -> eyre::Result; + + fn load_genesis(&self) -> eyre::Result; + + fn get_signing_provider(&self, private_key: PrivateKey) + -> Self::SigningProvider; +} + +#[derive(Copy, Clone, Debug)] +pub struct MakeConfigSettings { + pub runtime: RuntimeConfig, + pub transport: TransportProtocol, + pub discovery: DiscoveryConfig, + pub value_sync: ValueSyncConfig, +} + +pub trait CanMakeConfig: Node { + fn make_config(index: usize, total: usize, settings: MakeConfigSettings) -> Self::Config; +} + +pub trait CanMakeDistributedConfig: Node { + fn make_distributed_config( + index: usize, + total: usize, + machines: Vec, + bootstrap_set_size: usize, + settings: MakeConfigSettings, + ) -> Self::Config; +} + +pub trait CanGeneratePrivateKey: Node { + fn generate_private_key(&self, rng: R) -> PrivateKey + where + R: RngCore + CryptoRng; +} + +pub trait CanMakePrivateKeyFile: Node { + fn make_private_key_file(&self, private_key: PrivateKey) + -> Self::PrivateKeyFile; +} + +pub trait CanMakeGenesis: Node { + fn make_genesis( + &self, + validators: Vec<(PublicKey, VotingPower)>, + ) -> Self::Genesis; +} diff --git a/vendor/malachitebft-app/src/part_store.rs b/vendor/malachitebft-app/src/part_store.rs new file mode 100644 index 0000000..e194f41 --- /dev/null +++ b/vendor/malachitebft-app/src/part_store.rs @@ -0,0 +1,111 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use derive_where::derive_where; + +use malachitebft_core_types::{Context, Round, ValueId}; +use malachitebft_engine::util::streaming::StreamId; + +// This is a temporary store implementation for proposal parts +// +// TODO: Add Address to key +// NOTE: Not sure if this is required as consensus should verify that only the parts signed by the proposer for +// the height and round should be forwarded here (see the TODOs in consensus) + +type Key = (StreamId, Height, Round); + +/// Stores proposal parts for a given stream, height, and round. +/// `value_id` is the value id of the proposal as computed by the proposer. It is also included in one of the parts but stored here for convenience. +/// `parts` is a list of `ProposalPart`s, ordered by the sequence of the `StreamMessage` that delivered them. +#[derive_where(Clone, Debug, Default)] +pub struct Entry { + pub value_id: Option>, + pub parts: Vec::ProposalPart>>, +} +type Store = BTreeMap::Height>, Entry>; + +#[derive_where(Clone, Debug)] +pub struct PartStore { + store: Store, +} + +impl Default for PartStore { + fn default() -> Self { + Self::new() + } +} + +impl PartStore { + pub fn new() -> Self { + Self { + store: Default::default(), + } + } + + /// Return all the parts for the given `stream_id`, `height` and `round`. + /// Parts are already sorted by sequence in ascending order. + pub fn all_parts_by_stream_id( + &self, + stream_id: StreamId, + height: Ctx::Height, + round: Round, + ) -> Vec> { + self.store + .get(&(stream_id, height, round)) + .map(|entry| entry.parts.clone()) + .unwrap_or_default() + } + + /// Return all the parts for the given `value_id`. If multiple entries with same `value_id` are present, the parts of the first one are returned. + /// Parts are already sorted by sequence in ascending order. + pub fn all_parts_by_value_id(&self, value_id: &ValueId) -> Vec> { + self.store + .values() + .find(|entry| entry.value_id.as_ref() == Some(value_id)) + .map(|entry| entry.parts.clone()) + .unwrap_or_default() + } + + /// Store a part for the given `stream_id`, `height` and `round`. + /// The part is added to the end of the list of parts and is for the next sequence number after the last part. + pub fn store( + &mut self, + stream_id: &StreamId, + height: Ctx::Height, + round: Round, + proposal_part: Ctx::ProposalPart, + ) { + let existing = self + .store + .entry((stream_id.clone(), height, round)) + .or_default(); + existing.parts.push(Arc::new(proposal_part)); + } + + /// Store the `value_id` of the proposal, as computed by the proposer, for the given `stream_id`, `height` and `round`. + pub fn store_value_id( + &mut self, + stream_id: &StreamId, + height: Ctx::Height, + round: Round, + value_id: ValueId, + ) { + let existing = self + .store + .entry((stream_id.clone(), height, round)) + .or_default(); + existing.value_id = Some(value_id); + } + + /// Prune the parts for all heights lower than `min_height`. + /// This is used to prune the parts from the store when a min_height has been finalized. + /// Parts for higher heights may be present if the node is lagging and are kept. + pub fn prune(&mut self, min_height: Ctx::Height) { + self.store.retain(|(_, height, _), _| *height >= min_height); + } + + /// Return the number of blocks in the store. + pub fn blocks_count(&self) -> usize { + self.store.len() + } +} diff --git a/vendor/malachitebft-app/src/spawn.rs b/vendor/malachitebft-app/src/spawn.rs new file mode 100644 index 0000000..9f1def4 --- /dev/null +++ b/vendor/malachitebft-app/src/spawn.rs @@ -0,0 +1,231 @@ +//! Utility functions for spawning the actor system and connecting it to the application. + +use std::path::Path; +use std::time::Duration; + +use eyre::Result; +use tokio::task::JoinHandle; +use tracing::Span; + +use malachitebft_engine::consensus::{Consensus, ConsensusCodec, ConsensusParams, ConsensusRef}; +use malachitebft_engine::host::HostRef; +use malachitebft_engine::network::{Network, NetworkRef}; +use malachitebft_engine::node::{Node, NodeRef}; +use malachitebft_engine::sync::{Params as SyncParams, Sync, SyncCodec, SyncRef}; +use malachitebft_engine::util::events::TxEvent; +use malachitebft_engine::wal::{Wal, WalCodec, WalRef}; +use malachitebft_network::{Config as NetworkConfig, DiscoveryConfig, GossipSubConfig, Keypair}; +use malachitebft_sync as sync; + +use crate::config::{ConsensusConfig, PubSubProtocol, ValueSyncConfig}; +use crate::metrics::{Metrics, SharedRegistry}; +use crate::types::core::{Context, SigningProvider}; +use crate::types::ValuePayload; + +pub async fn spawn_node_actor( + ctx: Ctx, + network: NetworkRef, + consensus: ConsensusRef, + wal: WalRef, + sync: Option>, + host: HostRef, +) -> Result<(NodeRef, JoinHandle<()>)> +where + Ctx: Context, +{ + // Spawn the node actor + let node = Node::new( + ctx, + network, + consensus, + wal, + sync, + host, + tracing::Span::current(), + ); + + let (actor_ref, handle) = node.spawn().await?; + Ok((actor_ref, handle)) +} + +pub async fn spawn_network_actor( + cfg: &ConsensusConfig, + keypair: Keypair, + registry: &SharedRegistry, + codec: Codec, +) -> Result> +where + Ctx: Context, + Codec: ConsensusCodec, + Codec: SyncCodec, +{ + let config = make_gossip_config(cfg); + + Network::spawn(keypair, config, registry.clone(), codec, Span::current()) + .await + .map_err(Into::into) +} + +#[allow(clippy::too_many_arguments)] +pub async fn spawn_consensus_actor( + initial_height: Ctx::Height, + initial_validator_set: Ctx::ValidatorSet, + address: Ctx::Address, + ctx: Ctx, + mut cfg: ConsensusConfig, + sync_cfg: &ValueSyncConfig, + signing_provider: Box>, + network: NetworkRef, + host: HostRef, + wal: WalRef, + sync: Option>, + metrics: Metrics, + tx_event: TxEvent, +) -> Result> +where + Ctx: Context, +{ + use crate::config; + + let value_payload = match cfg.value_payload { + config::ValuePayload::PartsOnly => ValuePayload::PartsOnly, + config::ValuePayload::ProposalOnly => ValuePayload::ProposalOnly, + config::ValuePayload::ProposalAndParts => ValuePayload::ProposalAndParts, + }; + + let consensus_params = ConsensusParams { + initial_height, + initial_validator_set, + address, + threshold_params: Default::default(), + value_payload, + }; + + // Derive the consensus queue capacity from `sync.parallel_requests` + cfg.queue_capacity = sync_cfg.parallel_requests; + + Consensus::spawn( + ctx, + consensus_params, + cfg, + signing_provider, + network, + host, + wal, + sync, + metrics, + tx_event, + Span::current(), + ) + .await + .map_err(Into::into) +} + +pub async fn spawn_wal_actor( + ctx: &Ctx, + codec: Codec, + home_dir: &Path, + registry: &SharedRegistry, +) -> Result> +where + Ctx: Context, + Codec: WalCodec, +{ + let wal_dir = home_dir.join("wal"); + std::fs::create_dir_all(&wal_dir).unwrap(); + + let wal_file = wal_dir.join("consensus.wal"); + + Wal::spawn(ctx, codec, wal_file, registry.clone(), Span::current()) + .await + .map_err(Into::into) +} + +pub async fn spawn_sync_actor( + ctx: Ctx, + network: NetworkRef, + host: HostRef, + config: &ValueSyncConfig, + registry: &SharedRegistry, +) -> Result>> +where + Ctx: Context, +{ + if !config.enabled { + return Ok(None); + } + + let params = SyncParams { + status_update_interval: config.status_update_interval, + request_timeout: config.request_timeout, + }; + + let scoring_strategy = match config.scoring_strategy { + malachitebft_config::ScoringStrategy::Ema => sync::scoring::Strategy::Ema, + }; + + let sync_config = sync::Config { + enabled: config.enabled, + max_request_size: config.max_request_size.as_u64() as usize, + max_response_size: config.max_response_size.as_u64() as usize, + request_timeout: config.request_timeout, + parallel_requests: config.parallel_requests as u64, + scoring_strategy, + inactive_threshold: (!config.inactive_threshold.is_zero()) + .then_some(config.inactive_threshold), + // Tip-first sync: start syncing from near network tip for faster startup + // These fields use defaults as ValueSyncConfig doesn't expose them yet + tip_first_sync: false, + tip_first_buffer: 100, + }; + + let metrics = sync::Metrics::register(registry); + + let actor_ref = Sync::spawn( + ctx, + network, + host, + params, + sync_config, + metrics, + Span::current(), + ) + .await?; + + Ok(Some(actor_ref)) +} + +fn make_gossip_config(cfg: &ConsensusConfig) -> NetworkConfig { + NetworkConfig { + listen_addr: cfg.p2p.listen_addr.clone(), + persistent_peers: cfg.p2p.persistent_peers.clone(), + discovery: DiscoveryConfig { + enabled: cfg.p2p.discovery.enabled, + ..Default::default() + }, + idle_connection_timeout: Duration::from_secs(15 * 60), + transport: malachitebft_network::TransportProtocol::from_multiaddr(&cfg.p2p.listen_addr) + .unwrap_or_else(|| { + panic!( + "No valid transport protocol found in listen address: {}", + cfg.p2p.listen_addr + ) + }), + pubsub_protocol: match cfg.p2p.protocol { + PubSubProtocol::GossipSub(_) => malachitebft_network::PubSubProtocol::GossipSub, + PubSubProtocol::Broadcast => malachitebft_network::PubSubProtocol::Broadcast, + }, + gossipsub: match cfg.p2p.protocol { + PubSubProtocol::GossipSub(config) => GossipSubConfig { + mesh_n: config.mesh_n(), + mesh_n_high: config.mesh_n_high(), + mesh_n_low: config.mesh_n_low(), + mesh_outbound_min: config.mesh_outbound_min(), + }, + PubSubProtocol::Broadcast => GossipSubConfig::default(), + }, + rpc_max_size: cfg.p2p.rpc_max_size.as_u64() as usize, + pubsub_max_size: cfg.p2p.pubsub_max_size.as_u64() as usize, + enable_sync: true, + } +} diff --git a/vendor/malachitebft-app/src/types.rs b/vendor/malachitebft-app/src/types.rs new file mode 100644 index 0000000..4fff74c --- /dev/null +++ b/vendor/malachitebft-app/src/types.rs @@ -0,0 +1,28 @@ +//! Re-export of all types required to build a Malachite application. + +pub use libp2p_identity::Keypair; + +pub use malachitebft_core_consensus::{ + ConsensusMsg, ProposedValue, SignedConsensusMsg, ValuePayload, +}; +pub use malachitebft_engine::host::LocallyProposedValue; +pub use malachitebft_peer::PeerId; + +pub mod core { + pub use malachitebft_core_types::*; +} + +pub mod streaming { + pub use malachitebft_engine::util::streaming::{Sequence, StreamId, StreamMessage}; +} + +pub mod sync { + pub use malachitebft_sync::{Metrics, RawDecidedValue, Request, Response, Status}; +} + +pub mod codec { + pub use malachitebft_codec::Codec; + pub use malachitebft_engine::consensus::ConsensusCodec; + pub use malachitebft_engine::sync::SyncCodec; + pub use malachitebft_engine::wal::WalCodec; +} diff --git a/vendor/malachitebft-sync/src/config.rs b/vendor/malachitebft-sync/src/config.rs index 9f3aaa1..57fc2f0 100644 --- a/vendor/malachitebft-sync/src/config.rs +++ b/vendor/malachitebft-sync/src/config.rs @@ -3,6 +3,7 @@ use std::time::Duration; use crate::scoring::Strategy; const DEFAULT_PARALLEL_REQUESTS: u64 = 5; +const DEFAULT_TIP_FIRST_BUFFER: u64 = 100; #[derive(Copy, Clone, Debug)] pub struct Config { @@ -13,6 +14,12 @@ pub struct Config { pub parallel_requests: u64, pub scoring_strategy: Strategy, pub inactive_threshold: Option, + /// When enabled, sync starts from near the network tip instead of genesis. + /// This allows nodes to quickly join consensus without syncing full history. + pub tip_first_sync: bool, + /// Number of blocks before tip to start syncing from when tip_first_sync is enabled. + /// Default: 100 blocks. This ensures enough recent history for consensus participation. + pub tip_first_buffer: u64, } impl Config { @@ -52,6 +59,20 @@ impl Config { self.inactive_threshold = inactive_threshold; self } + + /// Enable tip-first sync mode where sync starts from near the network tip + /// instead of genesis. This allows faster consensus participation. + pub fn with_tip_first_sync(mut self, enabled: bool) -> Self { + self.tip_first_sync = enabled; + self + } + + /// Set the number of blocks before tip to start syncing from. + /// Only used when tip_first_sync is enabled. + pub fn with_tip_first_buffer(mut self, buffer: u64) -> Self { + self.tip_first_buffer = buffer; + self + } } impl Default for Config { @@ -64,6 +85,8 @@ impl Default for Config { parallel_requests: DEFAULT_PARALLEL_REQUESTS, scoring_strategy: Strategy::default(), inactive_threshold: None, + tip_first_sync: false, // Disabled by default for safety + tip_first_buffer: DEFAULT_TIP_FIRST_BUFFER, } } } diff --git a/vendor/malachitebft-sync/src/handle.rs b/vendor/malachitebft-sync/src/handle.rs index 45446a2..04ebae5 100644 --- a/vendor/malachitebft-sync/src/handle.rs +++ b/vendor/malachitebft-sync/src/handle.rs @@ -95,7 +95,7 @@ where pub async fn on_tick( co: Co, state: &mut State, - _metrics: &Metrics, + metrics: &Metrics, ) -> Result<(), Error> where Ctx: Context, @@ -114,6 +114,38 @@ where .reset_inactive_peers_scores(inactive_threshold); } + // Clear stale pending requests that have timed out + let cleared_count = state.clear_stale_pending_requests(); + if cleared_count > 0 { + warn!( + cleared_count, + height.sync = %state.sync_height, + pending_requests = state.pending_value_requests.len(), + "SYNC STALE CLEANUP: Cleared timed-out pending requests" + ); + + // After clearing stale requests, check if we need to skip ahead + // because peers can't serve the heights we're requesting + if let Some((skip_to, skip_peer)) = state.find_earliest_syncable_height(state.sync_height) { + if skip_to > state.sync_height { + warn!( + height.current = %state.sync_height, + height.skip_to = %skip_to, + peer = %skip_peer, + "SYNC SKIP AFTER CLEANUP: Peers cannot serve old heights, jumping to earliest available" + ); + + state.sync_height = skip_to; + state.tip_height = skip_to.decrement().unwrap_or_default(); + state.pending_value_requests.clear(); + state.height_per_request_id.clear(); + + // Request from the new height + request_value_from_peer(&co, state, metrics, skip_to, skip_peer).await?; + } + } + } + debug!("Peer scores: {:#?}", state.peer_scorer.get_scores()); Ok(()) @@ -147,6 +179,32 @@ where "SYNC REQUIRED: Falling behind" ); + // Check if tip-first sync should be triggered now that we have peer info + // Only trigger if we're significantly behind and haven't already jumped + if state.config.tip_first_sync { + if let Some((skip_to, skip_peer)) = + state.calculate_tip_first_start(state.sync_height, state.config.tip_first_buffer) + { + warn!( + height.current = %state.sync_height, + height.skip_to = %skip_to, + height.network_tip = ?state.get_network_tip(), + buffer = state.config.tip_first_buffer, + peer = %skip_peer, + "TIP-FIRST SYNC: Skipping to near network tip for fast consensus participation" + ); + + state.sync_height = skip_to; + state.tip_height = skip_to.decrement().unwrap_or_default(); + state.pending_value_requests.clear(); + state.height_per_request_id.clear(); + + // Request from the new height + request_value_from_peer(&co, state, metrics, skip_to, skip_peer).await?; + return Ok(()); + } + } + // We are lagging behind one of our peer at least, // request sync from any peer already at or above that peer's height. request_values(co, state, metrics).await?; @@ -170,9 +228,35 @@ where debug!(height.tip = %tip_height, height.sync = %height, %restart, "Starting new height"); state.started = true; - state.sync_height = height; state.tip_height = tip_height; + // Check if tip-first sync is enabled and applicable + if state.config.tip_first_sync && !state.peers.is_empty() { + if let Some((skip_to, peer)) = + state.calculate_tip_first_start(height, state.config.tip_first_buffer) + { + warn!( + height.local = %height, + height.skip_to = %skip_to, + height.network_tip = ?state.get_network_tip(), + buffer = state.config.tip_first_buffer, + peer = %peer, + "TIP-FIRST SYNC: Skipping to near network tip for fast consensus participation" + ); + + state.sync_height = skip_to; + state.pending_value_requests.clear(); + state.height_per_request_id.clear(); + + // Request the first block from the skip point + request_value_from_peer(&co, state, metrics, skip_to, peer).await?; + return Ok(()); + } + } + + // Normal sync: start from the given height + state.sync_height = height; + let height_to_remove = if restart { &height } else { &tip_height }; state.remove_pending_request_by_height(height_to_remove); @@ -190,9 +274,11 @@ pub async fn on_decided( where Ctx: Context, { - debug!(height.tip = %height, "Updating request state"); + debug!(height.tip = %height, "Decided value, removing pending request"); - state.validate_response(height); + // Remove the pending request immediately - it's been validated by consensus + // This frees the slot for new parallel sync requests + state.remove_pending_request_by_height(&height); Ok(()) } diff --git a/vendor/malachitebft-sync/src/state.rs b/vendor/malachitebft-sync/src/state.rs index da353ca..a0d1e90 100644 --- a/vendor/malachitebft-sync/src/state.rs +++ b/vendor/malachitebft-sync/src/state.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::time::Instant; use malachitebft_core_types::{Context, Height}; use malachitebft_peer::PeerId; @@ -39,7 +40,8 @@ where pub sync_height: Ctx::Height, /// Decided value requests for these heights have been sent out to peers. - pub pending_value_requests: BTreeMap, + /// Tuple contains: (request_id, state, timestamp_when_sent) + pub pending_value_requests: BTreeMap, /// Maps request ID to height for pending decided value requests. pub height_per_request_id: BTreeMap, @@ -132,15 +134,17 @@ where self.height_per_request_id .insert(request_id.clone(), height); - self.pending_value_requests - .insert(height, (request_id, RequestState::WaitingResponse)); + self.pending_value_requests.insert( + height, + (request_id, RequestState::WaitingResponse, Instant::now()), + ); } /// Mark that a response has been received for a height. /// /// State transition: WaitingResponse -> WaitingValidation pub fn response_received(&mut self, request_id: OutboundRequestId, height: Ctx::Height) { - if let Some((req_id, state)) = self.pending_value_requests.get_mut(&height) { + if let Some((req_id, state, _)) = self.pending_value_requests.get_mut(&height) { if req_id != &request_id { return; // A new request has been made in the meantime, ignore this response. } @@ -155,7 +159,7 @@ where /// State transition: WaitingValidation -> Validated /// It is also possible to have the following transition: WaitingResponse -> Validated. pub fn validate_response(&mut self, height: Ctx::Height) { - if let Some((_, state)) = self.pending_value_requests.get_mut(&height) { + if let Some((_, state, _)) = self.pending_value_requests.get_mut(&height) { *state = RequestState::Validated; } } @@ -167,7 +171,7 @@ where /// Remove the pending decided value request for a given height. pub fn remove_pending_request_by_height(&mut self, height: &Ctx::Height) { - if let Some((request_id, _)) = self.pending_value_requests.remove(height) { + if let Some((request_id, _, _)) = self.pending_value_requests.remove(height) { self.height_per_request_id.remove(&request_id); } } @@ -191,7 +195,7 @@ where /// Check if a pending decided value request for a given height is in the `Validated` state. pub fn is_pending_value_request_validated_by_height(&self, height: &Ctx::Height) -> bool { - if let Some((_, state)) = self.pending_value_requests.get(height) { + if let Some((_, state, _)) = self.pending_value_requests.get(height) { *state == RequestState::Validated } else { false @@ -207,6 +211,35 @@ where } } + /// Clear stale pending requests that have been waiting longer than the configured timeout, + /// as well as any validated requests that should have been cleaned up. + /// Returns the number of cleared requests. + pub fn clear_stale_pending_requests(&mut self) -> usize { + let timeout = self.config.request_timeout; + let now = Instant::now(); + + // Find heights with: + // 1. Validated requests (should have been removed by on_decided, safety net) + // 2. Timed-out pending requests (no response received in time) + let stale_heights: Vec = self + .pending_value_requests + .iter() + .filter(|(_, (_, state, sent_at))| { + *state == RequestState::Validated || now.duration_since(*sent_at) > timeout + }) + .map(|(height, _)| *height) + .collect(); + + let count = stale_heights.len(); + + // Remove stale requests + for height in stale_heights { + self.remove_pending_request_by_height(&height); + } + + count + } + /// Find the earliest height that any peer can serve, above the given height. /// Returns the height and a peer that can serve it. /// This is used when the requested height is not available from any peer. @@ -259,4 +292,41 @@ where None } + + /// Get the maximum tip height reported by any peer (network tip). + /// Returns None if no peers are connected. + pub fn get_network_tip(&self) -> Option { + self.peers.values().map(|status| status.tip_height).max() + } + + /// Calculate the starting height for tip-first sync. + /// Returns (start_height, peer) if tip-first sync is applicable. + /// Returns None if: + /// - No peers are connected + /// - Local height is already close to network tip + /// - No peer can serve the calculated start height + pub fn calculate_tip_first_start( + &mut self, + local_height: Ctx::Height, + buffer: u64, + ) -> Option<(Ctx::Height, PeerId)> { + // Get the network tip + let network_tip = self.get_network_tip()?; + + // Calculate the start height (tip - buffer) + // Use decrement_by to safely subtract, returns None if would underflow + let start_height = network_tip + .decrement_by(buffer) + .unwrap_or(Ctx::Height::ZERO); + + // If start_height is not significantly ahead of local_height, no benefit + // Use a threshold of buffer/2 to avoid unnecessary jumps + if start_height.as_u64() <= local_height.as_u64() + buffer / 2 { + return None; + } + + // Find a peer that can serve this height + self.random_peer_with_tip_at_or_above(start_height) + .map(|peer| (start_height, peer)) + } }