Merge branch 'main' into elmattic/api-run-subcommand

Author: elmattic

Cargo.lock

@@ -14,8 +14,10 @@ name = "forest"
 [workspace.dependencies]
 anyhow = "1"
 cid = { version = "0.11", default-features = false, features = ["std"] }
+dhat = "0.3"
 flume = "0.11"
 futures = "0.3"
+get-size2 = { version = "0.6", features = ["derive"] }
 libp2p = { version = "0.56", default-features = false }
 libp2p-swarm-test = { version = "0.6", default-features = false, features = ["tokio"] }
 multihash-codetable = { version = "0.1", features = ["blake2b", "blake2s", "blake3", "sha2", "sha3", "strobe"] }
@@ -92,7 +94,7 @@ fvm_ipld_encoding = "0.5.3"
 fvm_shared2 = { package = "fvm_shared", version = "~2.11" }
 fvm_shared3 = { package = "fvm_shared", version = "~3.13", features = ["proofs"] }
 fvm_shared4 = { package = "fvm_shared", version = "~4.7", features = ["proofs"] }
-get-size2 = { version = "0.6", features = ["derive"] }
+get-size2 = { workspace = true }
 gethostname = "1"
 git-version = "0.3"
 group = "0.13"

Cargo.toml

@@ -13,12 +13,15 @@ publish = false
 [dev-dependencies]
 anyhow = { workspace = true }
 cid = { workspace = true }
+dhat = { workspace = true }
 flume = { workspace = true }
 forest = { package = "forest-filecoin", path = "../", default-features = false, features = [
+  "rustalloc",
   "interop-tests-private",
   "no-f3-sidecar",
 ] }
 futures = { workspace = true }
+get-size2 = { workspace = true }
 libp2p = { workspace = true, features = [
   'kad',
   'identify',

interop-tests/Cargo.toml

@@ -0,0 +1,55 @@
+// Copyright 2019-2025 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use forest::interop_tests_private::beacon::BeaconEntry;
+use get_size2::GetSize;
+use std::mem::size_of;
+
+#[global_allocator]
+static ALLOC: dhat::Alloc = dhat::Alloc;
+
+#[test]
+fn test_get_size_beacon_entry() {
+    let _profiler = dhat::Profiler::builder().testing().build();
+    let b = BeaconEntry::new(1, vec![0; 10]);
+
+    let stats = dhat::HeapStats::get();
+    dhat::assert_eq!(stats.curr_bytes, 10);
+    dhat::assert_eq!(stats.curr_bytes, b.get_heap_size());
+
+    let mut v = vec![
+        b,
+        BeaconEntry::new(2, vec![0; 20]),
+        BeaconEntry::new(3, vec![0; 30]),
+    ];
+
+    let inner_bytes = v.iter().map(GetSize::get_heap_size).sum::<usize>();
+
+    let stats = dhat::HeapStats::get();
+    assert!(v.capacity() >= v.len());
+    dhat::assert_eq!(
+        stats.curr_bytes,
+        size_of::<BeaconEntry>() * v.capacity() + inner_bytes
+    );
+    dhat::assert_eq!(stats.curr_bytes, v.get_heap_size());
+
+    v.reserve_exact(100);
+
+    let stats = dhat::HeapStats::get();
+    assert!(v.capacity() >= 100 + v.len());
+    dhat::assert_eq!(
+        stats.curr_bytes,
+        size_of::<BeaconEntry>() * v.capacity() + inner_bytes
+    );
+    dhat::assert_eq!(stats.curr_bytes, v.get_heap_size());
+
+    v.shrink_to_fit();
+    let stats = dhat::HeapStats::get();
+    // `dhat::Alloc` works fine with `shrink_to_fit`
+    assert_eq!(v.capacity(), v.len());
+    dhat::assert_eq!(
+        stats.curr_bytes,
+        size_of::<BeaconEntry>() * v.capacity() + 60
+    );
+    dhat::assert_eq!(stats.curr_bytes, v.get_heap_size());
+}

interop-tests/tests/dhat_get_size_beacon_entry.rs

@@ -20,7 +20,6 @@ mod snapshot_cmd;
 mod state_cmd;
 mod sync_cmd;
 mod wait_api_cmd;
-use std::io::Write;
 
 pub(crate) use crate::cli_shared::cli::Config;
 use crate::cli_shared::cli::HELP_MESSAGE;

src/cli/subcommands/mod.rs

@@ -1,7 +1,6 @@
 // Copyright 2019-2025 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
-use super::*;
 use crate::chain::FilecoinSnapshotVersion;
 use crate::chain_sync::SyncConfig;
 use crate::cli_shared::snapshot::{self, TrustedVendor};
@@ -11,11 +10,10 @@ use crate::rpc::{self, chain::ForestChainExportParams, prelude::*, types::ApiTip
 use anyhow::Context as _;
 use chrono::DateTime;
 use clap::Subcommand;
-use human_repr::HumanCount;
-use num::Zero as _;
+use indicatif::{ProgressBar, ProgressStyle};
 use std::{
     path::{Path, PathBuf},
-    time::{Duration, Instant},
+    time::Duration,
 };
 use tokio::io::AsyncWriteExt;
 
@@ -99,48 +97,36 @@ impl SnapshotCommands {
                     dry_run,
                 };
 
+                let pb = ProgressBar::new_spinner().with_style(
+                    ProgressStyle::with_template(
+                        "{spinner} {msg} {binary_total_bytes} written in {elapsed} ({binary_bytes_per_sec})",
+                    )
+                    .expect("indicatif template must be valid"),
+                ).with_message(format!("Exporting {} ...", output_path.display()));
+                pb.enable_steady_tick(std::time::Duration::from_millis(80));
                 let handle = tokio::spawn({
-                    let start = Instant::now();
-                    let tmp_file = temp_path.to_owned();
-                    let output_path = output_path.clone();
+                    let path: PathBuf = (&temp_path).into();
+                    let pb = pb.clone();
+                    let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1));
                     async move {
-                        let mut interval =
-                            tokio::time::interval(tokio::time::Duration::from_secs_f32(0.5));
-                        println!("Getting ready to export...");
                         loop {
                             interval.tick().await;
-                            let snapshot_size = std::fs::metadata(&tmp_file)
-                                .map(|meta| meta.len())
-                                .unwrap_or(0);
-                            print!(
-                                "{}{}",
-                                anes::MoveCursorToPreviousLine(1),
-                                anes::ClearLine::All
-                            );
-                            let elapsed_secs = start.elapsed().as_secs_f64();
-                            println!(
-                                "{}: {} ({}/s)",
-                                &output_path.to_string_lossy(),
-                                snapshot_size.human_count_bytes(),
-                                if elapsed_secs.is_zero() {
-                                    0.
-                                } else {
-                                    (snapshot_size as f64) / elapsed_secs
-                                }
-                                .human_count_bytes(),
-                            );
-                            let _ = std::io::stdout().flush();
+                            if let Ok(meta) = std::fs::metadata(&path) {
+                                pb.set_position(meta.len());
+                            }
                         }
                     }
                 });
+
                 // Manually construct RpcRequest because snapshot export could
                 // take a few hours on mainnet
                 let hash_result = client
                     .call(ForestChainExport::request((params,))?.with_timeout(Duration::MAX))
                     .await?;
 
                 handle.abort();
-                let _ = handle.await;
+                pb.finish();
+                _ = handle.await;
 
                 if let Some(hash) = hash_result {
                     save_checksum(&output_path, hash).await?;

src/cli/subcommands/snapshot_cmd.rs

@@ -113,6 +113,9 @@ pub mod interop_tests_private {
     pub mod libp2p_bitswap {
         pub use crate::libp2p_bitswap::*;
     }
+    pub mod beacon {
+        pub use crate::beacon::BeaconEntry;
+    }
 }
 
 // These should be made private in https://github.com/ChainSafe/forest/issues/3013