feat: add --memory_profile_out flag for Chonk memory profiling

barretenberg/cpp/CLAUDE.md

@@ -155,3 +155,36 @@ Typical workflow
 2. Build native code: `cd barretenberg/cpp && ./bootstrap.sh build_native`
 3. Check VKs: `cd scripts && ./test_chonk_standalone_vks_havent_changed.sh`
 4. If VKs changed intentionally: `./test_chonk_standalone_vks_havent_changed.sh --update_inputs`
+
+## Example IVC inputs
+
+Example IVC inputs (msgpack files) for `bb prove --scheme chonk` are generated by e2e benchmark tests. Run the full bootstrap from the repo root to populate them:
+
+```bash
+cd $(git rev-parse --show-toplevel) && ./bootstrap.sh
+```
+
+This creates `yarn-project/end-to-end/example-app-ivc-inputs-out/<flow>/ivc-inputs.msgpack`. The inputs are generated by the `build_bench` function in `yarn-project/end-to-end/bootstrap.sh`, which runs client flow tests with `CAPTURE_IVC_FOLDER` set. In CI, these are cached as `bb-chonk-captures-<hash>.tar.gz`.
+
+## Memory profiling
+
+The `--memory_profile_out <file>` flag on `bb prove` outputs a JSON breakdown of memory consumption: per-circuit polynomial memory by category (wires, sigmas, selectors, etc.), CRS size, and RSS checkpoints at key proving stages (after polynomial allocation, after oink, after sumcheck, after PCS).
+
+```bash
+cd barretenberg/cpp
+./build/bin/bb prove \
+  --scheme chonk \
+  --ivc_inputs_path <path-to>/ivc-inputs.msgpack \
+  -o /tmp/proof-out \
+  -v \
+  --memory_profile_out /tmp/proof-out/memory_breakdown.json
+```
+
+The extraction script converts the JSON into dashboard-friendly benchmark entries:
+
+```bash
+echo '[]' > /tmp/proof-out/benchmarks.bench.json
+python3 scripts/extract_memory_benchmarks.py /tmp/proof-out "app-proving/flow/native"
+```
+
+This appends stacked chart entries (polynomial memory by category) and line chart entries (total polynomial MB, CRS MB, peak RSS) to `benchmarks.bench.json`. In CI, this is integrated into `ci_benchmark_ivc_flows.sh` and uploaded to the benchmark dashboard.

barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh

@@ -59,15 +59,15 @@ function run_bb_cli_bench {
 
   if [[ "$runtime" == "native" ]]; then
     # Add --bench_out_hierarchical flag for native builds to capture hierarchical op counts and timings
-    memusage "./$native_build_dir/bin/bb" "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" || {
-      echo "bb native failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json"
+    memusage "./$native_build_dir/bin/bb" "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" "--memory_profile_out" "$output/memory_breakdown.json" || {
+      echo "bb native failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json --memory_profile_out $output/memory_breakdown.json"
       exit 1
     }
   else # wasm
     export WASMTIME_ALLOWED_DIRS="--dir=$flow_folder --dir=$output"
     # Add --bench_out_hierarchical flag for wasm builds to capture hierarchical op counts and timings
-    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" || {
-      echo "bb wasm failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json"
+    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" "--memory_profile_out" "$output/memory_breakdown.json" || {
+      echo "bb wasm failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json --memory_profile_out $output/memory_breakdown.json"
       exit 1
     }
   fi
@@ -139,6 +139,12 @@ EOF
     echo "Extracting component timings from hierarchical breakdown..."
     python3 scripts/extract_component_benchmarks.py "$output" "$name_path"
   fi
+
+  # Extract memory breakdown metrics if available
+  if [[ -f "$output/memory_breakdown.json" ]]; then
+    echo "Extracting memory breakdown metrics..."
+    python3 scripts/extract_memory_benchmarks.py "$output" "$name_path"
+  fi
 }
 
 export -f verify_ivc_flow run_bb_cli_bench
@@ -178,4 +184,17 @@ if [[ "${CI:-}" == "1" ]] && [[ "${CI_USE_BUILD_INSTANCE_KEY:-0}" == "1" ]]; the
   else
     echo "Warning: benchmark breakdown file not found at $benchmark_breakdown_file"
   fi
+
+  # Upload memory breakdown to S3
+  memory_breakdown_file="bench-out/app-proving/$flow_name/$runtime/memory_breakdown.json"
+  if [[ -f "$memory_breakdown_file" ]]; then
+    tmp_memory_file="/tmp/memory_breakdown_${runtime}_${flow_name}_$$.json"
+    cp "$memory_breakdown_file" "$tmp_memory_file"
+    memory_disk_key="memory-${runtime}-${flow_name}-${current_sha}"
+    {
+      cat "$tmp_memory_file" | gzip | cache_s3_transfer_to "bench/bb-breakdown" "$memory_disk_key"
+      rm -f "$tmp_memory_file"
+    } &
+    echo "Uploaded memory breakdown to S3: bench/bb-breakdown/$memory_disk_key"
+  fi
 fi

barretenberg/cpp/scripts/extract_memory_benchmarks.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Extracts memory breakdown metrics from a memory profile JSON and appends
+them to the benchmark JSON file as dashboard entries.
+
+Usage: extract_memory_benchmarks.py <output_dir> <name_path>
+
+The output_dir must contain:
+  - memory_breakdown.json (memory profile data from bb --memory_profile_out)
+  - benchmarks.bench.json (existing benchmark results to append to)
+
+The memory profile JSON format is documented in memory_profile.cpp.
+"""
+import json
+import sys
+
+if len(sys.argv) != 3:
+    print(f"Usage: {sys.argv[0]} <output_dir> <name_path>", file=sys.stderr)
+    sys.exit(1)
+
+output_dir = sys.argv[1]
+name_path = sys.argv[2]
+
+try:
+    with open(f"{output_dir}/memory_breakdown.json", "r") as f:
+        data = json.load(f)
+
+    entries = []
+
+    # Stacked chart: polynomial memory by category (peak circuit)
+    peak_circuit = data.get("peak_circuit")
+    if peak_circuit:
+        for category, stats in peak_circuit.get("categories", {}).items():
+            entries.append({
+                "name": f"{name_path}/memory/{category}_MB",
+                "unit": "MB",
+                "value": round(stats["actual_mb"], 2),
+                "extra": f"stacked-area:{name_path}/memory/polynomial_categories"
+            })
+
+        # Total polynomial memory (peak circuit)
+        entries.append({
+            "name": f"{name_path}/memory/total_polynomial_MB",
+            "unit": "MB",
+            "value": round(peak_circuit.get("total_polynomial_mb", 0), 2)
+        })
+
+    # CRS memory
+    crs_mb = data.get("crs_mb", 0)
+    if crs_mb > 0:
+        entries.append({
+            "name": f"{name_path}/memory/crs_MB",
+            "unit": "MB",
+            "value": round(crs_mb, 2)
+        })
+
+    # Peak RSS from checkpoints
+    peak_rss = data.get("peak_rss", {})
+    if peak_rss.get("rss_mb", 0) > 0:
+        entries.append({
+            "name": f"{name_path}/memory/peak_rss_MB",
+            "unit": "MB",
+            "value": peak_rss["rss_mb"]
+        })
+
+    # Append to existing benchmarks file
+    with open(f"{output_dir}/benchmarks.bench.json", "r") as f:
+        existing = json.load(f)
+
+    existing.extend(entries)
+
+    with open(f"{output_dir}/benchmarks.bench.json", "w") as f:
+        json.dump(existing, f, indent=2)
+
+    print(f"Extracted {len(entries)} memory breakdown metrics")
+except Exception as e:
+    print(f"Warning: Could not extract memory breakdown: {e}", file=sys.stderr)

barretenberg/cpp/src/barretenberg/bb/cli.cpp

@@ -29,6 +29,7 @@
 #include "barretenberg/common/assert.hpp"
 #include "barretenberg/common/bb_bench.hpp"
 #include "barretenberg/common/get_bytecode.hpp"
+#include "barretenberg/common/memory_profile.hpp"
 #include "barretenberg/common/thread.hpp"
 #include "barretenberg/common/version.hpp"
 #include "barretenberg/dsl/acir_format/serde/index.hpp"
@@ -389,6 +390,15 @@ int parse_and_run_cli_command(int argc, char* argv[])
                          "parent-child relationships) as json.")
             ->group(advanced_group);
     };
+    std::string memory_profile_out;
+    const auto add_memory_profile_out_option = [&](CLI::App* subcommand) {
+        return subcommand
+            ->add_option("--memory_profile_out",
+                         memory_profile_out,
+                         "Path to write memory profile data (polynomial breakdown by category, RSS "
+                         "checkpoints, CRS size) as json.")
+            ->group(advanced_group);
+    };
 
     /***************************************************************************************************************
      * Top-level flags
@@ -482,6 +492,7 @@ int parse_and_run_cli_command(int argc, char* argv[])
     add_print_bench_flag(prove);
     add_bench_out_option(prove);
     add_bench_out_hierarchical_option(prove);
+    add_memory_profile_out_option(prove);
     add_storage_budget_option(prove);
     add_output_format_option(prove);
 
@@ -811,6 +822,10 @@ int parse_and_run_cli_command(int argc, char* argv[])
     if (!flags.storage_budget.empty()) {
         storage_budget = parse_size_string(flags.storage_budget);
     }
+    if (!memory_profile_out.empty()) {
+        bb::detail::use_memory_profile = true;
+        vinfo("Memory profiling enabled via --memory_profile_out");
+    }
     if (print_bench || !bench_out.empty() || !bench_out_hierarchical.empty()) {
         bb::detail::use_bb_bench = true;
         vinfo("BB_BENCH enabled via --print_bench or --bench_out");
@@ -987,6 +1002,11 @@ int parse_and_run_cli_command(int argc, char* argv[])
                     bb::detail::GLOBAL_BENCH_STATS.serialize_aggregate_data_json(file);
                 }
 #endif
+                if (!memory_profile_out.empty()) {
+                    std::ofstream file(memory_profile_out);
+                    bb::detail::GLOBAL_MEMORY_PROFILE.serialize_json(file);
+                    vinfo("Memory profile written to ", memory_profile_out);
+                }
                 return 0;
             }
             if (check->parsed()) {

barretenberg/cpp/src/barretenberg/chonk/chonk.cpp

@@ -7,6 +7,7 @@
 #include "barretenberg/chonk/chonk.hpp"
 #include "barretenberg/chonk/chonk_verifier.hpp"
 #include "barretenberg/common/bb_bench.hpp"
+#include "barretenberg/common/memory_profile.hpp"
 #include "barretenberg/common/streams.hpp"
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 #include "barretenberg/goblin/goblin_verifier.hpp"
@@ -538,6 +539,12 @@ void Chonk::accumulate_and_fold(ClientCircuit& circuit,
         break;
     }
 
+    if (detail::use_memory_profile) {
+        size_t circuit_idx =
+            detail::GLOBAL_MEMORY_PROFILE.circuits.empty() ? 0 : detail::GLOBAL_MEMORY_PROFILE.circuits.size() - 1;
+        detail::GLOBAL_MEMORY_PROFILE.add_rss_checkpoint("after_accumulate", circuit_idx);
+    }
+
     VerifierInputs queue_entry{ std::move(proof), precomputed_vk, queue_type, is_kernel };
     verification_queue.push_back(queue_entry);
 

barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp

@@ -11,6 +11,7 @@
  */
 
 #include "barretenberg/common/bb_bench.hpp"
+#include "barretenberg/common/memory_profile.hpp"
 #include "barretenberg/common/ref_span.hpp"
 #include "barretenberg/ecc/scalar_multiplication/scalar_multiplication.hpp"
 #include "barretenberg/polynomials/polynomial.hpp"
@@ -53,7 +54,11 @@ template <class Curve> class CommitmentKey {
     CommitmentKey(const size_t num_points)
         : srs(srs::get_crs_factory<Curve>()->get_crs(num_points))
         , srs_size(num_points)
-    {}
+    {
+        if (detail::use_memory_profile) {
+            detail::GLOBAL_MEMORY_PROFILE.set_crs_size(num_points);
+        }
+    }
     /**
      * @brief Checks the commitment key is properly initialized.
      *