Manage memory instrument with profiling cli

.github/workflows/ci.yml

@@ -66,13 +66,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: moonrepo/setup-rust@v1
+
+      - name: 'Install rust-toolchain.toml'
+        run: rustup toolchain install
+      - uses: Swatinem/rust-cache@v2
+      - name: Install cargo codspeed
+        uses: taiki-e/install-action@v2
         with:
-          bins: cargo-codspeed
+          tool: cargo-codspeed
+
       - name: Build benchmarks
         run: cargo codspeed build -p runner-shared
       - name: Run benchmarks
         uses: CodSpeedHQ/action@v4
         with:
-          mode: instrumentation
+          mode: simulation
           run: cargo codspeed run -p runner-shared

crates/exec-harness/Cargo.toml

@@ -5,10 +5,6 @@ edition = "2024"
 repository = "https://github.com/CodSpeedHQ/runner"
 publish = false
 
-[lib]
-name = "exec_harness"
-path = "src/lib.rs"
-
 [[bin]]
 name = "exec-harness"
 path = "src/main.rs"

crates/exec-harness/src/analysis.rs

@@ -0,0 +1,24 @@
+use crate::prelude::*;
+
+use crate::uri::NameAndUri;
+use codspeed::instrument_hooks::InstrumentHooks;
+use std::process::Command;
+
+pub fn perform(name_and_uri: NameAndUri, command: Vec<String>) -> Result<()> {
+    let hooks = InstrumentHooks::instance();
+
+    let mut cmd = Command::new(&command[0]);
+    cmd.args(&command[1..]);
+    hooks.start_benchmark().unwrap();
+    let status = cmd.status();
+    hooks.stop_benchmark().unwrap();
+    let status = status.context("Failed to execute command")?;
+
+    if !status.success() {
+        bail!("Command exited with non-zero status: {status}");
+    }
+
+    hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+
+    Ok(())
+}

crates/exec-harness/src/lib.rs

@@ -1,7 +1,15 @@
-//! CodSpeed exec-harness library
-//!
-//! This library provides the core functionality for wrapping commands
-//! with CodSpeed performance instrumentation.
+use clap::ValueEnum;
+use serde::{Deserialize, Serialize};
 
-mod prelude;
+pub mod analysis;
+pub mod prelude;
+pub mod uri;
 pub mod walltime;
+
+#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum MeasurementMode {
+    Walltime,
+    Memory,
+    Simulation,
+}

crates/exec-harness/src/main.rs

@@ -1,12 +1,9 @@
-use crate::prelude::*;
-use crate::walltime::WalltimeResults;
 use clap::Parser;
-use runner_shared::walltime_results::WalltimeBenchmark;
-use std::path::PathBuf;
-
-mod prelude;
-mod uri;
-mod walltime;
+use exec_harness::MeasurementMode;
+use exec_harness::analysis;
+use exec_harness::prelude::*;
+use exec_harness::uri;
+use exec_harness::walltime;
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
@@ -19,6 +16,10 @@ struct Args {
     #[arg(long)]
     name: Option<String>,
 
+    /// Set by the runner, should be coherent with the executor being used
+    #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE", hide = true)]
+    measurement_mode: Option<MeasurementMode>,
+
     #[command(flatten)]
     execution_args: walltime::WalltimeExecutionArgs,
 
@@ -33,43 +34,29 @@ fn main() -> Result<()> {
         .format_timestamp(None)
         .init();
 
+    debug!("Starting exec-harness with pid {}", std::process::id());
+
     let args = Args::parse();
 
     if args.command.is_empty() {
         bail!("Error: No command provided");
     }
 
-    let uri::NameAndUri {
-        name: bench_name,
-        uri: bench_uri,
-    } = uri::generate_name_and_uri(&args.name, &args.command);
-
-    // Build execution options from CLI args
-    let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
+    let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command);
 
-    let times_per_round_ns =
-        walltime::perform(bench_uri.clone(), args.command, &execution_options)?;
+    match args.measurement_mode {
+        Some(MeasurementMode::Walltime) | None => {
+            let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
 
-    // Collect walltime results
-    let max_time_ns = times_per_round_ns.iter().copied().max();
-    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
-        bench_name.clone(),
-        bench_uri.clone(),
-        vec![1; times_per_round_ns.len()],
-        times_per_round_ns,
-        max_time_ns,
-    );
-
-    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
-        .expect("Failed to create walltime results");
-
-    walltime_results
-        .save_to_file(
-            std::env::var("CODSPEED_PROFILE_FOLDER")
-                .map(PathBuf::from)
-                .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")),
-        )
-        .expect("Failed to save walltime results");
+            walltime::perform(bench_name_and_uri, args.command, &execution_options)?;
+        }
+        Some(MeasurementMode::Memory) => {
+            analysis::perform(bench_name_and_uri, args.command)?;
+        }
+        Some(MeasurementMode::Simulation) => {
+            bail!("Simulation measurement mode is not yet supported by exec-harness");
+        }
+    }
 
     Ok(())
 }

crates/exec-harness/src/uri.rs

@@ -1,6 +1,6 @@
 use crate::prelude::*;
 
-pub(crate) struct NameAndUri {
+pub struct NameAndUri {
     pub(crate) name: String,
     pub(crate) uri: String,
 }
@@ -9,7 +9,7 @@ pub(crate) struct NameAndUri {
 /// Should be removed once we have structured metadata around benchmarks
 const MAX_NAME_LENGTH: usize = 1024 - 100;
 
-pub(crate) fn generate_name_and_uri(name: &Option<String>, command: &[String]) -> NameAndUri {
+pub fn generate_name_and_uri(name: &Option<String>, command: &[String]) -> NameAndUri {
     let mut name = name.clone().unwrap_or_else(|| command.join(" "));
     let uri = format!("exec_harness::{name}");
 

crates/exec-harness/src/walltime/config.rs

@@ -81,7 +81,6 @@ impl WalltimeExecutionArgs {
     ///
     /// Unfortunately, clap does not provide a built-in way to serialize args back to CLI format,
     // Clippy is very confused since this is used in the runner, but not in the binary of exec-harness
-    #[allow(dead_code)]
     pub fn to_cli_args(&self) -> Vec<String> {
         let mut args = Vec::new();
 

crates/exec-harness/src/walltime/mod.rs

@@ -3,13 +3,52 @@ mod config;
 pub use config::ExecutionOptions;
 use config::RoundOrTime;
 pub use config::WalltimeExecutionArgs;
+use runner_shared::walltime_results::WalltimeBenchmark;
 pub use runner_shared::walltime_results::WalltimeResults;
 
 use crate::prelude::*;
+use crate::uri::NameAndUri;
 use codspeed::instrument_hooks::InstrumentHooks;
 use std::process::Command;
 
 pub fn perform(
+    name_and_uri: NameAndUri,
+    command: Vec<String>,
+    execution_options: &ExecutionOptions,
+) -> Result<()> {
+    let NameAndUri {
+        name: bench_name,
+        uri: bench_uri,
+    } = name_and_uri;
+
+    let times_per_round_ns = run_rounds(bench_uri.clone(), command, execution_options)?;
+
+    // Collect walltime results
+    let max_time_ns = times_per_round_ns.iter().copied().max();
+
+    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
+        bench_name.clone(),
+        bench_uri.clone(),
+        vec![1; times_per_round_ns.len()],
+        times_per_round_ns,
+        max_time_ns,
+    );
+
+    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
+        .expect("Failed to create walltime results");
+
+    walltime_results
+        .save_to_file(
+            std::env::var("CODSPEED_PROFILE_FOLDER")
+                .map(std::path::PathBuf::from)
+                .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")),
+        )
+        .context("Failed to save walltime results")?;
+
+    Ok(())
+}
+
+fn run_rounds(
     bench_uri: String,
     command: Vec<String>,
     config: &ExecutionOptions,
@@ -160,3 +199,6 @@ pub fn perform(
 
     Ok(times_per_round_ns)
 }
+
+#[cfg(test)]
+mod tests;