From 4d9b6b473c6204498bdfe787dcdc5896621f5bfb Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 15 Jan 2026 11:44:29 +0100 Subject: [PATCH 1/5] refactor(exec-harness): move all walltime benchmarking to its dedicated module Also removed the external tests as they can be scoped to the walltime module. --- crates/exec-harness/Cargo.toml | 4 - crates/exec-harness/src/lib.rs | 6 +- crates/exec-harness/src/main.rs | 41 +-- crates/exec-harness/src/uri.rs | 4 +- crates/exec-harness/src/walltime/mod.rs | 42 +++ .../walltime/tests.rs} | 273 ++++++++---------- 6 files changed, 174 insertions(+), 196 deletions(-) rename crates/exec-harness/{tests/integration_test.rs => src/walltime/tests.rs} (58%) diff --git a/crates/exec-harness/Cargo.toml b/crates/exec-harness/Cargo.toml index 8087cc50..27205893 100644 --- a/crates/exec-harness/Cargo.toml +++ b/crates/exec-harness/Cargo.toml @@ -5,10 +5,6 @@ edition = "2024" repository = "https://github.com/CodSpeedHQ/runner" publish = false -[lib] -name = "exec_harness" -path = "src/lib.rs" - [[bin]] name = "exec-harness" path = "src/main.rs" diff --git a/crates/exec-harness/src/lib.rs b/crates/exec-harness/src/lib.rs index 845065b7..eb9f61a3 100644 --- a/crates/exec-harness/src/lib.rs +++ b/crates/exec-harness/src/lib.rs @@ -1,7 +1,3 @@ -//! CodSpeed exec-harness library -//! -//! This library provides the core functionality for wrapping commands -//! with CodSpeed performance instrumentation. - mod prelude; +pub mod uri; pub mod walltime; diff --git a/crates/exec-harness/src/main.rs b/crates/exec-harness/src/main.rs index df574795..a80416e9 100644 --- a/crates/exec-harness/src/main.rs +++ b/crates/exec-harness/src/main.rs @@ -1,12 +1,8 @@ -use crate::prelude::*; -use crate::walltime::WalltimeResults; +use anyhow::Result; +use anyhow::bail; use clap::Parser; -use runner_shared::walltime_results::WalltimeBenchmark; -use std::path::PathBuf; - -mod prelude; -mod uri; -mod walltime; +use exec_harness::uri; +use exec_harness::walltime; #[derive(Parser, Debug)] #[command(name = "exec-harness")] @@ -39,37 +35,12 @@ fn main() -> Result<()> { bail!("Error: No command provided"); } - let uri::NameAndUri { - name: bench_name, - uri: bench_uri, - } = uri::generate_name_and_uri(&args.name, &args.command); + let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command); // Build execution options from CLI args let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?; - let times_per_round_ns = - walltime::perform(bench_uri.clone(), args.command, &execution_options)?; - - // Collect walltime results - let max_time_ns = times_per_round_ns.iter().copied().max(); - let walltime_benchmark = WalltimeBenchmark::from_runtime_data( - bench_name.clone(), - bench_uri.clone(), - vec![1; times_per_round_ns.len()], - times_per_round_ns, - max_time_ns, - ); - - let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark]) - .expect("Failed to create walltime results"); - - walltime_results - .save_to_file( - std::env::var("CODSPEED_PROFILE_FOLDER") - .map(PathBuf::from) - .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")), - ) - .expect("Failed to save walltime results"); + walltime::perform(bench_name_and_uri, args.command, &execution_options)?; Ok(()) } diff --git a/crates/exec-harness/src/uri.rs b/crates/exec-harness/src/uri.rs index 1e387a02..6e599217 100644 --- a/crates/exec-harness/src/uri.rs +++ b/crates/exec-harness/src/uri.rs @@ -1,6 +1,6 @@ use crate::prelude::*; -pub(crate) struct NameAndUri { +pub struct NameAndUri { pub(crate) name: String, pub(crate) uri: String, } @@ -9,7 +9,7 @@ pub(crate) struct NameAndUri { /// Should be removed once we have structured metadata around benchmarks const MAX_NAME_LENGTH: usize = 1024 - 100; -pub(crate) fn generate_name_and_uri(name: &Option, command: &[String]) -> NameAndUri { +pub fn generate_name_and_uri(name: &Option, command: &[String]) -> NameAndUri { let mut name = name.clone().unwrap_or_else(|| command.join(" ")); let uri = format!("exec_harness::{name}"); diff --git a/crates/exec-harness/src/walltime/mod.rs b/crates/exec-harness/src/walltime/mod.rs index 0fb9bcc4..7fadc1f9 100644 --- a/crates/exec-harness/src/walltime/mod.rs +++ b/crates/exec-harness/src/walltime/mod.rs @@ -3,13 +3,52 @@ mod config; pub use config::ExecutionOptions; use config::RoundOrTime; pub use config::WalltimeExecutionArgs; +use runner_shared::walltime_results::WalltimeBenchmark; pub use runner_shared::walltime_results::WalltimeResults; use crate::prelude::*; +use crate::uri::NameAndUri; use codspeed::instrument_hooks::InstrumentHooks; use std::process::Command; pub fn perform( + name_and_uri: NameAndUri, + command: Vec, + execution_options: &ExecutionOptions, +) -> Result<()> { + let NameAndUri { + name: bench_name, + uri: bench_uri, + } = name_and_uri; + + let times_per_round_ns = run_rounds(bench_uri.clone(), command, execution_options)?; + + // Collect walltime results + let max_time_ns = times_per_round_ns.iter().copied().max(); + + let walltime_benchmark = WalltimeBenchmark::from_runtime_data( + bench_name.clone(), + bench_uri.clone(), + vec![1; times_per_round_ns.len()], + times_per_round_ns, + max_time_ns, + ); + + let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark]) + .expect("Failed to create walltime results"); + + walltime_results + .save_to_file( + std::env::var("CODSPEED_PROFILE_FOLDER") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")), + ) + .context("Failed to save walltime results")?; + + Ok(()) +} + +fn run_rounds( bench_uri: String, command: Vec, config: &ExecutionOptions, @@ -160,3 +199,6 @@ pub fn perform( Ok(times_per_round_ns) } + +#[cfg(test)] +mod tests; diff --git a/crates/exec-harness/tests/integration_test.rs b/crates/exec-harness/src/walltime/tests.rs similarity index 58% rename from crates/exec-harness/tests/integration_test.rs rename to crates/exec-harness/src/walltime/tests.rs index b9f8839b..bec40a75 100644 --- a/crates/exec-harness/tests/integration_test.rs +++ b/crates/exec-harness/src/walltime/tests.rs @@ -1,6 +1,8 @@ use anyhow::Result; use tempfile::TempDir; +use super::*; + // Helper to create a simple sleep 100ms command fn sleep_cmd() -> Vec { vec!["sleep".to_string(), "0.1".to_string()] @@ -10,17 +12,15 @@ fn sleep_cmd() -> Vec { #[test] fn test_max_rounds_without_warmup() -> Result<()> { // Create execution options with no warmup and fixed rounds - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), // No warmup - max_time: None, - min_time: None, - max_rounds: Some(10), // Exactly 10 rounds - min_rounds: None, - }, - )?; - - let times = exec_harness::walltime::perform( + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), // No warmup + max_time: None, + min_time: None, + max_rounds: Some(10), // Exactly 10 rounds + min_rounds: None, + })?; + + let times = run_rounds( "test::max_rounds_no_warmup".to_string(), sleep_cmd(), &exec_opts, @@ -36,17 +36,15 @@ fn test_max_rounds_without_warmup() -> Result<()> { #[test] fn test_min_max_rounds_with_warmup() -> Result<()> { // Create execution options with warmup and min/max rounds - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("100ms".to_string()), // Short warmup - max_time: None, - min_time: None, - max_rounds: Some(50), // Max 50 rounds - min_rounds: Some(5), // Min 5 rounds - }, - )?; - - let times = exec_harness::walltime::perform( + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("100ms".to_string()), // Short warmup + max_time: None, + min_time: None, + max_rounds: Some(50), // Max 50 rounds + min_rounds: Some(5), // Min 5 rounds + })?; + + let times = run_rounds( "test::min_max_rounds_warmup".to_string(), sleep_cmd(), &exec_opts, @@ -71,18 +69,15 @@ fn test_min_max_rounds_with_warmup() -> Result<()> { #[test] fn test_max_time_constraint() -> Result<()> { // Use a very short max_time to ensure we don't run too many iterations - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("50ms".to_string()), // Short warmup - max_time: Some("500ms".to_string()), // Very short max time - min_time: None, - max_rounds: None, - min_rounds: None, - }, - )?; + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("50ms".to_string()), // Short warmup + max_time: Some("500ms".to_string()), // Very short max time + min_time: None, + max_rounds: None, + min_rounds: None, + })?; - let times = - exec_harness::walltime::perform("test::max_time".to_string(), sleep_cmd(), &exec_opts)?; + let times = run_rounds("test::max_time".to_string(), sleep_cmd(), &exec_opts)?; // Should have run at least 1 time, but not an excessive amount assert!(!times.is_empty(), "Expected at least 1 iteration"); @@ -99,17 +94,15 @@ fn test_max_time_constraint() -> Result<()> { #[test] fn test_min_rounds_and_min_time() -> Result<()> { // Set min_rounds and min_time - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("10ms".to_string()), // Very short warmup - max_time: None, - min_time: Some("1ms".to_string()), - max_rounds: None, - min_rounds: Some(15), - }, - )?; - - let times = exec_harness::walltime::perform( + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("10ms".to_string()), // Very short warmup + max_time: None, + min_time: Some("1ms".to_string()), + max_rounds: None, + min_rounds: Some(15), + })?; + + let times = run_rounds( "test::min_rounds_priority".to_string(), sleep_cmd(), &exec_opts, @@ -129,34 +122,30 @@ fn test_min_rounds_and_min_time() -> Result<()> { #[test] fn test_warmup_is_performed() -> Result<()> { // With warmup enabled - let exec_opts_with_warmup = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("200ms".to_string()), // Significant warmup time - max_time: Some("500ms".to_string()), - min_time: None, - max_rounds: None, - min_rounds: None, - }, - )?; - - let times_with_warmup = exec_harness::walltime::perform( + let exec_opts_with_warmup = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("200ms".to_string()), // Significant warmup time + max_time: Some("500ms".to_string()), + min_time: None, + max_rounds: None, + min_rounds: None, + })?; + + let times_with_warmup = run_rounds( "test::with_warmup".to_string(), sleep_cmd(), &exec_opts_with_warmup, )?; // With warmup disabled - let exec_opts_no_warmup = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), // No warmup - max_time: None, - min_time: None, - max_rounds: Some(5), // Fixed 5 rounds - min_rounds: None, - }, - )?; - - let times_no_warmup = exec_harness::walltime::perform( + let exec_opts_no_warmup = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), // No warmup + max_time: None, + min_time: None, + max_rounds: Some(5), // Fixed 5 rounds + min_rounds: None, + })?; + + let times_no_warmup = run_rounds( "test::no_warmup".to_string(), sleep_cmd(), &exec_opts_no_warmup, @@ -173,17 +162,15 @@ fn test_warmup_is_performed() -> Result<()> { #[test] fn test_with_sleep_command() -> Result<()> { // Use a command that takes a measurable amount of time - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), // No warmup for faster test - max_time: None, - min_time: None, - max_rounds: Some(3), // Just 3 rounds - min_rounds: None, - }, - )?; - - let times = exec_harness::walltime::perform( + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), // No warmup for faster test + max_time: None, + min_time: None, + max_rounds: Some(3), // Just 3 rounds + min_rounds: None, + })?; + + let times = run_rounds( "test::sleep_command".to_string(), vec!["sleep".to_string(), "0.01".to_string()], // 10ms sleep &exec_opts, @@ -206,19 +193,17 @@ fn test_with_sleep_command() -> Result<()> { /// Test that invalid command exits early #[test] fn test_invalid_command_fails() { - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), - max_time: None, - min_time: None, - max_rounds: Some(5), - min_rounds: None, - }, - ) + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), + max_time: None, + min_time: None, + max_rounds: Some(5), + min_rounds: None, + }) .unwrap(); // Try to run a command that doesn't exist - let result = exec_harness::walltime::perform( + let result = run_rounds( "test::invalid_command".to_string(), vec!["this_command_definitely_does_not_exist_12345".to_string()], &exec_opts, @@ -232,17 +217,15 @@ fn test_invalid_command_fails() { #[test] fn test_pure_numbers_as_seconds() -> Result<()> { // Use pure numbers which should be interpreted as seconds - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0.1".to_string()), // 0.1 seconds warmup - max_time: Some("1".to_string()), // 1 second max time - min_time: None, - max_rounds: None, - min_rounds: None, - }, - )?; - - let times = exec_harness::walltime::perform( + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0.1".to_string()), // 0.1 seconds warmup + max_time: Some("1".to_string()), // 1 second max time + min_time: None, + max_rounds: None, + min_rounds: None, + })?; + + let times = run_rounds( "test::pure_numbers_seconds".to_string(), sleep_cmd(), &exec_opts, @@ -252,17 +235,15 @@ fn test_pure_numbers_as_seconds() -> Result<()> { assert!(!times.is_empty(), "Expected at least one iteration"); // Test fractional seconds too - let exec_opts_fractional = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0.1".to_string()), // 0.1 seconds warmup - max_time: Some("0.5".to_string()), // 0.5 seconds max time - min_time: None, - max_rounds: None, - min_rounds: None, - }, - )?; - - let times_fractional = exec_harness::walltime::perform( + let exec_opts_fractional = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0.1".to_string()), // 0.1 seconds warmup + max_time: Some("0.5".to_string()), // 0.5 seconds max time + min_time: None, + max_rounds: None, + min_rounds: None, + })?; + + let times_fractional = run_rounds( "test::fractional_seconds".to_string(), sleep_cmd(), &exec_opts_fractional, @@ -280,15 +261,13 @@ fn test_pure_numbers_as_seconds() -> Result<()> { #[test] fn test_single_long_execution() -> Result<()> { // Set max_time very low and warmup time high to force single execution - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("100ms".to_string()), - max_time: Some("100ms".to_string()), // Low max time, shorter than command duration - min_time: None, - max_rounds: None, - min_rounds: None, - }, - )?; + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("100ms".to_string()), + max_time: Some("100ms".to_string()), // Low max time, shorter than command duration + min_time: None, + max_rounds: None, + min_rounds: None, + })?; // Create a temporary directory for the test let tmpdir = TempDir::new()?; @@ -298,7 +277,7 @@ fn test_single_long_execution() -> Result<()> { let test_dir = tmpdir.path().join("lock_file"); let cmd = format!("sleep 1 && mkdir {}", test_dir.display()); - let times = exec_harness::walltime::perform( + let times = run_rounds( "test::single_long_execution".to_string(), vec!["sh".to_string(), "-c".to_string(), cmd.clone()], &exec_opts, @@ -310,7 +289,7 @@ fn test_single_long_execution() -> Result<()> { // Sanity check: any subsequent run should fail due to directory existing, to avoid false // positives assert!( - exec_harness::walltime::perform( + run_rounds( "test::single_long_execution".to_string(), vec!["sh".to_string(), "-c".to_string(), cmd], &exec_opts, @@ -325,15 +304,13 @@ fn test_single_long_execution() -> Result<()> { /// Test that a command with shell operators (&&) works correctly when passed as a single argument #[test] fn test_command_with_shell_operators() -> Result<()> { - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), - max_time: None, - min_time: None, - max_rounds: Some(1), - min_rounds: None, - }, - )?; + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), + max_time: None, + min_time: None, + max_rounds: Some(1), + min_rounds: None, + })?; let tmpdir = TempDir::new()?; let marker_file = tmpdir.path().join("marker.txt"); @@ -342,7 +319,7 @@ fn test_command_with_shell_operators() -> Result<()> { // The entire "echo first && echo second > marker.txt" should be passed as one argument to -c let cmd = format!("echo first && echo second > {}", marker_file.display()); - let times = exec_harness::walltime::perform( + let times = run_rounds( "test::shell_operators".to_string(), vec!["bash".to_string(), "-c".to_string(), cmd], &exec_opts, @@ -369,15 +346,13 @@ fn test_command_with_shell_operators() -> Result<()> { /// Test that a command with pipes works correctly #[test] fn test_command_with_pipes() -> Result<()> { - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), - max_time: None, - min_time: None, - max_rounds: Some(1), - min_rounds: None, - }, - )?; + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), + max_time: None, + min_time: None, + max_rounds: Some(1), + min_rounds: None, + })?; let tmpdir = TempDir::new()?; let output_file = tmpdir.path().join("output.txt"); @@ -388,7 +363,7 @@ fn test_command_with_pipes() -> Result<()> { output_file.display() ); - let times = exec_harness::walltime::perform( + let times = run_rounds( "test::pipes".to_string(), vec!["bash".to_string(), "-c".to_string(), cmd], &exec_opts, @@ -410,15 +385,13 @@ fn test_command_with_pipes() -> Result<()> { /// Test that a command with quotes in the argument works correctly #[test] fn test_command_with_embedded_quotes() -> Result<()> { - let exec_opts = exec_harness::walltime::ExecutionOptions::try_from( - exec_harness::walltime::WalltimeExecutionArgs { - warmup_time: Some("0s".to_string()), - max_time: None, - min_time: None, - max_rounds: Some(1), - min_rounds: None, - }, - )?; + let exec_opts = ExecutionOptions::try_from(WalltimeExecutionArgs { + warmup_time: Some("0s".to_string()), + max_time: None, + min_time: None, + max_rounds: Some(1), + min_rounds: None, + })?; let tmpdir = TempDir::new()?; let output_file = tmpdir.path().join("output.txt"); @@ -426,7 +399,7 @@ fn test_command_with_embedded_quotes() -> Result<()> { // This simulates: bash -c "echo 'hello world' > output.txt" let cmd = format!("echo 'hello world' > {}", output_file.display()); - let times = exec_harness::walltime::perform( + let times = run_rounds( "test::embedded_quotes".to_string(), vec!["bash".to_string(), "-c".to_string(), cmd], &exec_opts, From e2705591a6d133143398ffda597c6ddc672bc5ba Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 16 Jan 2026 11:29:19 +0100 Subject: [PATCH 2/5] feat(exec-harness): add support for analysis mode with memory instrument --- crates/exec-harness/src/analysis.rs | 24 ++++++++++++++++++++++++ crates/exec-harness/src/lib.rs | 12 ++++++++++++ crates/exec-harness/src/main.rs | 24 +++++++++++++++++++++--- 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 crates/exec-harness/src/analysis.rs diff --git a/crates/exec-harness/src/analysis.rs b/crates/exec-harness/src/analysis.rs new file mode 100644 index 00000000..9ad46571 --- /dev/null +++ b/crates/exec-harness/src/analysis.rs @@ -0,0 +1,24 @@ +use crate::prelude::*; + +use crate::uri::NameAndUri; +use codspeed::instrument_hooks::InstrumentHooks; +use std::process::Command; + +pub fn perform(name_and_uri: NameAndUri, command: Vec) -> Result<()> { + let hooks = InstrumentHooks::instance(); + + let mut cmd = Command::new(&command[0]); + cmd.args(&command[1..]); + hooks.start_benchmark().unwrap(); + let status = cmd.status(); + hooks.stop_benchmark().unwrap(); + let status = status.context("Failed to execute command")?; + + if !status.success() { + bail!("Command exited with non-zero status: {status}"); + } + + hooks.set_executed_benchmark(&name_and_uri.uri).unwrap(); + + Ok(()) +} diff --git a/crates/exec-harness/src/lib.rs b/crates/exec-harness/src/lib.rs index eb9f61a3..16995983 100644 --- a/crates/exec-harness/src/lib.rs +++ b/crates/exec-harness/src/lib.rs @@ -1,3 +1,15 @@ +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +pub mod analysis; mod prelude; pub mod uri; pub mod walltime; + +#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum MeasurementMode { + Walltime, + Memory, + Simulation, +} diff --git a/crates/exec-harness/src/main.rs b/crates/exec-harness/src/main.rs index a80416e9..4c290f1f 100644 --- a/crates/exec-harness/src/main.rs +++ b/crates/exec-harness/src/main.rs @@ -1,8 +1,11 @@ use anyhow::Result; use anyhow::bail; use clap::Parser; +use exec_harness::MeasurementMode; +use exec_harness::analysis; use exec_harness::uri; use exec_harness::walltime; +use log::debug; #[derive(Parser, Debug)] #[command(name = "exec-harness")] @@ -15,6 +18,10 @@ struct Args { #[arg(long)] name: Option, + /// Set by the runner, should be coherent with the executor being used + #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE", hide = true)] + measurement_mode: Option, + #[command(flatten)] execution_args: walltime::WalltimeExecutionArgs, @@ -29,6 +36,8 @@ fn main() -> Result<()> { .format_timestamp(None) .init(); + debug!("Starting exec-harness with pid {}", std::process::id()); + let args = Args::parse(); if args.command.is_empty() { @@ -37,10 +46,19 @@ fn main() -> Result<()> { let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command); - // Build execution options from CLI args - let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?; + match args.measurement_mode { + Some(MeasurementMode::Walltime) | None => { + let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?; - walltime::perform(bench_name_and_uri, args.command, &execution_options)?; + walltime::perform(bench_name_and_uri, args.command, &execution_options)?; + } + Some(MeasurementMode::Memory) => { + analysis::perform(bench_name_and_uri, args.command)?; + } + Some(MeasurementMode::Simulation) => { + bail!("Simulation measurement mode is not yet supported by exec-harness"); + } + } Ok(()) } From 0dcf95f24e1d434211e56b31037305ce9351502b Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 16 Jan 2026 16:36:13 +0100 Subject: [PATCH 3/5] chore(exec-harness): add prelude and remove linter dead_code suppression --- crates/exec-harness/src/lib.rs | 2 +- crates/exec-harness/src/main.rs | 4 +--- crates/exec-harness/src/walltime/config.rs | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/exec-harness/src/lib.rs b/crates/exec-harness/src/lib.rs index 16995983..2bfb757b 100644 --- a/crates/exec-harness/src/lib.rs +++ b/crates/exec-harness/src/lib.rs @@ -2,7 +2,7 @@ use clap::ValueEnum; use serde::{Deserialize, Serialize}; pub mod analysis; -mod prelude; +pub mod prelude; pub mod uri; pub mod walltime; diff --git a/crates/exec-harness/src/main.rs b/crates/exec-harness/src/main.rs index 4c290f1f..1417f345 100644 --- a/crates/exec-harness/src/main.rs +++ b/crates/exec-harness/src/main.rs @@ -1,11 +1,9 @@ -use anyhow::Result; -use anyhow::bail; use clap::Parser; use exec_harness::MeasurementMode; use exec_harness::analysis; +use exec_harness::prelude::*; use exec_harness::uri; use exec_harness::walltime; -use log::debug; #[derive(Parser, Debug)] #[command(name = "exec-harness")] diff --git a/crates/exec-harness/src/walltime/config.rs b/crates/exec-harness/src/walltime/config.rs index fe12f057..ccb87001 100644 --- a/crates/exec-harness/src/walltime/config.rs +++ b/crates/exec-harness/src/walltime/config.rs @@ -81,7 +81,6 @@ impl WalltimeExecutionArgs { /// /// Unfortunately, clap does not provide a built-in way to serialize args back to CLI format, // Clippy is very confused since this is used in the runner, but not in the binary of exec-harness - #[allow(dead_code)] pub fn to_cli_args(&self) -> Vec { let mut args = Vec::new(); From 19ece4e433b322a12f6ecb9ede1c23588420bc4e Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 16 Jan 2026 16:45:27 +0100 Subject: [PATCH 4/5] ci: cache benchmark executable --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b552d6..e7dd5b5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,9 +66,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: moonrepo/setup-rust@v1 + + - name: 'Install rust-toolchain.toml' + run: rustup toolchain install + - uses: Swatinem/rust-cache@v2 + - name: Install cargo codspeed + uses: taiki-e/install-action@v2 with: - bins: cargo-codspeed + tool: cargo-codspeed + - name: Build benchmarks run: cargo codspeed build -p runner-shared - name: Run benchmarks From 8a39cf5756d9353b7c08b9801ff833065965bc2d Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 16 Jan 2026 16:50:21 +0100 Subject: [PATCH 5/5] chore: use simulation in codspeed action --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7dd5b5c..128a9cb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,5 +80,5 @@ jobs: - name: Run benchmarks uses: CodSpeedHQ/action@v4 with: - mode: instrumentation + mode: simulation run: cargo codspeed run -p runner-shared