diff --git a/crates/exec-harness/src/analysis/mod.rs b/crates/exec-harness/src/analysis/mod.rs index 46d56191..506f775c 100644 --- a/crates/exec-harness/src/analysis/mod.rs +++ b/crates/exec-harness/src/analysis/mod.rs @@ -27,7 +27,14 @@ pub fn perform(commands: Vec) -> Result<()> { let status = status.context("Failed to execute command")?; if !status.success() { - bail!("Command exited with non-zero status: {status}"); + if benchmark_cmd.ignore_failure { + warn!( + "Command exited with non-zero status: {status}; \ + continuing because --ignore-failure is set" + ); + } else { + bail!("Command exited with non-zero status: {status}"); + } } hooks.set_executed_benchmark(&name_and_uri.uri).unwrap(); @@ -68,7 +75,14 @@ pub fn perform_with_valgrind(commands: Vec) -> Result<()> { bail_if_command_spawned_subprocesses_under_valgrind(child.id())?; if !status.success() { - bail!("Command exited with non-zero status: {status}"); + if benchmark_cmd.ignore_failure { + warn!( + "Command exited with non-zero status: {status}; \ + continuing because --ignore-failure is set" + ); + } else { + bail!("Command exited with non-zero status: {status}"); + } } } diff --git a/crates/exec-harness/src/lib.rs b/crates/exec-harness/src/lib.rs index 30cb21b4..fb21b0c2 100644 --- a/crates/exec-harness/src/lib.rs +++ b/crates/exec-harness/src/lib.rs @@ -36,6 +36,11 @@ pub struct BenchmarkCommand { /// Walltime execution options (flattened into the JSON object) #[serde(default)] pub walltime_args: walltime::WalltimeExecutionArgs, + + /// When true, a non-zero exit from the command is logged as a warning + /// instead of aborting execution. + #[serde(default)] + pub ignore_failure: bool, } /// Read and parse benchmark commands from stdin as JSON diff --git a/crates/exec-harness/src/main.rs b/crates/exec-harness/src/main.rs index 99cbf7cd..b7d944ec 100644 --- a/crates/exec-harness/src/main.rs +++ b/crates/exec-harness/src/main.rs @@ -20,6 +20,13 @@ struct Args { #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE", hide = true)] measurement_mode: Option, + /// Allow the benchmarked command to exit with a non-zero status code. + /// + /// When set, a non-zero exit from the benchmarked process is logged as a + /// warning and measurement continues, instead of aborting. + #[arg(short = 'i', long, default_value = "false")] + ignore_failure: bool, + #[command(flatten)] walltime_args: WalltimeExecutionArgs, @@ -51,6 +58,7 @@ fn main() -> Result<()> { command: args.command, name: args.name, walltime_args: args.walltime_args, + ignore_failure: args.ignore_failure, }], }; diff --git a/crates/exec-harness/src/walltime/benchmark_loop.rs b/crates/exec-harness/src/walltime/benchmark_loop.rs index a66b075f..99178e7a 100644 --- a/crates/exec-harness/src/walltime/benchmark_loop.rs +++ b/crates/exec-harness/src/walltime/benchmark_loop.rs @@ -10,6 +10,7 @@ pub fn run_rounds( bench_uri: String, command: Vec, config: &ExecutionOptions, + ignore_failure: bool, ) -> Result> { let warmup_time_ns = config.warmup_time_ns; let hooks = InstrumentHooks::instance(INTEGRATION_NAME, INTEGRATION_VERSION); @@ -27,7 +28,14 @@ pub fn run_rounds( let bench_round_end_ts_ns = InstrumentHooks::current_timestamp(); if !status.success() { - bail!("Command exited with non-zero status: {status}"); + if ignore_failure { + warn!( + "Command exited with non-zero status: {status}; \ + continuing because --ignore-failure is set" + ); + } else { + bail!("Command exited with non-zero status: {status}"); + } } Ok((bench_round_start_ts_ns, bench_round_end_ts_ns)) diff --git a/crates/exec-harness/src/walltime/mod.rs b/crates/exec-harness/src/walltime/mod.rs index d52870c9..a02c6391 100644 --- a/crates/exec-harness/src/walltime/mod.rs +++ b/crates/exec-harness/src/walltime/mod.rs @@ -28,8 +28,12 @@ pub fn perform(commands: Vec) -> Result<()> { .. } = name_and_uri; - let times_per_round_ns = - benchmark_loop::run_rounds(bench_uri.clone(), cmd.command, &execution_options)?; + let times_per_round_ns = benchmark_loop::run_rounds( + bench_uri.clone(), + cmd.command, + &execution_options, + cmd.ignore_failure, + )?; // Collect walltime results let max_time_ns = times_per_round_ns.iter().copied().max(); diff --git a/src/cli/exec/mod.rs b/src/cli/exec/mod.rs index 8719128d..2e74c451 100644 --- a/src/cli/exec/mod.rs +++ b/src/cli/exec/mod.rs @@ -30,6 +30,14 @@ pub struct ExecArgs { #[arg(long)] pub name: Option, + /// Allow the benchmarked command to exit with a non-zero status code. + /// + /// When set, a non-zero exit from the benchmarked process is logged as a + /// warning and measurement continues, instead of aborting. Mirrors + /// hyperfine's `-i / --ignore-failure`. + #[arg(short = 'i', long, default_value = "false")] + pub ignore_failure: bool, + /// The command to execute with the exec harness pub command: Vec, } @@ -106,6 +114,7 @@ pub async fn run( command: merged_args.command.clone(), name: merged_args.name.clone(), walltime_args: merged_args.walltime_args.clone(), + ignore_failure: merged_args.ignore_failure, }; let config = build_orchestrator_config( merged_args, diff --git a/src/cli/exec/multi_targets.rs b/src/cli/exec/multi_targets.rs index d24c16b9..a315583d 100644 --- a/src/cli/exec/multi_targets.rs +++ b/src/cli/exec/multi_targets.rs @@ -59,6 +59,7 @@ pub fn build_benchmark_targets( command, name: target.name.clone(), walltime_args, + ignore_failure: false, }) } TargetCommand::Entrypoint { entrypoint } => Ok(BenchmarkTarget::Entrypoint { @@ -80,10 +81,12 @@ pub fn build_exec_targets_pipe_command( command, name, walltime_args, + ignore_failure, } => Ok(BenchmarkCommand { command: command.clone(), name: name.clone(), walltime_args: walltime_args.clone(), + ignore_failure: *ignore_failure, }), crate::executor::config::BenchmarkTarget::Entrypoint { .. } => { bail!("Entrypoint targets cannot be used with exec-harness pipe command") diff --git a/src/executor/config.rs b/src/executor/config.rs index 057fd2fd..04c26257 100644 --- a/src/executor/config.rs +++ b/src/executor/config.rs @@ -22,6 +22,8 @@ pub enum BenchmarkTarget { command: Vec, name: Option, walltime_args: exec_harness::walltime::WalltimeExecutionArgs, + /// When true, a non-zero exit from the command is tolerated. + ignore_failure: bool, }, /// A command with built-in harness (e.g. `pytest --codspeed src`) Entrypoint { @@ -286,11 +288,13 @@ mod tests { command: vec!["exec1".into()], name: None, walltime_args: Default::default(), + ignore_failure: false, }, BenchmarkTarget::Exec { command: vec!["exec2".into()], name: None, walltime_args: Default::default(), + ignore_failure: false, }, ], modes: vec![RunnerMode::Simulation], @@ -305,6 +309,7 @@ mod tests { command: vec!["exec1".into()], name: None, walltime_args: Default::default(), + ignore_failure: false, }, BenchmarkTarget::Entrypoint { command: "cmd".into(), @@ -336,6 +341,7 @@ mod tests { command: vec!["exec1".into()], name: None, walltime_args: Default::default(), + ignore_failure: false, }, BenchmarkTarget::Entrypoint { command: "cmd".into(),