From 83f04cdd31704c48175b47f1f1482193108036a7 Mon Sep 17 00:00:00 2001 From: David Peter Date: Sun, 10 Nov 2024 22:21:22 +0100 Subject: [PATCH] Add iteration information to failure error message --- src/benchmark/executor.rs | 49 +++++++++++++++++++++++++++++++++++---- src/benchmark/mod.rs | 29 ++++++++++++++++++----- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index 098451b..c60dbe2 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -16,11 +16,28 @@ use super::timing_result::TimingResult; use anyhow::{bail, Context, Result}; use statistical::mean; +pub enum BenchmarkIteration { + NonBenchmarkRun, + Warmup(u64), + Benchmark(u64), +} + +impl BenchmarkIteration { + pub fn to_env_var_value(&self) -> Option { + match self { + BenchmarkIteration::NonBenchmarkRun => None, + BenchmarkIteration::Warmup(i) => Some(format!("warmup-{}", i)), + BenchmarkIteration::Benchmark(i) => Some(format!("{}", i)), + } + } +} + pub trait Executor { /// Run the given command and measure the execution time fn run_command_and_measure( &self, command: &Command<'_>, + iteration: BenchmarkIteration, command_failure_action: Option, ) -> Result<(TimingResult, ExitStatus)>; @@ -39,6 +56,7 @@ pub trait Executor { fn run_command_and_measure_common( mut command: std::process::Command, + iteration: BenchmarkIteration, command_failure_action: CmdFailureAction, command_input_policy: &CommandInputPolicy, command_output_policy: &CommandOutputPolicy, @@ -53,17 +71,29 @@ fn run_command_and_measure_common( randomized_environment_offset::value(), ); + if let Some(value) = iteration.to_env_var_value() { + command.env("HYPERFINE_ITERATION", value); + } + let result = execute_and_measure(command) .with_context(|| format!("Failed to run command '{command_name}'"))?; if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() { + let when = match iteration { + BenchmarkIteration::NonBenchmarkRun => "a non-benchmark run".to_string(), + BenchmarkIteration::Warmup(0) => "the first warmup run".to_string(), + BenchmarkIteration::Warmup(i) => format!("warmup iteration {i}"), + BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(), + BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"), + }; bail!( - "{}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ + "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ Alternatively, use the '--show-output' option to debug what went wrong.", - result.status.code().map_or( + cause=result.status.code().map_or( "The process has been terminated by a signal".into(), - |c| format!("Command terminated with non-zero exit code: {c}") - ) + |c| format!("Command terminated with non-zero exit code {c}") + + ), ); } @@ -84,10 +114,12 @@ impl<'a> Executor for RawExecutor<'a> { fn run_command_and_measure( &self, command: &Command<'_>, + iteration: BenchmarkIteration, command_failure_action: Option, ) -> Result<(TimingResult, ExitStatus)> { let result = run_command_and_measure_common( command.get_command()?, + iteration, command_failure_action.unwrap_or(self.options.command_failure_action), &self.options.command_input_policy, &self.options.command_output_policy, @@ -133,6 +165,7 @@ impl<'a> Executor for ShellExecutor<'a> { fn run_command_and_measure( &self, command: &Command<'_>, + iteration: BenchmarkIteration, command_failure_action: Option, ) -> Result<(TimingResult, ExitStatus)> { let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); @@ -149,6 +182,7 @@ impl<'a> Executor for ShellExecutor<'a> { let mut result = run_command_and_measure_common( command_builder, + iteration, command_failure_action.unwrap_or(self.options.command_failure_action), &self.options.command_input_policy, &self.options.command_output_policy, @@ -191,7 +225,11 @@ impl<'a> Executor for ShellExecutor<'a> { for _ in 0..COUNT { // Just run the shell without any command - let res = self.run_command_and_measure(&Command::new(None, ""), None); + let res = self.run_command_and_measure( + &Command::new(None, ""), + BenchmarkIteration::NonBenchmarkRun, + None, + ); match res { Err(_) => { @@ -260,6 +298,7 @@ impl Executor for MockExecutor { fn run_command_and_measure( &self, command: &Command<'_>, + _iteration: BenchmarkIteration, _command_failure_action: Option, ) -> Result<(TimingResult, ExitStatus)> { #[cfg(unix)] diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 35c0e27..1d41c64 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -6,6 +6,7 @@ pub mod timing_result; use std::cmp; +use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{CmdFailureAction, ExecutorKind, Options, OutputStyleOption}; use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; @@ -57,7 +58,11 @@ impl<'a> Benchmark<'a> { error_output: &'static str, ) -> Result { self.executor - .run_command_and_measure(command, Some(CmdFailureAction::RaiseError)) + .run_command_and_measure( + command, + executor::BenchmarkIteration::NonBenchmarkRun, + Some(CmdFailureAction::RaiseError), + ) .map(|r| r.0) .map_err(|_| anyhow!(error_output)) } @@ -187,9 +192,13 @@ impl<'a> Benchmark<'a> { None }; - for _ in 0..self.options.warmup_count { + for i in 0..self.options.warmup_count { let _ = run_preparation_command()?; - let _ = self.executor.run_command_and_measure(self.command, None)?; + let _ = self.executor.run_command_and_measure( + self.command, + BenchmarkIteration::Warmup(i), + None, + )?; let _ = run_conclusion_command()?; if let Some(bar) = progress_bar.as_ref() { bar.inc(1) @@ -216,7 +225,11 @@ impl<'a> Benchmark<'a> { preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); // Initial timing run - let (res, status) = self.executor.run_command_and_measure(self.command, None)?; + let (res, status) = self.executor.run_command_and_measure( + self.command, + BenchmarkIteration::Benchmark(0), + None, + )?; let success = status.success(); let conclusion_result = run_conclusion_command()?; @@ -260,7 +273,7 @@ impl<'a> Benchmark<'a> { } // Gather statistics (perform the actual benchmark) - for _ in 0..count_remaining { + for i in 0..count_remaining { run_preparation_command()?; let msg = { @@ -272,7 +285,11 @@ impl<'a> Benchmark<'a> { bar.set_message(msg.to_owned()) } - let (res, status) = self.executor.run_command_and_measure(self.command, None)?; + let (res, status) = self.executor.run_command_and_measure( + self.command, + BenchmarkIteration::Benchmark(i + 1), + None, + )?; let success = status.success(); times_real.push(res.time_real);