From b48e85640dec10b04fab05c67be39d9697bef9e4 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:16:24 +0200 Subject: [PATCH 1/4] Added option to manually specify a reference to compare the results to. --- src/benchmark/relative_speed.rs | 41 +++++++++++++++++++++++++-------- src/benchmark/scheduler.rs | 40 ++++++++++++++++++++++++-------- src/cli.rs | 10 ++++++++ src/export/markup.rs | 2 +- src/options.rs | 6 +++++ 5 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 8bdc83b..b7c5a05 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: Scalar, pub relative_speed_stddev: Option, - pub is_fastest: bool, + pub is_reference: bool, + // Less means faster + pub relative_ordering: Ordering, } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal) } -fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { +pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { results .iter() .min_by(|&l, &r| compare_mean_time(l, r)) @@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], - fastest: &'a BenchmarkResult, + reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Vec> { let mut results: Vec<_> = results .iter() .map(|result| { - let is_fastest = result == fastest; + let is_reference = result == reference; + let relative_ordering = compare_mean_time(result, reference); if result.mean == 0.0 { return BenchmarkResultWithRelativeSpeed { result, - relative_speed: if is_fastest { 1.0 } else { f64::INFINITY }, + relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, relative_speed_stddev: None, - is_fastest, + is_reference, + relative_ordering, }; } - let ratio = result.mean / fastest.mean; + let ratio = match relative_ordering { + Ordering::Less => reference.mean / result.mean, + Ordering::Equal => 1.0, + Ordering::Greater => result.mean / reference.mean, + }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance asssumed to be 0, i.e. variables are assumed to be independent - let ratio_stddev = match (result.stddev, fastest.stddev) { + let ratio_stddev = match (result.stddev, reference.stddev) { (Some(result_stddev), Some(fastest_stddev)) => Some( ratio * ((result_stddev / result.mean).powi(2) - + (fastest_stddev / fastest.mean).powi(2)) + + (fastest_stddev / reference.mean).powi(2)) .sqrt(), ), _ => None, @@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>( result, relative_speed: ratio, relative_speed_stddev: ratio_stddev, - is_fastest, + is_reference, + relative_ordering, } }) .collect(); @@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>( results } +pub fn compute_with_check_from_reference<'a>( + results: &'a [BenchmarkResult], + reference: &'a BenchmarkResult, + sort_order: SortOrder, +) -> Option>> { + if fastest_of(results).mean == 0.0 || reference.mean == 0.0 { + return None; + } + + Some(compute_relative_speeds(results, reference, sort_order)) +} + pub fn compute_with_check( results: &[BenchmarkResult], sort_order: SortOrder, diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 941fa5a..c8ab49b 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -1,10 +1,10 @@ -use colored::*; - use super::benchmark_result::BenchmarkResult; use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor}; use super::{relative_speed, Benchmark}; +use colored::*; +use std::cmp::Ordering; -use crate::command::Commands; +use crate::command::{Command, Commands}; use crate::export::ExportManager; use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder}; @@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> { ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)), }; + let reference = self + .options + .reference_command + .as_ref() + .map(|cmd| Command::new(None, cmd)); + executor.calibrate()?; - for (number, cmd) in self.commands.iter().enumerate() { + for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() { self.results .push(Benchmark::new(number, cmd, self.options, &*executor).run()?); @@ -65,31 +71,45 @@ impl<'a> Scheduler<'a> { return; } - if let Some(annotated_results) = relative_speed::compute_with_check( + let reference = self + .options + .reference_command + .as_ref() + .map(|_| &self.results[0]) + .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); + + if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( &self.results, + reference, self.options.sort_order_speed_comparison, ) { match self.options.sort_order_speed_comparison { SortOrder::MeanTime => { println!("{}", "Summary".bold()); - let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap(); - let others = annotated_results.iter().filter(|r| !r.is_fastest); + let reference = annotated_results.iter().find(|r| r.is_reference).unwrap(); + let others = annotated_results.iter().filter(|r| !r.is_reference); println!( " {} ran", - fastest.result.command_with_unused_parameters.cyan() + reference.result.command_with_unused_parameters.cyan() ); for item in others { + let comparator = match item.relative_ordering { + Ordering::Less => "slower", + Ordering::Greater => "faster", + Ordering::Equal => "as fast", + }; println!( - "{}{} times faster than {}", + "{}{} times {} than {}", format!("{:8.2}", item.relative_speed).bold().green(), if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{stddev:.2}").green()) } else { "".into() }, + comparator, &item.result.command_with_unused_parameters.magenta() ); } @@ -101,7 +121,7 @@ impl<'a> Scheduler<'a> { println!( " {}{} {}", format!("{:10.2}", item.relative_speed).bold().green(), - if item.is_fastest { + if item.is_reference { " ".into() } else if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{stddev:5.2}").green()) diff --git a/src/cli.rs b/src/cli.rs index cc924cb..6684ebc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -86,6 +86,16 @@ fn build_command() -> Command { not every time as would happen with the --prepare option." ), ) + .arg( + Arg::new("reference") + .long("reference") + .action(ArgAction::Set) + .value_name("CMD") + .help( + "The reference command for the relative comparison of results. \ + If this is unset, results are compared with the fastest command as reference." + ) + ) .arg( Arg::new("prepare") .long("prepare") diff --git a/src/export/markup.rs b/src/export/markup.rs index abab3a0..eeeeb61 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -56,7 +56,7 @@ pub trait MarkupExporter { let min_str = format_duration_value(measurement.min, Some(unit)).0; let max_str = format_duration_value(measurement.max, Some(unit)).0; let rel_str = format!("{:.2}", entry.relative_speed); - let rel_stddev_str = if entry.is_fastest { + let rel_stddev_str = if entry.is_reference { "".into() } else if let Some(stddev) = entry.relative_speed_stddev { format!(" ± {stddev:.2}") diff --git a/src/options.rs b/src/options.rs index 7511bb7..5652b0a 100644 --- a/src/options.rs +++ b/src/options.rs @@ -204,6 +204,9 @@ pub struct Options { /// Whether or not to ignore non-zero exit codes pub command_failure_action: CmdFailureAction, + // Command to use as a reference for relative speed comparison + pub reference_command: Option, + /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -245,6 +248,7 @@ impl Default for Options { warmup_count: 0, min_benchmarking_time: 3.0, command_failure_action: CmdFailureAction::RaiseError, + reference_command: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -304,6 +308,8 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); + options.reference_command = matches.get_one::("reference").map(String::from); + options.preparation_command = matches .get_many::("prepare") .map(|values| values.map(String::from).collect::>()); From cd2fdbe410a65787921a3088cd1edc0a567a7da7 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Sun, 23 Jun 2024 18:38:07 +0200 Subject: [PATCH 2/4] Fix prepare and conclude behaviour and add tests. --- src/benchmark/relative_speed.rs | 14 +++ src/options.rs | 16 ++- tests/execution_order_tests.rs | 197 ++++++++++++++++++++++++++++++++ tests/integration_tests.rs | 85 ++++++++++++++ 4 files changed, 306 insertions(+), 6 deletions(-) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index b7c5a05..7449168 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -155,6 +155,20 @@ fn test_compute_relative_speed() { assert_relative_eq!(2.5, annotated_results[2].relative_speed); } +#[test] +fn test_compute_relative_speed_with_reference() { + use approx::assert_relative_eq; + + let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)]; + let reference = create_result("cmd2", 4.0); + + let annotated_results = + compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap(); + + assert_relative_eq!(2.0, annotated_results[0].relative_speed); + assert_relative_eq!(1.25, annotated_results[1].relative_speed); +} + #[test] fn test_compute_relative_speed_for_zero_times() { let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; diff --git a/src/options.rs b/src/options.rs index 5652b0a..72b1e1b 100644 --- a/src/options.rs +++ b/src/options.rs @@ -437,21 +437,25 @@ impl Options { } pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> { + let num_commands = commands.num_commands() + + if self.reference_command.is_some() { + 1 + } else { + 0 + }; if let Some(preparation_command) = &self.preparation_command { ensure!( - preparation_command.len() <= 1 - || commands.num_commands() == preparation_command.len(), + preparation_command.len() <= 1 || num_commands == preparation_command.len(), "The '--prepare' option has to be provided just once or N times, where N is the \ - number of benchmark commands." + number of benchmark commands including a potential reference." ); } if let Some(conclusion_command) = &self.conclusion_command { ensure!( - conclusion_command.len() <= 1 - || commands.num_commands() == conclusion_command.len(), + conclusion_command.len() <= 1 || num_commands == conclusion_command.len(), "The '--conclude' option has to be provided just once or N times, where N is the \ - number of benchmark commands." + number of benchmark commands including a potential reference." ); } diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 5404157..794cd22 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -54,6 +54,11 @@ impl ExecutionOrderTest { self.command(output) } + fn reference(&mut self, output: &str) -> &mut Self { + self.arg("--reference"); + self.command(output) + } + fn conclude(&mut self, output: &str) -> &mut Self { self.arg("--conclude"); self.command(output) @@ -364,3 +369,195 @@ fn multiple_parameter_values() { .expect_output("command 3 b") .run(); } + +#[test] +fn reference_is_executed_first() { + ExecutionOrderTest::new() + .arg("--runs=1") + .reference("reference") + .command("command 1") + .command("command 2") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 2") + .run(); +} + +#[test] +fn reference_is_executed_first_parameter_value() { + ExecutionOrderTest::new() + .arg("--runs=2") + .reference("reference") + .arg("--parameter-list") + .arg("number") + .arg("1,2,3") + .command("command {number}") + .expect_output("reference") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("command 3") + .expect_output("command 3") + .run(); +} + +#[test] +fn reference_is_executed_separately_from_commands() { + ExecutionOrderTest::new() + .arg("--runs=1") + .reference("command 1") + .command("command 1") + .command("command 2") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .run(); +} + +#[test] +fn setup_prepare_reference_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .prepare("prepare") + .reference("reference") + .command("command1") + .command("command2") + .conclude("conclude") + .cleanup("cleanup") + // reference + .expect_output("setup") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("cleanup") + .run(); +} + +#[test] +fn setup_separete_prepare_separate_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .cleanup("cleanup") + .prepare("prepare1") + .command("command1") + .conclude("conclude1") + .prepare("prepare2") + .command("command2") + .conclude("conclude2") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") + .run(); +} + +#[test] +fn setup_separete_prepare_reference_separate_conclude_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--warmup=1") + .arg("--runs=2") + .setup("setup") + .cleanup("cleanup") + .prepare("prepareref") + .reference("reference") + .conclude("concluderef") + .prepare("prepare1") + .command("command1") + .conclude("conclude1") + .prepare("prepare2") + .command("command2") + .conclude("conclude2") + // reference + .expect_output("setup") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") + .run(); +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2bbde41..3a58237 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() { .assert() .success(); + hyperfine() + .arg("--runs=1") + .arg("--prepare=echo ref") + .arg("--prepare=echo a") + .arg("--prepare=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .success(); + hyperfine() .arg("--runs=1") .arg("--prepare=echo a") @@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() { .stderr(predicate::str::contains( "The '--prepare' option has to be provided", )); + + hyperfine() + .arg("--runs=1") + .arg("--prepare=echo a") + .arg("--prepare=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .failure() + .stderr(predicate::str::contains( + "The '--prepare' option has to be provided", + )); } #[test] @@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() { .assert() .success(); + hyperfine() + .arg("--runs=1") + .arg("--conclude=echo ref") + .arg("--conclude=echo a") + .arg("--conclude=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .success(); + hyperfine() .arg("--runs=1") .arg("--conclude=echo a") @@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() { .stderr(predicate::str::contains( "The '--conclude' option has to be provided", )); + + hyperfine() + .arg("--runs=1") + .arg("--conclude=echo a") + .arg("--conclude=echo b") + .arg("--reference=echo ref") + .arg("echo a") + .arg("echo b") + .assert() + .failure() + .stderr(predicate::str::contains( + "The '--conclude' option has to be provided", + )); } #[test] @@ -413,6 +461,20 @@ fn shows_benchmark_comparison_with_relative_times() { ); } +#[test] +fn shows_benchmark_comparison_relative_to_reference() { + hyperfine_debug() + .arg("--reference=sleep 2.0") + .arg("sleep 1.0") + .arg("sleep 3.0") + .assert() + .success() + .stdout( + predicate::str::contains("2.00 ± 0.00 times slower") + .and(predicate::str::contains("1.50 ± 0.00 times faster")), + ); +} + #[test] fn performs_all_benchmarks_in_parameter_scan() { hyperfine_debug() @@ -434,6 +496,29 @@ fn performs_all_benchmarks_in_parameter_scan() { ); } +#[test] +fn performs_reference_and_all_benchmarks_in_parameter_scan() { + hyperfine_debug() + .arg("--reference=sleep 25") + .arg("--parameter-scan") + .arg("time") + .arg("30") + .arg("45") + .arg("--parameter-step-size") + .arg("5") + .arg("sleep {time}") + .assert() + .success() + .stdout( + predicate::str::contains("Benchmark 1: sleep 25") + .and(predicate::str::contains("Benchmark 2: sleep 30")) + .and(predicate::str::contains("Benchmark 3: sleep 35")) + .and(predicate::str::contains("Benchmark 4: sleep 40")) + .and(predicate::str::contains("Benchmark 5: sleep 45")) + .and(predicate::str::contains("Benchmark 6: sleep 50").not()), + ); +} + #[test] fn intermediate_results_are_not_exported_to_stdout() { hyperfine_debug() From f459aeeddf440392fbee276fb90e88fa02a354be Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:04:35 +0200 Subject: [PATCH 3/4] Fix typo --- tests/execution_order_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 794cd22..6ccc2e5 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -468,7 +468,7 @@ fn setup_prepare_reference_conclude_cleanup_combined() { } #[test] -fn setup_separete_prepare_separate_conclude_cleanup_combined() { +fn setup_separate_prepare_separate_conclude_cleanup_combined() { ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") @@ -508,7 +508,7 @@ fn setup_separete_prepare_separate_conclude_cleanup_combined() { } #[test] -fn setup_separete_prepare_reference_separate_conclude_cleanup_combined() { +fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") From 15e7fe3147a00e052d62ca7ea92e291f761acae0 Mon Sep 17 00:00:00 2001 From: Jan-Eric Nitschke <47750513+JanEricNitschke@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:48:18 +0200 Subject: [PATCH 4/4] Change message for identical time and add test for it --- src/benchmark/scheduler.rs | 31 +++++++++++++++++++++---------- tests/integration_tests.rs | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index c8ab49b..a88cda7 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -96,19 +96,30 @@ impl<'a> Scheduler<'a> { ); for item in others { + let stddev = if let Some(stddev) = item.relative_speed_stddev { + format!(" ± {}", format!("{:.2}", stddev).green()) + } else { + "".into() + }; let comparator = match item.relative_ordering { - Ordering::Less => "slower", - Ordering::Greater => "faster", - Ordering::Equal => "as fast", + Ordering::Less => format!( + "{}{} times slower than", + format!("{:8.2}", item.relative_speed).bold().green(), + stddev + ), + Ordering::Greater => format!( + "{}{} times faster than", + format!("{:8.2}", item.relative_speed).bold().green(), + stddev + ), + Ordering::Equal => format!( + " As fast ({}{}) as", + format!("{:.2}", item.relative_speed).bold().green(), + stddev + ), }; println!( - "{}{} times {} than {}", - format!("{:8.2}", item.relative_speed).bold().green(), - if let Some(stddev) = item.relative_speed_stddev { - format!(" ± {}", format!("{stddev:.2}").green()) - } else { - "".into() - }, + "{} {}", comparator, &item.result.command_with_unused_parameters.magenta() ); diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 3a58237..873f400 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -461,6 +461,24 @@ fn shows_benchmark_comparison_with_relative_times() { ); } +#[test] +fn shows_benchmark_comparison_with_same_time() { + hyperfine_debug() + .arg("--command-name=A") + .arg("--command-name=B") + .arg("sleep 1.0") + .arg("sleep 1.0") + .arg("sleep 2.0") + .arg("sleep 1000.0") + .assert() + .success() + .stdout( + predicate::str::contains("As fast (1.00 ± 0.00) as") + .and(predicate::str::contains("2.00 ± 0.00 times faster")) + .and(predicate::str::contains("1000.00 ± 0.00 times faster")), + ); +} + #[test] fn shows_benchmark_comparison_relative_to_reference() { hyperfine_debug()