diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 8bdc83b..b7c5a05 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: Scalar, pub relative_speed_stddev: Option, - pub is_fastest: bool, + pub is_reference: bool, + // Less means faster + pub relative_ordering: Ordering, } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal) } -fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { +pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { results .iter() .min_by(|&l, &r| compare_mean_time(l, r)) @@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], - fastest: &'a BenchmarkResult, + reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Vec> { let mut results: Vec<_> = results .iter() .map(|result| { - let is_fastest = result == fastest; + let is_reference = result == reference; + let relative_ordering = compare_mean_time(result, reference); if result.mean == 0.0 { return BenchmarkResultWithRelativeSpeed { result, - relative_speed: if is_fastest { 1.0 } else { f64::INFINITY }, + relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, relative_speed_stddev: None, - is_fastest, + is_reference, + relative_ordering, }; } - let ratio = result.mean / fastest.mean; + let ratio = match relative_ordering { + Ordering::Less => reference.mean / result.mean, + Ordering::Equal => 1.0, + Ordering::Greater => result.mean / reference.mean, + }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance asssumed to be 0, i.e. variables are assumed to be independent - let ratio_stddev = match (result.stddev, fastest.stddev) { + let ratio_stddev = match (result.stddev, reference.stddev) { (Some(result_stddev), Some(fastest_stddev)) => Some( ratio * ((result_stddev / result.mean).powi(2) - + (fastest_stddev / fastest.mean).powi(2)) + + (fastest_stddev / reference.mean).powi(2)) .sqrt(), ), _ => None, @@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>( result, relative_speed: ratio, relative_speed_stddev: ratio_stddev, - is_fastest, + is_reference, + relative_ordering, } }) .collect(); @@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>( results } +pub fn compute_with_check_from_reference<'a>( + results: &'a [BenchmarkResult], + reference: &'a BenchmarkResult, + sort_order: SortOrder, +) -> Option>> { + if fastest_of(results).mean == 0.0 || reference.mean == 0.0 { + return None; + } + + Some(compute_relative_speeds(results, reference, sort_order)) +} + pub fn compute_with_check( results: &[BenchmarkResult], sort_order: SortOrder, diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 941fa5a..c8ab49b 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -1,10 +1,10 @@ -use colored::*; - use super::benchmark_result::BenchmarkResult; use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor}; use super::{relative_speed, Benchmark}; +use colored::*; +use std::cmp::Ordering; -use crate::command::Commands; +use crate::command::{Command, Commands}; use crate::export::ExportManager; use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder}; @@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> { ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)), }; + let reference = self + .options + .reference_command + .as_ref() + .map(|cmd| Command::new(None, cmd)); + executor.calibrate()?; - for (number, cmd) in self.commands.iter().enumerate() { + for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() { self.results .push(Benchmark::new(number, cmd, self.options, &*executor).run()?); @@ -65,31 +71,45 @@ impl<'a> Scheduler<'a> { return; } - if let Some(annotated_results) = relative_speed::compute_with_check( + let reference = self + .options + .reference_command + .as_ref() + .map(|_| &self.results[0]) + .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); + + if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( &self.results, + reference, self.options.sort_order_speed_comparison, ) { match self.options.sort_order_speed_comparison { SortOrder::MeanTime => { println!("{}", "Summary".bold()); - let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap(); - let others = annotated_results.iter().filter(|r| !r.is_fastest); + let reference = annotated_results.iter().find(|r| r.is_reference).unwrap(); + let others = annotated_results.iter().filter(|r| !r.is_reference); println!( " {} ran", - fastest.result.command_with_unused_parameters.cyan() + reference.result.command_with_unused_parameters.cyan() ); for item in others { + let comparator = match item.relative_ordering { + Ordering::Less => "slower", + Ordering::Greater => "faster", + Ordering::Equal => "as fast", + }; println!( - "{}{} times faster than {}", + "{}{} times {} than {}", format!("{:8.2}", item.relative_speed).bold().green(), if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{stddev:.2}").green()) } else { "".into() }, + comparator, &item.result.command_with_unused_parameters.magenta() ); } @@ -101,7 +121,7 @@ impl<'a> Scheduler<'a> { println!( " {}{} {}", format!("{:10.2}", item.relative_speed).bold().green(), - if item.is_fastest { + if item.is_reference { " ".into() } else if let Some(stddev) = item.relative_speed_stddev { format!(" ± {}", format!("{stddev:5.2}").green()) diff --git a/src/cli.rs b/src/cli.rs index cc924cb..6684ebc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -86,6 +86,16 @@ fn build_command() -> Command { not every time as would happen with the --prepare option." ), ) + .arg( + Arg::new("reference") + .long("reference") + .action(ArgAction::Set) + .value_name("CMD") + .help( + "The reference command for the relative comparison of results. \ + If this is unset, results are compared with the fastest command as reference." + ) + ) .arg( Arg::new("prepare") .long("prepare") diff --git a/src/export/markup.rs b/src/export/markup.rs index abab3a0..eeeeb61 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -56,7 +56,7 @@ pub trait MarkupExporter { let min_str = format_duration_value(measurement.min, Some(unit)).0; let max_str = format_duration_value(measurement.max, Some(unit)).0; let rel_str = format!("{:.2}", entry.relative_speed); - let rel_stddev_str = if entry.is_fastest { + let rel_stddev_str = if entry.is_reference { "".into() } else if let Some(stddev) = entry.relative_speed_stddev { format!(" ± {stddev:.2}") diff --git a/src/options.rs b/src/options.rs index 7511bb7..5652b0a 100644 --- a/src/options.rs +++ b/src/options.rs @@ -204,6 +204,9 @@ pub struct Options { /// Whether or not to ignore non-zero exit codes pub command_failure_action: CmdFailureAction, + // Command to use as a reference for relative speed comparison + pub reference_command: Option, + /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -245,6 +248,7 @@ impl Default for Options { warmup_count: 0, min_benchmarking_time: 3.0, command_failure_action: CmdFailureAction::RaiseError, + reference_command: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -304,6 +308,8 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); + options.reference_command = matches.get_one::("reference").map(String::from); + options.preparation_command = matches .get_many::("prepare") .map(|values| values.map(String::from).collect::>());