Added option to manually specify a reference to compare the results to.

This commit is contained in:
Jan-Eric Nitschke 2024-04-27 20:16:24 +02:00 committed by David Peter
parent 4a00f1821c
commit 9806ed7694
5 changed files with 78 additions and 21 deletions

View File

@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub result: &'a BenchmarkResult,
pub relative_speed: Scalar,
pub relative_speed_stddev: Option<Scalar>,
pub is_fastest: bool,
pub is_reference: bool,
// Less means faster
pub relative_ordering: Ordering,
}
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
}
fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
results
.iter()
.min_by(|&l, &r| compare_mean_time(l, r))
@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
fn compute_relative_speeds<'a>(
results: &'a [BenchmarkResult],
fastest: &'a BenchmarkResult,
reference: &'a BenchmarkResult,
sort_order: SortOrder,
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
let mut results: Vec<_> = results
.iter()
.map(|result| {
let is_fastest = result == fastest;
let is_reference = result == reference;
let relative_ordering = compare_mean_time(result, reference);
if result.mean == 0.0 {
return BenchmarkResultWithRelativeSpeed {
result,
relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
relative_speed_stddev: None,
is_fastest,
is_reference,
relative_ordering,
};
}
let ratio = result.mean / fastest.mean;
let ratio = match relative_ordering {
Ordering::Less => reference.mean / result.mean,
Ordering::Equal => 1.0,
Ordering::Greater => result.mean / reference.mean,
};
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = match (result.stddev, fastest.stddev) {
let ratio_stddev = match (result.stddev, reference.stddev) {
(Some(result_stddev), Some(fastest_stddev)) => Some(
ratio
* ((result_stddev / result.mean).powi(2)
+ (fastest_stddev / fastest.mean).powi(2))
+ (fastest_stddev / reference.mean).powi(2))
.sqrt(),
),
_ => None,
@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>(
result,
relative_speed: ratio,
relative_speed_stddev: ratio_stddev,
is_fastest,
is_reference,
relative_ordering,
}
})
.collect();
@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>(
results
}
pub fn compute_with_check_from_reference<'a>(
results: &'a [BenchmarkResult],
reference: &'a BenchmarkResult,
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
return None;
}
Some(compute_relative_speeds(results, reference, sort_order))
}
pub fn compute_with_check(
results: &[BenchmarkResult],
sort_order: SortOrder,

View File

@ -1,10 +1,10 @@
use colored::*;
use super::benchmark_result::BenchmarkResult;
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
use super::{relative_speed, Benchmark};
use colored::*;
use std::cmp::Ordering;
use crate::command::Commands;
use crate::command::{Command, Commands};
use crate::export::ExportManager;
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> {
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
};
let reference = self
.options
.reference_command
.as_ref()
.map(|cmd| Command::new(None, cmd));
executor.calibrate()?;
for (number, cmd) in self.commands.iter().enumerate() {
for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
self.results
.push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
@ -65,31 +71,45 @@ impl<'a> Scheduler<'a> {
return;
}
if let Some(annotated_results) = relative_speed::compute_with_check(
let reference = self
.options
.reference_command
.as_ref()
.map(|_| &self.results[0])
.unwrap_or_else(|| relative_speed::fastest_of(&self.results));
if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
&self.results,
reference,
self.options.sort_order_speed_comparison,
) {
match self.options.sort_order_speed_comparison {
SortOrder::MeanTime => {
println!("{}", "Summary".bold());
let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
let others = annotated_results.iter().filter(|r| !r.is_fastest);
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
let others = annotated_results.iter().filter(|r| !r.is_reference);
println!(
" {} ran",
fastest.result.command_with_unused_parameters.cyan()
reference.result.command_with_unused_parameters.cyan()
);
for item in others {
let comparator = match item.relative_ordering {
Ordering::Less => "slower",
Ordering::Greater => "faster",
Ordering::Equal => "as fast",
};
println!(
"{}{} times faster than {}",
"{}{} times {} than {}",
format!("{:8.2}", item.relative_speed).bold().green(),
if let Some(stddev) = item.relative_speed_stddev {
format!(" ± {}", format!("{stddev:.2}").green())
} else {
"".into()
},
comparator,
&item.result.command_with_unused_parameters.magenta()
);
}
@ -101,7 +121,7 @@ impl<'a> Scheduler<'a> {
println!(
" {}{} {}",
format!("{:10.2}", item.relative_speed).bold().green(),
if item.is_fastest {
if item.is_reference {
" ".into()
} else if let Some(stddev) = item.relative_speed_stddev {
format!(" ± {}", format!("{stddev:5.2}").green())

View File

@ -86,6 +86,16 @@ fn build_command() -> Command {
not every time as would happen with the --prepare option."
),
)
.arg(
Arg::new("reference")
.long("reference")
.action(ArgAction::Set)
.value_name("CMD")
.help(
"The reference command for the relative comparison of results. \
If this is unset, results are compared with the fastest command as reference."
)
)
.arg(
Arg::new("prepare")
.long("prepare")

View File

@ -56,7 +56,7 @@ pub trait MarkupExporter {
let min_str = format_duration_value(measurement.min, Some(unit)).0;
let max_str = format_duration_value(measurement.max, Some(unit)).0;
let rel_str = format!("{:.2}", entry.relative_speed);
let rel_stddev_str = if entry.is_fastest {
let rel_stddev_str = if entry.is_reference {
"".into()
} else if let Some(stddev) = entry.relative_speed_stddev {
format!(" ± {stddev:.2}")

View File

@ -204,6 +204,9 @@ pub struct Options {
/// Whether or not to ignore non-zero exit codes
pub command_failure_action: CmdFailureAction,
// Command to use as a reference for relative speed comparison
pub reference_command: Option<String>,
/// Command(s) to run before each timing run
pub preparation_command: Option<Vec<String>>,
@ -245,6 +248,7 @@ impl Default for Options {
warmup_count: 0,
min_benchmarking_time: 3.0,
command_failure_action: CmdFailureAction::RaiseError,
reference_command: None,
preparation_command: None,
conclusion_command: None,
setup_command: None,
@ -304,6 +308,8 @@ impl Options {
options.setup_command = matches.get_one::<String>("setup").map(String::from);
options.reference_command = matches.get_one::<String>("reference").map(String::from);
options.preparation_command = matches
.get_many::<String>("prepare")
.map(|values| values.map(String::from).collect::<Vec<String>>());