mirror of
https://github.com/sharkdp/hyperfine.git
synced 2024-08-17 01:40:28 +03:00
Merge 6df4aedb28
into 981db9d102
This commit is contained in:
commit
b4a2eccdd1
@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
|
||||
pub result: &'a BenchmarkResult,
|
||||
pub relative_speed: Scalar,
|
||||
pub relative_speed_stddev: Option<Scalar>,
|
||||
pub is_fastest: bool,
|
||||
pub is_reference: bool,
|
||||
// Less means faster
|
||||
pub relative_ordering: Ordering,
|
||||
}
|
||||
|
||||
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
|
||||
l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
|
||||
}
|
||||
|
||||
fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
||||
pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
||||
results
|
||||
.iter()
|
||||
.min_by(|&l, &r| compare_mean_time(l, r))
|
||||
@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
||||
|
||||
fn compute_relative_speeds<'a>(
|
||||
results: &'a [BenchmarkResult],
|
||||
fastest: &'a BenchmarkResult,
|
||||
reference: &'a BenchmarkResult,
|
||||
sort_order: SortOrder,
|
||||
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
|
||||
let mut results: Vec<_> = results
|
||||
.iter()
|
||||
.map(|result| {
|
||||
let is_fastest = result == fastest;
|
||||
let is_reference = result == reference;
|
||||
let relative_ordering = compare_mean_time(result, reference);
|
||||
|
||||
if result.mean == 0.0 {
|
||||
return BenchmarkResultWithRelativeSpeed {
|
||||
result,
|
||||
relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
|
||||
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
|
||||
relative_speed_stddev: None,
|
||||
is_fastest,
|
||||
is_reference,
|
||||
relative_ordering,
|
||||
};
|
||||
}
|
||||
|
||||
let ratio = result.mean / fastest.mean;
|
||||
let ratio = match relative_ordering {
|
||||
Ordering::Less => reference.mean / result.mean,
|
||||
Ordering::Equal => 1.0,
|
||||
Ordering::Greater => result.mean / reference.mean,
|
||||
};
|
||||
|
||||
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
|
||||
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
|
||||
let ratio_stddev = match (result.stddev, fastest.stddev) {
|
||||
let ratio_stddev = match (result.stddev, reference.stddev) {
|
||||
(Some(result_stddev), Some(fastest_stddev)) => Some(
|
||||
ratio
|
||||
* ((result_stddev / result.mean).powi(2)
|
||||
+ (fastest_stddev / fastest.mean).powi(2))
|
||||
+ (fastest_stddev / reference.mean).powi(2))
|
||||
.sqrt(),
|
||||
),
|
||||
_ => None,
|
||||
@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>(
|
||||
result,
|
||||
relative_speed: ratio,
|
||||
relative_speed_stddev: ratio_stddev,
|
||||
is_fastest,
|
||||
is_reference,
|
||||
relative_ordering,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>(
|
||||
results
|
||||
}
|
||||
|
||||
pub fn compute_with_check_from_reference<'a>(
|
||||
results: &'a [BenchmarkResult],
|
||||
reference: &'a BenchmarkResult,
|
||||
sort_order: SortOrder,
|
||||
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
|
||||
if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(compute_relative_speeds(results, reference, sort_order))
|
||||
}
|
||||
|
||||
pub fn compute_with_check(
|
||||
results: &[BenchmarkResult],
|
||||
sort_order: SortOrder,
|
||||
|
@ -1,10 +1,10 @@
|
||||
use colored::*;
|
||||
|
||||
use super::benchmark_result::BenchmarkResult;
|
||||
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
||||
use super::{relative_speed, Benchmark};
|
||||
use colored::*;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use crate::command::Commands;
|
||||
use crate::command::{Command, Commands};
|
||||
use crate::export::ExportManager;
|
||||
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
||||
|
||||
@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> {
|
||||
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
|
||||
};
|
||||
|
||||
let reference = self
|
||||
.options
|
||||
.reference_command
|
||||
.as_ref()
|
||||
.map(|cmd| Command::new(None, cmd));
|
||||
|
||||
executor.calibrate()?;
|
||||
|
||||
for (number, cmd) in self.commands.iter().enumerate() {
|
||||
for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
|
||||
self.results
|
||||
.push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
|
||||
|
||||
@ -65,31 +71,45 @@ impl<'a> Scheduler<'a> {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(annotated_results) = relative_speed::compute_with_check(
|
||||
let reference = self
|
||||
.options
|
||||
.reference_command
|
||||
.as_ref()
|
||||
.map(|_| &self.results[0])
|
||||
.unwrap_or_else(|| relative_speed::fastest_of(&self.results));
|
||||
|
||||
if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
|
||||
&self.results,
|
||||
reference,
|
||||
self.options.sort_order_speed_comparison,
|
||||
) {
|
||||
match self.options.sort_order_speed_comparison {
|
||||
SortOrder::MeanTime => {
|
||||
println!("{}", "Summary".bold());
|
||||
|
||||
let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
|
||||
let others = annotated_results.iter().filter(|r| !r.is_fastest);
|
||||
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
|
||||
let others = annotated_results.iter().filter(|r| !r.is_reference);
|
||||
|
||||
println!(
|
||||
" {} ran",
|
||||
fastest.result.command_with_unused_parameters.cyan()
|
||||
reference.result.command_with_unused_parameters.cyan()
|
||||
);
|
||||
|
||||
for item in others {
|
||||
let comparator = match item.relative_ordering {
|
||||
Ordering::Less => "slower",
|
||||
Ordering::Greater => "faster",
|
||||
Ordering::Equal => "as fast",
|
||||
};
|
||||
println!(
|
||||
"{}{} times faster than {}",
|
||||
"{}{} times {} than {}",
|
||||
format!("{:8.2}", item.relative_speed).bold().green(),
|
||||
if let Some(stddev) = item.relative_speed_stddev {
|
||||
format!(" ± {}", format!("{stddev:.2}").green())
|
||||
} else {
|
||||
"".into()
|
||||
},
|
||||
comparator,
|
||||
&item.result.command_with_unused_parameters.magenta()
|
||||
);
|
||||
}
|
||||
@ -101,7 +121,7 @@ impl<'a> Scheduler<'a> {
|
||||
println!(
|
||||
" {}{} {}",
|
||||
format!("{:10.2}", item.relative_speed).bold().green(),
|
||||
if item.is_fastest {
|
||||
if item.is_reference {
|
||||
" ".into()
|
||||
} else if let Some(stddev) = item.relative_speed_stddev {
|
||||
format!(" ± {}", format!("{stddev:5.2}").green())
|
||||
|
10
src/cli.rs
10
src/cli.rs
@ -86,6 +86,16 @@ fn build_command() -> Command {
|
||||
not every time as would happen with the --prepare option."
|
||||
),
|
||||
)
|
||||
.arg(
|
||||
Arg::new("reference")
|
||||
.long("reference")
|
||||
.action(ArgAction::Set)
|
||||
.value_name("CMD")
|
||||
.help(
|
||||
"The reference command for the relative comparison of results. \
|
||||
If this is unset, results are compared with the fastest command as reference."
|
||||
)
|
||||
)
|
||||
.arg(
|
||||
Arg::new("prepare")
|
||||
.long("prepare")
|
||||
|
@ -56,7 +56,7 @@ pub trait MarkupExporter {
|
||||
let min_str = format_duration_value(measurement.min, Some(unit)).0;
|
||||
let max_str = format_duration_value(measurement.max, Some(unit)).0;
|
||||
let rel_str = format!("{:.2}", entry.relative_speed);
|
||||
let rel_stddev_str = if entry.is_fastest {
|
||||
let rel_stddev_str = if entry.is_reference {
|
||||
"".into()
|
||||
} else if let Some(stddev) = entry.relative_speed_stddev {
|
||||
format!(" ± {stddev:.2}")
|
||||
|
@ -204,6 +204,9 @@ pub struct Options {
|
||||
/// Whether or not to ignore non-zero exit codes
|
||||
pub command_failure_action: CmdFailureAction,
|
||||
|
||||
// Command to use as a reference for relative speed comparison
|
||||
pub reference_command: Option<String>,
|
||||
|
||||
/// Command(s) to run before each timing run
|
||||
pub preparation_command: Option<Vec<String>>,
|
||||
|
||||
@ -245,6 +248,7 @@ impl Default for Options {
|
||||
warmup_count: 0,
|
||||
min_benchmarking_time: 3.0,
|
||||
command_failure_action: CmdFailureAction::RaiseError,
|
||||
reference_command: None,
|
||||
preparation_command: None,
|
||||
conclusion_command: None,
|
||||
setup_command: None,
|
||||
@ -304,6 +308,8 @@ impl Options {
|
||||
|
||||
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
||||
|
||||
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
||||
|
||||
options.preparation_command = matches
|
||||
.get_many::<String>("prepare")
|
||||
.map(|values| values.map(String::from).collect::<Vec<String>>());
|
||||
|
Loading…
Reference in New Issue
Block a user