diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c1139f..9bfe473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ ## Bugfixes - Fix a bug in the outlier detection which would only detect "slow outliers" but not the fast ones (runs that are much faster than the rest of the benchmarking runs), see #329 +- Better error messages for very fast commands that would lead to inf/nan results in the relative + speed comparison, see #319 - Keep output colorized when the output is not interactive and `--style=full` or `--style=color` is used. ## Other diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index f8432a8..40b3a6a 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -5,7 +5,7 @@ use crate::hyperfine::internal::{compute_relative_speed, BenchmarkResultWithRela use crate::hyperfine::types::BenchmarkResult; use crate::hyperfine::units::Unit; -use std::io::Result; +use std::io::{Error, ErrorKind, Result}; #[derive(Default)] pub struct MarkdownExporter {} @@ -23,15 +23,20 @@ impl Exporter for MarkdownExporter { Unit::Second }; - let annotated_results = compute_relative_speed(results); + if let Some(annotated_results) = compute_relative_speed(results) { + let mut destination = start_table(unit); - let mut destination = start_table(unit); + for result in annotated_results { + add_table_row(&mut destination, &result, unit); + } - for result in annotated_results { - add_table_row(&mut destination, &result, unit); + Ok(destination) + } else { + Err(Error::new( + ErrorKind::Other, + "Relative speed comparison is not available for Markdown export.", + )) } - - Ok(destination) } } diff --git a/src/hyperfine/internal.rs b/src/hyperfine/internal.rs index 97961de..adeff63 100644 --- a/src/hyperfine/internal.rs +++ b/src/hyperfine/internal.rs @@ -52,6 +52,7 @@ pub fn min(vals: &[f64]) -> f64 { .unwrap() } +#[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: Scalar, @@ -65,31 +66,38 @@ fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { pub fn compute_relative_speed<'a>( results: &'a [BenchmarkResult], -) -> Vec> { +) -> Option>> { let fastest: &BenchmarkResult = results .iter() .min_by(|&l, &r| compare_mean_time(l, r)) .expect("at least one benchmark result"); - results - .iter() - .map(|result| { - let ratio = result.mean / fastest.mean; + if fastest.mean == 0.0 { + return None; + } - // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas - // Covariance asssumed to be 0, i.e. variables are assumed to be independent - let ratio_stddev = ratio - * ((result.stddev / result.mean).powi(2) + (fastest.stddev / fastest.mean).powi(2)) + Some( + results + .iter() + .map(|result| { + let ratio = result.mean / fastest.mean; + + // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas + // Covariance asssumed to be 0, i.e. variables are assumed to be independent + let ratio_stddev = ratio + * ((result.stddev / result.mean).powi(2) + + (fastest.stddev / fastest.mean).powi(2)) .sqrt(); - BenchmarkResultWithRelativeSpeed { - result, - relative_speed: ratio, - relative_speed_stddev: ratio_stddev, - is_fastest: result == fastest, - } - }) - .collect() + BenchmarkResultWithRelativeSpeed { + result, + relative_speed: ratio, + relative_speed_stddev: ratio_stddev, + is_fastest: result == fastest, + } + }) + .collect(), + ) } pub fn write_benchmark_comparison(results: &[BenchmarkResult]) { @@ -97,21 +105,31 @@ pub fn write_benchmark_comparison(results: &[BenchmarkResult]) { return; } - let mut annotated_results = compute_relative_speed(&results); - annotated_results.sort_by(|l, r| compare_mean_time(l.result, r.result)); + if let Some(mut annotated_results) = compute_relative_speed(&results) { + annotated_results.sort_by(|l, r| compare_mean_time(l.result, r.result)); - let fastest = &annotated_results[0]; - let others = &annotated_results[1..]; + let fastest = &annotated_results[0]; + let others = &annotated_results[1..]; - println!("{}", "Summary".bold()); - println!(" '{}' ran", fastest.result.command.cyan()); + println!("{}", "Summary".bold()); + println!(" '{}' ran", fastest.result.command.cyan()); - for item in others { - println!( - "{} ± {} times faster than '{}'", - format!("{:8.2}", item.relative_speed).bold().green(), - format!("{:.2}", item.relative_speed_stddev).green(), - &item.result.command.magenta() + for item in others { + println!( + "{} ± {} times faster than '{}'", + format!("{:8.2}", item.relative_speed).bold().green(), + format!("{:.2}", item.relative_speed_stddev).green(), + &item.result.command.magenta() + ); + } + } else { + eprintln!( + "{}: The benchmark comparison could not be computed as some benchmark times are zero. \ + This could be caused by background interference during the initial calibration phase \ + of hyperfine, in combination with very fast commands (faster than a few milliseconds). \ + Try to re-run the benchmark on a quiet system. If it does not help, you command is \ + most likely too fast to be accurately benchmarked by hyperfine.", + "Note".bold().red() ); } } @@ -125,12 +143,11 @@ fn test_max() { assert_eq!(1.0, max(&[-1.0, 1.0, 0.0])); } -#[test] -fn test_compute_relative_speed() { - use approx::assert_relative_eq; +#[cfg(test)] +fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { use std::collections::BTreeMap; - let create_result = |name: &str, mean| BenchmarkResult { + BenchmarkResult { command: name.into(), mean, stddev: 1.0, @@ -141,7 +158,12 @@ fn test_compute_relative_speed() { max: mean, times: None, parameters: BTreeMap::new(), - }; + } +} + +#[test] +fn test_compute_relative_speed() { + use approx::assert_relative_eq; let results = vec![ create_result("cmd1", 3.0), @@ -149,13 +171,22 @@ fn test_compute_relative_speed() { create_result("cmd3", 5.0), ]; - let annotated_results = compute_relative_speed(&results); + let annotated_results = compute_relative_speed(&results).unwrap(); assert_relative_eq!(1.5, annotated_results[0].relative_speed); assert_relative_eq!(1.0, annotated_results[1].relative_speed); assert_relative_eq!(2.5, annotated_results[2].relative_speed); } +#[test] +fn test_compute_relative_speed_for_zero_times() { + let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; + + let annotated_results = compute_relative_speed(&results); + + assert!(annotated_results.is_none()); +} + pub fn tokenize<'a>(values: &'a str) -> Vec { let mut tokens = vec![]; let mut buf = String::new(); diff --git a/src/main.rs b/src/main.rs index a0502e4..45e929b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ use crate::hyperfine::export::{ExportManager, ExportType}; use crate::hyperfine::internal::{tokenize, write_benchmark_comparison}; use crate::hyperfine::parameter_range::get_parameterized_commands; use crate::hyperfine::types::{ - BenchmarkResult, CmdFailureAction, Command, HyperfineOptions, OutputStyleOption, ParameterValue, + CmdFailureAction, Command, HyperfineOptions, OutputStyleOption, ParameterValue, }; use crate::hyperfine::units::Unit; @@ -27,7 +27,11 @@ pub fn error(message: &str) -> ! { } /// Runs the benchmark for the given commands -fn run(commands: &[Command<'_>], options: &HyperfineOptions) -> io::Result> { +fn run( + commands: &[Command<'_>], + options: &HyperfineOptions, + export_manager: &ExportManager, +) -> io::Result<()> { let shell_spawning_time = mean_shell_spawning_time(&options.shell, options.output_style, options.show_output)?; @@ -47,7 +51,21 @@ fn run(commands: &[Command<'_>], options: &HyperfineOptions) -> io::Result run(&commands, &opts), + Ok(ref opts) => run(&commands, &opts, &export_manager), Err(ref e) => error(&e.to_string()), }; match res { - Ok(timing_results) => { - let options = options.unwrap(); - - if options.output_style != OutputStyleOption::Disabled { - write_benchmark_comparison(&timing_results); - } - - let ans = export_manager.write_results(timing_results, options.time_unit); - if let Err(e) = ans { - error(&format!( - "The following error occurred while exporting: {}", - e - )); - } - } + Ok(_) => {} Err(e) => error(&e.to_string()), } }