diff --git a/Cargo.lock b/Cargo.lock index b8ef62f..65ab3be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,6 +88,7 @@ dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)", "indicatif 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -133,6 +134,72 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-bigint" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-complex" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-integer" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-iter" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-rational" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "owning_ref" version = "0.3.3" @@ -200,6 +267,11 @@ name = "regex-syntax" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rustc-serialize" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "smallvec" version = "0.6.0" @@ -210,6 +282,15 @@ name = "stable_deref_trait" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "statistical" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.6.0" @@ -329,6 +410,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cc4083e14b542ea3eb9b5f33ff48bd373a92d78687e74f4cc0a30caeb754f0ca" +"checksum num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "bdc1494b5912f088f260b775799468d9b9209ac60885d8186a547a0476289e23" +"checksum num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "58de7b4bf7cf5dbecb635a5797d489864eadd03b107930cbccf9e0fd7428b47c" +"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba" +"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01" +"checksum num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0c7cb72a95250d8a370105c828f388932373e0e94414919891a0f945222310fe" +"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070" "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" "checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412" "checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8" @@ -337,8 +425,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa" "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" +"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum smallvec 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44db0ecb22921ef790d17ae13a3f6d15784183ff5f2a01aa32098c7498d2b4b9" "checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b" +"checksum statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c139942f46d96c53b28420a2cdfb374629f122656bd9daef7fc221ed4d8ec228" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" diff --git a/Cargo.toml b/Cargo.toml index eef64ed..5cbd6cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ version = "0.2.0" [dependencies] ansi_term = "0.10" indicatif = "0.8" +statistical = "0.1" [dependencies.clap] version = "2" diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index d3653b0..fca2920 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -3,31 +3,35 @@ use std::process::{Command, Stdio}; use std::time::Instant; use ansi_term::Colour::{Cyan, Green, White, Yellow}; +use statistical::{mean, standard_deviation}; use hyperfine::internal::{get_progress_bar, HyperfineOptions, Second, Warnings, MIN_EXECUTION_TIME}; use hyperfine::format::{format_duration, format_duration_unit, Unit}; -use hyperfine::statistics::mean; /// Results from timing a single shell command pub struct TimingResult { /// Execution time in seconds - pub execution_time_sec: Second, + pub execution_time: Second, /// True if the command finished with exit code zero pub success: bool, } impl TimingResult { - fn new(execution_time_sec: Second, success: bool) -> TimingResult { + fn new(execution_time: Second, success: bool) -> TimingResult { TimingResult { - execution_time_sec, + execution_time, success, } } } /// Run the given shell command and measure the execution time -pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result { +pub fn time_shell_command( + shell_cmd: &str, + ignore_failure: bool, + shell_spawning_time: Option, +) -> io::Result { let start = Instant::now(); let status = Command::new("sh") @@ -48,9 +52,23 @@ pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result io::Result { let mut times: Vec = vec![]; for _ in 0..COUNT { - let res = time_shell_command("", false); + // Just run the shell without any command + let res = time_shell_command("", false, None); match res { Err(_) => { @@ -71,15 +90,14 @@ pub fn mean_shell_spawning_time() -> io::Result { )) } Ok(r) => { - times.push(r.execution_time_sec); + times.push(r.execution_time); } } bar.inc(1); } bar.finish_and_clear(); - let mean: f64 = mean(times.iter().cloned()); // TODO: get rid of .cloned() - Ok(mean) + Ok(mean(×)) } /// Run the benchmark for a single shell command @@ -89,15 +107,6 @@ pub fn run_benchmark( shell_spawning_time: Second, options: &HyperfineOptions, ) -> io::Result<()> { - // Helper function to compute corrected execution times - let get_execution_time = |r: &TimingResult| -> Second { - if r.execution_time_sec < shell_spawning_time { - 0.0 - } else { - r.execution_time_sec - shell_spawning_time - } - }; - println!( "{}{}: {}", White.bold().paint("Benchmark #"), @@ -106,14 +115,15 @@ pub fn run_benchmark( ); println!(); - let mut results = vec![]; + let mut execution_times: Vec = vec![]; + let mut all_succeeded = true; // Warmup phase if options.warmup_count > 0 { let bar = get_progress_bar(options.warmup_count, "Performing warmup runs"); for _ in 0..options.warmup_count { - let _ = time_shell_command(cmd, options.ignore_failure)?; + let _ = time_shell_command(cmd, options.ignore_failure, None)?; bar.inc(1); } bar.finish_and_clear(); @@ -123,10 +133,11 @@ pub fn run_benchmark( let bar = get_progress_bar(options.min_runs, "Initial time measurement"); // Initial timing run - let res = time_shell_command(cmd, options.ignore_failure)?; + let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?; // Determine number of benchmark runs - let runs_in_min_time = (options.min_time_sec / res.execution_time_sec) as u64; + let runs_in_min_time = + (options.min_time_sec / (res.execution_time + shell_spawning_time)) as u64; let count = if runs_in_min_time >= options.min_runs { runs_in_min_time @@ -137,7 +148,8 @@ pub fn run_benchmark( let count_remaining = count - 1; // Save the first result - results.push(res); + execution_times.push(res.execution_time); + all_succeeded = all_succeeded && res.success; // Re-configure the progress bar bar.set_length(count_remaining); @@ -145,14 +157,16 @@ pub fn run_benchmark( // Gather statistics for _ in 0..count_remaining { let msg = { - let execution_times = results.iter().map(&get_execution_time); - let mean = format_duration(mean(execution_times), Unit::Auto); + let mean = format_duration(mean(&execution_times), Unit::Auto); format!("Current estimate: {}", Green.paint(mean)) }; bar.set_message(&msg); - let res = time_shell_command(cmd, options.ignore_failure)?; - results.push(res); + let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?; + + execution_times.push(res.execution_time); + all_succeeded = all_succeeded && res.success; + bar.inc(1); } bar.finish_and_clear(); @@ -160,15 +174,11 @@ pub fn run_benchmark( // Compute statistical quantities // Corrected execution times - let execution_times = results.iter().map(&get_execution_time); - - let t_mean = mean(execution_times.clone()); - let t2_mean = mean(execution_times.clone().map(|t| t.powi(2))); - - let stddev = (t2_mean - t_mean.powi(2)).sqrt(); + let mean = mean(&execution_times); + let stddev = standard_deviation(&execution_times, Some(mean)); // Formatting and console output - let (mean_str, unit_mean) = format_duration_unit(t_mean, Unit::Auto); + let (mean_str, unit_mean) = format_duration_unit(mean, Unit::Auto); let stddev_str = format_duration(stddev, unit_mean); let time_fmt = format!("{} ± {}", mean_str, stddev_str); @@ -178,12 +188,12 @@ pub fn run_benchmark( let mut warnings = vec![]; // Check execution time - if execution_times.clone().any(|t| t < MIN_EXECUTION_TIME) { + if execution_times.iter().any(|&t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } // Check programm exit codes - if results.iter().any(|r| !r.success) { + if !all_succeeded { warnings.push(Warnings::NonZeroExitCode); } diff --git a/src/hyperfine/mod.rs b/src/hyperfine/mod.rs index e15af48..b9caf45 100644 --- a/src/hyperfine/mod.rs +++ b/src/hyperfine/mod.rs @@ -1,4 +1,3 @@ pub mod benchmark; pub mod format; pub mod internal; -pub mod statistics; diff --git a/src/hyperfine/statistics.rs b/src/hyperfine/statistics.rs deleted file mode 100644 index 4758025..0000000 --- a/src/hyperfine/statistics.rs +++ /dev/null @@ -1,13 +0,0 @@ -/// Calculate statistical average -pub fn mean(values: I) -> f64 -where - I: IntoIterator, -{ - let mut sum: f64 = 0.0; - let mut len: u64 = 0; - for v in values { - sum += v; - len += 1; - } - sum / (len as f64) -} diff --git a/src/main.rs b/src/main.rs index f94256f..3741b0e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ extern crate ansi_term; #[macro_use] extern crate clap; extern crate indicatif; +extern crate statistical; use std::cmp; use std::error::Error; @@ -81,7 +82,8 @@ fn main() { .unwrap_or(0); if let Some(min_runs) = matches.value_of("min-runs").and_then(&str_to_u64) { - options.min_runs = cmp::max(1, min_runs); + // we need at least two runs to compute a variance + options.min_runs = cmp::max(2, min_runs); } options.ignore_failure = matches.is_present("ignore-failure");