mirror of
https://github.com/sharkdp/hyperfine.git
synced 2024-11-29 06:56:57 +03:00
Use statistical crate
This commit is contained in:
sharkdp
parent
c432fecd36
commit
05e5899643
90
Cargo.lock
generated
90
Cargo.lock
generated
@ -88,6 +88,7 @@ dependencies = [
|
||||
"ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"indicatif 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -133,6 +134,72 @@ dependencies = [
|
||||
"libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-iter"
|
||||
version = "0.1.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.1.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "owning_ref"
|
||||
version = "0.3.3"
|
||||
@ -200,6 +267,11 @@ name = "regex-syntax"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-serialize"
|
||||
version = "0.3.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "0.6.0"
|
||||
@ -210,6 +282,15 @@ name = "stable_deref_trait"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "statistical"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.6.0"
|
||||
@ -329,6 +410,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
|
||||
"checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121"
|
||||
"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
|
||||
"checksum num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cc4083e14b542ea3eb9b5f33ff48bd373a92d78687e74f4cc0a30caeb754f0ca"
|
||||
"checksum num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "bdc1494b5912f088f260b775799468d9b9209ac60885d8186a547a0476289e23"
|
||||
"checksum num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "58de7b4bf7cf5dbecb635a5797d489864eadd03b107930cbccf9e0fd7428b47c"
|
||||
"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba"
|
||||
"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01"
|
||||
"checksum num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0c7cb72a95250d8a370105c828f388932373e0e94414919891a0f945222310fe"
|
||||
"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070"
|
||||
"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
|
||||
"checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412"
|
||||
"checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8"
|
||||
@ -337,8 +425,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
|
||||
"checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa"
|
||||
"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
|
||||
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
|
||||
"checksum smallvec 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44db0ecb22921ef790d17ae13a3f6d15784183ff5f2a01aa32098c7498d2b4b9"
|
||||
"checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b"
|
||||
"checksum statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c139942f46d96c53b28420a2cdfb374629f122656bd9daef7fc221ed4d8ec228"
|
||||
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
|
||||
"checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327"
|
||||
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
|
||||
|
||||
@ -12,6 +12,7 @@ version = "0.2.0"
|
||||
[dependencies]
|
||||
ansi_term = "0.10"
|
||||
indicatif = "0.8"
|
||||
statistical = "0.1"
|
||||
|
||||
[dependencies.clap]
|
||||
version = "2"
|
||||
|
||||
@ -3,31 +3,35 @@ use std::process::{Command, Stdio};
|
||||
use std::time::Instant;
|
||||
|
||||
use ansi_term::Colour::{Cyan, Green, White, Yellow};
|
||||
use statistical::{mean, standard_deviation};
|
||||
|
||||
use hyperfine::internal::{get_progress_bar, HyperfineOptions, Second, Warnings, MIN_EXECUTION_TIME};
|
||||
use hyperfine::format::{format_duration, format_duration_unit, Unit};
|
||||
use hyperfine::statistics::mean;
|
||||
|
||||
/// Results from timing a single shell command
|
||||
pub struct TimingResult {
|
||||
/// Execution time in seconds
|
||||
pub execution_time_sec: Second,
|
||||
pub execution_time: Second,
|
||||
|
||||
/// True if the command finished with exit code zero
|
||||
pub success: bool,
|
||||
}
|
||||
|
||||
impl TimingResult {
|
||||
fn new(execution_time_sec: Second, success: bool) -> TimingResult {
|
||||
fn new(execution_time: Second, success: bool) -> TimingResult {
|
||||
TimingResult {
|
||||
execution_time_sec,
|
||||
execution_time,
|
||||
success,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the given shell command and measure the execution time
|
||||
pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result<TimingResult> {
|
||||
pub fn time_shell_command(
|
||||
shell_cmd: &str,
|
||||
ignore_failure: bool,
|
||||
shell_spawning_time: Option<Second>,
|
||||
) -> io::Result<TimingResult> {
|
||||
let start = Instant::now();
|
||||
|
||||
let status = Command::new("sh")
|
||||
@ -48,9 +52,23 @@ pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result<T
|
||||
|
||||
let duration = start.elapsed();
|
||||
|
||||
let execution_time_sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 * 1e-9;
|
||||
let execution_time = duration.as_secs() as f64 + duration.subsec_nanos() as f64 * 1e-9;
|
||||
|
||||
Ok(TimingResult::new(execution_time_sec, status.success()))
|
||||
// Correct for shell spawning time
|
||||
let execution_time_corrected = if let Some(spawning_time) = shell_spawning_time {
|
||||
if execution_time < spawning_time {
|
||||
0.0
|
||||
} else {
|
||||
execution_time - spawning_time
|
||||
}
|
||||
} else {
|
||||
execution_time
|
||||
};
|
||||
|
||||
Ok(TimingResult::new(
|
||||
execution_time_corrected,
|
||||
status.success(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Measure the average shell spawning time
|
||||
@ -60,7 +78,8 @@ pub fn mean_shell_spawning_time() -> io::Result<Second> {
|
||||
|
||||
let mut times: Vec<Second> = vec![];
|
||||
for _ in 0..COUNT {
|
||||
let res = time_shell_command("", false);
|
||||
// Just run the shell without any command
|
||||
let res = time_shell_command("", false, None);
|
||||
|
||||
match res {
|
||||
Err(_) => {
|
||||
@ -71,15 +90,14 @@ pub fn mean_shell_spawning_time() -> io::Result<Second> {
|
||||
))
|
||||
}
|
||||
Ok(r) => {
|
||||
times.push(r.execution_time_sec);
|
||||
times.push(r.execution_time);
|
||||
}
|
||||
}
|
||||
bar.inc(1);
|
||||
}
|
||||
bar.finish_and_clear();
|
||||
|
||||
let mean: f64 = mean(times.iter().cloned()); // TODO: get rid of .cloned()
|
||||
Ok(mean)
|
||||
Ok(mean(×))
|
||||
}
|
||||
|
||||
/// Run the benchmark for a single shell command
|
||||
@ -89,15 +107,6 @@ pub fn run_benchmark(
|
||||
shell_spawning_time: Second,
|
||||
options: &HyperfineOptions,
|
||||
) -> io::Result<()> {
|
||||
// Helper function to compute corrected execution times
|
||||
let get_execution_time = |r: &TimingResult| -> Second {
|
||||
if r.execution_time_sec < shell_spawning_time {
|
||||
0.0
|
||||
} else {
|
||||
r.execution_time_sec - shell_spawning_time
|
||||
}
|
||||
};
|
||||
|
||||
println!(
|
||||
"{}{}: {}",
|
||||
White.bold().paint("Benchmark #"),
|
||||
@ -106,14 +115,15 @@ pub fn run_benchmark(
|
||||
);
|
||||
println!();
|
||||
|
||||
let mut results = vec![];
|
||||
let mut execution_times: Vec<Second> = vec![];
|
||||
let mut all_succeeded = true;
|
||||
|
||||
// Warmup phase
|
||||
if options.warmup_count > 0 {
|
||||
let bar = get_progress_bar(options.warmup_count, "Performing warmup runs");
|
||||
|
||||
for _ in 0..options.warmup_count {
|
||||
let _ = time_shell_command(cmd, options.ignore_failure)?;
|
||||
let _ = time_shell_command(cmd, options.ignore_failure, None)?;
|
||||
bar.inc(1);
|
||||
}
|
||||
bar.finish_and_clear();
|
||||
@ -123,10 +133,11 @@ pub fn run_benchmark(
|
||||
let bar = get_progress_bar(options.min_runs, "Initial time measurement");
|
||||
|
||||
// Initial timing run
|
||||
let res = time_shell_command(cmd, options.ignore_failure)?;
|
||||
let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?;
|
||||
|
||||
// Determine number of benchmark runs
|
||||
let runs_in_min_time = (options.min_time_sec / res.execution_time_sec) as u64;
|
||||
let runs_in_min_time =
|
||||
(options.min_time_sec / (res.execution_time + shell_spawning_time)) as u64;
|
||||
|
||||
let count = if runs_in_min_time >= options.min_runs {
|
||||
runs_in_min_time
|
||||
@ -137,7 +148,8 @@ pub fn run_benchmark(
|
||||
let count_remaining = count - 1;
|
||||
|
||||
// Save the first result
|
||||
results.push(res);
|
||||
execution_times.push(res.execution_time);
|
||||
all_succeeded = all_succeeded && res.success;
|
||||
|
||||
// Re-configure the progress bar
|
||||
bar.set_length(count_remaining);
|
||||
@ -145,14 +157,16 @@ pub fn run_benchmark(
|
||||
// Gather statistics
|
||||
for _ in 0..count_remaining {
|
||||
let msg = {
|
||||
let execution_times = results.iter().map(&get_execution_time);
|
||||
let mean = format_duration(mean(execution_times), Unit::Auto);
|
||||
let mean = format_duration(mean(&execution_times), Unit::Auto);
|
||||
format!("Current estimate: {}", Green.paint(mean))
|
||||
};
|
||||
bar.set_message(&msg);
|
||||
|
||||
let res = time_shell_command(cmd, options.ignore_failure)?;
|
||||
results.push(res);
|
||||
let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?;
|
||||
|
||||
execution_times.push(res.execution_time);
|
||||
all_succeeded = all_succeeded && res.success;
|
||||
|
||||
bar.inc(1);
|
||||
}
|
||||
bar.finish_and_clear();
|
||||
@ -160,15 +174,11 @@ pub fn run_benchmark(
|
||||
// Compute statistical quantities
|
||||
|
||||
// Corrected execution times
|
||||
let execution_times = results.iter().map(&get_execution_time);
|
||||
|
||||
let t_mean = mean(execution_times.clone());
|
||||
let t2_mean = mean(execution_times.clone().map(|t| t.powi(2)));
|
||||
|
||||
let stddev = (t2_mean - t_mean.powi(2)).sqrt();
|
||||
let mean = mean(&execution_times);
|
||||
let stddev = standard_deviation(&execution_times, Some(mean));
|
||||
|
||||
// Formatting and console output
|
||||
let (mean_str, unit_mean) = format_duration_unit(t_mean, Unit::Auto);
|
||||
let (mean_str, unit_mean) = format_duration_unit(mean, Unit::Auto);
|
||||
let stddev_str = format_duration(stddev, unit_mean);
|
||||
let time_fmt = format!("{} ± {}", mean_str, stddev_str);
|
||||
|
||||
@ -178,12 +188,12 @@ pub fn run_benchmark(
|
||||
let mut warnings = vec![];
|
||||
|
||||
// Check execution time
|
||||
if execution_times.clone().any(|t| t < MIN_EXECUTION_TIME) {
|
||||
if execution_times.iter().any(|&t| t < MIN_EXECUTION_TIME) {
|
||||
warnings.push(Warnings::FastExecutionTime);
|
||||
}
|
||||
|
||||
// Check programm exit codes
|
||||
if results.iter().any(|r| !r.success) {
|
||||
if !all_succeeded {
|
||||
warnings.push(Warnings::NonZeroExitCode);
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
pub mod benchmark;
|
||||
pub mod format;
|
||||
pub mod internal;
|
||||
pub mod statistics;
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
/// Calculate statistical average
|
||||
pub fn mean<I>(values: I) -> f64
|
||||
where
|
||||
I: IntoIterator<Item = f64>,
|
||||
{
|
||||
let mut sum: f64 = 0.0;
|
||||
let mut len: u64 = 0;
|
||||
for v in values {
|
||||
sum += v;
|
||||
len += 1;
|
||||
}
|
||||
sum / (len as f64)
|
||||
}
|
||||
@ -2,6 +2,7 @@ extern crate ansi_term;
|
||||
#[macro_use]
|
||||
extern crate clap;
|
||||
extern crate indicatif;
|
||||
extern crate statistical;
|
||||
|
||||
use std::cmp;
|
||||
use std::error::Error;
|
||||
@ -81,7 +82,8 @@ fn main() {
|
||||
.unwrap_or(0);
|
||||
|
||||
if let Some(min_runs) = matches.value_of("min-runs").and_then(&str_to_u64) {
|
||||
options.min_runs = cmp::max(1, min_runs);
|
||||
// we need at least two runs to compute a variance
|
||||
options.min_runs = cmp::max(2, min_runs);
|
||||
}
|
||||
|
||||
options.ignore_failure = matches.is_present("ignore-failure");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user