mirror of
https://github.com/sharkdp/hyperfine.git
synced 2024-08-17 01:40:28 +03:00
Compare commits
10 Commits
d634d8e3b2
...
92c930de3c
Author | SHA1 | Date | |
---|---|---|---|
|
92c930de3c | ||
|
df34f32dca | ||
|
4c3f6ad4f8 | ||
|
13547e973c | ||
|
9134337b62 | ||
|
d4749b015f | ||
|
9806ed7694 | ||
|
02975df2ad | ||
|
c83b4126ea | ||
|
4e8cb3fec0 |
153
Cargo.lock
generated
153
Cargo.lock
generated
@ -13,17 +13,6 @@ dependencies = [
|
|||||||
"version_check",
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ahash"
|
|
||||||
version = "0.8.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"once_cell",
|
|
||||||
"version_check",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aho-corasick"
|
name = "aho-corasick"
|
||||||
version = "1.1.1"
|
version = "1.1.1"
|
||||||
@ -158,47 +147,26 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "borsh"
|
name = "borsh"
|
||||||
version = "0.10.3"
|
version = "1.5.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
|
checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"borsh-derive",
|
"borsh-derive",
|
||||||
"hashbrown 0.13.2",
|
"cfg_aliases",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "borsh-derive"
|
name = "borsh-derive"
|
||||||
version = "0.10.3"
|
version = "1.5.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7"
|
checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"borsh-derive-internal",
|
"once_cell",
|
||||||
"borsh-schema-derive-internal",
|
|
||||||
"proc-macro-crate",
|
"proc-macro-crate",
|
||||||
"proc-macro2",
|
|
||||||
"syn 1.0.109",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "borsh-derive-internal"
|
|
||||||
version = "0.10.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 1.0.109",
|
"syn 2.0.37",
|
||||||
]
|
"syn_derive",
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "borsh-schema-derive-internal"
|
|
||||||
version = "0.10.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 1.0.109",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -246,6 +214,12 @@ version = "1.0.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg_aliases"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.4.12"
|
version = "4.4.12"
|
||||||
@ -366,6 +340,12 @@ version = "0.3.6"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "equivalent"
|
||||||
|
version = "1.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "errno"
|
name = "errno"
|
||||||
version = "0.3.8"
|
version = "0.3.8"
|
||||||
@ -420,17 +400,14 @@ version = "0.12.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash 0.7.6",
|
"ahash",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.13.2"
|
version = "0.14.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
|
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||||
dependencies = [
|
|
||||||
"ahash 0.8.3",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyperfine"
|
name = "hyperfine"
|
||||||
@ -459,6 +436,16 @@ dependencies = [
|
|||||||
"windows-sys 0.48.0",
|
"windows-sys 0.48.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "indexmap"
|
||||||
|
version = "2.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
|
||||||
|
dependencies = [
|
||||||
|
"equivalent",
|
||||||
|
"hashbrown 0.14.5",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "indicatif"
|
name = "indicatif"
|
||||||
version = "0.17.4"
|
version = "0.17.4"
|
||||||
@ -622,9 +609,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.18.0"
|
version = "1.19.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "portable-atomic"
|
name = "portable-atomic"
|
||||||
@ -671,11 +658,34 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro-crate"
|
name = "proc-macro-crate"
|
||||||
version = "0.1.5"
|
version = "3.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
|
checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"toml",
|
"toml_edit",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-error-attr",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error-attr"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -944,9 +954,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rust_decimal"
|
name = "rust_decimal"
|
||||||
version = "1.32.0"
|
version = "1.35.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd"
|
checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrayvec",
|
"arrayvec",
|
||||||
"borsh",
|
"borsh",
|
||||||
@ -1064,6 +1074,18 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "syn_derive"
|
||||||
|
version = "0.1.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-error",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.37",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tap"
|
name = "tap"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
@ -1135,12 +1157,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml"
|
name = "toml_datetime"
|
||||||
version = "0.5.11"
|
version = "0.6.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
|
checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "toml_edit"
|
||||||
|
version = "0.21.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde",
|
"indexmap",
|
||||||
|
"toml_datetime",
|
||||||
|
"winnow",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@ -1408,6 +1438,15 @@ version = "0.52.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
|
checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "winnow"
|
||||||
|
version = "0.5.40"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wyz"
|
name = "wyz"
|
||||||
version = "0.5.1"
|
version = "0.5.1"
|
||||||
|
@ -23,7 +23,7 @@ statistical = "1.0"
|
|||||||
csv = "1.1"
|
csv = "1.1"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
rust_decimal = "1.31"
|
rust_decimal = "1.35"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
shell-words = "1.0"
|
shell-words = "1.0"
|
||||||
thiserror = "1.0"
|
thiserror = "1.0"
|
||||||
@ -42,7 +42,7 @@ windows-sys = { version = "0.48", features = [
|
|||||||
] }
|
] }
|
||||||
|
|
||||||
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
|
[target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
|
||||||
once_cell = "1.17"
|
once_cell = "1.19"
|
||||||
|
|
||||||
[target.'cfg(target_os="linux")'.dependencies]
|
[target.'cfg(target_os="linux")'.dependencies]
|
||||||
nix = { version = "0.27.1", features = ["zerocopy"] }
|
nix = { version = "0.27.1", features = ["zerocopy"] }
|
||||||
|
@ -180,6 +180,7 @@ impl<'a> Executor for ShellExecutor<'a> {
|
|||||||
COUNT,
|
COUNT,
|
||||||
"Measuring shell spawning time",
|
"Measuring shell spawning time",
|
||||||
self.options.output_style,
|
self.options.output_style,
|
||||||
|
self.options.show_elapsed,
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@ -214,7 +215,8 @@ impl<'a> Executor for ShellExecutor<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(bar) = progress_bar.as_ref() {
|
if let Some(bar) = progress_bar.as_ref() {
|
||||||
bar.inc(1)
|
bar.inc(1);
|
||||||
|
bar.reset_elapsed();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -182,6 +182,7 @@ impl<'a> Benchmark<'a> {
|
|||||||
self.options.warmup_count,
|
self.options.warmup_count,
|
||||||
"Performing warmup runs",
|
"Performing warmup runs",
|
||||||
self.options.output_style,
|
self.options.output_style,
|
||||||
|
self.options.show_elapsed,
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@ -192,7 +193,8 @@ impl<'a> Benchmark<'a> {
|
|||||||
let _ = self.executor.run_command_and_measure(self.command, None)?;
|
let _ = self.executor.run_command_and_measure(self.command, None)?;
|
||||||
let _ = run_conclusion_command()?;
|
let _ = run_conclusion_command()?;
|
||||||
if let Some(bar) = progress_bar.as_ref() {
|
if let Some(bar) = progress_bar.as_ref() {
|
||||||
bar.inc(1)
|
bar.inc(1);
|
||||||
|
bar.reset_elapsed();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(bar) = progress_bar.as_ref() {
|
if let Some(bar) = progress_bar.as_ref() {
|
||||||
@ -206,6 +208,7 @@ impl<'a> Benchmark<'a> {
|
|||||||
self.options.run_bounds.min,
|
self.options.run_bounds.min,
|
||||||
"Initial time measurement",
|
"Initial time measurement",
|
||||||
self.options.output_style,
|
self.options.output_style,
|
||||||
|
self.options.show_elapsed,
|
||||||
))
|
))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@ -256,7 +259,8 @@ impl<'a> Benchmark<'a> {
|
|||||||
bar.set_length(count)
|
bar.set_length(count)
|
||||||
}
|
}
|
||||||
if let Some(bar) = progress_bar.as_ref() {
|
if let Some(bar) = progress_bar.as_ref() {
|
||||||
bar.inc(1)
|
bar.inc(1);
|
||||||
|
bar.reset_elapsed();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gather statistics (perform the actual benchmark)
|
// Gather statistics (perform the actual benchmark)
|
||||||
@ -283,7 +287,8 @@ impl<'a> Benchmark<'a> {
|
|||||||
all_succeeded = all_succeeded && success;
|
all_succeeded = all_succeeded && success;
|
||||||
|
|
||||||
if let Some(bar) = progress_bar.as_ref() {
|
if let Some(bar) = progress_bar.as_ref() {
|
||||||
bar.inc(1)
|
bar.inc(1);
|
||||||
|
bar.reset_elapsed();
|
||||||
}
|
}
|
||||||
|
|
||||||
run_conclusion_command()?;
|
run_conclusion_command()?;
|
||||||
|
@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
|
|||||||
pub result: &'a BenchmarkResult,
|
pub result: &'a BenchmarkResult,
|
||||||
pub relative_speed: Scalar,
|
pub relative_speed: Scalar,
|
||||||
pub relative_speed_stddev: Option<Scalar>,
|
pub relative_speed_stddev: Option<Scalar>,
|
||||||
pub is_fastest: bool,
|
pub is_reference: bool,
|
||||||
|
// Less means faster
|
||||||
|
pub relative_ordering: Ordering,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
|
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
|
||||||
l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
|
l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
||||||
results
|
results
|
||||||
.iter()
|
.iter()
|
||||||
.min_by(|&l, &r| compare_mean_time(l, r))
|
.min_by(|&l, &r| compare_mean_time(l, r))
|
||||||
@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
|
|||||||
|
|
||||||
fn compute_relative_speeds<'a>(
|
fn compute_relative_speeds<'a>(
|
||||||
results: &'a [BenchmarkResult],
|
results: &'a [BenchmarkResult],
|
||||||
fastest: &'a BenchmarkResult,
|
reference: &'a BenchmarkResult,
|
||||||
sort_order: SortOrder,
|
sort_order: SortOrder,
|
||||||
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
|
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
|
||||||
let mut results: Vec<_> = results
|
let mut results: Vec<_> = results
|
||||||
.iter()
|
.iter()
|
||||||
.map(|result| {
|
.map(|result| {
|
||||||
let is_fastest = result == fastest;
|
let is_reference = result == reference;
|
||||||
|
let relative_ordering = compare_mean_time(result, reference);
|
||||||
|
|
||||||
if result.mean == 0.0 {
|
if result.mean == 0.0 {
|
||||||
return BenchmarkResultWithRelativeSpeed {
|
return BenchmarkResultWithRelativeSpeed {
|
||||||
result,
|
result,
|
||||||
relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
|
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
|
||||||
relative_speed_stddev: None,
|
relative_speed_stddev: None,
|
||||||
is_fastest,
|
is_reference,
|
||||||
|
relative_ordering,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let ratio = result.mean / fastest.mean;
|
let ratio = match relative_ordering {
|
||||||
|
Ordering::Less => reference.mean / result.mean,
|
||||||
|
Ordering::Equal => 1.0,
|
||||||
|
Ordering::Greater => result.mean / reference.mean,
|
||||||
|
};
|
||||||
|
|
||||||
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
|
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
|
||||||
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
|
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
|
||||||
let ratio_stddev = match (result.stddev, fastest.stddev) {
|
let ratio_stddev = match (result.stddev, reference.stddev) {
|
||||||
(Some(result_stddev), Some(fastest_stddev)) => Some(
|
(Some(result_stddev), Some(fastest_stddev)) => Some(
|
||||||
ratio
|
ratio
|
||||||
* ((result_stddev / result.mean).powi(2)
|
* ((result_stddev / result.mean).powi(2)
|
||||||
+ (fastest_stddev / fastest.mean).powi(2))
|
+ (fastest_stddev / reference.mean).powi(2))
|
||||||
.sqrt(),
|
.sqrt(),
|
||||||
),
|
),
|
||||||
_ => None,
|
_ => None,
|
||||||
@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>(
|
|||||||
result,
|
result,
|
||||||
relative_speed: ratio,
|
relative_speed: ratio,
|
||||||
relative_speed_stddev: ratio_stddev,
|
relative_speed_stddev: ratio_stddev,
|
||||||
is_fastest,
|
is_reference,
|
||||||
|
relative_ordering,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>(
|
|||||||
results
|
results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn compute_with_check_from_reference<'a>(
|
||||||
|
results: &'a [BenchmarkResult],
|
||||||
|
reference: &'a BenchmarkResult,
|
||||||
|
sort_order: SortOrder,
|
||||||
|
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
|
||||||
|
if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(compute_relative_speeds(results, reference, sort_order))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn compute_with_check(
|
pub fn compute_with_check(
|
||||||
results: &[BenchmarkResult],
|
results: &[BenchmarkResult],
|
||||||
sort_order: SortOrder,
|
sort_order: SortOrder,
|
||||||
@ -134,6 +155,20 @@ fn test_compute_relative_speed() {
|
|||||||
assert_relative_eq!(2.5, annotated_results[2].relative_speed);
|
assert_relative_eq!(2.5, annotated_results[2].relative_speed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_relative_speed_with_reference() {
|
||||||
|
use approx::assert_relative_eq;
|
||||||
|
|
||||||
|
let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
|
||||||
|
let reference = create_result("cmd2", 4.0);
|
||||||
|
|
||||||
|
let annotated_results =
|
||||||
|
compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
|
||||||
|
|
||||||
|
assert_relative_eq!(2.0, annotated_results[0].relative_speed);
|
||||||
|
assert_relative_eq!(1.25, annotated_results[1].relative_speed);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_compute_relative_speed_for_zero_times() {
|
fn test_compute_relative_speed_for_zero_times() {
|
||||||
let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
|
let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
use colored::*;
|
|
||||||
|
|
||||||
use super::benchmark_result::BenchmarkResult;
|
use super::benchmark_result::BenchmarkResult;
|
||||||
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
|
||||||
use super::{relative_speed, Benchmark};
|
use super::{relative_speed, Benchmark};
|
||||||
|
use colored::*;
|
||||||
|
use std::cmp::Ordering;
|
||||||
|
|
||||||
use crate::command::Commands;
|
use crate::command::{Command, Commands};
|
||||||
use crate::export::ExportManager;
|
use crate::export::ExportManager;
|
||||||
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
|
||||||
|
|
||||||
@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> {
|
|||||||
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
|
ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let reference = self
|
||||||
|
.options
|
||||||
|
.reference_command
|
||||||
|
.as_ref()
|
||||||
|
.map(|cmd| Command::new(None, cmd));
|
||||||
|
|
||||||
executor.calibrate()?;
|
executor.calibrate()?;
|
||||||
|
|
||||||
for (number, cmd) in self.commands.iter().enumerate() {
|
for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
|
||||||
self.results
|
self.results
|
||||||
.push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
|
.push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
|
||||||
|
|
||||||
@ -65,31 +71,56 @@ impl<'a> Scheduler<'a> {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(annotated_results) = relative_speed::compute_with_check(
|
let reference = self
|
||||||
|
.options
|
||||||
|
.reference_command
|
||||||
|
.as_ref()
|
||||||
|
.map(|_| &self.results[0])
|
||||||
|
.unwrap_or_else(|| relative_speed::fastest_of(&self.results));
|
||||||
|
|
||||||
|
if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
|
||||||
&self.results,
|
&self.results,
|
||||||
|
reference,
|
||||||
self.options.sort_order_speed_comparison,
|
self.options.sort_order_speed_comparison,
|
||||||
) {
|
) {
|
||||||
match self.options.sort_order_speed_comparison {
|
match self.options.sort_order_speed_comparison {
|
||||||
SortOrder::MeanTime => {
|
SortOrder::MeanTime => {
|
||||||
println!("{}", "Summary".bold());
|
println!("{}", "Summary".bold());
|
||||||
|
|
||||||
let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
|
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
|
||||||
let others = annotated_results.iter().filter(|r| !r.is_fastest);
|
let others = annotated_results.iter().filter(|r| !r.is_reference);
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
" {} ran",
|
" {} ran",
|
||||||
fastest.result.command_with_unused_parameters.cyan()
|
reference.result.command_with_unused_parameters.cyan()
|
||||||
);
|
);
|
||||||
|
|
||||||
for item in others {
|
for item in others {
|
||||||
|
let stddev = if let Some(stddev) = item.relative_speed_stddev {
|
||||||
|
format!(" ± {}", format!("{:.2}", stddev).green())
|
||||||
|
} else {
|
||||||
|
"".into()
|
||||||
|
};
|
||||||
|
let comparator = match item.relative_ordering {
|
||||||
|
Ordering::Less => format!(
|
||||||
|
"{}{} times slower than",
|
||||||
|
format!("{:8.2}", item.relative_speed).bold().green(),
|
||||||
|
stddev
|
||||||
|
),
|
||||||
|
Ordering::Greater => format!(
|
||||||
|
"{}{} times faster than",
|
||||||
|
format!("{:8.2}", item.relative_speed).bold().green(),
|
||||||
|
stddev
|
||||||
|
),
|
||||||
|
Ordering::Equal => format!(
|
||||||
|
" As fast ({}{}) as",
|
||||||
|
format!("{:.2}", item.relative_speed).bold().green(),
|
||||||
|
stddev
|
||||||
|
),
|
||||||
|
};
|
||||||
println!(
|
println!(
|
||||||
"{}{} times faster than {}",
|
"{} {}",
|
||||||
format!("{:8.2}", item.relative_speed).bold().green(),
|
comparator,
|
||||||
if let Some(stddev) = item.relative_speed_stddev {
|
|
||||||
format!(" ± {}", format!("{stddev:.2}").green())
|
|
||||||
} else {
|
|
||||||
"".into()
|
|
||||||
},
|
|
||||||
&item.result.command_with_unused_parameters.magenta()
|
&item.result.command_with_unused_parameters.magenta()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -101,7 +132,7 @@ impl<'a> Scheduler<'a> {
|
|||||||
println!(
|
println!(
|
||||||
" {}{} {}",
|
" {}{} {}",
|
||||||
format!("{:10.2}", item.relative_speed).bold().green(),
|
format!("{:10.2}", item.relative_speed).bold().green(),
|
||||||
if item.is_fastest {
|
if item.is_reference {
|
||||||
" ".into()
|
" ".into()
|
||||||
} else if let Some(stddev) = item.relative_speed_stddev {
|
} else if let Some(stddev) = item.relative_speed_stddev {
|
||||||
format!(" ± {}", format!("{stddev:5.2}").green())
|
format!(" ± {}", format!("{stddev:5.2}").green())
|
||||||
|
21
src/cli.rs
21
src/cli.rs
@ -86,6 +86,16 @@ fn build_command() -> Command {
|
|||||||
not every time as would happen with the --prepare option."
|
not every time as would happen with the --prepare option."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::new("reference")
|
||||||
|
.long("reference")
|
||||||
|
.action(ArgAction::Set)
|
||||||
|
.value_name("CMD")
|
||||||
|
.help(
|
||||||
|
"The reference command for the relative comparison of results. \
|
||||||
|
If this is unset, results are compared with the fastest command as reference."
|
||||||
|
)
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::new("prepare")
|
Arg::new("prepare")
|
||||||
.long("prepare")
|
.long("prepare")
|
||||||
@ -244,6 +254,17 @@ fn build_command() -> Command {
|
|||||||
* 'mean-time': order benchmarks by mean runtime\n"
|
* 'mean-time': order benchmarks by mean runtime\n"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
.arg(
|
||||||
|
Arg::new("show-elapsed")
|
||||||
|
.long("show-elapsed")
|
||||||
|
.short('E')
|
||||||
|
.action(ArgAction::SetTrue)
|
||||||
|
.help(
|
||||||
|
"Show time elapsed since the current run was started. \
|
||||||
|
This is useful for especially long benchmarks to see \
|
||||||
|
the progress the benchmark has made"
|
||||||
|
)
|
||||||
|
)
|
||||||
.arg(
|
.arg(
|
||||||
Arg::new("time-unit")
|
Arg::new("time-unit")
|
||||||
.long("time-unit")
|
.long("time-unit")
|
||||||
|
@ -56,7 +56,7 @@ pub trait MarkupExporter {
|
|||||||
let min_str = format_duration_value(measurement.min, Some(unit)).0;
|
let min_str = format_duration_value(measurement.min, Some(unit)).0;
|
||||||
let max_str = format_duration_value(measurement.max, Some(unit)).0;
|
let max_str = format_duration_value(measurement.max, Some(unit)).0;
|
||||||
let rel_str = format!("{:.2}", entry.relative_speed);
|
let rel_str = format!("{:.2}", entry.relative_speed);
|
||||||
let rel_stddev_str = if entry.is_fastest {
|
let rel_stddev_str = if entry.is_reference {
|
||||||
"".into()
|
"".into()
|
||||||
} else if let Some(stddev) = entry.relative_speed_stddev {
|
} else if let Some(stddev) = entry.relative_speed_stddev {
|
||||||
format!(" ± {stddev:.2}")
|
format!(" ± {stddev:.2}")
|
||||||
|
@ -204,6 +204,9 @@ pub struct Options {
|
|||||||
/// Whether or not to ignore non-zero exit codes
|
/// Whether or not to ignore non-zero exit codes
|
||||||
pub command_failure_action: CmdFailureAction,
|
pub command_failure_action: CmdFailureAction,
|
||||||
|
|
||||||
|
// Command to use as a reference for relative speed comparison
|
||||||
|
pub reference_command: Option<String>,
|
||||||
|
|
||||||
/// Command(s) to run before each timing run
|
/// Command(s) to run before each timing run
|
||||||
pub preparation_command: Option<Vec<String>>,
|
pub preparation_command: Option<Vec<String>>,
|
||||||
|
|
||||||
@ -236,6 +239,9 @@ pub struct Options {
|
|||||||
|
|
||||||
/// Which time unit to use when displaying results
|
/// Which time unit to use when displaying results
|
||||||
pub time_unit: Option<Unit>,
|
pub time_unit: Option<Unit>,
|
||||||
|
|
||||||
|
/// Show elapsed time since current run start.
|
||||||
|
pub show_elapsed: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for Options {
|
impl Default for Options {
|
||||||
@ -245,6 +251,7 @@ impl Default for Options {
|
|||||||
warmup_count: 0,
|
warmup_count: 0,
|
||||||
min_benchmarking_time: 3.0,
|
min_benchmarking_time: 3.0,
|
||||||
command_failure_action: CmdFailureAction::RaiseError,
|
command_failure_action: CmdFailureAction::RaiseError,
|
||||||
|
reference_command: None,
|
||||||
preparation_command: None,
|
preparation_command: None,
|
||||||
conclusion_command: None,
|
conclusion_command: None,
|
||||||
setup_command: None,
|
setup_command: None,
|
||||||
@ -256,6 +263,7 @@ impl Default for Options {
|
|||||||
command_output_policy: CommandOutputPolicy::Null,
|
command_output_policy: CommandOutputPolicy::Null,
|
||||||
time_unit: None,
|
time_unit: None,
|
||||||
command_input_policy: CommandInputPolicy::Null,
|
command_input_policy: CommandInputPolicy::Null,
|
||||||
|
show_elapsed: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -304,6 +312,8 @@ impl Options {
|
|||||||
|
|
||||||
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
options.setup_command = matches.get_one::<String>("setup").map(String::from);
|
||||||
|
|
||||||
|
options.reference_command = matches.get_one::<String>("reference").map(String::from);
|
||||||
|
|
||||||
options.preparation_command = matches
|
options.preparation_command = matches
|
||||||
.get_many::<String>("prepare")
|
.get_many::<String>("prepare")
|
||||||
.map(|values| values.map(String::from).collect::<Vec<String>>());
|
.map(|values| values.map(String::from).collect::<Vec<String>>());
|
||||||
@ -427,25 +437,33 @@ impl Options {
|
|||||||
CommandInputPolicy::Null
|
CommandInputPolicy::Null
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if matches.get_flag("show-elapsed") {
|
||||||
|
options.show_elapsed = true;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(options)
|
Ok(options)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
|
pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
|
||||||
|
let num_commands = commands.num_commands()
|
||||||
|
+ if self.reference_command.is_some() {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
if let Some(preparation_command) = &self.preparation_command {
|
if let Some(preparation_command) = &self.preparation_command {
|
||||||
ensure!(
|
ensure!(
|
||||||
preparation_command.len() <= 1
|
preparation_command.len() <= 1 || num_commands == preparation_command.len(),
|
||||||
|| commands.num_commands() == preparation_command.len(),
|
|
||||||
"The '--prepare' option has to be provided just once or N times, where N is the \
|
"The '--prepare' option has to be provided just once or N times, where N is the \
|
||||||
number of benchmark commands."
|
number of benchmark commands including a potential reference."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(conclusion_command) = &self.conclusion_command {
|
if let Some(conclusion_command) = &self.conclusion_command {
|
||||||
ensure!(
|
ensure!(
|
||||||
conclusion_command.len() <= 1
|
conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
|
||||||
|| commands.num_commands() == conclusion_command.len(),
|
|
||||||
"The '--conclude' option has to be provided just once or N times, where N is the \
|
"The '--conclude' option has to be provided just once or N times, where N is the \
|
||||||
number of benchmark commands."
|
number of benchmark commands including a potential reference."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -10,12 +10,22 @@ const TICK_SETTINGS: (&str, u64) = ("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ ", 80);
|
|||||||
const TICK_SETTINGS: (&str, u64) = (r"+-x| ", 200);
|
const TICK_SETTINGS: (&str, u64) = (r"+-x| ", 200);
|
||||||
|
|
||||||
/// Return a pre-configured progress bar
|
/// Return a pre-configured progress bar
|
||||||
pub fn get_progress_bar(length: u64, msg: &str, option: OutputStyleOption) -> ProgressBar {
|
pub fn get_progress_bar(
|
||||||
|
length: u64,
|
||||||
|
msg: &str,
|
||||||
|
option: OutputStyleOption,
|
||||||
|
show_elapsed: bool,
|
||||||
|
) -> ProgressBar {
|
||||||
|
let template_str = match show_elapsed {
|
||||||
|
true => " {spinner} {msg:<30} {wide_bar} ET {elapsed_precise} ETA {eta_precise} ",
|
||||||
|
false => " {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ",
|
||||||
|
};
|
||||||
|
|
||||||
let progressbar_style = match option {
|
let progressbar_style = match option {
|
||||||
OutputStyleOption::Basic | OutputStyleOption::Color => ProgressStyle::default_bar(),
|
OutputStyleOption::Basic | OutputStyleOption::Color => ProgressStyle::default_bar(),
|
||||||
_ => ProgressStyle::default_spinner()
|
_ => ProgressStyle::default_spinner()
|
||||||
.tick_chars(TICK_SETTINGS.0)
|
.tick_chars(TICK_SETTINGS.0)
|
||||||
.template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ")
|
.template(template_str)
|
||||||
.expect("no template error"),
|
.expect("no template error"),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -54,6 +54,11 @@ impl ExecutionOrderTest {
|
|||||||
self.command(output)
|
self.command(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn reference(&mut self, output: &str) -> &mut Self {
|
||||||
|
self.arg("--reference");
|
||||||
|
self.command(output)
|
||||||
|
}
|
||||||
|
|
||||||
fn conclude(&mut self, output: &str) -> &mut Self {
|
fn conclude(&mut self, output: &str) -> &mut Self {
|
||||||
self.arg("--conclude");
|
self.arg("--conclude");
|
||||||
self.command(output)
|
self.command(output)
|
||||||
@ -364,3 +369,195 @@ fn multiple_parameter_values() {
|
|||||||
.expect_output("command 3 b")
|
.expect_output("command 3 b")
|
||||||
.run();
|
.run();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reference_is_executed_first() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.reference("reference")
|
||||||
|
.command("command 1")
|
||||||
|
.command("command 2")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("command 1")
|
||||||
|
.expect_output("command 2")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reference_is_executed_first_parameter_value() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--runs=2")
|
||||||
|
.reference("reference")
|
||||||
|
.arg("--parameter-list")
|
||||||
|
.arg("number")
|
||||||
|
.arg("1,2,3")
|
||||||
|
.command("command {number}")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("command 1")
|
||||||
|
.expect_output("command 1")
|
||||||
|
.expect_output("command 2")
|
||||||
|
.expect_output("command 2")
|
||||||
|
.expect_output("command 3")
|
||||||
|
.expect_output("command 3")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reference_is_executed_separately_from_commands() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.reference("command 1")
|
||||||
|
.command("command 1")
|
||||||
|
.command("command 2")
|
||||||
|
.expect_output("command 1")
|
||||||
|
.expect_output("command 1")
|
||||||
|
.expect_output("command 2")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn setup_prepare_reference_conclude_cleanup_combined() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--warmup=1")
|
||||||
|
.arg("--runs=2")
|
||||||
|
.setup("setup")
|
||||||
|
.prepare("prepare")
|
||||||
|
.reference("reference")
|
||||||
|
.command("command1")
|
||||||
|
.command("command2")
|
||||||
|
.conclude("conclude")
|
||||||
|
.cleanup("cleanup")
|
||||||
|
// reference
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
// 1
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
// 2
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("prepare")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn setup_separate_prepare_separate_conclude_cleanup_combined() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--warmup=1")
|
||||||
|
.arg("--runs=2")
|
||||||
|
.setup("setup")
|
||||||
|
.cleanup("cleanup")
|
||||||
|
.prepare("prepare1")
|
||||||
|
.command("command1")
|
||||||
|
.conclude("conclude1")
|
||||||
|
.prepare("prepare2")
|
||||||
|
.command("command2")
|
||||||
|
.conclude("conclude2")
|
||||||
|
// 1
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
// 2
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
|
||||||
|
ExecutionOrderTest::new()
|
||||||
|
.arg("--warmup=1")
|
||||||
|
.arg("--runs=2")
|
||||||
|
.setup("setup")
|
||||||
|
.cleanup("cleanup")
|
||||||
|
.prepare("prepareref")
|
||||||
|
.reference("reference")
|
||||||
|
.conclude("concluderef")
|
||||||
|
.prepare("prepare1")
|
||||||
|
.command("command1")
|
||||||
|
.conclude("conclude1")
|
||||||
|
.prepare("prepare2")
|
||||||
|
.command("command2")
|
||||||
|
.conclude("conclude2")
|
||||||
|
// reference
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepareref")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("concluderef")
|
||||||
|
.expect_output("prepareref")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("concluderef")
|
||||||
|
.expect_output("prepareref")
|
||||||
|
.expect_output("reference")
|
||||||
|
.expect_output("concluderef")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
// 1
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("prepare1")
|
||||||
|
.expect_output("command1")
|
||||||
|
.expect_output("conclude1")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
// 2
|
||||||
|
.expect_output("setup")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("prepare2")
|
||||||
|
.expect_output("command2")
|
||||||
|
.expect_output("conclude2")
|
||||||
|
.expect_output("cleanup")
|
||||||
|
.run();
|
||||||
|
}
|
||||||
|
@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() {
|
|||||||
.assert()
|
.assert()
|
||||||
.success();
|
.success();
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--prepare=echo ref")
|
||||||
|
.arg("--prepare=echo a")
|
||||||
|
.arg("--prepare=echo b")
|
||||||
|
.arg("--reference=echo ref")
|
||||||
|
.arg("echo a")
|
||||||
|
.arg("echo b")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
hyperfine()
|
hyperfine()
|
||||||
.arg("--runs=1")
|
.arg("--runs=1")
|
||||||
.arg("--prepare=echo a")
|
.arg("--prepare=echo a")
|
||||||
@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() {
|
|||||||
.stderr(predicate::str::contains(
|
.stderr(predicate::str::contains(
|
||||||
"The '--prepare' option has to be provided",
|
"The '--prepare' option has to be provided",
|
||||||
));
|
));
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--prepare=echo a")
|
||||||
|
.arg("--prepare=echo b")
|
||||||
|
.arg("--reference=echo ref")
|
||||||
|
.arg("echo a")
|
||||||
|
.arg("echo b")
|
||||||
|
.assert()
|
||||||
|
.failure()
|
||||||
|
.stderr(predicate::str::contains(
|
||||||
|
"The '--prepare' option has to be provided",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() {
|
|||||||
.assert()
|
.assert()
|
||||||
.success();
|
.success();
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--conclude=echo ref")
|
||||||
|
.arg("--conclude=echo a")
|
||||||
|
.arg("--conclude=echo b")
|
||||||
|
.arg("--reference=echo ref")
|
||||||
|
.arg("echo a")
|
||||||
|
.arg("echo b")
|
||||||
|
.assert()
|
||||||
|
.success();
|
||||||
|
|
||||||
hyperfine()
|
hyperfine()
|
||||||
.arg("--runs=1")
|
.arg("--runs=1")
|
||||||
.arg("--conclude=echo a")
|
.arg("--conclude=echo a")
|
||||||
@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() {
|
|||||||
.stderr(predicate::str::contains(
|
.stderr(predicate::str::contains(
|
||||||
"The '--conclude' option has to be provided",
|
"The '--conclude' option has to be provided",
|
||||||
));
|
));
|
||||||
|
|
||||||
|
hyperfine()
|
||||||
|
.arg("--runs=1")
|
||||||
|
.arg("--conclude=echo a")
|
||||||
|
.arg("--conclude=echo b")
|
||||||
|
.arg("--reference=echo ref")
|
||||||
|
.arg("echo a")
|
||||||
|
.arg("echo b")
|
||||||
|
.assert()
|
||||||
|
.failure()
|
||||||
|
.stderr(predicate::str::contains(
|
||||||
|
"The '--conclude' option has to be provided",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -413,6 +461,38 @@ fn shows_benchmark_comparison_with_relative_times() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shows_benchmark_comparison_with_same_time() {
|
||||||
|
hyperfine_debug()
|
||||||
|
.arg("--command-name=A")
|
||||||
|
.arg("--command-name=B")
|
||||||
|
.arg("sleep 1.0")
|
||||||
|
.arg("sleep 1.0")
|
||||||
|
.arg("sleep 2.0")
|
||||||
|
.arg("sleep 1000.0")
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(
|
||||||
|
predicate::str::contains("As fast (1.00 ± 0.00) as")
|
||||||
|
.and(predicate::str::contains("2.00 ± 0.00 times faster"))
|
||||||
|
.and(predicate::str::contains("1000.00 ± 0.00 times faster")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn shows_benchmark_comparison_relative_to_reference() {
|
||||||
|
hyperfine_debug()
|
||||||
|
.arg("--reference=sleep 2.0")
|
||||||
|
.arg("sleep 1.0")
|
||||||
|
.arg("sleep 3.0")
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(
|
||||||
|
predicate::str::contains("2.00 ± 0.00 times slower")
|
||||||
|
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn performs_all_benchmarks_in_parameter_scan() {
|
fn performs_all_benchmarks_in_parameter_scan() {
|
||||||
hyperfine_debug()
|
hyperfine_debug()
|
||||||
@ -434,6 +514,29 @@ fn performs_all_benchmarks_in_parameter_scan() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||||
|
hyperfine_debug()
|
||||||
|
.arg("--reference=sleep 25")
|
||||||
|
.arg("--parameter-scan")
|
||||||
|
.arg("time")
|
||||||
|
.arg("30")
|
||||||
|
.arg("45")
|
||||||
|
.arg("--parameter-step-size")
|
||||||
|
.arg("5")
|
||||||
|
.arg("sleep {time}")
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(
|
||||||
|
predicate::str::contains("Benchmark 1: sleep 25")
|
||||||
|
.and(predicate::str::contains("Benchmark 2: sleep 30"))
|
||||||
|
.and(predicate::str::contains("Benchmark 3: sleep 35"))
|
||||||
|
.and(predicate::str::contains("Benchmark 4: sleep 40"))
|
||||||
|
.and(predicate::str::contains("Benchmark 5: sleep 45"))
|
||||||
|
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn intermediate_results_are_not_exported_to_stdout() {
|
fn intermediate_results_are_not_exported_to_stdout() {
|
||||||
hyperfine_debug()
|
hyperfine_debug()
|
||||||
|
Loading…
Reference in New Issue
Block a user