Merge 02975df2ad into df34f32dca

Bumps [once_cell](https://github.com/matklad/once_cell) from 1.18.0 to 1.19.0.
- [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md)
- [Commits](https://github.com/matklad/once_cell/compare/v1.18.0...v1.19.0)

---
updated-dependencies:
- dependency-name: once_cell
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Cargo.lock

@@ -13,17 +13,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "ahash"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
-dependencies = [
- "cfg-if",
- "once_cell",
- "version_check",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "1.1.1"
@@ -158,47 +147,26 @@ dependencies = [
 
 [[package]]
 name = "borsh"
-version = "0.10.3"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
+checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed"
 dependencies = [
  "borsh-derive",
- "hashbrown 0.13.2",
+ "cfg_aliases",
 ]
 
 [[package]]
 name = "borsh-derive"
-version = "0.10.3"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7"
+checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
 dependencies = [
- "borsh-derive-internal",
- "borsh-schema-derive-internal",
+ "once_cell",
  "proc-macro-crate",
- "proc-macro2",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "borsh-derive-internal"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb"
-dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "borsh-schema-derive-internal"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
+ "syn 2.0.37",
+ "syn_derive",
 ]
 
 [[package]]
@@ -246,6 +214,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
 [[package]]
 name = "clap"
 version = "4.4.12"
@@ -366,6 +340,12 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
 version = "0.3.8"
@@ -420,17 +400,14 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash 0.7.6",
+ "ahash",
 ]
 
 [[package]]
 name = "hashbrown"
-version = "0.13.2"
+version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
-dependencies = [
- "ahash 0.8.3",
-]
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 
 [[package]]
 name = "hyperfine"
@@ -459,6 +436,16 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.5",
+]
+
 [[package]]
 name = "indicatif"
 version = "0.17.4"
@@ -622,9 +609,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "once_cell"
-version = "1.18.0"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "portable-atomic"
@@ -671,11 +658,34 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "0.1.5"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
+checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
 dependencies = [
- "toml",
+ "toml_edit",
+]
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
 ]
 
 [[package]]
@@ -944,9 +954,9 @@ dependencies = [
 
 [[package]]
 name = "rust_decimal"
-version = "1.32.0"
+version = "1.35.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd"
+checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a"
 dependencies = [
  "arrayvec",
  "borsh",
@@ -1064,6 +1074,18 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn_derive"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b"
+dependencies = [
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -1135,12 +1157,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
-name = "toml"
-version = "0.5.11"
+name = "toml_datetime"
+version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
+checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+
+[[package]]
+name = "toml_edit"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
 dependencies = [
- "serde",
+ "indexmap",
+ "toml_datetime",
+ "winnow",
 ]
 
 [[package]]
@@ -1408,6 +1438,15 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
 
+[[package]]
+name = "winnow"
+version = "0.5.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wyz"
 version = "0.5.1"

Cargo.toml

@@ -23,7 +23,7 @@ statistical = "1.0"
 csv = "1.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-rust_decimal = "1.31"
+rust_decimal = "1.35"
 rand = "0.8"
 shell-words = "1.0"
 thiserror = "1.0"
@@ -42,7 +42,7 @@ windows-sys = { version = "0.48", features = [
 ] }
 
 [target.'cfg(all(windows, not(windows_process_extensions_main_thread_handle)))'.dependencies]
-once_cell = "1.17"
+once_cell = "1.19"
 
 [target.'cfg(target_os="linux")'.dependencies]
 nix = { version = "0.27.1", features = ["zerocopy"] }

src/benchmark/executor.rs

@@ -180,6 +180,7 @@ impl<'a> Executor for ShellExecutor<'a> {
                 COUNT,
                 "Measuring shell spawning time",
                 self.options.output_style,
+                self.options.show_elapsed,
             ))
         } else {
             None
@@ -214,7 +215,8 @@ impl<'a> Executor for ShellExecutor<'a> {
             }
 
             if let Some(bar) = progress_bar.as_ref() {
-                bar.inc(1)
+                bar.inc(1);
+                bar.reset_elapsed();
             }
         }
 

src/benchmark/mod.rs

@@ -182,6 +182,7 @@ impl<'a> Benchmark<'a> {
                     self.options.warmup_count,
                     "Performing warmup runs",
                     self.options.output_style,
+                    self.options.show_elapsed,
                 ))
             } else {
                 None
@@ -192,7 +193,8 @@ impl<'a> Benchmark<'a> {
                 let _ = self.executor.run_command_and_measure(self.command, None)?;
                 let _ = run_conclusion_command()?;
                 if let Some(bar) = progress_bar.as_ref() {
-                    bar.inc(1)
+                    bar.inc(1);
+                    bar.reset_elapsed();
                 }
             }
             if let Some(bar) = progress_bar.as_ref() {
@@ -206,6 +208,7 @@ impl<'a> Benchmark<'a> {
                 self.options.run_bounds.min,
                 "Initial time measurement",
                 self.options.output_style,
+                self.options.show_elapsed,
             ))
         } else {
             None
@@ -256,7 +259,8 @@ impl<'a> Benchmark<'a> {
             bar.set_length(count)
         }
         if let Some(bar) = progress_bar.as_ref() {
-            bar.inc(1)
+            bar.inc(1);
+            bar.reset_elapsed();
         }
 
         // Gather statistics (perform the actual benchmark)
@@ -283,7 +287,8 @@ impl<'a> Benchmark<'a> {
             all_succeeded = all_succeeded && success;
 
             if let Some(bar) = progress_bar.as_ref() {
-                bar.inc(1)
+                bar.inc(1);
+                bar.reset_elapsed();
             }
 
             run_conclusion_command()?;

src/benchmark/relative_speed.rs

@@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
     pub result: &'a BenchmarkResult,
     pub relative_speed: Scalar,
     pub relative_speed_stddev: Option<Scalar>,
-    pub is_fastest: bool,
+    pub is_reference: bool,
+    // Less means faster
+    pub relative_ordering: Ordering,
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
     l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
 }
 
-fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
+pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
     results
         .iter()
         .min_by(|&l, &r| compare_mean_time(l, r))
@@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
 
 fn compute_relative_speeds<'a>(
     results: &'a [BenchmarkResult],
-    fastest: &'a BenchmarkResult,
+    reference: &'a BenchmarkResult,
     sort_order: SortOrder,
 ) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
     let mut results: Vec<_> = results
         .iter()
         .map(|result| {
-            let is_fastest = result == fastest;
+            let is_reference = result == reference;
+            let relative_ordering = compare_mean_time(result, reference);
 
             if result.mean == 0.0 {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
-                    relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
+                    relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
                     relative_speed_stddev: None,
-                    is_fastest,
+                    is_reference,
+                    relative_ordering,
                 };
             }
 
-            let ratio = result.mean / fastest.mean;
+            let ratio = match relative_ordering {
+                Ordering::Less => reference.mean / result.mean,
+                Ordering::Equal => 1.0,
+                Ordering::Greater => result.mean / reference.mean,
+            };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
             // Covariance asssumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (result.stddev, fastest.stddev) {
+            let ratio_stddev = match (result.stddev, reference.stddev) {
                 (Some(result_stddev), Some(fastest_stddev)) => Some(
                     ratio
                         * ((result_stddev / result.mean).powi(2)
-                            + (fastest_stddev / fastest.mean).powi(2))
+                            + (fastest_stddev / reference.mean).powi(2))
                         .sqrt(),
                 ),
                 _ => None,
@@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>(
                 result,
                 relative_speed: ratio,
                 relative_speed_stddev: ratio_stddev,
-                is_fastest,
+                is_reference,
+                relative_ordering,
             }
         })
         .collect();
@@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>(
     results
 }
 
+pub fn compute_with_check_from_reference<'a>(
+    results: &'a [BenchmarkResult],
+    reference: &'a BenchmarkResult,
+    sort_order: SortOrder,
+) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
+    if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
+        return None;
+    }
+
+    Some(compute_relative_speeds(results, reference, sort_order))
+}
+
 pub fn compute_with_check(
     results: &[BenchmarkResult],
     sort_order: SortOrder,
@@ -134,6 +155,20 @@ fn test_compute_relative_speed() {
     assert_relative_eq!(2.5, annotated_results[2].relative_speed);
 }
 
+#[test]
+fn test_compute_relative_speed_with_reference() {
+    use approx::assert_relative_eq;
+
+    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
+    let reference = create_result("cmd2", 4.0);
+
+    let annotated_results =
+        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
+
+    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
+    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
+}
+
 #[test]
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];

src/benchmark/scheduler.rs

@@ -1,10 +1,10 @@
-use colored::*;
-
 use super::benchmark_result::BenchmarkResult;
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
 use super::{relative_speed, Benchmark};
+use colored::*;
+use std::cmp::Ordering;
 
-use crate::command::Commands;
+use crate::command::{Command, Commands};
 use crate::export::ExportManager;
 use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
 
@@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> {
             ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
         };
 
+        let reference = self
+            .options
+            .reference_command
+            .as_ref()
+            .map(|cmd| Command::new(None, cmd));
+
         executor.calibrate()?;
 
-        for (number, cmd) in self.commands.iter().enumerate() {
+        for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
             self.results
                 .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
 
@@ -65,31 +71,56 @@ impl<'a> Scheduler<'a> {
             return;
         }
 
-        if let Some(annotated_results) = relative_speed::compute_with_check(
+        let reference = self
+            .options
+            .reference_command
+            .as_ref()
+            .map(|_| &self.results[0])
+            .unwrap_or_else(|| relative_speed::fastest_of(&self.results));
+
+        if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
             &self.results,
+            reference,
             self.options.sort_order_speed_comparison,
         ) {
             match self.options.sort_order_speed_comparison {
                 SortOrder::MeanTime => {
                     println!("{}", "Summary".bold());
 
-                    let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
-                    let others = annotated_results.iter().filter(|r| !r.is_fastest);
+                    let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
+                    let others = annotated_results.iter().filter(|r| !r.is_reference);
 
                     println!(
                         "  {} ran",
-                        fastest.result.command_with_unused_parameters.cyan()
+                        reference.result.command_with_unused_parameters.cyan()
                     );
 
                     for item in others {
+                        let stddev = if let Some(stddev) = item.relative_speed_stddev {
+                            format!(" ± {}", format!("{:.2}", stddev).green())
+                        } else {
+                            "".into()
+                        };
+                        let comparator = match item.relative_ordering {
+                            Ordering::Less => format!(
+                                "{}{} times slower than",
+                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                            Ordering::Greater => format!(
+                                "{}{} times faster than",
+                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                            Ordering::Equal => format!(
+                                "    As fast ({}{}) as",
+                                format!("{:.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                        };
                         println!(
-                            "{}{} times faster than {}",
-                            format!("{:8.2}", item.relative_speed).bold().green(),
-                            if let Some(stddev) = item.relative_speed_stddev {
-                                format!(" ± {}", format!("{stddev:.2}").green())
-                            } else {
-                                "".into()
-                            },
+                            "{} {}",
+                            comparator,
                             &item.result.command_with_unused_parameters.magenta()
                         );
                     }
@@ -101,7 +132,7 @@ impl<'a> Scheduler<'a> {
                         println!(
                             "  {}{}  {}",
                             format!("{:10.2}", item.relative_speed).bold().green(),
-                            if item.is_fastest {
+                            if item.is_reference {
                                 "        ".into()
                             } else if let Some(stddev) = item.relative_speed_stddev {
                                 format!(" ± {}", format!("{stddev:5.2}").green())

src/cli.rs

@@ -86,6 +86,16 @@ fn build_command() -> Command {
                      not every time as would happen with the --prepare option."
                 ),
         )
+        .arg(
+            Arg::new("reference")
+                .long("reference")
+                .action(ArgAction::Set)
+                .value_name("CMD")
+                .help(
+                    "The reference command for the relative comparison of results. \
+                    If this is unset, results are compared with the fastest command as reference."
+                )
+        )
         .arg(
             Arg::new("prepare")
                 .long("prepare")
@@ -244,6 +254,17 @@ fn build_command() -> Command {
                    * 'mean-time': order benchmarks by mean runtime\n"
             ),
         )
+        .arg(
+            Arg::new("show-elapsed")
+                .long("show-elapsed")
+                .short('E')
+                .action(ArgAction::SetTrue)
+                .help(
+                    "Show time elapsed since the current run was started. \
+                     This is useful for especially long benchmarks to see \
+                     the progress the benchmark has made"
+                )
+        )
         .arg(
             Arg::new("time-unit")
                 .long("time-unit")

src/export/markup.rs

@@ -56,7 +56,7 @@ pub trait MarkupExporter {
             let min_str = format_duration_value(measurement.min, Some(unit)).0;
             let max_str = format_duration_value(measurement.max, Some(unit)).0;
             let rel_str = format!("{:.2}", entry.relative_speed);
-            let rel_stddev_str = if entry.is_fastest {
+            let rel_stddev_str = if entry.is_reference {
                 "".into()
             } else if let Some(stddev) = entry.relative_speed_stddev {
                 format!(" ± {stddev:.2}")

src/options.rs

@@ -204,6 +204,9 @@ pub struct Options {
     /// Whether or not to ignore non-zero exit codes
     pub command_failure_action: CmdFailureAction,
 
+    // Command to use as a reference for relative speed comparison
+    pub reference_command: Option<String>,
+
     /// Command(s) to run before each timing run
     pub preparation_command: Option<Vec<String>>,
 
@@ -236,6 +239,9 @@ pub struct Options {
 
     /// Which time unit to use when displaying results
     pub time_unit: Option<Unit>,
+
+    /// Show elapsed time since current run start.
+    pub show_elapsed: bool,
 }
 
 impl Default for Options {
@@ -245,6 +251,7 @@ impl Default for Options {
             warmup_count: 0,
             min_benchmarking_time: 3.0,
             command_failure_action: CmdFailureAction::RaiseError,
+            reference_command: None,
             preparation_command: None,
             conclusion_command: None,
             setup_command: None,
@@ -256,6 +263,7 @@ impl Default for Options {
             command_output_policy: CommandOutputPolicy::Null,
             time_unit: None,
             command_input_policy: CommandInputPolicy::Null,
+            show_elapsed: false,
         }
     }
 }
@@ -304,6 +312,8 @@ impl Options {
 
         options.setup_command = matches.get_one::<String>("setup").map(String::from);
 
+        options.reference_command = matches.get_one::<String>("reference").map(String::from);
+
         options.preparation_command = matches
             .get_many::<String>("prepare")
             .map(|values| values.map(String::from).collect::<Vec<String>>());
@@ -427,25 +437,33 @@ impl Options {
             CommandInputPolicy::Null
         };
 
+        if matches.get_flag("show-elapsed") {
+            options.show_elapsed = true;
+        }
+
         Ok(options)
     }
 
     pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
+        let num_commands = commands.num_commands()
+            + if self.reference_command.is_some() {
+                1
+            } else {
+                0
+            };
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
-                preparation_command.len() <= 1
-                    || commands.num_commands() == preparation_command.len(),
+                preparation_command.len() <= 1 || num_commands == preparation_command.len(),
                 "The '--prepare' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 
         if let Some(conclusion_command) = &self.conclusion_command {
             ensure!(
-                conclusion_command.len() <= 1
-                    || commands.num_commands() == conclusion_command.len(),
+                conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
                 "The '--conclude' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 

src/output/progress_bar.rs

@@ -10,12 +10,22 @@ const TICK_SETTINGS: (&str, u64) = ("⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ ", 80);
 const TICK_SETTINGS: (&str, u64) = (r"+-x| ", 200);
 
 /// Return a pre-configured progress bar
-pub fn get_progress_bar(length: u64, msg: &str, option: OutputStyleOption) -> ProgressBar {
+pub fn get_progress_bar(
+    length: u64,
+    msg: &str,
+    option: OutputStyleOption,
+    show_elapsed: bool,
+) -> ProgressBar {
+    let template_str = match show_elapsed {
+        true => " {spinner} {msg:<30} {wide_bar} ET {elapsed_precise} ETA {eta_precise} ",
+        false => " {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ",
+    };
+
     let progressbar_style = match option {
         OutputStyleOption::Basic | OutputStyleOption::Color => ProgressStyle::default_bar(),
         _ => ProgressStyle::default_spinner()
             .tick_chars(TICK_SETTINGS.0)
-            .template(" {spinner} {msg:<30} {wide_bar} ETA {eta_precise} ")
+            .template(template_str)
             .expect("no template error"),
     };
 

tests/execution_order_tests.rs

@@ -54,6 +54,11 @@ impl ExecutionOrderTest {
         self.command(output)
     }
 
+    fn reference(&mut self, output: &str) -> &mut Self {
+        self.arg("--reference");
+        self.command(output)
+    }
+
     fn conclude(&mut self, output: &str) -> &mut Self {
         self.arg("--conclude");
         self.command(output)
@@ -364,3 +369,195 @@ fn multiple_parameter_values() {
         .expect_output("command 3 b")
         .run();
 }
+
+#[test]
+fn reference_is_executed_first() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("reference")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_first_parameter_value() {
+    ExecutionOrderTest::new()
+        .arg("--runs=2")
+        .reference("reference")
+        .arg("--parameter-list")
+        .arg("number")
+        .arg("1,2,3")
+        .command("command {number}")
+        .expect_output("reference")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .expect_output("command 2")
+        .expect_output("command 3")
+        .expect_output("command 3")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_separately_from_commands() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("command 1")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn setup_prepare_reference_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .prepare("prepare")
+        .reference("reference")
+        .command("command1")
+        .command("command2")
+        .conclude("conclude")
+        .cleanup("cleanup")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separate_prepare_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepareref")
+        .reference("reference")
+        .conclude("concluderef")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}

tests/integration_tests.rs

@@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo ref")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--prepare=echo a")
@@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() {
         .stderr(predicate::str::contains(
             "The '--prepare' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--prepare' option has to be provided",
+        ));
 }
 
 #[test]
@@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo ref")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--conclude=echo a")
@@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() {
         .stderr(predicate::str::contains(
             "The '--conclude' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--conclude' option has to be provided",
+        ));
 }
 
 #[test]
@@ -413,6 +461,38 @@ fn shows_benchmark_comparison_with_relative_times() {
         );
 }
 
+#[test]
+fn shows_benchmark_comparison_with_same_time() {
+    hyperfine_debug()
+        .arg("--command-name=A")
+        .arg("--command-name=B")
+        .arg("sleep 1.0")
+        .arg("sleep 1.0")
+        .arg("sleep 2.0")
+        .arg("sleep 1000.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("As fast (1.00 ± 0.00) as")
+                .and(predicate::str::contains("2.00 ± 0.00 times faster"))
+                .and(predicate::str::contains("1000.00 ± 0.00 times faster")),
+        );
+}
+
+#[test]
+fn shows_benchmark_comparison_relative_to_reference() {
+    hyperfine_debug()
+        .arg("--reference=sleep 2.0")
+        .arg("sleep 1.0")
+        .arg("sleep 3.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("2.00 ± 0.00 times slower")
+                .and(predicate::str::contains("1.50 ± 0.00 times faster")),
+        );
+}
+
 #[test]
 fn performs_all_benchmarks_in_parameter_scan() {
     hyperfine_debug()
@@ -434,6 +514,29 @@ fn performs_all_benchmarks_in_parameter_scan() {
         );
 }
 
+#[test]
+fn performs_reference_and_all_benchmarks_in_parameter_scan() {
+    hyperfine_debug()
+        .arg("--reference=sleep 25")
+        .arg("--parameter-scan")
+        .arg("time")
+        .arg("30")
+        .arg("45")
+        .arg("--parameter-step-size")
+        .arg("5")
+        .arg("sleep {time}")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Benchmark 1: sleep 25")
+                .and(predicate::str::contains("Benchmark 2: sleep 30"))
+                .and(predicate::str::contains("Benchmark 3: sleep 35"))
+                .and(predicate::str::contains("Benchmark 4: sleep 40"))
+                .and(predicate::str::contains("Benchmark 5: sleep 45"))
+                .and(predicate::str::contains("Benchmark 6: sleep 50").not()),
+        );
+}
+
 #[test]
 fn intermediate_results_are_not_exported_to_stdout() {
     hyperfine_debug()