Change message for identical time and add test for it

README.md

@@ -240,6 +240,14 @@ On NixOS, hyperfine can be installed [from the official repositories](https://ni
 nix-env -i hyperfine
 ```
 
+### On Flox
+
+On Flox, hyperfine can be installed as follows.
+```
+flox install hyperfine
+```
+Hyperfine's version in Flox follows that of Nix.
+
 ### On openSUSE
 
 On openSUSE, hyperfine can be installed [from the official repositories](https://software.opensuse.org/package/hyperfine):

src/benchmark/executor.rs

@@ -54,7 +54,7 @@ fn run_command_and_measure_common(
     );
 
     let result = execute_and_measure(command)
-        .with_context(|| format!("Failed to run command '{}'", command_name))?;
+        .with_context(|| format!("Failed to run command '{command_name}'"))?;
 
     if command_failure_action == CmdFailureAction::RaiseError && !result.status.success() {
         bail!(
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
             Alternatively, use the '--show-output' option to debug what went wrong.",
             result.status.code().map_or(
                 "The process has been terminated by a signal".into(),
-                |c| format!("Command terminated with non-zero exit code: {}", c)
+                |c| format!("Command terminated with non-zero exit code: {c}")
             )
         );
     }

src/benchmark/mod.rs

@@ -312,7 +312,7 @@ impl<'a> Benchmark<'a> {
         let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
         let min_str = format_duration(t_min, Some(time_unit));
         let max_str = format_duration(t_max, Some(time_unit));
-        let num_str = format!("{} runs", t_num);
+        let num_str = format!("{t_num} runs");
 
         let user_str = format_duration(user_mean, Some(time_unit));
         let system_str = format_duration(system_mean, Some(time_unit));

src/benchmark/relative_speed.rs

@@ -8,14 +8,16 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
     pub result: &'a BenchmarkResult,
     pub relative_speed: Scalar,
     pub relative_speed_stddev: Option<Scalar>,
-    pub is_fastest: bool,
+    pub is_reference: bool,
+    // Less means faster
+    pub relative_ordering: Ordering,
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
     l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
 }
 
-fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
+pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
     results
         .iter()
         .min_by(|&l, &r| compare_mean_time(l, r))
@@ -24,32 +26,38 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
 
 fn compute_relative_speeds<'a>(
     results: &'a [BenchmarkResult],
-    fastest: &'a BenchmarkResult,
+    reference: &'a BenchmarkResult,
     sort_order: SortOrder,
 ) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
     let mut results: Vec<_> = results
         .iter()
         .map(|result| {
-            let is_fastest = result == fastest;
+            let is_reference = result == reference;
+            let relative_ordering = compare_mean_time(result, reference);
 
             if result.mean == 0.0 {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
-                    relative_speed: if is_fastest { 1.0 } else { f64::INFINITY },
+                    relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
                     relative_speed_stddev: None,
-                    is_fastest,
+                    is_reference,
+                    relative_ordering,
                 };
             }
 
-            let ratio = result.mean / fastest.mean;
+            let ratio = match relative_ordering {
+                Ordering::Less => reference.mean / result.mean,
+                Ordering::Equal => 1.0,
+                Ordering::Greater => result.mean / reference.mean,
+            };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
             // Covariance asssumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (result.stddev, fastest.stddev) {
+            let ratio_stddev = match (result.stddev, reference.stddev) {
                 (Some(result_stddev), Some(fastest_stddev)) => Some(
                     ratio
                         * ((result_stddev / result.mean).powi(2)
-                            + (fastest_stddev / fastest.mean).powi(2))
+                            + (fastest_stddev / reference.mean).powi(2))
                         .sqrt(),
                 ),
                 _ => None,
@@ -59,7 +67,8 @@ fn compute_relative_speeds<'a>(
                 result,
                 relative_speed: ratio,
                 relative_speed_stddev: ratio_stddev,
-                is_fastest,
+                is_reference,
+                relative_ordering,
             }
         })
         .collect();
@@ -74,6 +83,18 @@ fn compute_relative_speeds<'a>(
     results
 }
 
+pub fn compute_with_check_from_reference<'a>(
+    results: &'a [BenchmarkResult],
+    reference: &'a BenchmarkResult,
+    sort_order: SortOrder,
+) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
+    if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
+        return None;
+    }
+
+    Some(compute_relative_speeds(results, reference, sort_order))
+}
+
 pub fn compute_with_check(
     results: &[BenchmarkResult],
     sort_order: SortOrder,
@@ -134,6 +155,20 @@ fn test_compute_relative_speed() {
     assert_relative_eq!(2.5, annotated_results[2].relative_speed);
 }
 
+#[test]
+fn test_compute_relative_speed_with_reference() {
+    use approx::assert_relative_eq;
+
+    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
+    let reference = create_result("cmd2", 4.0);
+
+    let annotated_results =
+        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
+
+    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
+    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
+}
+
 #[test]
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];

src/benchmark/scheduler.rs

@@ -1,10 +1,10 @@
-use colored::*;
-
 use super::benchmark_result::BenchmarkResult;
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
 use super::{relative_speed, Benchmark};
+use colored::*;
+use std::cmp::Ordering;
 
-use crate::command::Commands;
+use crate::command::{Command, Commands};
 use crate::export::ExportManager;
 use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
 
@@ -38,9 +38,15 @@ impl<'a> Scheduler<'a> {
             ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
         };
 
+        let reference = self
+            .options
+            .reference_command
+            .as_ref()
+            .map(|cmd| Command::new(None, cmd));
+
         executor.calibrate()?;
 
-        for (number, cmd) in self.commands.iter().enumerate() {
+        for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
             self.results
                 .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
 
@@ -65,31 +71,56 @@ impl<'a> Scheduler<'a> {
             return;
         }
 
-        if let Some(annotated_results) = relative_speed::compute_with_check(
+        let reference = self
+            .options
+            .reference_command
+            .as_ref()
+            .map(|_| &self.results[0])
+            .unwrap_or_else(|| relative_speed::fastest_of(&self.results));
+
+        if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
             &self.results,
+            reference,
             self.options.sort_order_speed_comparison,
         ) {
             match self.options.sort_order_speed_comparison {
                 SortOrder::MeanTime => {
                     println!("{}", "Summary".bold());
 
-                    let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
+                    let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
-                    let others = annotated_results.iter().filter(|r| !r.is_fastest);
+                    let others = annotated_results.iter().filter(|r| !r.is_reference);
 
                     println!(
                         "  {} ran",
-                        fastest.result.command_with_unused_parameters.cyan()
+                        reference.result.command_with_unused_parameters.cyan()
                     );
 
                     for item in others {
+                        let stddev = if let Some(stddev) = item.relative_speed_stddev {
+                            format!(" ± {}", format!("{:.2}", stddev).green())
+                        } else {
+                            "".into()
+                        };
+                        let comparator = match item.relative_ordering {
+                            Ordering::Less => format!(
+                                "{}{} times slower than",
+                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                            Ordering::Greater => format!(
+                                "{}{} times faster than",
+                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                            Ordering::Equal => format!(
+                                "    As fast ({}{}) as",
+                                format!("{:.2}", item.relative_speed).bold().green(),
+                                stddev
+                            ),
+                        };
                         println!(
-                            "{}{} times faster than {}",
+                            "{} {}",
-                            format!("{:8.2}", item.relative_speed).bold().green(),
+                            comparator,
-                            if let Some(stddev) = item.relative_speed_stddev {
-                                format!(" ± {}", format!("{:.2}", stddev).green())
-                            } else {
-                                "".into()
-                            },
                             &item.result.command_with_unused_parameters.magenta()
                         );
                     }
@@ -101,10 +132,10 @@ impl<'a> Scheduler<'a> {
                         println!(
                             "  {}{}  {}",
                             format!("{:10.2}", item.relative_speed).bold().green(),
-                            if item.is_fastest {
+                            if item.is_reference {
                                 "        ".into()
                             } else if let Some(stddev) = item.relative_speed_stddev {
-                                format!(" ± {}", format!("{:5.2}", stddev).green())
+                                format!(" ± {}", format!("{stddev:5.2}").green())
                             } else {
                                 "        ".into()
                             },

src/cli.rs

@@ -86,6 +86,16 @@ fn build_command() -> Command {
                      not every time as would happen with the --prepare option."
                 ),
         )
+        .arg(
+            Arg::new("reference")
+                .long("reference")
+                .action(ArgAction::Set)
+                .value_name("CMD")
+                .help(
+                    "The reference command for the relative comparison of results. \
+                    If this is unset, results are compared with the fastest command as reference."
+                )
+        )
         .arg(
             Arg::new("prepare")
                 .long("prepare")

src/command.rs

@@ -62,13 +62,13 @@ impl<'a> Command<'a> {
         let parameters = self
             .get_unused_parameters()
             .fold(String::new(), |output, (parameter, value)| {
-                output + &format!("{} = {}, ", parameter, value)
+                output + &format!("{parameter} = {value}, ")
             });
         let parameters = parameters.trim_end_matches(", ");
         let parameters = if parameters.is_empty() {
             "".into()
         } else {
-            format!(" ({})", parameters)
+            format!(" ({parameters})")
         };
 
         format!("{}{}", self.get_name(), parameters)
@@ -81,7 +81,7 @@ impl<'a> Command<'a> {
     pub fn get_command(&self) -> Result<std::process::Command> {
         let command_line = self.get_command_line();
         let mut tokens = shell_words::split(&command_line)
-            .with_context(|| format!("Failed to parse command '{}'", command_line))?
+            .with_context(|| format!("Failed to parse command '{command_line}'"))?
             .into_iter();
 
         if let Some(program_name) = tokens.next() {
@@ -100,17 +100,14 @@ impl<'a> Command<'a> {
     pub fn get_unused_parameters(&self) -> impl Iterator<Item = &(&'a str, ParameterValue)> {
         self.parameters
             .iter()
-            .filter(move |(parameter, _)| !self.expression.contains(&format!("{{{}}}", parameter)))
+            .filter(move |(parameter, _)| !self.expression.contains(&format!("{{{parameter}}}")))
     }
 
     fn replace_parameters_in(&self, original: &str) -> String {
         let mut result = String::new();
         let mut replacements = BTreeMap::<String, String>::new();
         for (param_name, param_value) in &self.parameters {
-            replacements.insert(
+            replacements.insert(format!("{{{param_name}}}"), param_value.to_string());
-                format!("{{{param_name}}}", param_name = param_name),
-                param_value.to_string(),
-            );
         }
         let mut remaining = original;
         // Manually replace consecutive occurrences to avoid double-replacing: e.g.,

src/export/asciidoc.rs

@@ -32,7 +32,7 @@ impl MarkupExporter for AsciidocExporter {
     }
 
     fn command(&self, cmd: &str) -> String {
-        format!("`{}`", cmd)
+        format!("`{cmd}`")
     }
 }
 
@@ -71,8 +71,7 @@ fn test_asciidoc_exporter_table_header() {
 #[cfg(test)]
 fn cfg_test_table_header(unit_short_name: &str) -> String {
     format!(
-        "[cols=\"<,>,>,>,>\"]\n|===\n| Command \n| Mean [{unit}] \n| Min [{unit}] \n| Max [{unit}] \n| Relative \n",
+        "[cols=\"<,>,>,>,>\"]\n|===\n| Command \n| Mean [{unit_short_name}] \n| Min [{unit_short_name}] \n| Max [{unit_short_name}] \n| Relative \n"
-        unit = unit_short_name
     )
 }
 

src/export/csv.rs

@@ -31,7 +31,7 @@ impl Exporter for CsvExporter {
             .collect();
             if let Some(res) = results.first() {
                 for param_name in res.parameters.keys() {
-                    headers.push(Cow::Owned(format!("parameter_{}", param_name).into_bytes()));
+                    headers.push(Cow::Owned(format!("parameter_{param_name}").into_bytes()));
                 }
             }
             writer.write_record(headers)?;

src/export/markdown.rs

@@ -24,7 +24,7 @@ impl MarkupExporter for MarkdownExporter {
     }
 
     fn command(&self, cmd: &str) -> String {
-        format!("`{}`", cmd)
+        format!("`{cmd}`")
     }
 }
 
@@ -53,8 +53,7 @@ fn test_markdown_formatter_table_divider() {
 #[cfg(test)]
 fn cfg_test_table_header(unit_short_name: String) -> String {
     format!(
-        "| Command | Mean [{unit}] | Min [{unit}] | Max [{unit}] | Relative |\n|:---|---:|---:|---:|---:|\n",
+        "| Command | Mean [{unit_short_name}] | Min [{unit_short_name}] | Max [{unit_short_name}] | Relative |\n|:---|---:|---:|---:|---:|\n"
-        unit = unit_short_name
     )
 }
 

src/export/markup.rs

@@ -32,9 +32,9 @@ pub trait MarkupExporter {
         // emit table header data
         table.push_str(&self.table_row(&[
             "Command",
-            &format!("Mean {}", notation),
+            &format!("Mean {notation}"),
-            &format!("Min {}", notation),
+            &format!("Min {notation}"),
-            &format!("Max {}", notation),
+            &format!("Max {notation}"),
             "Relative",
         ]));
 
@@ -56,10 +56,10 @@ pub trait MarkupExporter {
             let min_str = format_duration_value(measurement.min, Some(unit)).0;
             let max_str = format_duration_value(measurement.max, Some(unit)).0;
             let rel_str = format!("{:.2}", entry.relative_speed);
-            let rel_stddev_str = if entry.is_fastest {
+            let rel_stddev_str = if entry.is_reference {
                 "".into()
             } else if let Some(stddev) = entry.relative_speed_stddev {
-                format!(" ± {:.2}", stddev)
+                format!(" ± {stddev:.2}")
             } else {
                 "".into()
             };
@@ -67,10 +67,10 @@ pub trait MarkupExporter {
             // prepare table row entries
             table.push_str(&self.table_row(&[
                 &self.command(&cmd_str),
-                &format!("{}{}", mean_str, stddev_str),
+                &format!("{mean_str}{stddev_str}"),
                 &min_str,
                 &max_str,
-                &format!("{}{}", rel_str, rel_stddev_str),
+                &format!("{rel_str}{rel_stddev_str}"),
             ]))
         }
 

src/export/mod.rs

@@ -108,7 +108,7 @@ impl ExportManager {
                 ExportTarget::Stdout
             } else {
                 let _ = File::create(filename)
-                    .with_context(|| format!("Could not create export file '{}'", filename))?;
+                    .with_context(|| format!("Could not create export file '{filename}'"))?;
                 ExportTarget::File(filename.to_string())
             },
         });
@@ -153,5 +153,5 @@ impl ExportManager {
 fn write_to_file(filename: &str, content: &[u8]) -> Result<()> {
     let mut file = OpenOptions::new().write(true).open(filename)?;
     file.write_all(content)
-        .with_context(|| format!("Failed to export results to '{}'", filename))
+        .with_context(|| format!("Failed to export results to '{filename}'"))
 }

src/export/orgmode.rs

@@ -18,7 +18,7 @@ impl MarkupExporter for OrgmodeExporter {
     }
 
     fn command(&self, cmd: &str) -> String {
-        format!("={}=", cmd)
+        format!("={cmd}=")
     }
 }
 
@@ -58,8 +58,7 @@ fn test_orgmode_formatter_table_line() {
 #[cfg(test)]
 fn cfg_test_table_header(unit_short_name: String) -> String {
     format!(
-        "| Command  |  Mean [{unit}] |  Min [{unit}] |  Max [{unit}] |  Relative |\n|--+--+--+--+--|\n",
+        "| Command  |  Mean [{unit_short_name}] |  Min [{unit_short_name}] |  Max [{unit_short_name}] |  Relative |\n|--+--+--+--+--|\n"
-        unit = unit_short_name
     )
 }
 

src/options.rs

@@ -38,7 +38,7 @@ impl Default for Shell {
 impl fmt::Display for Shell {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Shell::Default(cmd) => write!(f, "{}", cmd),
+            Shell::Default(cmd) => write!(f, "{cmd}"),
             Shell::Custom(cmdline) => write!(f, "{}", shell_words::join(cmdline)),
         }
     }
@@ -204,6 +204,9 @@ pub struct Options {
     /// Whether or not to ignore non-zero exit codes
     pub command_failure_action: CmdFailureAction,
 
+    // Command to use as a reference for relative speed comparison
+    pub reference_command: Option<String>,
+
     /// Command(s) to run before each timing run
     pub preparation_command: Option<Vec<String>>,
 
@@ -245,6 +248,7 @@ impl Default for Options {
             warmup_count: 0,
             min_benchmarking_time: 3.0,
             command_failure_action: CmdFailureAction::RaiseError,
+            reference_command: None,
             preparation_command: None,
             conclusion_command: None,
             setup_command: None,
@@ -304,6 +308,8 @@ impl Options {
 
         options.setup_command = matches.get_one::<String>("setup").map(String::from);
 
+        options.reference_command = matches.get_one::<String>("reference").map(String::from);
+
         options.preparation_command = matches
             .get_many::<String>("prepare")
             .map(|values| values.map(String::from).collect::<Vec<String>>());
@@ -431,21 +437,25 @@ impl Options {
     }
 
     pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
+        let num_commands = commands.num_commands()
+            + if self.reference_command.is_some() {
+                1
+            } else {
+                0
+            };
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
-                preparation_command.len() <= 1
+                preparation_command.len() <= 1 || num_commands == preparation_command.len(),
-                    || commands.num_commands() == preparation_command.len(),
                 "The '--prepare' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 
         if let Some(conclusion_command) = &self.conclusion_command {
             ensure!(
-                conclusion_command.len() <= 1
+                conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
-                    || commands.num_commands() == conclusion_command.len(),
                 "The '--conclude' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 
@@ -457,7 +467,7 @@ impl Options {
 fn test_default_shell() {
     let shell = Shell::default();
 
-    let s = format!("{}", shell);
+    let s = format!("{shell}");
     assert_eq!(&s, DEFAULT_SHELL);
 
     let cmd = shell.command();
@@ -468,7 +478,7 @@ fn test_default_shell() {
 fn test_can_parse_shell_command_line_from_str() {
     let shell = Shell::parse_from_str("shell -x 'aaa bbb'").unwrap();
 
-    let s = format!("{}", shell);
+    let s = format!("{shell}");
     assert_eq!(&s, "shell -x 'aaa bbb'");
 
     let cmd = shell.command();

src/parameter/mod.rs

@@ -16,7 +16,7 @@ impl Display for ParameterValue {
             ParameterValue::Text(ref value) => value.clone(),
             ParameterValue::Numeric(value) => value.to_string(),
         };
-        write!(f, "{}", str)
+        write!(f, "{str}")
     }
 }
 

src/util/units.rs

@@ -26,7 +26,7 @@ impl Unit {
     /// Returns the Second value formatted for the Unit.
     pub fn format(self, value: Second) -> String {
         match self {
-            Unit::Second => format!("{:.3}", value),
+            Unit::Second => format!("{value:.3}"),
             Unit::MilliSecond => format!("{:.1}", value * 1e3),
             Unit::MicroSecond => format!("{:.1}", value * 1e6),
         }

tests/execution_order_tests.rs

@@ -54,6 +54,11 @@ impl ExecutionOrderTest {
         self.command(output)
     }
 
+    fn reference(&mut self, output: &str) -> &mut Self {
+        self.arg("--reference");
+        self.command(output)
+    }
+
     fn conclude(&mut self, output: &str) -> &mut Self {
         self.arg("--conclude");
         self.command(output)
@@ -364,3 +369,195 @@ fn multiple_parameter_values() {
         .expect_output("command 3 b")
         .run();
 }
+
+#[test]
+fn reference_is_executed_first() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("reference")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_first_parameter_value() {
+    ExecutionOrderTest::new()
+        .arg("--runs=2")
+        .reference("reference")
+        .arg("--parameter-list")
+        .arg("number")
+        .arg("1,2,3")
+        .command("command {number}")
+        .expect_output("reference")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .expect_output("command 2")
+        .expect_output("command 3")
+        .expect_output("command 3")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_separately_from_commands() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("command 1")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn setup_prepare_reference_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .prepare("prepare")
+        .reference("reference")
+        .command("command1")
+        .command("command2")
+        .conclude("conclude")
+        .cleanup("cleanup")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separate_prepare_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepareref")
+        .reference("reference")
+        .conclude("concluderef")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}

tests/integration_tests.rs

@@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo ref")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--prepare=echo a")
@@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() {
         .stderr(predicate::str::contains(
             "The '--prepare' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--prepare' option has to be provided",
+        ));
 }
 
 #[test]
@@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo ref")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--conclude=echo a")
@@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() {
         .stderr(predicate::str::contains(
             "The '--conclude' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--conclude' option has to be provided",
+        ));
 }
 
 #[test]
@@ -413,6 +461,38 @@ fn shows_benchmark_comparison_with_relative_times() {
         );
 }
 
+#[test]
+fn shows_benchmark_comparison_with_same_time() {
+    hyperfine_debug()
+        .arg("--command-name=A")
+        .arg("--command-name=B")
+        .arg("sleep 1.0")
+        .arg("sleep 1.0")
+        .arg("sleep 2.0")
+        .arg("sleep 1000.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("As fast (1.00 ± 0.00) as")
+                .and(predicate::str::contains("2.00 ± 0.00 times faster"))
+                .and(predicate::str::contains("1000.00 ± 0.00 times faster")),
+        );
+}
+
+#[test]
+fn shows_benchmark_comparison_relative_to_reference() {
+    hyperfine_debug()
+        .arg("--reference=sleep 2.0")
+        .arg("sleep 1.0")
+        .arg("sleep 3.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("2.00 ± 0.00 times slower")
+                .and(predicate::str::contains("1.50 ± 0.00 times faster")),
+        );
+}
+
 #[test]
 fn performs_all_benchmarks_in_parameter_scan() {
     hyperfine_debug()
@@ -434,6 +514,29 @@ fn performs_all_benchmarks_in_parameter_scan() {
         );
 }
 
+#[test]
+fn performs_reference_and_all_benchmarks_in_parameter_scan() {
+    hyperfine_debug()
+        .arg("--reference=sleep 25")
+        .arg("--parameter-scan")
+        .arg("time")
+        .arg("30")
+        .arg("45")
+        .arg("--parameter-step-size")
+        .arg("5")
+        .arg("sleep {time}")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Benchmark 1: sleep 25")
+                .and(predicate::str::contains("Benchmark 2: sleep 30"))
+                .and(predicate::str::contains("Benchmark 3: sleep 35"))
+                .and(predicate::str::contains("Benchmark 4: sleep 40"))
+                .and(predicate::str::contains("Benchmark 5: sleep 45"))
+                .and(predicate::str::contains("Benchmark 6: sleep 50").not()),
+        );
+}
+
 #[test]
 fn intermediate_results_are_not_exported_to_stdout() {
     hyperfine_debug()