Add new --sort option

This adds a new `--sort` option to choose the way in which the results
in the speed comparison and the markup exports are ordered.

closes #614
closes #601

src/benchmark/relative_speed.rs

@@ -1,7 +1,7 @@
 use std::cmp::Ordering;
 
 use super::benchmark_result::BenchmarkResult;
-use crate::util::units::Scalar;
+use crate::{options::SortOrder, util::units::Scalar};
 
 #[derive(Debug)]
 pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -25,8 +25,9 @@ fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
 fn compute_relative_speeds<'a>(
     results: &'a [BenchmarkResult],
     fastest: &'a BenchmarkResult,
+    sort_order: SortOrder,
 ) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    results
+    let mut results: Vec<_> = results
         .iter()
         .map(|result| {
             let is_fastest = result == fastest;
@@ -61,11 +62,21 @@ fn compute_relative_speeds<'a>(
                 is_fastest,
             }
         })
-        .collect()
+        .collect();
+
+    match sort_order {
+        SortOrder::Command => {}
+        SortOrder::MeanTime => {
+            results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));
+        }
+    }
+
+    results
 }
 
 pub fn compute_with_check(
     results: &[BenchmarkResult],
+    sort_order: SortOrder,
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
     let fastest = fastest_of(results);
 
@@ -73,14 +84,17 @@ pub fn compute_with_check(
         return None;
     }
 
-    Some(compute_relative_speeds(results, fastest))
+    Some(compute_relative_speeds(results, fastest, sort_order))
 }
 
 /// Same as compute_with_check, potentially resulting in relative speeds of infinity
-pub fn compute(results: &[BenchmarkResult]) -> Vec<BenchmarkResultWithRelativeSpeed> {
+pub fn compute(
+    results: &[BenchmarkResult],
+    sort_order: SortOrder,
+) -> Vec<BenchmarkResultWithRelativeSpeed> {
     let fastest = fastest_of(results);
 
-    compute_relative_speeds(results, fastest)
+    compute_relative_speeds(results, fastest, sort_order)
 }
 
 #[cfg(test)]
@@ -113,7 +127,7 @@ fn test_compute_relative_speed() {
         create_result("cmd3", 5.0),
     ];
 
-    let annotated_results = compute_with_check(&results).unwrap();
+    let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();
 
     assert_relative_eq!(1.5, annotated_results[0].relative_speed);
     assert_relative_eq!(1.0, annotated_results[1].relative_speed);
@@ -124,7 +138,7 @@ fn test_compute_relative_speed() {
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
 
-    let annotated_results = compute_with_check(&results);
+    let annotated_results = compute_with_check(&results, SortOrder::Command);
 
     assert!(annotated_results.is_none());
 }

src/benchmark/scheduler.rs

@@ -6,7 +6,7 @@ use super::{relative_speed, Benchmark};
 
 use crate::command::Commands;
 use crate::export::ExportManager;
-use crate::options::{ExecutorKind, Options, OutputStyleOption};
+use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
 
 use anyhow::Result;
 
@@ -46,7 +46,11 @@ impl<'a> Scheduler<'a> {
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.
-            self.export_manager.write_results(&self.results, true)?;
+            self.export_manager.write_results(
+                &self.results,
+                self.options.sort_order_exports,
+                true,
+            )?;
         }
 
         Ok(())
@@ -61,29 +65,55 @@ impl<'a> Scheduler<'a> {
             return;
         }
 
-        if let Some(mut annotated_results) = relative_speed::compute_with_check(&self.results) {
-            annotated_results.sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result));
+        if let Some(annotated_results) = relative_speed::compute_with_check(
+            &self.results,
+            self.options.sort_order_speed_comparison,
+        ) {
+            match self.options.sort_order_speed_comparison {
+                SortOrder::MeanTime => {
+                    println!("{}", "Summary".bold());
 
-            let fastest = &annotated_results[0];
-            let others = &annotated_results[1..];
+                    let fastest = annotated_results.iter().find(|r| r.is_fastest).unwrap();
+                    let others = annotated_results.iter().filter(|r| !r.is_fastest);
 
-            println!("{}", "Summary".bold());
-            println!(
-                "  {} ran",
-                fastest.result.command_with_unused_parameters.cyan()
-            );
+                    println!(
+                        "  {} ran",
+                        fastest.result.command_with_unused_parameters.cyan()
+                    );
 
-            for item in others {
-                println!(
-                    "{}{} times faster than {}",
-                    format!("{:8.2}", item.relative_speed).bold().green(),
-                    if let Some(stddev) = item.relative_speed_stddev {
-                        format!(" ± {}", format!("{:.2}", stddev).green())
-                    } else {
-                        "".into()
-                    },
-                    &item.result.command_with_unused_parameters.magenta()
-                );
+                    for item in others {
+                        println!(
+                            "{}{} times faster than {}",
+                            format!("{:8.2}", item.relative_speed).bold().green(),
+                            if let Some(stddev) = item.relative_speed_stddev {
+                                format!(" ± {}", format!("{:.2}", stddev).green())
+                            } else {
+                                "".into()
+                            },
+                            &item.result.command_with_unused_parameters.magenta()
+                        );
+                    }
+                }
+                SortOrder::Command => {
+                    println!("{}", "Relative speed comparison".bold());
+
+                    for item in annotated_results {
+                        println!(
+                            "  {}{}  {}",
+                            format!("{:10.2}", item.relative_speed).bold().green(),
+                            if item.is_fastest {
+                                "        ".into()
+                            } else {
+                                if let Some(stddev) = item.relative_speed_stddev {
+                                    format!(" ± {}", format!("{:5.2}", stddev).green())
+                                } else {
+                                    "        ".into()
+                                }
+                            },
+                            &item.result.command_with_unused_parameters,
+                        );
+                    }
+                }
             }
         } else {
             eprintln!(
@@ -99,6 +129,7 @@ impl<'a> Scheduler<'a> {
     }
 
     pub fn final_export(&self) -> Result<()> {
-        self.export_manager.write_results(&self.results, false)
+        self.export_manager
+            .write_results(&self.results, self.options.sort_order_exports, false)
     }
 }

src/cli.rs

@@ -160,21 +160,6 @@ fn build_command() -> Command {
                      possible parameter combinations.\n"
                 ),
         )
-        .arg(
-            Arg::new("style")
-                .long("style")
-                .action(ArgAction::Set)
-                .value_name("TYPE")
-                .value_parser(["auto", "basic", "full", "nocolor", "color", "none"])
-                .help(
-                    "Set output style type (default: auto). Set this to 'basic' to disable output \
-                     coloring and interactive elements. Set it to 'full' to enable all effects \
-                     even if no interactive terminal was detected. Set this to 'nocolor' to \
-                     keep the interactive output without any colors. Set this to 'color' to keep \
-                     the colors without any interactive output. Set this to 'none' to disable all \
-                     the output of the tool.",
-                ),
-        )
         .arg(
             Arg::new("shell")
                 .long("shell")
@@ -204,6 +189,38 @@ fn build_command() -> Command {
                 .short('i')
                 .help("Ignore non-zero exit codes of the benchmarked programs."),
         )
+        .arg(
+            Arg::new("style")
+                .long("style")
+                .action(ArgAction::Set)
+                .value_name("TYPE")
+                .value_parser(["auto", "basic", "full", "nocolor", "color", "none"])
+                .help(
+                    "Set output style type (default: auto). Set this to 'basic' to disable output \
+                     coloring and interactive elements. Set it to 'full' to enable all effects \
+                     even if no interactive terminal was detected. Set this to 'nocolor' to \
+                     keep the interactive output without any colors. Set this to 'color' to keep \
+                     the colors without any interactive output. Set this to 'none' to disable all \
+                     the output of the tool.",
+                ),
+        )
+        .arg(
+            Arg::new("sort")
+            .long("sort")
+            .action(ArgAction::Set)
+            .value_name("METHOD")
+            .value_parser(["auto", "command", "mean-time"])
+            .default_value("auto")
+            .hide_default_value(true)
+            .help(
+                "Specify the sort order of the speed comparison summary and the exported tables for \
+                 markup formats (Markdown, AsciiDoc, org-mode):\n  \
+                   * 'auto' (default): the speed comparison will be ordered by time and\n    \
+                     the markup tables will be ordered by command (input order).\n  \
+                   * 'command': order benchmarks in the way they were specified\n  \
+                   * 'mean-time': order benchmarks by mean runtime\n"
+            ),
+        )
         .arg(
             Arg::new("time-unit")
                 .long("time-unit")

src/export/asciidoc.rs

@@ -76,6 +76,9 @@ fn cfg_test_table_header(unit_short_name: &str) -> String {
     )
 }
 
+#[cfg(test)]
+use crate::options::SortOrder;
+
 #[cfg(test)]
 use crate::util::units::Unit;
 
@@ -132,8 +135,12 @@ fn test_asciidoc_format_s() {
         },
     ];
 
-    let actual =
-        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&results, Some(Unit::Second), SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}
 | `FOO=1 BAR=2 command \\| 1` 
@@ -205,7 +212,7 @@ fn test_asciidoc_format_ms() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(Unit::MilliSecond))
+            .serialize(&results, Some(Unit::MilliSecond), SortOrder::Command)
             .unwrap(),
     )
     .unwrap();

src/export/csv.rs

@@ -4,6 +4,7 @@ use csv::WriterBuilder;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::options::SortOrder;
 use crate::util::units::Unit;
 
 use anyhow::Result;
@@ -12,7 +13,12 @@ use anyhow::Result;
 pub struct CsvExporter {}
 
 impl Exporter for CsvExporter {
-    fn serialize(&self, results: &[BenchmarkResult], _unit: Option<Unit>) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        _unit: Option<Unit>,
+        _sort_order: SortOrder,
+    ) -> Result<Vec<u8>> {
         let mut writer = WriterBuilder::new().from_writer(vec![]);
 
         {
@@ -105,8 +111,12 @@ fn test_csv() {
         FOO=one BAR=seven command | 2,11,12,11,13,14,15,16.5,seven,one\n\
         ",
     );
-    let gens =
-        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+    let gens = String::from_utf8(
+        exporter
+            .serialize(&results, Some(Unit::Second), SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
 
     assert_eq!(exps, gens);
 }

src/export/json.rs

@@ -3,6 +3,7 @@ use serde_json::to_vec_pretty;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::options::SortOrder;
 use crate::util::units::Unit;
 
 use anyhow::Result;
@@ -16,7 +17,12 @@ struct HyperfineSummary<'a> {
 pub struct JsonExporter {}
 
 impl Exporter for JsonExporter {
-    fn serialize(&self, results: &[BenchmarkResult], _unit: Option<Unit>) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        _unit: Option<Unit>,
+        _sort_order: SortOrder,
+    ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary { results });
         if let Ok(ref mut content) = output {
             content.push(b'\n');

src/export/markdown.rs

@@ -28,6 +28,9 @@ impl MarkupExporter for MarkdownExporter {
     }
 }
 
+#[cfg(test)]
+use crate::options::SortOrder;
+
 /// Check Markdown-based data row formatting
 #[test]
 fn test_markdown_formatter_table_data() {
@@ -99,7 +102,12 @@ fn test_markdown_format_ms() {
         },
     ];
 
-    let actual = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&timing_results, None, SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}\
 | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 |
@@ -151,7 +159,12 @@ fn test_markdown_format_s() {
         },
     ];
 
-    let actual = String::from_utf8(exporter.serialize(&timing_results, None).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&timing_results, None, SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}\
 | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
@@ -173,20 +186,6 @@ fn test_markdown_format_time_unit_s() {
     let exporter = MarkdownExporter::default();
 
     let timing_results = vec![
-        BenchmarkResult {
-            command: String::from("sleep 0.1"),
-            command_with_unused_parameters: String::from("sleep 0.1"),
-            mean: 0.1057,
-            stddev: Some(0.0016),
-            median: 0.1057,
-            user: 0.0009,
-            system: 0.0011,
-            min: 0.1023,
-            max: 0.1080,
-            times: Some(vec![0.1, 0.1, 0.1]),
-            exit_codes: vec![Some(0), Some(0), Some(0)],
-            parameters: BTreeMap::new(),
-        },
         BenchmarkResult {
             command: String::from("sleep 2"),
             command_with_unused_parameters: String::from("sleep 2"),
@@ -201,23 +200,57 @@ fn test_markdown_format_time_unit_s() {
             exit_codes: vec![Some(0), Some(0), Some(0)],
             parameters: BTreeMap::new(),
         },
+        BenchmarkResult {
+            command: String::from("sleep 0.1"),
+            command_with_unused_parameters: String::from("sleep 0.1"),
+            mean: 0.1057,
+            stddev: Some(0.0016),
+            median: 0.1057,
+            user: 0.0009,
+            system: 0.0011,
+            min: 0.1023,
+            max: 0.1080,
+            times: Some(vec![0.1, 0.1, 0.1]),
+            exit_codes: vec![Some(0), Some(0), Some(0)],
+            parameters: BTreeMap::new(),
+        },
     ];
 
-    let actual = String::from_utf8(
-        exporter
-            .serialize(&timing_results, Some(Unit::Second))
-            .unwrap(),
-    )
-    .unwrap();
-    let expect = format!(
-        "{}\
+    {
+        let actual = String::from_utf8(
+            exporter
+                .serialize(&timing_results, Some(Unit::Second), SortOrder::Command)
+                .unwrap(),
+        )
+        .unwrap();
+        let expect = format!(
+            "{}\
+| `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
+| `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 |
+",
+            cfg_test_table_header("s".to_string())
+        );
+
+        assert_eq!(expect, actual);
+    }
+
+    {
+        let actual = String::from_utf8(
+            exporter
+                .serialize(&timing_results, Some(Unit::Second), SortOrder::MeanTime)
+                .unwrap(),
+        )
+        .unwrap();
+        let expect = format!(
+            "{}\
 | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 |
 | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
 ",
-        cfg_test_table_header("s".to_string())
-    );
+            cfg_test_table_header("s".to_string())
+        );
 
-    assert_eq!(expect, actual);
+        assert_eq!(expect, actual);
+    }
 }
 
 /// This (again) demonstrates that the given time unit (ms) is used to set
@@ -263,7 +296,7 @@ fn test_markdown_format_time_unit_ms() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&timing_results, Some(Unit::MilliSecond))
+            .serialize(&timing_results, Some(Unit::MilliSecond), SortOrder::Command)
             .unwrap(),
     )
     .unwrap();

src/export/markup.rs

@@ -1,5 +1,6 @@
 use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
 use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
+use crate::options::SortOrder;
 use crate::output::format::format_duration_value;
 use crate::util::units::Unit;
 
@@ -105,9 +106,14 @@ fn determine_unit_from_results(results: &[BenchmarkResult]) -> Unit {
 }
 
 impl<T: MarkupExporter> Exporter for T {
-    fn serialize(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        unit: Option<Unit>,
+        sort_order: SortOrder,
+    ) -> Result<Vec<u8>> {
         let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
-        let entries = relative_speed::compute(results);
+        let entries = relative_speed::compute(results, sort_order);
 
         let table = self.table_results(&entries, unit);
         Ok(table.as_bytes().to_vec())

src/export/mod.rs

@@ -15,6 +15,7 @@ use self::markdown::MarkdownExporter;
 use self::orgmode::OrgmodeExporter;
 
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::options::SortOrder;
 use crate::util::units::Unit;
 
 use anyhow::{Context, Result};
@@ -42,7 +43,12 @@ pub enum ExportType {
 /// Interface for different exporters.
 trait Exporter {
     /// Export the given entries in the serialized form.
-    fn serialize(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<Vec<u8>>;
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        unit: Option<Unit>,
+        sort_order: SortOrder,
+    ) -> Result<Vec<u8>>;
 }
 
 pub enum ExportTarget {
@@ -116,9 +122,14 @@ impl ExportManager {
     /// results are written to all file targets (to always have them up to date, even
     /// if a benchmark fails). In the latter case, we only print to stdout targets (in
     /// order not to clutter the output of hyperfine with intermediate results).
-    pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
+    pub fn write_results(
+        &self,
+        results: &[BenchmarkResult],
+        sort_order: SortOrder,
+        intermediate: bool,
+    ) -> Result<()> {
         for e in &self.exporters {
-            let content = || e.exporter.serialize(results, self.time_unit);
+            let content = || e.exporter.serialize(results, self.time_unit, sort_order);
 
             match e.target {
                 ExportTarget::File(ref filename) => {

src/export/orgmode.rs

@@ -22,6 +22,9 @@ impl MarkupExporter for OrgmodeExporter {
     }
 }
 
+#[cfg(test)]
+use crate::options::SortOrder;
+
 /// Check Emacs org-mode data row formatting
 #[test]
 fn test_orgmode_formatter_table_data() {
@@ -105,7 +108,12 @@ fn test_orgmode_format_ms() {
         },
     ];
 
-    let actual = String::from_utf8(exporter.serialize(&results, None).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&results, None, SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}\
 | =sleep 0.1=  |  105.7 ± 1.6 |  102.3 |  108.0 |  1.00 |
@@ -159,8 +167,12 @@ fn test_orgmode_format_s() {
         },
     ];
 
-    let actual =
-        String::from_utf8(exporter.serialize(&results, Some(Unit::Second)).unwrap()).unwrap();
+    let actual = String::from_utf8(
+        exporter
+            .serialize(&results, Some(Unit::Second), SortOrder::Command)
+            .unwrap(),
+    )
+    .unwrap();
     let expect = format!(
         "{}\
 | =sleep 2=  |  2.005 ± 0.002 |  2.002 |  2.008 |  18.97 ± 0.29 |

src/options.rs

@@ -95,6 +95,12 @@ pub enum OutputStyleOption {
     Disabled,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SortOrder {
+    Command,
+    MeanTime,
+}
+
 /// Bounds for the number of benchmark runs
 pub struct RunBounds {
     /// Minimum number of benchmark runs
@@ -210,6 +216,12 @@ pub struct Options {
     /// What color mode to use for the terminal output
     pub output_style: OutputStyleOption,
 
+    /// How to order benchmarks in the relative speed comparison
+    pub sort_order_speed_comparison: SortOrder,
+
+    /// How to order benchmarks in the markup format exports
+    pub sort_order_exports: SortOrder,
+
     /// Determines how we run commands
     pub executor_kind: ExecutorKind,
 
@@ -234,6 +246,8 @@ impl Default for Options {
             setup_command: None,
             cleanup_command: None,
             output_style: OutputStyleOption::Full,
+            sort_order_speed_comparison: SortOrder::MeanTime,
+            sort_order_exports: SortOrder::Command,
             executor_kind: ExecutorKind::default(),
             command_output_policy: CommandOutputPolicy::Null,
             time_unit: None,
@@ -346,6 +360,16 @@ impl Options {
             OutputStyleOption::Disabled => {}
         };
 
+        (
+            options.sort_order_speed_comparison,
+            options.sort_order_exports,
+        ) = match matches.get_one::<String>("sort").map(|s| s.as_str()) {
+            None | Some("auto") => (SortOrder::MeanTime, SortOrder::Command),
+            Some("command") => (SortOrder::Command, SortOrder::Command),
+            Some("mean-time") => (SortOrder::MeanTime, SortOrder::MeanTime),
+            Some(_) => unreachable!("Unknown sort order"),
+        };
+
         options.executor_kind = if matches.get_flag("no-shell") {
             ExecutorKind::Raw
         } else {

tests/integration_tests.rs

@@ -398,3 +398,28 @@ fn unused_parameters_are_shown_in_benchmark_name() {
                 .and(predicate::str::contains("echo test (branch = feature)")),
         );
 }
+
+#[test]
+fn speed_comparison_sort_order() {
+    for sort_order in ["auto", "mean-time"] {
+        hyperfine_debug()
+            .arg("sleep 2")
+            .arg("sleep 1")
+            .arg(format!("--sort={sort_order}"))
+            .assert()
+            .success()
+            .stdout(predicate::str::contains(
+                "sleep 1 ran\n    2.00 ± 0.00 times faster than sleep 2",
+            ));
+    }
+
+    hyperfine_debug()
+        .arg("sleep 2")
+        .arg("sleep 1")
+        .arg("--sort=command")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains(
+            "2.00 ±  0.00  sleep 2\n        1.00          sleep 1",
+        ));
+}