Add --setup (-s) option, like --prepare but runs once per batch

Add a --setup option for the use-case of wanting to do one-off setup
before a set of benchmarks, not the once-per-test setup that --prepare
does. This is useful for the cases noted in the updated documentation.

Per the feedback on https://github.com/sharkdp/hyperfine/pull/448 this
new "--setup" option will steal the "-s" short option from
"--style" (initially this was called "--build" and used the
non-conflicting "-b").

Potential future work:

I'd prefer if this and --cleanup took N number of commands, so you
could do e.g.:

    hyperfine -L rev master,next \
        -b 'git worktree add /tmp/test-{r}'  \
        -b 'make -C /tmp/test-{r} all'  \
	'make -C /tmp/test-{r} test' \
	-c 'git worktree remove /tmp/test-{r}'

I.e. a shortcut around not providing these with &&, which makes things
a bit more readable.

But the --cleanup option doesn't do that, so let's just go with what
it's doing for consistency, so for this you'll now need to do:

    hyperfine -L rev master,next \
        -b 'git worktree add /tmp/test-{r} &&
	    make -C /tmp/test-{r} all'  \
	'make -C /tmp/test-{r} test' \
	-c 'git worktree remove /tmp/test-{r}'

CHANGELOG.md

@@ -2,8 +2,14 @@
 
 ## Features
 
+- Added `--setup` (`s`) option that can be used to run `make all` or
+  similar. It runs once per set of tests, like `--cleanup` (`c`).
+
 ## Changes
 
+- The `-s` short option for `--style` has been usurped by the new
+  `--setup` option.
+
 ## Bugfixes
 
 ## Other

doc/hyperfine.1

@@ -62,6 +62,12 @@ Perform at most \fImaxruns\fP (number) runs for each command. Default: no limit.
 Perform exactly \fIruns\fP (number) runs for each command. If this option is not specified,
 \fBhyperfine\fR automatically determines the number of runs.
 .HP
+\fB\-s\fR, \fB\-\-setup\fR \fIcmd\fP
+.IP
+Execute \fIcmd\fP once before each set of timing runs. This is useful
+for compiling software or doing other one-off setup.
+The \fB\-\-setup\fR option can only be specified once.
+.HP
 \fB\-p\fR, \fB\-\-prepare\fR \fIcmd...\fP
 .IP
 Execute \fIcmd\fP before each timing run. This is useful for clearing disk caches,
@@ -119,14 +125,15 @@ Example:
 .IP
 This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'.
 .HP
-\fB\-s\fR, \fB\-\-style\fR \fItype\fP
+\fB\-\-style\fR \fItype\fP
 .IP
 Set output style \fItype\fP (default: auto). Set this to 'basic' to disable output
 coloring and interactive elements. Set it to 'full' to enable all effects even
 if no interactive terminal was detected. Set this to 'nocolor' to keep the
 interactive output without any colors. Set this to 'color' to keep the colors
 without any interactive output. Set this to 'none' to disable all the output
-of the tool.
+of the tool. In hyperfine versions v0.4.0..v1.12.0 this option took the \fB\-s\fR,
+short option, which is now used by \fB\--setup\fR.
 .HP
 \fB\-S\fR, \fB\-\-shell\fR \fIshell\fP
 .IP
@@ -207,6 +214,18 @@ Export the results of a parameter scan benchmark to a markdown table:
 \fBhyperfine\fR \fB\-\-export\-markdown\fR output.md \fB\-\-parameter-scan\fR time 1 5 'sleep {time}'
 .fi
 .RE
+.LP
+Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fIcmd\fP and \fB\-\-cleanup\fR will run:
+.RS
+.nf
+\fBhyperfine\fR \fB\-L\fR n 1,2 \fB\-r\fR 2 \fB\-\-show-output\fR \\
+	\fB\-\-setup\fR 'echo setup n={n}' \\
+	\fB\-\-prepare\fR 'echo prepare={n}' \\
+	\fB\-\-cleanup\fR 'echo cleanup n={n}' \\
+	'echo command n={n}'
+.fi
+.RE
+.RE
 .SH AUTHOR
 .LP
 David Peter (sharkdp)

src/app.rs

@@ -73,6 +73,20 @@ fn build_app() -> App<'static, 'static> {
                 .help("Perform exactly NUM runs for each command. If this option is not specified, \
                        hyperfine automatically determines the number of runs."),
         )
+        .arg(
+            Arg::with_name("setup")
+                .long("setup")
+                .short("s")
+                .takes_value(true)
+                .number_of_values(1)
+                .value_name("CMD")
+                .help(
+                    "Execute CMD before each set of timing runs. This is useful for \
+                     compiling your software with the provided parameters, or to do any \
+                     other work that should happen once before a series of benchmark runs,\
+                     not every time as would happen with the --prepare option."
+                ),
+        )
         .arg(
             Arg::with_name("prepare")
                 .long("prepare")
@@ -155,7 +169,6 @@ fn build_app() -> App<'static, 'static> {
         .arg(
             Arg::with_name("style")
                 .long("style")
-                .short("s")
                 .takes_value(true)
                 .value_name("TYPE")
                 .possible_values(&["auto", "basic", "full", "nocolor", "color", "none"])

src/benchmark.rs

@@ -185,6 +185,18 @@ fn run_intermediate_command(
     })
 }
 
+/// Run the command specified by `--setup`.
+fn run_setup_command(
+    shell: &Shell,
+    command: &Option<Command<'_>>,
+    show_output: bool,
+) -> io::Result<TimingResult> {
+    let error_output = "The setup command terminated with a non-zero exit code. \
+                        Append ' || true' to the command if you are sure that this can be ignored.";
+
+    run_intermediate_command(shell, command, show_output, error_output)
+}
+
 /// Run the command specified by `--prepare`.
 fn run_preparation_command(
     shell: &Shell,
@@ -263,6 +275,12 @@ pub fn run_benchmark(
         Command::new_parametrized(None, preparation_command, cmd.get_parameters().clone())
     });
 
+    // Run setup command
+    let setup_cmd = options.setup_command.as_ref().map(|setup_command| {
+        Command::new_parametrized(None, setup_command, cmd.get_parameters().clone())
+    });
+    run_setup_command(&options.shell, &setup_cmd, options.show_output)?;
+
     // Warmup phase
     if options.warmup_count > 0 {
         let progress_bar = if options.output_style != OutputStyleOption::Disabled {

src/main.rs

@@ -193,6 +193,8 @@ fn build_hyperfine_options<'a>(
         (None, None) => {}
     };
 
+    options.setup_command = matches.value_of("setup").map(String::from);
+
     options.preparation_command = matches
         .values_of("prepare")
         .map(|values| values.map(String::from).collect::<Vec<String>>());

src/options.rs

@@ -114,6 +114,9 @@ pub struct HyperfineOptions {
     /// Whether or not to ignore non-zero exit codes
     pub failure_action: CmdFailureAction,
 
+    /// Command to run before each batch of timing runs
+    pub setup_command: Option<String>,
+
     /// Command to run before each timing run
     pub preparation_command: Option<Vec<String>>,
 
@@ -146,6 +149,7 @@ impl Default for HyperfineOptions {
             runs: Runs::default(),
             min_time_sec: 3.0,
             failure_action: CmdFailureAction::RaiseError,
+            setup_command: None,
             preparation_command: None,
             cleanup_command: None,
             output_style: OutputStyleOption::Full,

tests/integration_tests.rs

@@ -70,6 +70,11 @@ impl ExecutionOrderTest {
         self
     }
 
+    fn setup(&mut self, output: &str) -> &mut Self {
+        self.arg("--setup");
+        self.command(output)
+    }
+
     fn prepare(&mut self, output: &str) -> &mut Self {
         self.arg("--prepare");
         self.command(output)
@@ -147,6 +152,22 @@ fn warmup_runs_are_executed_before_benchmarking_runs() {
         .run();
 }
 
+#[test]
+fn setup_commands_are_executed_before_each_series_of_timing_runs() {
+    ExecutionOrderTest::new()
+        .arg("--runs=2")
+        .setup("setup")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("setup")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("setup")
+        .expect_output("command 2")
+        .expect_output("command 2")
+        .run();
+}
+
 #[test]
 fn prepare_commands_are_executed_before_each_timing_run() {
     ExecutionOrderTest::new()
@@ -181,6 +202,23 @@ fn cleanup_commands_are_executed_once_after_each_benchmark() {
         .run();
 }
 
+#[test]
+fn setup_prepare_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--runs=2")
+        .setup("make")
+        .prepare("make testclean")
+        .command("make test")
+        .cleanup("make clean")
+        .expect_output("make")
+        .expect_output("make testclean")
+        .expect_output("make test")
+        .expect_output("make testclean")
+        .expect_output("make test")
+        .expect_output("make clean")
+        .run();
+}
+
 #[test]
 fn single_parameter_value() {
     ExecutionOrderTest::new()