sharkdp/hyperfine
1
1
Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2024-10-05 18:07:24 +03:00
Code Issues Projects Releases Wiki Activity

Fix prepare and conclude behaviour and add tests.

Browse Source
This commit is contained in:
Jan-Eric Nitschke 2024-06-23 18:38:07 +02:00
parent 6df4aedb28
commit c900f7f6af
4 changed files with 306 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/benchmark/relative_speed.rs
View File

@ -155,6 +155,20 @@ fn test_compute_relative_speed() {
assert_relative_eq!(2.5, annotated_results[2].relative_speed);
}
#[test]
fn test_compute_relative_speed_with_reference() {
use approx::assert_relative_eq;
let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
let reference = create_result("cmd2", 4.0);
let annotated_results =
compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
assert_relative_eq!(2.0, annotated_results[0].relative_speed);
assert_relative_eq!(1.25, annotated_results[1].relative_speed);
}
#[test]
fn test_compute_relative_speed_for_zero_times() {
let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];

16
src/options.rs
View File

@ -437,21 +437,25 @@ impl Options {
}
pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
let num_commands = commands.num_commands()
+ if self.reference_command.is_some() {
1
} else {
0
};
if let Some(preparation_command) = &self.preparation_command {
ensure!(
preparation_command.len() <= 1
|| commands.num_commands() == preparation_command.len(),
preparation_command.len() <= 1 || num_commands == preparation_command.len(),
"The '--prepare' option has to be provided just once or N times, where N is the \
number of benchmark commands."
number of benchmark commands including a potential reference."
);
}
if let Some(conclusion_command) = &self.conclusion_command {
ensure!(
conclusion_command.len() <= 1
|| commands.num_commands() == conclusion_command.len(),
conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
"The '--conclude' option has to be provided just once or N times, where N is the \
number of benchmark commands."
number of benchmark commands including a potential reference."
);
}

197
tests/execution_order_tests.rs
View File

@ -54,6 +54,11 @@ impl ExecutionOrderTest {
self.command(output)
}
fn reference(&mut self, output: &str) -> &mut Self {
self.arg("--reference");
self.command(output)
}
fn conclude(&mut self, output: &str) -> &mut Self {
self.arg("--conclude");
self.command(output)
@ -364,3 +369,195 @@ fn multiple_parameter_values() {
.expect_output("command 3 b")
.run();
}
#[test]
fn reference_is_executed_first() {
ExecutionOrderTest::new()
.arg("--runs=1")
.reference("reference")
.command("command 1")
.command("command 2")
.expect_output("reference")
.expect_output("command 1")
.expect_output("command 2")
.run();
}
#[test]
fn reference_is_executed_first_parameter_value() {
ExecutionOrderTest::new()
.arg("--runs=2")
.reference("reference")
.arg("--parameter-list")
.arg("number")
.arg("1,2,3")
.command("command {number}")
.expect_output("reference")
.expect_output("reference")
.expect_output("command 1")
.expect_output("command 1")
.expect_output("command 2")
.expect_output("command 2")
.expect_output("command 3")
.expect_output("command 3")
.run();
}
#[test]
fn reference_is_executed_separately_from_commands() {
ExecutionOrderTest::new()
.arg("--runs=1")
.reference("command 1")
.command("command 1")
.command("command 2")
.expect_output("command 1")
.expect_output("command 1")
.expect_output("command 2")
.run();
}
#[test]
fn setup_prepare_reference_conclude_cleanup_combined() {
ExecutionOrderTest::new()
.arg("--warmup=1")
.arg("--runs=2")
.setup("setup")
.prepare("prepare")
.reference("reference")
.command("command1")
.command("command2")
.conclude("conclude")
.cleanup("cleanup")
// reference
.expect_output("setup")
.expect_output("prepare")
.expect_output("reference")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("reference")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("reference")
.expect_output("conclude")
.expect_output("cleanup")
// 1
.expect_output("setup")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("cleanup")
// 2
.expect_output("setup")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("cleanup")
.run();
}
#[test]
fn setup_separete_prepare_separate_conclude_cleanup_combined() {
ExecutionOrderTest::new()
.arg("--warmup=1")
.arg("--runs=2")
.setup("setup")
.cleanup("cleanup")
.prepare("prepare1")
.command("command1")
.conclude("conclude1")
.prepare("prepare2")
.command("command2")
.conclude("conclude2")
// 1
.expect_output("setup")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("cleanup")
// 2
.expect_output("setup")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("cleanup")
.run();
}
#[test]
fn setup_separete_prepare_reference_separate_conclude_cleanup_combined() {
ExecutionOrderTest::new()
.arg("--warmup=1")
.arg("--runs=2")
.setup("setup")
.cleanup("cleanup")
.prepare("prepareref")
.reference("reference")
.conclude("concluderef")
.prepare("prepare1")
.command("command1")
.conclude("conclude1")
.prepare("prepare2")
.command("command2")
.conclude("conclude2")
// reference
.expect_output("setup")
.expect_output("prepareref")
.expect_output("reference")
.expect_output("concluderef")
.expect_output("prepareref")
.expect_output("reference")
.expect_output("concluderef")
.expect_output("prepareref")
.expect_output("reference")
.expect_output("concluderef")
.expect_output("cleanup")
// 1
.expect_output("setup")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("prepare1")
.expect_output("command1")
.expect_output("conclude1")
.expect_output("cleanup")
// 2
.expect_output("setup")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("prepare2")
.expect_output("command2")
.expect_output("conclude2")
.expect_output("cleanup")
.run();
}

85
tests/integration_tests.rs
View File

@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() {
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--prepare=echo ref")
.arg("--prepare=echo a")
.arg("--prepare=echo b")
.arg("--reference=echo ref")
.arg("echo a")
.arg("echo b")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--prepare=echo a")
@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() {
.stderr(predicate::str::contains(
"The '--prepare' option has to be provided",
));
hyperfine()
.arg("--runs=1")
.arg("--prepare=echo a")
.arg("--prepare=echo b")
.arg("--reference=echo ref")
.arg("echo a")
.arg("echo b")
.assert()
.failure()
.stderr(predicate::str::contains(
"The '--prepare' option has to be provided",
));
}
#[test]
@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() {
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--conclude=echo ref")
.arg("--conclude=echo a")
.arg("--conclude=echo b")
.arg("--reference=echo ref")
.arg("echo a")
.arg("echo b")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--conclude=echo a")
@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() {
.stderr(predicate::str::contains(
"The '--conclude' option has to be provided",
));
hyperfine()
.arg("--runs=1")
.arg("--conclude=echo a")
.arg("--conclude=echo b")
.arg("--reference=echo ref")
.arg("echo a")
.arg("echo b")
.assert()
.failure()
.stderr(predicate::str::contains(
"The '--conclude' option has to be provided",
));
}
#[test]
@ -413,6 +461,20 @@ fn shows_benchmark_comparison_with_relative_times() {
);
}
#[test]
fn shows_benchmark_comparison_relative_to_reference() {
hyperfine_debug()
.arg("--reference=sleep 2.0")
.arg("sleep 1.0")
.arg("sleep 3.0")
.assert()
.success()
.stdout(
predicate::str::contains("2.00 ± 0.00 times slower")
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
);
}
#[test]
fn performs_all_benchmarks_in_parameter_scan() {
hyperfine_debug()
@ -434,6 +496,29 @@ fn performs_all_benchmarks_in_parameter_scan() {
);
}
#[test]
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
hyperfine_debug()
.arg("--reference=sleep 25")
.arg("--parameter-scan")
.arg("time")
.arg("30")
.arg("45")
.arg("--parameter-step-size")
.arg("5")
.arg("sleep {time}")
.assert()
.success()
.stdout(
predicate::str::contains("Benchmark 1: sleep 25")
.and(predicate::str::contains("Benchmark 2: sleep 30"))
.and(predicate::str::contains("Benchmark 3: sleep 35"))
.and(predicate::str::contains("Benchmark 4: sleep 40"))
.and(predicate::str::contains("Benchmark 5: sleep 45"))
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
);
}
#[test]
fn intermediate_results_are_not_exported_to_stdout() {
hyperfine_debug()