Merge pull request #1314 from rtfeldman/benchmarks

criterion benchmarks for `examples/benchmarks`

.github/workflows/benchmarks.yml

@@ -0,0 +1,24 @@
+on: [pull_request]
+
+name: Benchmarks
+
+env:
+  RUST_BACKTRACE: 1
+
+jobs:
+  prep-dependency-container:
+    name: cd cli; cargo criterion
+    runs-on: [self-hosted, i7-6700K]
+    timeout-minutes: 60
+    env:
+      FORCE_COLOR: 1
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          clean: "true"
+
+      - name: Earthly version
+        run: earthly --version
+
+      - name: install dependencies, build, cd cli, benchmark with criterion
+        run: ./ci/safe-earthly.sh +bench-roc

.github/workflows/ci.yml

@@ -21,5 +21,5 @@ jobs:
         run: earthly --version
 
       - name: install dependencies, build, run zig tests, rustfmt, clippy, cargo test --release
-        run: ./ci/safe-earthly-test-all.sh
+        run: ./ci/safe-earthly.sh +test-all
 

Cargo.lock

@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "ab_glyph"
 version = "0.2.11"
@@ -384,6 +386,23 @@ dependencies = [
  "syn 1.0.72",
 ]
 
+[[package]]
+name = "cli_utils"
+version = "0.1.0"
+dependencies = [
+ "bumpalo",
+ "criterion",
+ "inlinable_string",
+ "roc_cli",
+ "roc_collections",
+ "roc_load",
+ "roc_module",
+ "serde",
+ "serde-xml-rs",
+ "strip-ansi-escapes",
+ "tempfile",
+]
+
 [[package]]
 name = "clipboard-win"
 version = "3.1.1"
@@ -607,9 +626,8 @@ dependencies = [
 
 [[package]]
 name = "criterion"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab327ed7354547cc2ef43cbe20ef68b988e70b4b593cbd66a2a61733123a3d23"
+version = "0.3.5"
+source = "git+https://github.com/Anton-4/criterion.rs#9bd532e35486a3b321d4012534d3e97751a53f85"
 dependencies = [
  "atty",
  "cast",
@@ -634,8 +652,7 @@ dependencies = [
 [[package]]
 name = "criterion-plot"
 version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d"
+source = "git+https://github.com/Anton-4/criterion.rs#9bd532e35486a3b321d4012534d3e97751a53f85"
 dependencies = [
  "cast",
  "itertools 0.9.0",
@@ -2393,9 +2410,8 @@ checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
 
 [[package]]
 name = "plotters"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45ca0ae5f169d0917a7c7f5a9c1a3d3d9598f18f529dd2b8373ed988efea307a"
+version = "0.3.1"
+source = "git+https://github.com/Anton-4/plotters#d043988179b61db714ad60f678637ee145e363d3"
 dependencies = [
  "num-traits",
  "plotters-backend",
@@ -2986,7 +3002,9 @@ version = "0.1.0"
 dependencies = [
  "bumpalo",
  "clap 3.0.0-beta.1",
+ "cli_utils",
  "const_format",
+ "criterion",
  "im 14.3.0",
  "im-rc 14.3.0",
  "indoc 0.3.6",
@@ -3019,10 +3037,7 @@ dependencies = [
  "roc_unify",
  "rustyline",
  "rustyline-derive",
- "serde",
- "serde-xml-rs",
  "serial_test",
- "strip-ansi-escapes",
  "target-lexicon",
  "tempfile",
 ]
@@ -3083,7 +3098,6 @@ dependencies = [
  "colored",
  "confy",
  "copypasta",
- "criterion",
  "env_logger 0.8.3",
  "futures",
  "glyph_brush",

Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "vendor/pretty",
     "editor",
     "cli",
+    "cli/cli_utils",
     "roc_std",
     "docs"
 ]

Earthfile

@@ -38,6 +38,8 @@ install-zig-llvm-valgrind-clippy-rustfmt:
     RUN rustup component add clippy
     # rustfmt
     RUN rustup component add rustfmt
+    # criterion
+    RUN cargo install --git https://github.com/Anton-4/cargo-criterion --branch main
     # sccache
     RUN apt -y install libssl-dev
     RUN cargo install sccache
@@ -108,4 +110,11 @@ test-all:
     BUILD +check-rustfmt
     BUILD +check-clippy
     BUILD +test-rust
+
+bench-roc:
+    FROM +copy-dirs-and-cache
+    ENV RUST_BACKTRACE=full
+    RUN cargo criterion -V
+    RUN --privileged --mount=type=cache,target=$SCCACHE_DIR \
+        cd cli && cargo criterion && sccache --show-stats
     

ci/safe-earthly.sh

@@ -3,7 +3,13 @@
 LOG_FILE="earthly_log.txt"
 touch $LOG_FILE
 
-script -efq $LOG_FILE -c "earthly --config ci/earthly-conf.yml +test-all"
+ARGS=$1
+
+if [[ $ARGS == *"bench"* ]]; then
+  ARGS="--allow-privileged $ARGS"
+fi
+
+script -efq $LOG_FILE -c "earthly --config ci/earthly-conf.yml $ARGS"
 EXIT_CODE=$?
 
 if grep -q "failed to mount" "$LOG_FILE"; then
@@ -14,7 +20,7 @@ if grep -q "failed to mount" "$LOG_FILE"; then
   echo "------<<<<<<!!!!!!>>>>>>------"
   echo ""
   echo ""
-  earthly --config ci/earthly-conf.yml --no-cache +test-all
+  earthly --config ci/earthly-conf.yml --no-cache $ARGS
 else
   exit $EXIT_CODE
 fi

cli/Cargo.toml

@@ -72,8 +72,19 @@ maplit = "1.0.1"
 indoc = "0.3.3"
 quickcheck = "0.8"
 quickcheck_macros = "0.8"
-strip-ansi-escapes = "0.1"
-serde = { version = "1.0", features = ["derive"] }
-serde-xml-rs = "0.4"
 serial_test = "0.5"
 tempfile = "3.1.0"
+criterion = { git = "https://github.com/Anton-4/criterion.rs"} 
+cli_utils = { path = "cli_utils" }
+# Keep the commented deps, they are commented because they require nightly rust
+# criterion-perf-events = "0.1.3"
+# perfcnt = "0.7.1"
+
+[[bench]]
+name = "time_bench"
+harness = false
+
+# Keep this benchmark, it's commented because it requires nightly
+# [[bench]]
+# name = "events_bench"
+# harness = false

cli/benches/events_bench.rs

@@ -0,0 +1,94 @@
+// Keep this benchmark. It's commented because it requires nightly rust.
+/*use cli_utils::bench_utils::{
+    bench_cfold, bench_deriv, bench_nqueens, bench_rbtree_ck, bench_rbtree_delete,
+};
+use criterion_perf_events::Perf;
+use perfcnt::linux::HardwareEventType as Hardware;
+use perfcnt::linux::PerfCounterBuilderLinux as Builder;
+
+use criterion::{criterion_group, criterion_main, BenchmarkGroup, Criterion, SamplingMode};
+
+fn bench_group(c: &mut Criterion<Perf>, hw_event_str: &str) {
+    let mut group = c.benchmark_group(format!("bench-group_no-opt_{}", hw_event_str));
+    // calculate statistics based on a fixed(flat) 100 runs
+    group.sampling_mode(SamplingMode::Flat);
+
+    let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<Perf>>) -> ()> = vec![
+        bench_nqueens,
+        bench_cfold,
+        bench_deriv,
+        bench_rbtree_ck,
+        bench_rbtree_delete,
+        // TODO quicksort
+    ];
+
+    for bench_func in bench_funcs.iter() {
+        bench_func(Some(&mut group))
+    }
+
+    group.finish();
+}
+
+use perfcnt::linux::HardwareEventType;
+
+fn init_criterion(event: HardwareEventType) -> Criterion<Perf> {
+    Criterion::default().with_measurement(Perf::new(Builder::from_hardware_event(event)))
+}
+
+fn bench_instructions(c: &mut Criterion<Perf>) {
+    bench_group(c, "instructions")
+}
+
+criterion_group!(
+    name = benches_instructions;
+    config = init_criterion(Hardware::Instructions);
+    targets = bench_instructions
+);
+
+fn bench_cache_refs(c: &mut Criterion<Perf>) {
+    bench_group(c, "cache_refs")
+}
+
+criterion_group!(
+    name = benches_cache_refs;
+    config = init_criterion(Hardware::CacheReferences);
+    targets = bench_cache_refs
+);
+
+fn bench_cache_misses(c: &mut Criterion<Perf>) {
+    bench_group(c, "cache_misses")
+}
+
+criterion_group!(
+    name = benches_cache_misses;
+    config = init_criterion(Hardware::CacheMisses);
+    targets = bench_cache_misses
+);
+
+fn bench_branch_instructions(c: &mut Criterion<Perf>) {
+    bench_group(c, "branch_instructions")
+}
+
+criterion_group!(
+    name = benches_branch_instructions;
+    config = init_criterion(Hardware::BranchInstructions);
+    targets = bench_branch_instructions
+);
+
+fn bench_branch_misses(c: &mut Criterion<Perf>) {
+    bench_group(c, "branch_misses")
+}
+
+criterion_group!(
+    name = benches_branch_misses;
+    config = init_criterion(Hardware::BranchMisses);
+    targets = bench_branch_misses
+);
+
+criterion_main!(
+    benches_instructions,
+    benches_cache_refs,
+    benches_cache_misses,
+    benches_branch_instructions,
+    benches_branch_misses
+);*/

cli/benches/time_bench.rs

@@ -0,0 +1,31 @@
+use cli_utils::bench_utils::{
+    bench_cfold, bench_deriv, bench_nqueens, bench_rbtree_ck, bench_rbtree_delete,
+};
+use criterion::{
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion, SamplingMode,
+};
+
+fn bench_group_wall_time(c: &mut Criterion) {
+    let mut group = c.benchmark_group("bench-group_wall-time");
+    // calculate statistics based on a fixed(flat) 100 runs
+    group.sampling_mode(SamplingMode::Flat);
+    group.sample_size(200);
+
+    let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![
+        bench_nqueens,   // queens 11
+        bench_cfold,     // e = mkExpr 12 1
+        bench_deriv,     // nest deriv 7 f
+        bench_rbtree_ck, // ms = makeMap 5 5600
+        bench_rbtree_delete, // m = makeMap 6000
+                         // TODO quicksort
+    ];
+
+    for bench_func in bench_funcs.iter() {
+        bench_func(Some(&mut group))
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_group_wall_time);
+criterion_main!(benches);

cli/cli_utils/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "cli_utils"
+version = "0.1.0"
+authors = ["The Roc Contributors"]
+license = "UPL-1.0"
+repository = "https://github.com/rtfeldman/roc"
+edition = "2018"
+description = "Shared code for cli tests and benchmarks"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+roc_cli = { path = "../../cli" }
+roc_collections = { path = "../../compiler/collections" }
+roc_load = { path = "../../compiler/load" }
+roc_module = { path = "../../compiler/module" }
+bumpalo = { version = "3.6.1", features = ["collections"] }
+criterion = { git = "https://github.com/Anton-4/criterion.rs"} 
+inlinable_string = "0.1"
+serde = { version = "1.0", features = ["derive"] }
+serde-xml-rs = "0.4"
+strip-ansi-escapes = "0.1"
+tempfile = "3.1.0"

cli/cli_utils/src/bench_utils.rs

@@ -0,0 +1,127 @@
+use crate::helpers::{example_file, run_cmd, run_roc};
+use criterion::{black_box, measurement::Measurement, BenchmarkGroup};
+use std::path::Path;
+
+fn exec_bench_w_input<T: Measurement>(
+    file: &Path,
+    stdin_str: &str,
+    executable_filename: &str,
+    expected_ending: &str,
+    bench_group_opt: Option<&mut BenchmarkGroup<T>>,
+) {
+    let flags: &[&str] = &["--optimize"];
+
+    let compile_out = run_roc(&[&["build", file.to_str().unwrap()], flags].concat());
+
+    if !compile_out.stderr.is_empty() {
+        panic!("{}", compile_out.stderr);
+    }
+
+    assert!(
+        compile_out.status.success(),
+        "build ended with bad status {:?}",
+        compile_out
+    );
+
+    check_cmd_output(file, stdin_str, executable_filename, expected_ending);
+
+    bench_cmd(file, stdin_str, executable_filename, bench_group_opt);
+}
+
+fn check_cmd_output(
+    file: &Path,
+    stdin_str: &str,
+    executable_filename: &str,
+    expected_ending: &str,
+) {
+    let out = run_cmd(
+        file.with_file_name(executable_filename).to_str().unwrap(),
+        stdin_str,
+        &[],
+    );
+
+    if !&out.stdout.ends_with(expected_ending) {
+        panic!(
+            "expected output to end with {:?} but instead got {:#?}",
+            expected_ending, out
+        );
+    }
+    assert!(out.status.success());
+}
+
+fn bench_cmd<T: Measurement>(
+    file: &Path,
+    stdin_str: &str,
+    executable_filename: &str,
+    bench_group_opt: Option<&mut BenchmarkGroup<T>>,
+) {
+    if let Some(bench_group) = bench_group_opt {
+        bench_group.bench_function(&format!("Benchmarking {:?}", executable_filename), |b| {
+            b.iter(|| {
+                run_cmd(
+                    black_box(file.with_file_name(executable_filename).to_str().unwrap()),
+                    black_box(stdin_str),
+                    &[],
+                )
+            })
+        });
+    } else {
+        run_cmd(
+            black_box(file.with_file_name(executable_filename).to_str().unwrap()),
+            black_box(stdin_str),
+            &[],
+        );
+    }
+}
+
+pub fn bench_nqueens<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup<T>>) {
+    exec_bench_w_input(
+        &example_file("benchmarks", "NQueens.roc"),
+        "11",
+        "nqueens",
+        "2680\n",
+        bench_group_opt,
+    );
+}
+
+pub fn bench_cfold<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup<T>>) {
+    exec_bench_w_input(
+        &example_file("benchmarks", "CFold.roc"),
+        "12",
+        "cfold",
+        "10426 & 10426\n",
+        bench_group_opt,
+    );
+}
+
+pub fn bench_deriv<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup<T>>) {
+    exec_bench_w_input(
+        &example_file("benchmarks", "Deriv.roc"),
+        "7",
+        "deriv",
+        "1 count: 6\n2 count: 22\n3 count: 90\n4 count: 420\n5 count: 2202\n6 count: 12886\n7 count: 83648\n",
+        bench_group_opt,
+    );
+}
+
+pub fn bench_rbtree_ck<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup<T>>) {
+    exec_bench_w_input(
+        &example_file("benchmarks", "RBTreeCk.roc"),
+        "5600",
+        "rbtree-ck",
+        "560\n",
+        bench_group_opt,
+    );
+}
+
+pub fn bench_rbtree_delete<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup<T>>) {
+    exec_bench_w_input(
+        &example_file("benchmarks", "RBTreeDel.roc"),
+        "6000",
+        "rbtree-del",
+        "420\n",
+        bench_group_opt,
+    );
+}
+
+// TODO quicksort

cli/cli_utils/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod bench_utils;
+pub mod helpers;

cli/tests/cli_run.rs

@@ -7,11 +7,9 @@ extern crate roc_collections;
 extern crate roc_load;
 extern crate roc_module;
 
-mod helpers;
-
 #[cfg(test)]
 mod cli_run {
-    use crate::helpers::{
+    use cli_utils::helpers::{
         example_file, extract_valgrind_errors, fixture_file, run_cmd, run_roc, run_with_valgrind,
         ValgrindError, ValgrindErrorXWhat,
     };
@@ -181,7 +179,7 @@ mod cli_run {
     fn run_nqueens_not_optimized() {
         check_output_with_stdin(
             &example_file("benchmarks", "NQueens.roc"),
-            "",
+            "6",
             "nqueens",
             &[],
             "4\n",
@@ -192,8 +190,9 @@ mod cli_run {
     #[test]
     #[serial(cfold)]
     fn run_cfold_not_optimized() {
-        check_output(
+        check_output_with_stdin(
             &example_file("benchmarks", "CFold.roc"),
+            "3",
             "cfold",
             &[],
             "11 & 11\n",
@@ -204,8 +203,9 @@ mod cli_run {
     #[test]
     #[serial(deriv)]
     fn run_deriv_not_optimized() {
-        check_output(
+        check_output_with_stdin(
             &example_file("benchmarks", "Deriv.roc"),
+            "2",
             "deriv",
             &[],
             "1 count: 6\n2 count: 22\n",
@@ -228,8 +228,9 @@ mod cli_run {
     #[test]
     #[serial(deriv)]
     fn run_rbtree_delete_not_optimized() {
-        check_output(
+        check_output_with_stdin(
             &example_file("benchmarks", "RBTreeDel.roc"),
+            "420",
             "rbtree-del",
             &[],
             "30\n",

cli/tests/repl_eval.rs

@@ -4,11 +4,9 @@ extern crate pretty_assertions;
 #[macro_use]
 extern crate indoc;
 
-mod helpers;
-
 #[cfg(test)]
 mod repl_eval {
-    use crate::helpers;
+    use cli_utils::helpers;
     use roc_gen::run_roc::RocCallResult;
 
     #[test]

editor/Cargo.toml

@@ -60,5 +60,4 @@ pretty_assertions = "0.6"
 maplit = "1.0.1"
 quickcheck = "1.0"
 quickcheck_macros = "1.0"
-criterion = "0.3"
 rand = "0.8.2"

examples/benchmarks/CFold.roc

@@ -7,15 +7,16 @@ app "cfold"
 
 main : Task.Task {} []
 main =
-    e = mkExpr 3 1
-    unoptimized = eval e
-    optimized = eval (constFolding (reassoc e))
+    Task.after Task.getInt \n ->
+        e = mkExpr n 1 # original koka n = 20 (set `ulimit -s unlimited` to avoid stack overflow for n = 20)
+        unoptimized = eval e
+        optimized = eval (constFolding (reassoc e))
 
-    unoptimized
-        |> Str.fromInt
-        |> Str.concat " & "
-        |> Str.concat (Str.fromInt optimized)
-        |> Task.putLine
+        unoptimized
+            |> Str.fromInt
+            |> Str.concat " & "
+            |> Str.concat (Str.fromInt optimized)
+            |> Task.putLine
 
 Expr : [
     Add Expr Expr,

examples/benchmarks/Deriv.roc

@@ -9,14 +9,15 @@ IO a : Task.Task a []
 
 main : IO {}
 main =
-    x : Expr
-    x = Var "x"
+    Task.after Task.getInt \n ->
+        x : Expr
+        x = Var "x"
 
-    f : Expr
-    f = pow x x
+        f : Expr
+        f = pow x x
 
-    nest deriv 2 f
-        |> Task.map (\_ -> {})
+        nest deriv n f # original koka n = 10
+            |> Task.map (\_ -> {})
 
 Expr : [ Val I64, Var Str, Add Expr Expr, Mul Expr Expr, Pow Expr Expr, Ln Expr ]
 

examples/benchmarks/NQueens.roc

@@ -5,10 +5,10 @@ app "nqueens"
 
 main : Task.Task {} []
 main =
-    # Task.after Task.getInt \n ->
-    queens 6
-        |> Str.fromInt
-        |> Task.putLine
+    Task.after Task.getInt \n ->
+        queens n # original koka 13
+            |> Str.fromInt
+            |> Task.putLine
 
 ConsList a : [ Nil, Cons a (ConsList a) ]
 

examples/benchmarks/RBTreeCk.roc

@@ -47,18 +47,19 @@ resultWithDefault = \res, default ->
 
 main : Task.Task {} []
 main =
-    ms : ConsList Map
-    ms = makeMap 5 5 # 42_000_00
+    Task.after Task.getInt \n ->
+        ms : ConsList Map
+        ms = makeMap 5 n # original koka n = 4_200_000
 
-    when ms is
-        Cons head _ ->
-            val = fold (\_, v, r -> if v then r + 1 else r) head 0
-            val
-                |> Str.fromInt
-                |> Task.putLine
+        when ms is
+            Cons head _ ->
+                val = fold (\_, v, r -> if v then r + 1 else r) head 0
+                val
+                    |> Str.fromInt
+                    |> Task.putLine
 
-        Nil ->
-            Task.putLine "fail"
+            Nil ->
+                Task.putLine "fail"
 
 insert : Tree (Num k) v, (Num k), v -> Tree (Num k) v
 insert = \t, k, v -> if isRed t then setBlack (ins t k v) else ins t k v

examples/benchmarks/RBTreeDel.roc

@@ -14,14 +14,14 @@ ConsList a : [ Nil, Cons a (ConsList a) ]
 
 main : Task.Task {} []
 main =
-    # benchmarks use 4_200_000
-    m = makeMap 420
+    Task.after Task.getInt \n ->
+        m = makeMap n # koka original n = 4_200_000
 
-    val = fold (\_, v, r -> if v then r + 1 else r) m 0
+        val = fold (\_, v, r -> if v then r + 1 else r) m 0
 
-    val
-        |> Str.fromInt
-        |> Task.putLine
+        val
+            |> Str.fromInt
+            |> Task.putLine
 
 boom : Str -> a
 boom = \_ -> boom ""

vendor/pretty/Cargo.toml

@@ -21,4 +21,4 @@ termcolor = { version = "1.1.0", optional = true }
 [dev-dependencies]
 tempfile = "3.1.0"
 difference = "2"
-criterion = "0.3"
+criterion = { git = "https://github.com/Anton-4/criterion.rs"}