Merge pull request #4347 from roc-lang/benchmarks-to-nix

moved earthly benchmarks to nix

.earthignore

@@ -1,4 +0,0 @@
-AUTHORS
-nix
-.envrc
-.gitignore

.github/workflows/benchmarks.yml

@@ -20,28 +20,21 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          ref: "main"
+          # ref: "main" # TODO set this to main once merged, can't do this now because main does not yet have the right files.
           clean: "true"
 
-      - name: Earthly version
-        run: earthly --version
-
       - name: on main; prepare a self-contained benchmark folder
-        run: ./ci/safe-earthly.sh --build-arg BENCH_SUFFIX=main +prep-bench-folder
+        run: nix develop -c ./ci/benchmarks/prep_folder.sh main
 
       - uses: actions/checkout@v3
         with:
           clean: "false" # we want to keep the benchmark folder
 
       - name: on current branch; prepare a self-contained benchmark folder
-        run: ./ci/safe-earthly.sh +prep-bench-folder
-
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+        run: nix develop -c ./ci/benchmarks/prep_folder.sh branch
 
       - name: build benchmark runner
-        run: cd ci/bench-runner && cargo build --release && cd ../..
+        run: nix develop -c bash -c "cd ci/benchmarks/bench-runner && cargo build --release && cd ../../.."
 
       - name: run benchmarks with regression check
-        run: ./ci/bench-runner/target/release/bench-runner --check-executables-changed
+        run: nix develop -c ./ci/benchmarks/bench-runner/target/release/bench-runner --check-executables-changed

.github/workflows/nix_linux_x86_64.yml

@@ -20,7 +20,7 @@ jobs:
           clean: "true"
 
       - name: execute tests with --release
-        run: /home/big-ci-user/.nix-profile/bin/nix develop -c cargo test --locked --release
+        run: nix develop -c cargo test --locked --release
 
       - name: test wasm32 cli_run
-        run: /home/big-ci-user/.nix-profile/bin/nix develop -c cargo test --locked --release --features="wasm32-cli-run"
+        run: nix develop -c cargo test --locked --release --features="wasm32-cli-run"

Cargo.toml

@@ -58,7 +58,7 @@ members = [
     "crates/wasi-libc-sys",
 ]
 exclude = [
-    "ci/bench-runner",
+    "ci/benchmarks/bench-runner",
     # Examples sometimes have Rust hosts in their platforms. The compiler should ignore those.
     "crates/cli_testing_examples",
     "examples",

Earthfile

@@ -1,72 +0,0 @@
-FROM rust:1.61.0-slim-bullseye # make sure to update rust-toolchain.toml too so that everything uses the same rust version
-WORKDIR /earthbuild
-
-prep-debian:
-    RUN apt -y update
-
-install-other-libs:
-    FROM +prep-debian
-    RUN apt -y install wget git
-    RUN apt -y install libxcb-shape0-dev libxcb-xfixes0-dev # for editor clipboard
-    RUN apt -y install libasound2-dev # for editor sounds
-    RUN apt -y install libunwind-dev pkg-config libx11-dev zlib1g-dev
-    RUN apt -y install unzip # for www/build.sh
-
-install-zig-llvm-valgrind:
-    FROM +install-other-libs
-    # editor
-    RUN apt -y install libxkbcommon-dev
-    # zig
-    RUN wget -c https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz --no-check-certificate
-    RUN tar -xf zig-linux-x86_64-0.9.1.tar.xz
-    RUN ln -s /earthbuild/zig-linux-x86_64-0.9.1/zig /bin/zig
-    # zig builtins wasm tests
-    RUN apt -y install build-essential
-    RUN cargo install wasmer-cli --features "singlepass"
-    RUN cargo install bindgen
-    # llvm
-    RUN apt -y install lsb-release software-properties-common gnupg
-    RUN wget https://apt.llvm.org/llvm.sh
-    RUN chmod +x llvm.sh
-    RUN ./llvm.sh 13
-    RUN ln -s /usr/bin/clang-13 /usr/bin/clang
-    # use lld as linker
-    RUN ln -s /usr/bin/lld-13 /usr/bin/ld.lld
-    ENV RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=native"
-    # valgrind
-    RUN apt -y install valgrind
-    # wasm repl & tests
-    RUN rustup target add wasm32-unknown-unknown wasm32-wasi
-    RUN apt -y install libssl-dev
-    RUN OPENSSL_NO_VENDOR=1 cargo install wasm-pack
-    # criterion
-    RUN cargo install cargo-criterion
-    # sccache
-    RUN cargo install sccache
-    RUN sccache -V
-    ENV RUSTC_WRAPPER=/usr/local/cargo/bin/sccache
-    ENV SCCACHE_DIR=/earthbuild/sccache_dir
-    ENV CARGO_INCREMENTAL=0 # no need to recompile package when using new function
-
-copy-dirs:
-    FROM +install-zig-llvm-valgrind
-    COPY --dir crates Cargo.toml Cargo.lock version.txt www ./
-
-# compile everything needed for benchmarks and output a self-contained dir from which benchmarks can be run.
-prep-bench-folder:
-    FROM +copy-dirs
-    # to make use of avx, avx2, sse2, sse4.2... instructions
-    ENV RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=native"
-    ARG BENCH_SUFFIX=branch
-    RUN cargo criterion -V
-    RUN --mount=type=cache,target=$SCCACHE_DIR cd crates/cli && cargo criterion --no-run
-    RUN mkdir -p bench-folder/crates/cli_testing_examples/benchmarks
-    RUN mkdir -p bench-folder/crates/compiler/builtins/bitcode/src
-    RUN mkdir -p bench-folder/target/release/deps
-    RUN cp crates/cli_testing_examples/benchmarks/*.roc bench-folder/crates/cli_testing_examples/benchmarks/
-    RUN cp -r crates/cli_testing_examples/benchmarks/platform bench-folder/crates/cli_testing_examples/benchmarks/
-    RUN cp crates/compiler/builtins/bitcode/src/str.zig bench-folder/crates/compiler/builtins/bitcode/src
-    RUN cp target/release/roc bench-folder/target/release
-    # copy the most recent time bench to bench-folder
-    RUN cp target/release/deps/`ls -t target/release/deps/ | grep time_bench | head -n 1` bench-folder/target/release/deps/time_bench
-    SAVE ARTIFACT bench-folder AS LOCAL bench-folder-$BENCH_SUFFIX

ci/benchmarks/bench-runner/src/main.rs

@@ -33,7 +33,7 @@ fn main() {
 
             if check_if_bench_executables_changed() {
                 println!(
-                    "Comparison of sha256 of executables reveals changes, doing full benchmarks..."
+                    "\n\nComparison of sha256 of executables reveals changes, doing full benchmarks...\n\n"
                 );
 
                 let all_regressed_benches = do_all_benches(optional_args.nr_repeat_benchmarks);
@@ -51,8 +51,8 @@ fn main() {
         eprintln!(
             r#"I can't find bench-folder-main and bench-folder-branch from the current directory.
         I should be executed from the repo root.
-        Use `./ci/safe-earthly.sh --build-arg BENCH_SUFFIX=main +prep-bench-folder` to generate bench-folder-main.
-        Use `./ci/safe-earthly.sh +prep-bench-folder` to generate bench-folder-branch."#
+        Use `./ci/benchmarks/prep_folder.sh main` to generate bench-folder-main.
+        Use `./ci/benchmarks/prep_folder.sh branch` to generate bench-folder-branch."#
         );
 
         process::exit(1)
@@ -85,6 +85,8 @@ fn do_all_benches(nr_repeat_benchmarks: usize) -> HashSet<String> {
         return HashSet::new();
     }
 
+    println!("\n\nDoing benchmarks {:?} times to reduce flukes.\n\n", nr_repeat_benchmarks);
+
     for _ in 1..nr_repeat_benchmarks {
         delete_old_bench_results();
         do_benchmark("main");
@@ -112,7 +114,7 @@ fn do_benchmark(branch_name: &'static str) -> HashSet<String> {
     ))
     .args(&["--bench", "--noplot"])
     .stdout(Stdio::piped())
-    .stderr(Stdio::piped())
+    .stderr(Stdio::inherit())
     .spawn()
     .unwrap_or_else(|_| panic!("Failed to benchmark {}.", branch_name));
 
@@ -133,14 +135,14 @@ fn do_benchmark(branch_name: &'static str) -> HashSet<String> {
                 "Failed to get line that contains benchmark name from last_three_lines_queue.",
             );
 
-            let regex_match = bench_name_regex.find(regressed_bench_name_line).expect("This line should hoave the benchmark name between double quotes but I could not match it");
+            let regex_match = bench_name_regex.find(regressed_bench_name_line).expect("This line should have the benchmark name between double quotes but I could not match it");
 
             regressed_benches.insert(regex_match.as_str().to_string().replace("\"", ""));
         }
 
         last_three_lines_queue.push_front(line_str.clone());
 
-        println!("bench {:?}: {:?}", branch_name, line_str);
+        println!(">>bench {:?}: {:?}", branch_name, line_str);
     }
 
     regressed_benches
@@ -186,8 +188,20 @@ fn sha256_digest<R: Read>(mut reader: R) -> Result<Digest, io::Error> {
 }
 
 fn sha_file(file_path: &Path) -> Result<String, io::Error> {
-    let input = File::open(file_path)?;
-    let reader = BufReader::new(input);
+    // Debug info is dependent on the dir in which executable was created,
+    // so we need to strip that to be able to compare binaries.
+    let no_debug_info_file_path = file_path.to_str().unwrap().to_string() + ("_no_debug_info");
+    std::fs::copy(file_path, &no_debug_info_file_path)?;
+
+    let strip_output = Command::new("strip")
+            .args(["--strip-debug", &no_debug_info_file_path])
+            .output()
+            .expect("failed to execute process");
+
+    assert!(strip_output.status.success());
+
+    let no_debug_info_file = File::open(no_debug_info_file_path)?;
+    let reader = BufReader::new(no_debug_info_file);
     let digest = sha256_digest(reader)?;
 
     Ok(HEXUPPER.encode(digest.as_ref()))

ci/benchmarks/prep_folder.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# compile everything needed for benchmarks and output a self-contained dir from which benchmarks can be run.
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# to make use of avx, avx2, sse2, sse4.2... instructions
+RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=native"
+BENCH_SUFFIX=$1
+
+cargo criterion -V
+cd crates/cli && cargo criterion --no-run && cd ../..
+mkdir -p bench-folder/crates/cli_testing_examples/benchmarks
+mkdir -p bench-folder/crates/compiler/builtins/bitcode/src
+mkdir -p bench-folder/target/release/deps
+mkdir -p bench-folder/target/release/lib
+cp "crates/cli_testing_examples/benchmarks/"*".roc" bench-folder/crates/cli_testing_examples/benchmarks/
+cp -r crates/cli_testing_examples/benchmarks/platform bench-folder/crates/cli_testing_examples/benchmarks/
+cp crates/compiler/builtins/bitcode/src/str.zig bench-folder/crates/compiler/builtins/bitcode/src
+cp target/release/roc bench-folder/target/release
+cp -r target/release/lib bench-folder/target/release
+
+# copy the most recent time bench to bench-folder
+cp target/release/deps/`ls -t target/release/deps/ | grep time_bench | head -n 1` bench-folder/target/release/deps/time_bench
+mv bench-folder bench-folder-$BENCH_SUFFIX

ci/enable-lld.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 mkdir -p $HOME/.cargo
 echo -e "[build]\nrustflags = [\"-C\", \"link-arg=-fuse-ld=lld\", \"-C\", \"target-cpu=native\"]" > $HOME/.cargo/config
 

ci/get_latest_release_url.sh

@@ -2,6 +2,9 @@
 
 # assumes roc_releases.json is present
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 LATEST_RELEASE_URL=`cat roc_releases.json | jq --arg arch $1 --arg today $(date +'%Y-%m-%d') '.[0] | .assets | map(.browser_download_url) | map(select(. | contains("\($arch)-\($today)"))) | .[0]'`
 
 if [[ "$LATEST_RELEASE_URL" == "null" ]]

ci/package_release.sh

@@ -1,4 +1,8 @@
 #!/usr/bin/env bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 cp target/release/roc ./roc # to be able to exclude "target" later in the tar command
 cp -r target/release/lib ./lib
 tar -czvf $1 --exclude="target" --exclude="zig-cache" roc lib LICENSE LEGAL_DETAILS examples/helloWorld.roc examples/platform-switching examples/cli crates/roc_std

ci/safe-earthly.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-LOG_FILE="earthly_log.txt"
-touch $LOG_FILE
-
-# first arg + everything after
-ARGS=${@:1}
-FULL_CMD="earthly --config ci/earthly-conf.yml $ARGS"
-echo $FULL_CMD
-script -efq $LOG_FILE -c "$FULL_CMD"
-EXIT_CODE=$?
-
-if grep -q "failed to mount" "$LOG_FILE"; then
-  echo ""
-  echo ""
-  echo "------<<<<<<!!!!!!>>>>>>------"
-  echo "DETECTED FAILURE TO MOUNT ERROR: running without cache"
-  echo "------<<<<<<!!!!!!>>>>>>------"
-  echo ""
-  echo ""
-  earthly --config ci/earthly-conf.yml --no-cache $ARGS
-else
-  exit $EXIT_CODE
-fi

ci/write_version.sh

@@ -1,3 +1,7 @@
 #!/usr/bin/env bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 # version.txt is used by the CLI: roc --version
 printf 'nightly pre-release, built from commit ' > version.txt && git log --pretty=format:'%h' -n 1 >> version.txt && printf ' on ' >> version.txt && date -u >> version.txt

ci/www-repl.sh

@@ -1,3 +1,6 @@
 #!/usr/bin/env bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 crates/repl_wasm/build-www.sh `pwd`/roc_repl_wasm.tar.gz

crates/compiler/build/src/link.rs

@@ -137,7 +137,7 @@ pub fn build_zig_host_native(
 
     zig_cmd.args(&[
         zig_host_src,
-        emit_bin,
+        &format!("-femit-bin={}", emit_bin),
         "--pkg-begin",
         "str",
         zig_str_path,
@@ -205,7 +205,7 @@ pub fn build_zig_host_native(
 
     zig_cmd.args(&[
         zig_host_src,
-        emit_bin,
+        &format!("-femit-bin={}", emit_bin),
         "--pkg-begin",
         "str",
         zig_str_path,
@@ -299,7 +299,7 @@ pub fn build_zig_host_native(
     }
     zig_cmd.args(&[
         zig_host_src,
-        emit_bin,
+        &format!("-femit-bin={}", emit_bin),
         "--pkg-begin",
         "str",
         zig_str_path,
@@ -579,8 +579,6 @@ pub fn rebuild_host(
                 )
             }
             Architecture::X86_64 => {
-                let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap());
-
                 let target = match target.operating_system {
                     OperatingSystem::Windows => "x86_64-windows-gnu",
                     _ => "native",
@@ -589,7 +587,7 @@ pub fn rebuild_host(
                 build_zig_host_native(
                     &env_path,
                     &env_home,
-                    &emit_bin,
+                    host_dest.to_str().unwrap(),
                     zig_host_src.to_str().unwrap(),
                     zig_str_path.to_str().unwrap(),
                     target,
@@ -597,33 +595,27 @@ pub fn rebuild_host(
                     shared_lib_path,
                 )
             }
-            Architecture::X86_32(_) => {
-                let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap());
-                build_zig_host_native(
-                    &env_path,
-                    &env_home,
-                    &emit_bin,
-                    zig_host_src.to_str().unwrap(),
-                    zig_str_path.to_str().unwrap(),
-                    "i386-linux-musl",
-                    opt_level,
-                    shared_lib_path,
-                )
-            }
+            Architecture::X86_32(_) => build_zig_host_native(
+                &env_path,
+                &env_home,
+                host_dest.to_str().unwrap(),
+                zig_host_src.to_str().unwrap(),
+                zig_str_path.to_str().unwrap(),
+                "i386-linux-musl",
+                opt_level,
+                shared_lib_path,
+            ),
 
-            Architecture::Aarch64(_) => {
-                let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap());
-                build_zig_host_native(
-                    &env_path,
-                    &env_home,
-                    &emit_bin,
-                    zig_host_src.to_str().unwrap(),
-                    zig_str_path.to_str().unwrap(),
-                    target_zig_str(target),
-                    opt_level,
-                    shared_lib_path,
-                )
-            }
+            Architecture::Aarch64(_) => build_zig_host_native(
+                &env_path,
+                &env_home,
+                host_dest.to_str().unwrap(),
+                zig_host_src.to_str().unwrap(),
+                zig_str_path.to_str().unwrap(),
+                target_zig_str(target),
+                opt_level,
+                shared_lib_path,
+            ),
             _ => internal_error!("Unsupported architecture {:?}", target.architecture),
         };
 

crates/compiler/builtins/bitcode/benchmark.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
 zig build-exe benchmark/dec.zig -O ReleaseFast --main-pkg-path .

crates/repl_test/test_wasm.sh

@@ -1,4 +1,5 @@
-set -eux
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
 
 # Build the compiler for WebAssembly target
 # We *could* write a build.rs to do this but we'd have nested cargo processes with different targets.

crates/repl_wasm/build-www.sh

@@ -8,6 +8,7 @@
 # Our website deployment script downloads that zipfile and copies the files into www/build/repl/
 # We use this two-step process because Netlify times out if we try to build the Web REPL there.
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
 if ! which wasm-pack

examples/swiftui/run.sh

@@ -1,5 +1,8 @@
 #! /usr/bin/env bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
 cargo run -- build
 mkdir -p SwiftUIDemo.app/Contents/MacOS/
 mv swiftui SwiftUIDemo.app/Contents/MacOS/SwiftUIDemo

flake.nix

@@ -105,6 +105,7 @@
           debugir
           rust
           rust-bindgen
+          cargo-criterion
         ]);
       in {
 

www/build.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
 # cd into the directory where this script lives.

www/netlify.sh

@@ -2,6 +2,7 @@
 
 # Runs on every Netlify build, to set up the Netlify server.
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
 rustup update