Track benchmarks in CI (#2012)

* bencher

* backslash

* rust_criterion

* try out the tweag runner

* nix it

* try using the bencher secret

* run it on ubuntu-latest

* try the master/pr split

* huh?

* Rewrite it

* why isn't the upload running?

* test

* test

* Better workflow name

* test

* test

* test

* test

* put back the conditional

* Add docs

.github/workflows/benchmark-master.yaml

@@ -0,0 +1,29 @@
+name: benchmark master
+on:
+  push:
+    branches: master
+jobs:
+  benchmark-master:
+    name: benchmark master
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: cachix/install-nix-action@V27
+        name: "Installing Nix"
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+            accept-flake-config = true
+          nix_path: "nixpkgs=channel:nixos-unstable"
+      - uses: bencherdev/bencher@main
+      - name: Track benchmarks on master
+        run: |
+          bencher run \
+          --project nickel \
+          --token '${{ secrets.BENCHER_API_TOKEN }}' \
+          --branch master \
+          --testbed ubuntu-latest \
+          --adapter rust_criterion \
+          nix develop --command cargo bench --package nickel-lang-core --bench numeric

.github/workflows/benchmark-pr.yaml

@@ -0,0 +1,39 @@
+# This is the part of the benchmark runner that runs on pull requests.
+# It's separate from the master runner for two reasons:
+# - to track it differently on bencher.dev, because we only want to use master as
+#   the historical baseline for comparison
+# - because it might want to run on PRs from forks, and those don't get access to the
+#   bencher API key.
+#
+# See the [bencher docs] for more details.
+#
+# [bencher docs]:  https://bencher.dev/docs/how-to/github-actions/#benchmark-fork-pr-and-upload-from-default-branch
+
+name: benchmark pull requests
+on:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+
+jobs:
+  benchmark-pr:
+    name: benchmark pr
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: cachix/install-nix-action@V27
+        name: "Installing Nix"
+        with:
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+            accept-flake-config = true
+          nix_path: "nixpkgs=channel:nixos-unstable"
+      - name: run benchmarks
+        run: |
+          nix develop --command cargo bench --package nickel-lang-core --bench numeric > criterion-output.txt
+      - name: upload results
+        uses: actions/upload-artifact@v4
+        with:
+          name: criterion-output
+          path: criterion-output.txt

.github/workflows/release-artifacts.yaml

@@ -1,4 +1,4 @@
-name: Upload release artifacts
+name: Test Docker manifest
 on:
   release:
     types: [published]

.github/workflows/run-benchmark.yaml

@@ -1,17 +0,0 @@
-name: benchmark pull requests
-on:
-  pull_request:
-    types: [labeled]
-jobs:
-  runBenchmark:
-    if: ${{ github.event.label.name == 'run-benchmark' && github.event_name == 'pull_request' }}
-    name: run benchmark
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: jasonwilliams/criterion-compare-action@move_to_actions
-        with:
-          branchName: ${{ github.base_ref }}
-          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/upload-pr-benchmarks.yaml

@@ -0,0 +1,72 @@
+name: Track Benchmarks with Bencher
+
+on:
+  workflow_run:
+    workflows: [benchmark pull requests]
+    types: [completed]
+
+jobs:
+  track_fork_pr_branch:
+    name: track pr benchmarks
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    env:
+      BENCHMARK_RESULTS: criterion-output
+      PR_EVENT: event.json
+    steps:
+      - name: Download Benchmark Results
+        uses: actions/github-script@v6
+        with:
+          script: |
+            async function downloadArtifact(artifactName) {
+              let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id,
+              });
+              let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == artifactName
+              })[0];
+              if (!matchArtifact) {
+                core.setFailed(`Failed to find artifact: ${artifactName}`);
+              }
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
+            }
+            await downloadArtifact(process.env.BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.PR_EVENT);
+      - name: Unzip Benchmark Results
+        run: |
+          unzip $BENCHMARK_RESULTS.zip
+          unzip $PR_EVENT.zip
+      - name: Export PR Event Data
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let fs = require('fs');
+            let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
+            core.exportVariable("PR_HEAD", `${prEvent.number}/merge`);
+            core.exportVariable("PR_BASE", prEvent.pull_request.base.ref);
+            core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha);
+            core.exportVariable("PR_NUMBER", prEvent.number);
+      - uses: bencherdev/bencher@main
+      - name: Track Benchmarks with Bencher
+        run: |
+          bencher run \
+          --project nickel \
+          --token '${{ secrets.BENCHER_API_TOKEN }}' \
+          --branch '${{ env.PR_HEAD }}' \
+          --branch-start-point '${{ env.PR_BASE }}' \
+          --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \
+          --testbed ubuntu-latest \
+          --adapter criterion_rust \
+          --err \
+          --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+          --ci-number '${{ env.PR_NUMBER }}' \
+          --file "$BENCHMARK_RESULTS.txt"