diff --git a/.github/workflows/benchmark-master.yaml b/.github/workflows/benchmark-master.yaml new file mode 100644 index 00000000..7e317448 --- /dev/null +++ b/.github/workflows/benchmark-master.yaml @@ -0,0 +1,29 @@ +name: benchmark master +on: + push: + branches: master +jobs: + benchmark-master: + name: benchmark master + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@V27 + name: "Installing Nix" + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + nix_path: "nixpkgs=channel:nixos-unstable" + - uses: bencherdev/bencher@main + - name: Track benchmarks on master + run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch master \ + --testbed ubuntu-latest \ + --adapter rust_criterion \ + nix develop --command cargo bench --package nickel-lang-core --bench numeric diff --git a/.github/workflows/benchmark-pr.yaml b/.github/workflows/benchmark-pr.yaml new file mode 100644 index 00000000..73ac2a2a --- /dev/null +++ b/.github/workflows/benchmark-pr.yaml @@ -0,0 +1,39 @@ +# This is the part of the benchmark runner that runs on pull requests. +# It's separate from the master runner for two reasons: +# - to track it differently on bencher.dev, because we only want to use master as +# the historical baseline for comparison +# - because it might want to run on PRs from forks, and those don't get access to the +# bencher API key. +# +# See the [bencher docs] for more details. +# +# [bencher docs]: https://bencher.dev/docs/how-to/github-actions/#benchmark-fork-pr-and-upload-from-default-branch + +name: benchmark pull requests +on: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + benchmark-pr: + name: benchmark pr + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@V27 + name: "Installing Nix" + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = true + nix_path: "nixpkgs=channel:nixos-unstable" + - name: run benchmarks + run: | + nix develop --command cargo bench --package nickel-lang-core --bench numeric > criterion-output.txt + - name: upload results + uses: actions/upload-artifact@v4 + with: + name: criterion-output + path: criterion-output.txt diff --git a/.github/workflows/release-artifacts.yaml b/.github/workflows/release-artifacts.yaml index d9cee164..b9eb3e7d 100644 --- a/.github/workflows/release-artifacts.yaml +++ b/.github/workflows/release-artifacts.yaml @@ -1,4 +1,4 @@ -name: Upload release artifacts +name: Test Docker manifest on: release: types: [published] diff --git a/.github/workflows/run-benchmark.yaml b/.github/workflows/run-benchmark.yaml deleted file mode 100644 index b1a0ee3a..00000000 --- a/.github/workflows/run-benchmark.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: benchmark pull requests -on: - pull_request: - types: [labeled] -jobs: - runBenchmark: - if: ${{ github.event.label.name == 'run-benchmark' && github.event_name == 'pull_request' }} - name: run benchmark - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: jasonwilliams/criterion-compare-action@move_to_actions - with: - branchName: ${{ github.base_ref }} - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/upload-pr-benchmarks.yaml b/.github/workflows/upload-pr-benchmarks.yaml new file mode 100644 index 00000000..f8f7f3dc --- /dev/null +++ b/.github/workflows/upload-pr-benchmarks.yaml @@ -0,0 +1,72 @@ +name: Track Benchmarks with Bencher + +on: + workflow_run: + workflows: [benchmark pull requests] + types: [completed] + +jobs: + track_fork_pr_branch: + name: track pr benchmarks + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + env: + BENCHMARK_RESULTS: criterion-output + PR_EVENT: event.json + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + async function downloadArtifact(artifactName) { + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == artifactName + })[0]; + if (!matchArtifact) { + core.setFailed(`Failed to find artifact: ${artifactName}`); + } + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); + } + await downloadArtifact(process.env.BENCHMARK_RESULTS); + await downloadArtifact(process.env.PR_EVENT); + - name: Unzip Benchmark Results + run: | + unzip $BENCHMARK_RESULTS.zip + unzip $PR_EVENT.zip + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", `${prEvent.number}/merge`); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --project nickel \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch '${{ env.PR_HEAD }}' \ + --branch-start-point '${{ env.PR_BASE }}' \ + --branch-start-point-hash '${{ env.PR_BASE_SHA }}' \ + --testbed ubuntu-latest \ + --adapter criterion_rust \ + --err \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number '${{ env.PR_NUMBER }}' \ + --file "$BENCHMARK_RESULTS.txt"