graphql-engine/v3/benchmark.sh

#!/usr/bin/env bash

# Benchmark script to compare the benchmarks caused by the changes of the
# current PR against the main branch

set -e
set -u
set -o pipefail

set -x

# Build without needing to read the Git context
export RELEASE_VERSION='bench'

# Run the benchmarks on the current PR and save it in a JSON file
#
# TODO: naveen, Benchmark seperate targets and merge their JSON. Currently,
# `--save-baseline` only supports saving benchmarks for a single target. For
# now, we'll benchmark only the `execute` target
# See: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
#
# cargo bench --bench execute --bench generate_ir --bench validation --bench parser -- --save-baseline current-benchmarks
cargo bench --bench execute -- --save-baseline current-benchmarks
critcmp --export current-benchmarks > current-benchmarks.json

# Clone into the main branch of v3-engine
# https://$2@github.com/hasura/v3-engine.git
git clone --branch main "https://${2}@github.com/hasura/v3-engine.git" main-copy
pushd main-copy
# Run the benchmarks on the main branch and save it in a JSON file
cargo bench --bench execute -- --save-baseline main-benchmarks
critcmp --export main-benchmarks > /app/main-benchmarks.json
popd

# Compare between the benchmarks of main and the benchmarks generated by the PR
critcmp main-benchmarks.json current-benchmarks.json > diff.txt

# Format the result of benchmark into a Code comment, so that when we
# comment on the PR, the benchmark results are displayed in a nice format
sed -i '1s/^/Benchmark report\n ```\n/' diff.txt
echo '```' >> diff.txt

# Format the results into a json format
# { "body": "<BENCHMARK AS STRING>""}
jq -n --rawfile diff diff.txt '.body=$diff' > bench_comment.json

curl -s -H "Authorization: token $2" \
  -X GET "https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" > /tmp/comments.json
comment_id=$(jq 'map(select(.body | contains ("Benchmark report"))) | last | .id' /tmp/comments.json)
echo "Comment ID: ${comment_id}"

# Post the benchmark as comment on PR or update existing one
if [ "$comment_id" = 'null' ]; then
  curl -L \
    -X POST \
    -H "Accept: application/vnd.github+json" \
    -H "Authorization: Bearer $2" \
    -H "X-GitHub-Api-Version: 2022-11-28" \
    "https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" \
    -d @bench_comment.json
else
  curl -L \
    -X PATCH \
    -H "Accept: application/vnd.github+json" \
    -H "Authorization: Bearer $2" \
    -H "X-GitHub-Api-Version: 2022-11-28" \
    "https://api.github.com/repos/hasura/v3-engine/issues/comments/$comment_id" \
    -d @bench_comment.json
fi