mirror of
https://github.com/hasura/graphql-engine.git
synced 2024-12-14 17:02:49 +03:00
29408c7854
The benchmarks were not working because they couldn't access Git information. While this is fair, they didn't actually fail, they kept going, so I have improved the script to fail hard, and fixed the bug by setting `RELEASE_VERSION` so the build script doesn't bother trying to read Git. V3_GIT_ORIGIN_REV_ID: 9573ea32371d7a4d7b99c87017a8d5d77815832c
70 lines
2.6 KiB
Bash
Executable File
70 lines
2.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Benchmark script to compare the benchmarks caused by the changes of the
|
|
# current PR against the main branch
|
|
|
|
set -e
|
|
set -u
|
|
set -o pipefail
|
|
|
|
set -x
|
|
|
|
# Build without needing to read the Git context
|
|
export RELEASE_VERSION='bench'
|
|
|
|
# Run the benchmarks on the current PR and save it in a JSON file
|
|
#
|
|
# TODO: naveen, Benchmark seperate targets and merge their JSON. Currently,
|
|
# `--save-baseline` only supports saving benchmarks for a single target. For
|
|
# now, we'll benchmark only the `execute` target
|
|
# See: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
|
|
#
|
|
# cargo bench --bench execute --bench generate_ir --bench validation --bench parser -- --save-baseline current-benchmarks
|
|
cargo bench --bench execute -- --save-baseline current-benchmarks
|
|
critcmp --export current-benchmarks > current-benchmarks.json
|
|
|
|
# Clone into the main branch of v3-engine
|
|
# https://$2@github.com/hasura/v3-engine.git
|
|
git clone --branch main "https://${2}@github.com/hasura/v3-engine.git" main-copy
|
|
pushd main-copy
|
|
# Run the benchmarks on the main branch and save it in a JSON file
|
|
cargo bench --bench execute -- --save-baseline main-benchmarks
|
|
critcmp --export main-benchmarks > /app/main-benchmarks.json
|
|
popd
|
|
|
|
# Compare between the benchmarks of main and the benchmarks generated by the PR
|
|
critcmp main-benchmarks.json current-benchmarks.json > diff.txt
|
|
|
|
# Format the result of benchmark into a Code comment, so that when we
|
|
# comment on the PR, the benchmark results are displayed in a nice format
|
|
sed -i '1s/^/Benchmark report\n ```\n/' diff.txt
|
|
echo '```' >> diff.txt
|
|
|
|
# Format the results into a json format
|
|
# { "body": "<BENCHMARK AS STRING>""}
|
|
jq -n --rawfile diff diff.txt '.body=$diff' > bench_comment.json
|
|
|
|
curl -s -H "Authorization: token $2" \
|
|
-X GET "https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" > /tmp/comments.json
|
|
comment_id=$(jq 'map(select(.body | contains ("Benchmark report"))) | last | .id' /tmp/comments.json)
|
|
echo "Comment ID: ${comment_id}"
|
|
|
|
# Post the benchmark as comment on PR or update existing one
|
|
if [ "$comment_id" = 'null' ]; then
|
|
curl -L \
|
|
-X POST \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer $2" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"https://api.github.com/repos/hasura/v3-engine/issues/$1/comments" \
|
|
-d @bench_comment.json
|
|
else
|
|
curl -L \
|
|
-X PATCH \
|
|
-H "Accept: application/vnd.github+json" \
|
|
-H "Authorization: Bearer $2" \
|
|
-H "X-GitHub-Api-Version: 2022-11-28" \
|
|
"https://api.github.com/repos/hasura/v3-engine/issues/comments/$comment_id" \
|
|
-d @bench_comment.json
|
|
fi
|