mirror of
https://github.com/ilyakooo0/streamly.git
synced 2024-10-06 13:17:10 +03:00
Add benchmark comparison script
This commit is contained in:
parent
9c35804edc
commit
32049bdd73
154
bench.sh
Executable file
154
bench.sh
Executable file
@ -0,0 +1,154 @@
|
||||
#!/bin/bash
|
||||
|
||||
print_help () {
|
||||
echo "Usage: $0 [--quick] [--append] [--no-graphs] [--no-measure] \
|
||||
[--no-compare] [--base commit] [--candidate commit] -- <gauge options>"
|
||||
echo "Any arguments after a '--' are passed directly to guage"
|
||||
echo "You can omit '--' if the gauge args used do not start with a '-'."
|
||||
exit
|
||||
}
|
||||
|
||||
# $1: message
|
||||
die () {
|
||||
>&2 echo -e "Error: $1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
COMPARE=1
|
||||
|
||||
while test -n "$1"
|
||||
do
|
||||
case $1 in
|
||||
-h|--help|help) print_help ;;
|
||||
--quick) QUICK=1; shift ;;
|
||||
--append) APPEND=1; shift ;;
|
||||
--base) shift; BASE=$1; shift ;;
|
||||
--candidate) shift; CANDIDATE=$1; shift ;;
|
||||
--no-compare) COMPARE=0; shift ;;
|
||||
--no-graphs) GRAPH=0; shift ;;
|
||||
--no-measure) MEASURE=0; shift ;;
|
||||
--) shift; break ;;
|
||||
-*|--*) print_help ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
|
||||
GAUGE_ARGS=$*
|
||||
|
||||
STACK=stack
|
||||
echo "Using stack command [$STACK]"
|
||||
|
||||
# We build it first at the current commit before checking out any other commit
|
||||
# for benchmarking.
|
||||
if test "$GRAPH" != "0"
|
||||
then
|
||||
CHARTS=$($STACK exec which charts)
|
||||
if test ! -x $CHARTS
|
||||
then
|
||||
echo "Building charts executable"
|
||||
$STACK build --flag "streamly:dev" || die "build failed"
|
||||
fi
|
||||
|
||||
CHARTS=$($STACK exec which charts)
|
||||
if test ! -x $CHARTS
|
||||
then
|
||||
die "Could not find 'charts' executable"
|
||||
fi
|
||||
fi
|
||||
|
||||
# We run the benchmarks in isolation in a separate process so that different
|
||||
# benchmarks do not interfere with other. To enable that we need to pass the
|
||||
# benchmark exe path to guage as an argument. Unfortunately it cannot find its
|
||||
# own path currently.
|
||||
|
||||
# The path is dependent on the architecture and cabal version.
|
||||
# Use this command to find the exe if this script fails with an error:
|
||||
# find .stack-work/ -type f -name "benchmarks"
|
||||
|
||||
find_bench_prog () {
|
||||
BENCH_PROG=`$STACK path --dist-dir`/build/benchOps/benchOps
|
||||
if test ! -x "$BENCH_PROG"
|
||||
then
|
||||
echo
|
||||
echo "WARNING! benchmark binary [$BENCH_PROG] not found or not executable"
|
||||
echo "WARNING! not using isolated measurement."
|
||||
echo
|
||||
fi
|
||||
}
|
||||
|
||||
# --min-duration 0 means exactly one iteration per sample. We use a million
|
||||
# iterations in the benchmarking code explicitly and do not use the iterations
|
||||
# done by the benchmarking tool.
|
||||
#
|
||||
# Benchmarking tool by default discards the first iteration to remove
|
||||
# aberrations due to initial evaluations etc. We do not discard it because we
|
||||
# are anyway doing iterations in the benchmarking code and many of them so that
|
||||
# any constant factor gets amortized and anyway it is a cost that we pay in
|
||||
# real life.
|
||||
#
|
||||
# We can pass --min-samples value from the command line as second argument
|
||||
# after the benchmark name in case we want to use more than one sample.
|
||||
|
||||
if test "$QUICK" = "1"
|
||||
then
|
||||
ENABLE_QUICK="--quick"
|
||||
fi
|
||||
|
||||
OUTPUT_FILE="charts/results.csv"
|
||||
|
||||
run_bench () {
|
||||
$STACK build --bench --no-run-benchmarks || die "build failed"
|
||||
find_bench_prog
|
||||
mkdir -p charts
|
||||
|
||||
# We set min-samples to 1 so that we run with default benchmark duration of 5
|
||||
# seconds, whatever number of samples are possible in that.
|
||||
# We run just one iteration for each sample. Anyway the default is to run
|
||||
# for 30 ms and most our benchmarks are close to that or more.
|
||||
# If we use less than three samples, statistical analysis crashes
|
||||
$BENCH_PROG $ENABLE_QUICK \
|
||||
--include-first-iter \
|
||||
--min-samples 3 \
|
||||
--min-duration 0 \
|
||||
--csvraw=$OUTPUT_FILE \
|
||||
-v 2 \
|
||||
--measure-with $BENCH_PROG $GAUGE_ARGS || die "Benchmarking failed"
|
||||
}
|
||||
|
||||
if test "$MEASURE" != "0"
|
||||
then
|
||||
if test -e $OUTPUT_FILE -a "$APPEND" != 1
|
||||
then
|
||||
mv -f -v $OUTPUT_FILE ${OUTPUT_FILE}.prev
|
||||
fi
|
||||
|
||||
if test "$COMPARE" = "0"
|
||||
then
|
||||
run_bench
|
||||
else
|
||||
if test -z "$CANDIDATE"
|
||||
then
|
||||
CANDIDATE=HEAD
|
||||
fi
|
||||
if test -z "$BASE"
|
||||
then
|
||||
# XXX Should be where the current branch is forked from master
|
||||
BASE="$CANDIDATE^"
|
||||
fi
|
||||
echo "Checking out base commit for benchmarking"
|
||||
git checkout "$BASE" || die "Checkout of base commit failed"
|
||||
run_bench
|
||||
echo "Checking out candidate commit for benchmarking"
|
||||
git checkout "$CANDIDATE" || die "Checkout of candidate commit failed"
|
||||
run_bench
|
||||
fi
|
||||
fi
|
||||
|
||||
if test "$GRAPH" != "0"
|
||||
then
|
||||
echo
|
||||
echo "Generating charts from ${OUTPUT_FILE}..."
|
||||
$CHARTS
|
||||
fi
|
||||
|
||||
# XXX reset back to the original commit
|
Loading…
Reference in New Issue
Block a user