1
1
mirror of https://github.com/idris-lang/Idris2.git synced 2024-12-25 12:42:02 +03:00
Idris2/benchmark/bench.sh
2021-01-22 15:08:49 +00:00

122 lines
3.2 KiB
Bash
Executable File

#!/bin/bash
# potential TODO: make tests recursive to allow nicer organisation of tests
TESTS_ROOT=benchmarks
# timeout threshold in seconds (this may be nice to have as a named param)
TIMEOUT=15
# defaults for the optionals
FILENAME="bench-results"
BACKEND="chez"
BUILD=false
FAST=false
function usage {
echo 'Perform all benchmarks in the Idris2 benchmark suite. Command output is written to a csv file.'
echo
echo "Usage: $(basename $0) [OPTION]..." 2>&1
echo ' -b build each test first'
echo ' -c specify the backend to use during compilation (requires -b flag)'
echo ' must be one of chez, refc, racket, gambit'
echo ' (default is chez)'
echo ' -f perform a faster version of the benchmark'
echo ' -o specify name of output csv file'
echo ' (default is `bench-results_$BACKEND[_fast]`)'
echo ' -h shows this message'
echo
echo 'e.g. bench -fbc refc -o myrefcoutput'
exit 1
}
# begin args parsing
optstring=":bc:fo:h"
while getopts ${optstring} arg; do
case "${arg}" in
# whether to build
b) BUILD=true ;;
# set codegen (redundant unless -b passed)
c) case $2 in
chez|refc|racket|gambit)
BACKEND="$2"
export IDRIS2_CG=$BACKEND ;;
*)
echo "$BACKEND is not a recognised backend! You can check the available backends using $0 -h"
exit 1 ;;
esac
;;
# test speed flag
f) FAST=true ;;
# custom output file name
o)
[[ ! -z "$2" ]] && FILENAME="$2"
echo "-o option passed, with value $FILENAME"
;;
# help message
h) usage ;;
\?)
echo "Invalid option: -${OPTARG}."
echo
usage
;;
:) echo "Option -$OPTARG requires an argument" >&2; exit 1 ;;
esac
done
# end args parsing
#begin tests
cwd=$(pwd)
if [ $FILENAME != "bench-results" ];
then
echo here
output_path="$FILENAME.csv";
else
output_path="$cwd/${FILENAME}_${BACKEND}"
if [[ $FAST = true ]]; then output_path+="_fast"; fi
output_path+=".csv"
fi
echo "benchmark,elapsed,user,system" > $output_path
cd $TESTS_ROOT
dashes='-------------------------------'
tests=`ls -d ./*/`
ntests=$(echo $tests | wc -w)
n=1
for test in $tests
do
# clean test name
test=${test%/}
test=${test#./}
echo $dashes
echo "[$n/$ntests] current test:" $test
echo $dashes
cd $test
# build test if necessary
if ! test -f "build/exec/${test}" || [ "$BUILD" = true ]; then
echo building $test...
[[ -f $infile.in ]] && rm -r build
idris2 -o $test $test.idr # - currently do not have a way to handle compilation errors (todo?)
fi
# begin test
echo executing build/exec/$test
# set in stream to relevant input file if present
[[ $FAST = true ]] && infile="${test}_fast" || infile="$test"
[[ -f $infile.in ]] && exec 3<"${infile}.in" || exec 3<&0
# time it, eat stdout
timeout -s SIGINT $TIMEOUT /usr/bin/time -f "${test},%e,%U,%S" -o ${output_path} -a build/exec/${test} <&3 > /dev/null 2>&1
if [ $? -eq 124 ]; then
echo "benchmark failed! (timeout after ${TIMEOUT}s)"
fi
# end test
cd ..
((n+=1))
done
echo $dashes
echo "benchmarks complete! data written to $output_path"
# end tests