diff --git a/bin/bench-exec-one.sh b/bin/bench-exec-one.sh index 9ee9ef1e5..e193ccc55 100755 --- a/bin/bench-exec-one.sh +++ b/bin/bench-exec-one.sh @@ -10,6 +10,9 @@ set -e set -o pipefail +SCRIPT_DIR=$(cd `dirname $0`; pwd) +source $SCRIPT_DIR/bench-exec-options.sh + # $1: message die () { >&2 echo -e "Error: $1" @@ -23,94 +26,6 @@ warn () { test -n "$BENCH_EXEC_PATH" || die "BENCH_EXEC_PATH env var must be set" test -n "$QUICK_MODE" || warn "QUICK_MODE env var not set (to 0 or 1)" -#------------------------------------------------------------------------------ -# RTS Options -#------------------------------------------------------------------------------ - -# RTS options based on the benchmark executable -bench_exe_rts_opts () { - case "$1" in - Prelude.Concurrent*) echo -n "-K256K -M384M" ;; - *) echo -n "" ;; - esac -} - -# General RTS options for different classes of benchmarks -bench_rts_opts_default () { - case "$1" in - */o-1-sp*) echo -n "-K36K -M16M" ;; - */o-n-h*) echo -n "-K36K -M32M" ;; - */o-n-st*) echo -n "-K1M -M16M" ;; - */o-n-sp*) echo -n "-K1M -M32M" ;; - *) echo -n "" ;; - esac -} - -# Overrides for specific benchmarks -# XXX Note: for tasty-bench we replace the "." separator in the benchmark names -# with "/" for matching with this. It may not work reliably if the benchmark -# name already contains ".". -bench_rts_opts_specific () { - case "$1" in - Data.Stream.StreamD/o-n-space/elimination/toList) echo -n "-K2M" ;; - Data.Stream.StreamK/o-n-space/elimination/toList) echo -n "-K2M" ;; - - Prelude.Parallel/o-n-heap/mapping/mapM) echo -n "-M256M" ;; - Prelude.Parallel/o-n-heap/monad-outer-product/*) echo -n "-M256M" ;; - Prelude.Parallel/o-n-space/monad-outer-product/*) echo -n "-K4M -M256M" ;; - - Prelude.Rate/o-1-space/*) echo -n "-K128K" ;; - Prelude.Rate/o-1-space/asyncly/*) echo -n "-K128K" ;; - - # XXX For GHC-9.0 - Prelude.Serial/o-1-space/mixed/sum-product-fold) echo -n "-K64M" ;; - - # XXX These should be moved to o-n-space? - Prelude.Serial/o-n-heap/grouping/classifySessionsOf) echo -n "-K1M -M32M" ;; - Prelude.Serial/o-n-heap/Functor/*) echo -n "-K4M -M32M" ;; - Prelude.Serial/o-n-heap/transformer/*) echo -n "-K8M -M64M" ;; - - Prelude.Serial/o-n-space/Functor/*) echo -n "-K4M -M64M" ;; - Prelude.Serial/o-n-space/Applicative/*) echo -n "-K8M -M128M" ;; - Prelude.Serial/o-n-space/Monad/*) echo -n "-K8M -M64M" ;; - - # Use -K4M for o-n-space except for grouping - Prelude.Serial/o-n-space/grouping/*) echo -n "" ;; - Prelude.Serial/o-n-space/*) echo -n "-K4M" ;; - - Prelude.WSerial/o-n-space/*) echo -n "-K4M" ;; - - Prelude.Async/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; - Prelude.Ahead/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; - Prelude.Ahead/o-1-space/*) echo -n "-K128K" ;; - - Prelude.WAsync/o-n-heap/monad-outer-product/toNull3) echo -n "-M64M" ;; - Prelude.WAsync/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; - - # XXX need to investigate these, taking too much stack - Data.Parser.ParserD/o-1-space/some) echo -n "-K8M" ;; - Data.Parser/o-1-space/some) echo -n "-K8M" ;; - Data.Parser.ParserD/o-1-space/manyTill) echo -n "-K4M" ;; - Data.Parser/o-1-space/manyTill) echo -n "-K4M" ;; - Data.Parser/o-n-heap/manyAlt) echo -n "-K4M -M128M" ;; - Data.Parser/o-n-heap/someAlt) echo -n "-K4M -M128M" ;; - Data.Parser.ParserK/o-n-heap/manyAlt) echo -n "-K4M -M128M" ;; - Data.Parser.ParserK/o-n-heap/someAlt) echo -n "-K4M -M128M" ;; - Data.Parser.ParserK/o-n-heap/sequence) echo -n "-M64M";; - Data.Parser.ParserK/o-n-heap/sequenceA) echo -n "-M64M";; - - Data.SmallArray/o-1-sp*) echo -n "-K128K" ;; - # For tasty-bench - Data.Array*/o-1-space/generation/show) echo -n "-M32M" ;; - # XXX For GHC-8.10 - Data.Array/o-1-space/transformationX4/map) echo -n "-M32M" ;; - # DEVBUILD only benchmarks - array foldable instance - Data.Array.Foreign/o-1-space/elimination/foldable/foldl*) echo -n "-K8M" ;; - Data.Array.Foreign/o-1-space/elimination/foldable/sum) echo -n "-K8M" ;; - *) echo -n "" ;; - esac -} - #------------------------------------------------------------------------------ # Speed options #------------------------------------------------------------------------------ @@ -139,43 +54,6 @@ use_quicker_mode () { fi } -bench_exe_quick_opts () { - case "$1" in - Prelude.Concurrent) set_super_quick_mode ;; - Prelude.Rate) set_super_quick_mode ;; - Prelude.Adaptive) set_super_quick_mode;; - *) echo -n "" ;; - esac -} - -# XXX Note: for tasty-bench we replace the "." separator in the benchmark names -# with "/" for matching with this. It may not work reliably if the benchmark -# name already contains ".". - -# Use quick options for benchmarks that take too long -bench_quick_opts () { - case "$1" in - Prelude.Parallel/o-n-heap/mapping/mapM) set_super_quick_mode ;; - Prelude.Parallel/o-n-heap/monad-outer-product/*) set_super_quick_mode ;; - Prelude.Parallel/o-n-space/monad-outer-product/*) set_super_quick_mode ;; - Prelude.Parallel/o-n-heap/generation/*) use_quicker_mode ;; - Prelude.Parallel/o-n-heap/mapping/*) use_quicker_mode ;; - Prelude.Parallel/o-n-heap/concat-foldable/*) use_quicker_mode ;; - - Prelude.Async/o-1-space/monad-outer-product/*) use_quicker_mode ;; - Prelude.Async/o-n-space/monad-outer-product/*) use_quicker_mode ;; - - Prelude.Ahead/o-1-space/monad-outer-product/*) use_quicker_mode ;; - Prelude.Ahead/o-n-space/monad-outer-product/*) use_quicker_mode ;; - - Prelude.WAsync/o-n-heap/monad-outer-product/*) use_quicker_mode ;; - Prelude.WAsync/o-n-space/monad-outer-product/*) use_quicker_mode ;; - - FileSystem.Handle/*) use_quicker_mode ;; - *) echo -n "" ;; - esac -} - bench_output_file() { local bench_name=$1 echo "charts/$bench_name/results.csv" @@ -203,16 +81,13 @@ fi RTS_OPTIONS=\ "+RTS -T \ -$(bench_exe_rts_opts $(basename $BENCH_EXEC_PATH)) \ -$(bench_rts_opts_default $BENCH_NAME) \ -$(bench_rts_opts_specific $BENCH_NAME) \ +$(bench_rts_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME) \ $RTS_OPTIONS \ -RTS" QUICK_MODE_TYPE="\ $(if test "$QUICK_MODE" -ne 0; then set_super_quick_mode; fi) \ -$(bench_exe_quick_opts $(basename $BENCH_EXEC_PATH)) \ -$(bench_quick_opts $BENCH_NAME)" +$(bench_speed_options $(basename $BENCH_EXEC_PATH) $BENCH_NAME)" for i in $QUICK_MODE_TYPE do diff --git a/bin/bench-exec-options.sh b/bin/bench-exec-options.sh new file mode 100644 index 000000000..cf0e0c6a4 --- /dev/null +++ b/bin/bench-exec-options.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------ +# RTS options based on the benchmark executable name and benchmark name +#------------------------------------------------------------------------------ + +bench_rts_options () { + local exe_name + local bench_name + + exe_name="$1" + bench_name="$2" + + # Based on benchmark class + case "$bench_name" in + */o-1-sp*) echo -n "-K36K -M16M" ;; + */o-n-h*) echo -n "-K36K -M32M" ;; + */o-n-st*) echo -n "-K1M -M16M" ;; + */o-n-sp*) echo -n "-K1M -M32M" ;; + *) echo -n "" ;; + esac + + echo " " + + # Based on specific benchmark + # XXX Note: for tasty-bench we replace the "." separator in the benchmark names + # with "/" for matching with this. It may not work reliably if the benchmark + # name already contains ".". + case "$bench_name" in + Data.Stream.StreamD/o-n-space/elimination/toList) echo -n "-K2M" ;; + Data.Stream.StreamK/o-n-space/elimination/toList) echo -n "-K2M" ;; + + Prelude.Parallel/o-n-heap/mapping/mapM) echo -n "-M256M" ;; + Prelude.Parallel/o-n-heap/monad-outer-product/*) echo -n "-M256M" ;; + Prelude.Parallel/o-n-space/monad-outer-product/*) echo -n "-K4M -M256M" ;; + + Prelude.Rate/o-1-space/*) echo -n "-K128K" ;; + Prelude.Rate/o-1-space/asyncly/*) echo -n "-K128K" ;; + + # XXX For GHC-9.0 + Prelude.Serial/o-1-space/mixed/sum-product-fold) echo -n "-K64M" ;; + + # XXX These should be moved to o-n-space? + Prelude.Serial/o-n-heap/grouping/classifySessionsOf) echo -n "-K1M -M32M" ;; + Prelude.Serial/o-n-heap/Functor/*) echo -n "-K4M -M32M" ;; + Prelude.Serial/o-n-heap/transformer/*) echo -n "-K8M -M64M" ;; + + Prelude.Serial/o-n-space/Functor/*) echo -n "-K4M -M64M" ;; + Prelude.Serial/o-n-space/Applicative/*) echo -n "-K8M -M128M" ;; + Prelude.Serial/o-n-space/Monad/*) echo -n "-K8M -M64M" ;; + + # Use -K4M for o-n-space except for grouping + Prelude.Serial/o-n-space/grouping/*) echo -n "" ;; + Prelude.Serial/o-n-space/*) echo -n "-K4M" ;; + + Prelude.WSerial/o-n-space/*) echo -n "-K4M" ;; + + Prelude.Async/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; + Prelude.Ahead/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; + Prelude.Ahead/o-1-space/*) echo -n "-K128K" ;; + + Prelude.WAsync/o-n-heap/monad-outer-product/toNull3) echo -n "-M64M" ;; + Prelude.WAsync/o-n-space/monad-outer-product/*) echo -n "-K4M" ;; + + # XXX need to investigate these, taking too much stack + Data.Parser.ParserD/o-1-space/some) echo -n "-K8M" ;; + Data.Parser/o-1-space/some) echo -n "-K8M" ;; + Data.Parser.ParserD/o-1-space/manyTill) echo -n "-K4M" ;; + Data.Parser/o-1-space/manyTill) echo -n "-K4M" ;; + Data.Parser/o-n-heap/manyAlt) echo -n "-K4M -M128M" ;; + Data.Parser/o-n-heap/someAlt) echo -n "-K4M -M128M" ;; + Data.Parser.ParserK/o-n-heap/manyAlt) echo -n "-K4M -M128M" ;; + Data.Parser.ParserK/o-n-heap/someAlt) echo -n "-K4M -M128M" ;; + Data.Parser.ParserK/o-n-heap/sequence) echo -n "-M64M";; + Data.Parser.ParserK/o-n-heap/sequenceA) echo -n "-M64M";; + + Data.SmallArray/o-1-sp*) echo -n "-K128K" ;; + # For tasty-bench + Data.Array*/o-1-space/generation/show) echo -n "-M32M" ;; + # XXX For GHC-8.10 + Data.Array/o-1-space/transformationX4/map) echo -n "-M32M" ;; + # DEVBUILD only benchmarks - array foldable instance + Data.Array.Foreign/o-1-space/elimination/foldable/foldl*) echo -n "-K8M" ;; + Data.Array.Foreign/o-1-space/elimination/foldable/sum) echo -n "-K8M" ;; + *) echo -n "" ;; + esac + + case "$exe_name" in + Prelude.Concurrent*) echo -n " -K256K -M384M" ;; + *) echo -n "" ;; + esac +} + +#------------------------------------------------------------------------------ +# Speed options +#------------------------------------------------------------------------------ + +bench_speed_options () { + local exe_name + local bench_name + + exe_name="$1" + bench_name="$2" + + case "$exe_name" in + Prelude.Concurrent) set_super_quick_mode ;; + Prelude.Rate) set_super_quick_mode ;; + Prelude.Adaptive) set_super_quick_mode;; + *) echo -n "" ;; + esac + + # XXX Note: for tasty-bench we replace the "." separator in the benchmark names + # with "/" for matching with this. It may not work reliably if the benchmark + # name already contains ".". + + # Use quick options for benchmarks that take too long + case "$bench_name" in + Prelude.Parallel/o-n-heap/mapping/mapM) set_super_quick_mode ;; + Prelude.Parallel/o-n-heap/monad-outer-product/*) set_super_quick_mode ;; + Prelude.Parallel/o-n-space/monad-outer-product/*) set_super_quick_mode ;; + Prelude.Parallel/o-n-heap/generation/*) use_quicker_mode ;; + Prelude.Parallel/o-n-heap/mapping/*) use_quicker_mode ;; + Prelude.Parallel/o-n-heap/concat-foldable/*) use_quicker_mode ;; + + Prelude.Async/o-1-space/monad-outer-product/*) use_quicker_mode ;; + Prelude.Async/o-n-space/monad-outer-product/*) use_quicker_mode ;; + + Prelude.Ahead/o-1-space/monad-outer-product/*) use_quicker_mode ;; + Prelude.Ahead/o-n-space/monad-outer-product/*) use_quicker_mode ;; + + Prelude.WAsync/o-n-heap/monad-outer-product/*) use_quicker_mode ;; + Prelude.WAsync/o-n-space/monad-outer-product/*) use_quicker_mode ;; + + FileSystem.Handle/*) use_quicker_mode ;; + *) echo -n "" ;; + esac +} diff --git a/bin/bench.sh b/bin/bench.sh index 130c3aaa3..8c84b4986 100755 --- a/bin/bench.sh +++ b/bin/bench.sh @@ -1,5 +1,19 @@ #!/usr/bin/env bash +# Note that this script is used in the "streamly" package as well as +# in "streaming-benchmarks" package. Any changes to the script should be +# generic enough so that it works in both the cases. + +# Customization options +BENCHMARK_DIR=benchmark +BENCHMARK_PACKAGE_NAME=streamly-benchmarks +BENCHMARK_PACKAGE_VERSION=0.0.0 + +USE_GAUGE=0 +DEFAULT_FIELDS="allocated cputime bytescopied" + +#------------------------------------------------------------------------------ + set -o pipefail SCRIPT_DIR=$(cd `dirname $0`; pwd) @@ -13,6 +27,7 @@ print_help () { echo " [--prefix " echo " [--fields <"field1 field2 ..." | help>]" echo " [--sort-by-name]" + echo " [--compare]" echo " [--graphs]" echo " [--no-measure]" echo " [--append]" @@ -25,10 +40,11 @@ print_help () { echo " [--with-compiler ]" echo " [--cabal-build-options ]" echo " [--rtsopts ]" - echo " [--compare] [--base ] [--candidate ]" + echo " [--commit-compare] [--base ] [--candidate ]" echo " -- " echo echo "--benchmarks: benchmarks to run, use 'help' for list of benchmarks" + echo "--compare: compare the specified benchmarks with each other" echo "--fields: measurement fields to report, use 'help' for a list" echo "--graphs: Generate graphical reports" echo "--no-measure: Don't run benchmarks, run reports from previous results" @@ -46,7 +62,7 @@ print_help () { echo "bench.sh --benchmarks Data.Parser -- Data.Parser/o-1-space " echo "restricts Heap/Stack space for O(1) characterstics" echo - echo "When using --compare, by default comparative chart of HEAD^ vs HEAD" + echo "When using --commit-compare, by default comparative chart of HEAD^ vs HEAD" echo "commit is generated, in the 'charts' directory." echo "Use --base and --candidate to select the commits to compare." echo @@ -99,9 +115,9 @@ build_report_prog() { then echo "Building bench-report executables" BUILD_ONCE=1 - pushd benchmark/bench-report + pushd $BENCHMARK_DIR/bench-report local cmd - cmd="$CABAL_EXECUTABLE install --installdir ../../bin bench-report" + cmd="$CABAL_EXECUTABLE install --installdir $SCRIPT_DIR bench-report" if test "$USE_NIX" -eq 0 then $cmd || die "bench-report build failed" @@ -240,14 +256,14 @@ run_benches_comparing() { git checkout "$BASE" || die "Checkout of base commit [$BASE] failed" $BUILD_BENCH || die "build failed" - run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args + run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args echo "Checking out candidate commit [$CANDIDATE] for benchmarking" git checkout "$CANDIDATE" || \ die "Checkout of candidate [$CANDIDATE] commit failed" $BUILD_BENCH || die "build failed" - run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args + run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args # XXX reset back to the original commit } @@ -269,9 +285,9 @@ run_measurements() { backup_output_file $i done - if test "$COMPARE" = "0" + if test "$COMMIT_COMPARE" = "0" then - run_bench_targets streamly-benchmarks b "$bench_list" target_exe_extra_args + run_bench_targets $BENCHMARK_PACKAGE_NAME b "$bench_list" target_exe_extra_args else run_benches_comparing "$bench_list" fi @@ -300,16 +316,21 @@ run_reports() { cd $SCRIPT_DIR/.. -USE_GAUGE=0 USE_GIT_CABAL=1 USE_NIX=0 set_common_vars -DEFAULT_FIELDS="allocated bytescopied cputime" -ALL_FIELDS="$FIELDS time cycles utime stime minflt majflt nvcsw nivcsw" +COMMON_FIELDS="allocated bytescopied cputime maxrss" +if test "$USE_GAUGE" -eq 1 +then + ALL_FIELDS="$COMMON_FIELDS time cycles utime stime minflt majflt nvcsw nivcsw" +else + ALL_FIELDS="$COMMON_FIELDS" +fi FIELDS=$DEFAULT_FIELDS COMPARE=0 +COMMIT_COMPARE=0 BASE= CANDIDATE= @@ -348,6 +369,7 @@ do --slow) SLOW=1; shift ;; --quick) QUICK_MODE=1; shift ;; --compare) COMPARE=1; shift ;; + --commit-compare) COMMIT_COMPARE=1; shift ;; --raw) RAW=1; shift ;; --append) APPEND=1; shift ;; --long) LONG=1; shift ;; @@ -443,7 +465,7 @@ fi BUILD_BENCH="$CABAL_EXECUTABLE v2-build $BUILD_FLAGS $CABAL_BUILD_OPTIONS --enable-benchmarks" if test "$MEASURE" = "1" then - run_build "$BUILD_BENCH" streamly-benchmarks bench "$TARGETS" + run_build "$BUILD_BENCH" $BENCHMARK_PACKAGE_NAME bench "$TARGETS" run_measurements "$TARGETS" fi @@ -451,20 +473,38 @@ fi # Run reports #----------------------------------------------------------------------------- -COMPARISON_REPORTS="" -for i in $COMPARISONS -do - if test "$(has_item "$TARGETS_ORIG" $i)" = $i - then - COMPARISON_REPORTS="$COMPARISON_REPORTS $i" - mkdir -p "charts/$i" - constituents=$(eval "echo -n \$${i}") - dest_file="charts/$i/results.csv" +# $1: var name +build_comparison_results () { + local name + local constituents + + name=$1 + constituents=$(eval "echo -n \$${name}") + mkdir -p "charts/$name" + dest_file="charts/$name/results.csv" : > $dest_file for j in $constituents do cat "charts/$j/results.csv" >> $dest_file done +} + +if test "$COMPARE" -eq 1 +then + DYN_CMP_GRP="$(echo "$TARGETS" | sed -e 's/ /_/g')_cmp" + eval "$DYN_CMP_GRP=\"$TARGETS\"" + COMPARISON_REPORTS=$DYN_CMP_GRP + build_comparison_results $DYN_CMP_GRP +else + COMPARISON_REPORTS="" +fi + +for i in $COMPARISONS +do + if test "$(has_item "$TARGETS_ORIG" $i)" = $i + then + COMPARISON_REPORTS="$COMPARISON_REPORTS $i" + build_comparison_results $i fi done @@ -472,4 +512,8 @@ if test "$RAW" = "0" then run_reports "$TARGETS" run_reports "$COMPARISON_REPORTS" + if test -n "$DYN_CMP_GRP" + then + rm -rf "charts/$DYN_CMP_GRP" + fi fi