on master help sign in
fix benchmark comparison graphs

Generate graphs for both linear and nested benchmark ops

commit4b5abb0915c4be2c6810f4e3e8ef0063c4f17e3a authorHarendra Kumar <harendra.kumar@gmail.com> dateWed 25 Apr 2018, 14:21 (8 years ago) parent12b970c
5 files changed+93−19
bench.sh+31−16 benchmark/Charts.hs → benchmark/ChartLinear.hs benchmark/ChartNested.hs+46 benchmark/Nested.hs+1−1 streamly.cabal+15−2
bench.sh+31−16
@@ -1,13 +1,16 @@
11#!/bin/bash
22
33print_help () {
4− echo "Usage: $0 [--quick] [--append] [--no-graphs] [--no-measure] \
5−[--no-compare] [--base commit] [--candidate commit] -- <gauge options>"
4+ echo "Usage: $0 "
5+ echo " [--quick] [--append] "
6+ echo " [--no-graphs] [--no-measure]"
7+ echo " [--benchmark]"
8+ echo " [--compare] [--base commit] [--candidate commit]"
9+ echo " -- <gauge options>"
610 echo
7− echo "Default generates comparative chart of HEAD^ vs HEAD commit, "
8− echo "in the 'charts' directory. Use --base and --candidate to select "
9− echo "the commits to compare. Use --no-compare to just benchmark the "
10− echo "current workspace only."
11+ echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"
12+ echo "commit is generated, in the 'charts' directory."
13+ echo "Use --base and --candidate to select the commits to compare."
1114 echo
1215 echo "Any arguments after a '--' are passed directly to guage"
1316 echo "You can omit '--' if the gauge args used do not start with a '-'."
@@ -20,7 +23,8 @@ die () {
2023 exit 1
2124}
2225
23−COMPARE=1
26+DEFAULT_BENCHMARK=linear
27+COMPARE=0
2428
2529while test -n "$1"
2630do
@@ -28,9 +32,10 @@ do
2832 -h|--help|help) print_help ;;
2933 --quick) QUICK=1; shift ;;
3034 --append) APPEND=1; shift ;;
35+ --benchmark) shift; BENCHMARK=$1; shift ;;
3136 --base) shift; BASE=$1; shift ;;
3237 --candidate) shift; CANDIDATE=$1; shift ;;
33− --no-compare) COMPARE=0; shift ;;
38+ --compare) COMPARE=1; shift ;;
3439 --no-graphs) GRAPH=0; shift ;;
3540 --no-measure) MEASURE=0; shift ;;
3641 --) shift; break ;;
@@ -48,18 +53,21 @@ echo "Using stack command [$STACK]"
4853# for benchmarking.
4954if test "$GRAPH" != "0"
5055then
51− CHARTS=$($STACK exec which charts)
52− if test ! -x $CHARTS
56+ CHART_PROG="chart-$BENCHMARK"
57+ prog=$($STACK exec which $CHART_PROG)
58+ if test ! -x "$prog"
5359 then
54− echo "Building charts executable"
60+ echo "Building charting executable"
5561 $STACK build --flag "streamly:dev" || die "build failed"
5662 fi
5763
58− CHARTS=$($STACK exec which charts)
59− if test ! -x $CHARTS
64+ prog=$($STACK exec which $CHART_PROG)
65+ if test ! -x "$prog"
6066 then
61− die "Could not find 'charts' executable"
67+ die "Could not find [$CHART_PROG] executable"
6268 fi
69+ CHART_PROG=$prog
70+ echo "Using chart executable [$CHART_PROG]"
6371fi
6472
6573# We run the benchmarks in isolation in a separate process so that different
@@ -72,7 +80,14 @@ fi
7280# find .stack-work/ -type f -name "benchmarks"
7381
7482find_bench_prog () {
75− BENCH_PROG=`$STACK path --dist-dir`/build/benchOps/benchOps
83+ if test -z "$BENCHMARK"
84+ then
85+ BENCHMARK=$DEFAULT_BENCHMARK
86+ echo "Using default benchmark suite [$BENCHMARK], use --benchmark to specify another"
87+ else
88+ echo "Using benchmark suite [$BENCHMARK]"
89+ fi
90+ BENCH_PROG=`$STACK path --dist-dir`/build/$BENCHMARK/$BENCHMARK
7691 if test ! -x "$BENCH_PROG"
7792 then
7893 echo
@@ -154,7 +169,7 @@ if test "$GRAPH" != "0"
154169then
155170 echo
156171 echo "Generating charts from ${OUTPUT_FILE}..."
157− $CHARTS
172+ $CHART_PROG
158173fi
159174
160175# XXX reset back to the original commit
benchmark/Charts.hs → benchmark/ChartLinear.hsrenamed

The content is the same.

benchmark/ChartNested.hsnew file+46
@@ -0,0 +1,46 @@
1+{-# LANGUAGE TupleSections #-}
2+{-# LANGUAGE ScopedTypeVariables #-}
3+
4+module Main where
5+
6+import Data.List
7+import Data.List.Split
8+import BenchGraph (bgraph, defaultConfig, Config(..), ComparisonStyle(..))
9+import Control.Exception (handle, catch, SomeException, ErrorCall)
10+
11+main :: IO ()
12+main = do
13+ let cfg = defaultConfig
14+ { outputDir = "charts"
15+ , comparisonStyle = CompareFull
16+ }
17+
18+ ignoringErr a = catch a (\(_ :: ErrorCall) ->
19+ putStrLn "Failed. Skipping.")
20+ -- bgraph <input> <output> <field in csv file to be plotted>
21+ -- other interesting fields to plot are:
22+ -- allocated
23+ -- bytesCopied
24+ -- mutatorCpuSeconds
25+ -- gcCpuSeconds
26+ ignoringErr $ bgraph "charts/results.csv" "nested-ops" "time" $ cfg
27+ { chartTitle = Just "Nested operations (time)"
28+ , classifyBenchmark = \b ->
29+ let ls = splitOn "/" b
30+ in case head ls of
31+ "linear" -> Nothing
32+ _ -> Just (head ls, last ls)
33+ , sortBenchmarks = nub
34+ , comparisonStyle = CompareFull
35+ }
36+
37+ ignoringErr $ bgraph "charts/results.csv" "nested-serial-comparative" "time" $ cfg
38+ { chartTitle = Just "Nested serial comparative (time)"
39+ , classifyBenchmark = \b ->
40+ let ls = splitOn "/" b
41+ in case head ls of
42+ "streamly" -> Just (head ls, last ls)
43+ _ -> Nothing
44+ , sortBenchmarks = nub
45+ , comparisonStyle = CompareDelta
46+ }
benchmark/Nested.hs+1−1
@@ -22,7 +22,7 @@ main :: IO ()
2222main = do
2323 -- TBD Study scaling with 10, 100, 1000 loop iterations
2424 defaultMain
25− [ bgroup "Linear"
25+ [ bgroup "linear"
2626 [ benchIO "toNullLinear" Ops.toNullLinear
2727 , benchIO "toListLinear" Ops.toListLinear
2828 ]
streamly.cabal+15−2
@@ -231,10 +231,23 @@ benchmark nested
231231-- Examples
232232-------------------------------------------------------------------------------
233233
234−executable charts
234+executable chart-linear
235235 default-language: Haskell2010
236236 hs-source-dirs: benchmark
237− main-is: Charts.hs
237+ main-is: ChartLinear.hs
238+ if flag(dev)
239+ buildable: True
240+ build-Depends:
241+ base >= 4.8 && < 5
242+ , bench-graph
243+ , split
244+ else
245+ buildable: False
246+
247+executable chart-nested
248+ default-language: Haskell2010
249+ hs-source-dirs: benchmark
250+ main-is: ChartNested.hs
238251 if flag(dev)
239252 buildable: True
240253 build-Depends: