4b5abb0 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

fix benchmark comparison graphs

4b5abb0Harendra Kumar8 years ago

bench.sh+31−16

compare with master

unifiedsplit

@@ -1,13 +1,16 @@

1#!/bin/bash

3print_help () {

4+ echo "Usage: $0 "

5+ echo " [--quick] [--append] "

4− echo "Usage: $0 [--quick] [--append] [--no-graphs] [--no-measure] \

6+ echo " [--no-graphs] [--no-measure]"

7+ echo " [--benchmark]"

5−[--no-compare] [--base commit] [--candidate commit] -- <gauge options>"

8+ echo " [--compare] [--base commit] [--candidate commit]"

9+ echo " -- <gauge options>"

6 echo

10 echo

7− echo "Default generates comparative chart of HEAD^ vs HEAD commit, "

11+ echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

12+ echo "commit is generated, in the 'charts' directory."

8− echo "in the 'charts' directory. Use --base and --candidate to select "

13+ echo "Use --base and --candidate to select the commits to compare."

9− echo "the commits to compare. Use --no-compare to just benchmark the "

10− echo "current workspace only."

11 echo

14 echo

12 echo "Any arguments after a '--' are passed directly to guage"

15 echo "Any arguments after a '--' are passed directly to guage"

13 echo "You can omit '--' if the gauge args used do not start with a '-'."

16 echo "You can omit '--' if the gauge args used do not start with a '-'."

@@ -20,7 +23,8 @@ die () {

20 exit 1

23 exit 1

21}

24}

26+DEFAULT_BENCHMARK=linear

23−COMPARE=1

27+COMPARE=0

25while test -n "$1"

29while test -n "$1"

26do

30do

@@ -28,9 +32,10 @@ do

28 -h|--help|help) print_help ;;

32 -h|--help|help) print_help ;;

29 --quick) QUICK=1; shift ;;

33 --quick) QUICK=1; shift ;;

30 --append) APPEND=1; shift ;;

34 --append) APPEND=1; shift ;;

35+ --benchmark) shift; BENCHMARK=$1; shift ;;

31 --base) shift; BASE=$1; shift ;;

36 --base) shift; BASE=$1; shift ;;

32 --candidate) shift; CANDIDATE=$1; shift ;;

37 --candidate) shift; CANDIDATE=$1; shift ;;

33− --no-compare) COMPARE=0; shift ;;

38+ --compare) COMPARE=1; shift ;;

34 --no-graphs) GRAPH=0; shift ;;

39 --no-graphs) GRAPH=0; shift ;;

35 --no-measure) MEASURE=0; shift ;;

40 --no-measure) MEASURE=0; shift ;;

36 --) shift; break ;;

41 --) shift; break ;;

@@ -48,18 +53,21 @@ echo "Using stack command [$STACK]"

48# for benchmarking.

53# for benchmarking.

49if test "$GRAPH" != "0"

54if test "$GRAPH" != "0"

50then

55then

56+ CHART_PROG="chart-$BENCHMARK"

51− CHARTS=$($STACK exec which charts)

57+ prog=$($STACK exec which $CHART_PROG)

52− if test ! -x $CHARTS

58+ if test ! -x "$prog"

53 then

59 then

54− echo "Building charts executable"

60+ echo "Building charting executable"

55 $STACK build --flag "streamly:dev" || die "build failed"

61 $STACK build --flag "streamly:dev" || die "build failed"

56 fi

62 fi

58− CHARTS=$($STACK exec which charts)

64+ prog=$($STACK exec which $CHART_PROG)

59− if test ! -x $CHARTS

65+ if test ! -x "$prog"

60 then

66 then

61− die "Could not find 'charts' executable"

67+ die "Could not find [$CHART_PROG] executable"

62 fi

68 fi

69+ CHART_PROG=$prog

70+ echo "Using chart executable [$CHART_PROG]"

63fi

71fi

65# We run the benchmarks in isolation in a separate process so that different

73# We run the benchmarks in isolation in a separate process so that different

@@ -72,7 +80,14 @@ fi

72# find .stack-work/ -type f -name "benchmarks"

80# find .stack-work/ -type f -name "benchmarks"

74find_bench_prog () {

82find_bench_prog () {

83+ if test -z "$BENCHMARK"

84+ then

85+ BENCHMARK=$DEFAULT_BENCHMARK

86+ echo "Using default benchmark suite [$BENCHMARK], use --benchmark to specify another"

87+ else

88+ echo "Using benchmark suite [$BENCHMARK]"

89+ fi

75− BENCH_PROG=`$STACK path --dist-dir`/build/benchOps/benchOps

90+ BENCH_PROG=`$STACK path --dist-dir`/build/$BENCHMARK/$BENCHMARK

76 if test ! -x "$BENCH_PROG"

91 if test ! -x "$BENCH_PROG"

77 then

92 then

78 echo

93 echo

@@ -154,7 +169,7 @@ if test "$GRAPH" != "0"

154then

169then

155 echo

170 echo

156 echo "Generating charts from ${OUTPUT_FILE}..."

171 echo "Generating charts from ${OUTPUT_FILE}..."

157− $CHARTS

172+ $CHART_PROG

158fi

173fi

159

174

160# XXX reset back to the original commit

175# XXX reset back to the original commit

benchmark/Charts.hs → benchmark/ChartLinear.hsrenamed

The content is the same.

benchmark/ChartNested.hsnew file+46

@@ -0,0 +1,46 @@

1+{-# LANGUAGE TupleSections #-}

2+{-# LANGUAGE ScopedTypeVariables #-}

4+module Main where

6+import Data.List

7+import Data.List.Split

8+import BenchGraph (bgraph, defaultConfig, Config(..), ComparisonStyle(..))

9+import Control.Exception (handle, catch, SomeException, ErrorCall)

10+

11+main :: IO ()

12+main = do

13+ let cfg = defaultConfig

14+ { outputDir = "charts"

15+ , comparisonStyle = CompareFull

16+ }

17+

18+ ignoringErr a = catch a (\(_ :: ErrorCall) ->

19+ putStrLn "Failed. Skipping.")

20+ -- bgraph <input> <output> <field in csv file to be plotted>

21+ -- other interesting fields to plot are:

22+ -- allocated

23+ -- bytesCopied

24+ -- mutatorCpuSeconds

25+ -- gcCpuSeconds

26+ ignoringErr $ bgraph "charts/results.csv" "nested-ops" "time" $ cfg

27+ { chartTitle = Just "Nested operations (time)"

28+ , classifyBenchmark = \b ->

29+ let ls = splitOn "/" b

30+ in case head ls of

31+ "linear" -> Nothing

32+ _ -> Just (head ls, last ls)

33+ , sortBenchmarks = nub

34+ , comparisonStyle = CompareFull

35+ }

36+

37+ ignoringErr $ bgraph "charts/results.csv" "nested-serial-comparative" "time" $ cfg

38+ { chartTitle = Just "Nested serial comparative (time)"

39+ , classifyBenchmark = \b ->

40+ let ls = splitOn "/" b

41+ in case head ls of

42+ "streamly" -> Just (head ls, last ls)

43+ _ -> Nothing

44+ , sortBenchmarks = nub

45+ , comparisonStyle = CompareDelta

46+ }

benchmark/Nested.hs+1−1

@@ -22,7 +22,7 @@ main :: IO ()

22main = do

23 -- TBD Study scaling with 10, 100, 1000 loop iterations

24 defaultMain

25− [ bgroup "Linear"

25+ [ bgroup "linear"

26 [ benchIO "toNullLinear" Ops.toNullLinear

27 , benchIO "toListLinear" Ops.toListLinear

28 ]

streamly.cabal+15−2

@@ -231,10 +231,23 @@ benchmark nested

231-- Examples

232-------------------------------------------------------------------------------

233

234−executable charts

234+executable chart-linear

235 default-language: Haskell2010

236 hs-source-dirs: benchmark

237− main-is: Charts.hs

237+ main-is: ChartLinear.hs

238+ if flag(dev)

239+ buildable: True

240+ build-Depends:

241+ base >= 4.8 && < 5

242+ , bench-graph

243+ , split

244+ else

245+ buildable: False

246+

247+executable chart-nested

248+ default-language: Haskell2010

249+ hs-source-dirs: benchmark

250+ main-is: ChartNested.hs

238 if flag(dev)

251 if flag(dev)

239 buildable: True

252 buildable: True

240 build-Depends:

253 build-Depends: