bench.sh · streamly · git.iko.soy

git.iko.soy/streamly

on 4172ad9 help sign in

streamly/bench.sh

340 lines 8.9 kB history raw

1#!/bin/bash

3print_help () {

4 echo "Usage: $0 "

6 echo " [--group-diff]"

7 echo " [--graphs]"

8 echo " [--no-measure]"

9 echo " [--append] "

10 echo " [--compare] [--base commit] [--candidate commit]"

11 echo " [--slow]"

12 echo " -- <gauge options>"

13 echo

14 echo "Multiple benchmarks can be specified as a space separate list"

15 echo " e.g. --benchmarks \"linear nested\""

16 echo

17 echo "--group-diff is used to compare groups within a single benchmark"

18 echo " e.g. StreamD vs StreamK in base benchmark."

19 echo

20 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

21 echo "commit is generated, in the 'charts' directory."

22 echo "Use --base and --candidate to select the commits to compare."

23 echo

24 echo "Any arguments after a '--' are passed directly to gauge"

25 exit

26}

28# $1: message

29die () {

30 >&2 echo -e "Error: $1"

31 exit 1

32}

34set_benchmarks() {

35 if test -z "$BENCHMARKS"

36 then

37 BENCHMARKS=$DEFAULT_BENCHMARKS

38 elif test "$BENCHMARKS" = "all"

39 then

40 BENCHMARKS=$ALL_BENCHMARKS

41 fi

42 echo "Using benchmark suites [$BENCHMARKS]"

43}

45# $1: benchmark name (linear, nested, base)

46find_report_prog() {

47 local prog_name="chart"

48 hash -r

49 local prog_path=$($WHICH_COMMAND $prog_name)

50 if test -x "$prog_path"

51 then

52 echo $prog_path

53 else

54 return 1

55 fi

56}

58# $1: benchmark name (linear, nested, base)

59build_report_prog() {

60 local prog_name="chart"

61 local prog_path=$($WHICH_COMMAND $prog_name)

63 hash -r

64 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"

65 then

66 echo "Building bench-graph executables"

67 BUILD_ONCE=1

68 $BUILD_CHART_EXE || die "build failed"

69 elif test ! -x "$prog_path"

70 then

71 return 1

72 fi

73 return 0

74}

76build_report_progs() {

77 if test "$RAW" = "0"

78 then

79 build_report_prog || exit 1

80 local prog

81 prog=$(find_report_prog) || \

82 die "Cannot find bench-graph executable"

83 echo "Using bench-graph executable [$prog]"

84 fi

85}

87# We run the benchmarks in isolation in a separate process so that different

88# benchmarks do not interfere with other. To enable that we need to pass the

89# benchmark exe path to gauge as an argument. Unfortunately it cannot find its

90# own path currently.

92# The path is dependent on the architecture and cabal version.

93# Use this command to find the exe if this script fails with an error:

94# find .stack-work/ -type f -name "benchmarks"

96stack_bench_prog () {

97 local bench_name=$1

98 local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name

99 if test -x "$bench_prog"

100 then

101 echo $bench_prog

102 else

103 return 1

104 fi

105}

106

107cabal_bench_prog () {

108 local bench_name=$1

109 local bench_prog=`$WHICH_COMMAND $1`

110 if test -x "$bench_prog"

111 then

112 echo $bench_prog

113 else

114 return 1

115 fi

116}

117

118bench_output_file() {

119 local bench_name=$1

120 echo "charts/$bench_name/results.csv"

121}

122

123# --min-duration 0 means exactly one iteration per sample. We use a million

124# iterations in the benchmarking code explicitly and do not use the iterations

125# done by the benchmarking tool.

126#

127# Benchmarking tool by default discards the first iteration to remove

128# aberrations due to initial evaluations etc. We do not discard it because we

129# are anyway doing iterations in the benchmarking code and many of them so that

130# any constant factor gets amortized and anyway it is a cost that we pay in

131# real life.

132#

133# We can pass --min-samples value from the command line as second argument

134# after the benchmark name in case we want to use more than one sample.

135

136run_bench () {

137 local bench_name=$1

138 local output_file=$(bench_output_file $bench_name)

139 local bench_prog

140 bench_prog=$($GET_BENCH_PROG $bench_name) || \

141 die "Cannot find benchmark executable for benchmark $bench_name"

142

143 mkdir -p `dirname $output_file`

144

145 echo "Running benchmark $bench_name ..."

146

147 $bench_prog $SPEED_OPTIONS \

148 --csvraw=$output_file \

149 -v 2 \

150 --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"

151}

152

153run_benches() {

154 for i in $1

155 do

156 run_bench $i

157 done

158}

159

160run_benches_comparing() {

161 local bench_list=$1

162

163 if test -z "$CANDIDATE"

164 then

165 CANDIDATE=$(git rev-parse HEAD)

166 fi

167 if test -z "$BASE"

168 then

169 # XXX Should be where the current branch is forked from master

170 BASE="$CANDIDATE^"

171 fi

172 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"

173 echo "Checking out base commit [$BASE] for benchmarking"

174 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"

175

176 $BUILD_BENCH || die "build failed"

177 run_benches "$bench_list"

178

179 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"

180 git checkout "$CANDIDATE" || \

181 die "Checkout of candidate [$CANDIDATE] commit failed"

182

183 $BUILD_BENCH || die "build failed"

184 run_benches "$bench_list"

185 # XXX reset back to the original commit

186}

187

188backup_output_file() {

189 local bench_name=$1

190 local output_file=$(bench_output_file $bench_name)

191

192 if test -e $output_file -a "$APPEND" != 1

193 then

194 mv -f -v $output_file ${output_file}.prev

195 fi

196}

197

198run_measurements() {

199 local bench_list=$1

200

201 for i in $bench_list

202 do

203 backup_output_file $i

204 done

205

206 if test "$COMPARE" = "0"

207 then

208 run_benches "$bench_list"

209 else

210 run_benches_comparing "$bench_list"

211 fi

212}

213

214run_reports() {

215 local prog

216 prog=$(find_report_prog) || \

217 die "Cannot find bench-graph executable"

218 echo

219

220 for i in $1

221 do

222 echo "Generating reports for ${i}..."

223 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

224 $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \

225 --benchmark $i

226 done

227}

228

229#-----------------------------------------------------------------------------

230# Execution starts here

231#-----------------------------------------------------------------------------

232

233DEFAULT_BENCHMARKS="linear"

234ALL_BENCHMARKS="linear linear-async linear-rate nested concurrrent fileio array base"

235GROUP_DIFF=0

236

237COMPARE=0

238BASE=

239CANDIDATE=

240

241APPEND=0

242RAW=0

243GRAPH=0

244MEASURE=1

245SPEED_OPTIONS="--quick --min-samples 10 --time-limit 1 --min-duration 0"

246

247GAUGE_ARGS=

248BUILD_ONCE=0

249USE_STACK=0

250

251GHC_VERSION=$(ghc --numeric-version)

252

253cabal_which() {

254 find dist-newstyle -type f -path "*${GHC_VERSION}*/$1"

255}

256

257#-----------------------------------------------------------------------------

258# Read command line

259#-----------------------------------------------------------------------------

260

261while test -n "$1"

262do

263 case $1 in

264 -h|--help|help) print_help ;;

265 # options with arguments

266 --slow) SPEED_OPTIONS="--min-duration 0"; shift ;;

267 --benchmarks) shift; BENCHMARKS=$1; shift ;;

268 --base) shift; BASE=$1; shift ;;

269 --candidate) shift; CANDIDATE=$1; shift ;;

270 # flags

271 --compare) COMPARE=1; shift ;;

272 --raw) RAW=1; shift ;;

273 --append) APPEND=1; shift ;;

274 --group-diff) GROUP_DIFF=1; shift ;;

275 --graphs) GRAPH=1; shift ;;

276 --no-measure) MEASURE=0; shift ;;

277 --) shift; break ;;

278 -*|--*) print_help ;;

279 *) break ;;

280 esac

281done

282GAUGE_ARGS=$*

283

284echo "Using stack command [$STACK]"

285set_benchmarks

286

287if echo "$BENCHMARKS" | grep -q base

288then

289 STACK_BUILD_FLAGS="--flag streamly:dev"

290 CABAL_BUILD_FLAGS="--flags dev"

291fi

292

293if echo "$BENCHMARKS" | grep -q concurrent

294then

295 STACK_BUILD_FLAGS="--flag streamly:dev"

296 CABAL_BUILD_FLAGS="--flags dev"

297fi

298

299if test "$USE_STACK" = "1"

300then

301 WHICH_COMMAND="stack exec which"

302 BUILD_CHART_EXE="stack build --flag streamly:dev"

303 GET_BENCH_PROG=stack_bench_prog

304 BUILD_BENCH="stack build $STACK_BUILD_FLAGS --flags "streamly:benchmark" --bench --no-run-benchmarks"

305else

306 # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables

307 #WHICH_COMMAND="cabal v2-exec which"

308 WHICH_COMMAND=cabal_which

309 BUILD_CHART_EXE="cabal v2-build --flags dev chart"

310 GET_BENCH_PROG=cabal_bench_prog

311 BUILD_BENCH="cabal v2-build $CABAL_BUILD_FLAGS --flag benchmark --enable-benchmarks"

312fi

313

314#-----------------------------------------------------------------------------

315# Build stuff

316#-----------------------------------------------------------------------------

317

318# We need to build the report progs first at the current (latest) commit before

319# checking out any other commit for benchmarking.

320build_report_progs "$BENCHMARKS"

321

322#-----------------------------------------------------------------------------

323# Run benchmarks

324#-----------------------------------------------------------------------------

325

326if test "$MEASURE" = "1"

327then

328 echo $BUILD_BENCH

329 $BUILD_BENCH || die "build failed"

330 run_measurements "$BENCHMARKS"

331fi

332

333#-----------------------------------------------------------------------------

334# Run reports

335#-----------------------------------------------------------------------------

336

337if test "$RAW" = "0"

338then

339 run_reports "$BENCHMARKS"

340fi