bench.sh · streamly · git.iko.soy

git.iko.soy/streamly

streamly/bench.sh

380 lines 9.8 kB history raw

1#!/bin/bash

3ALL_BENCHMARKS="linear linear-async linear-rate nested nested-concurrent nested-unfold concurrent fileio array base"

5print_help () {

6 echo "Usage: $0 "

8 echo " [--group-diff]"

9 echo " [--graphs]"

10 echo " [--no-measure]"

11 echo " [--append]"

12 echo " [--long]"

13 echo " [--slow]"

14 echo " [--quick]"

15 echo " [--compare] [--base commit] [--candidate commit]"

16 echo " [--cabal-build-flags]"

17 echo " -- <gauge options>"

18 echo

19 echo "Multiple benchmarks can be specified as a space separate list"

20 echo " e.g. --benchmarks \"linear nested\""

21 echo

22 echo "--group-diff is used to compare groups within a single benchmark"

23 echo " e.g. StreamD vs StreamK in base benchmark."

24 echo

25 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

26 echo "commit is generated, in the 'charts' directory."

27 echo "Use --base and --candidate to select the commits to compare."

28 echo

29 echo "Any arguments after a '--' are passed directly to gauge"

30 exit

31}

33# $1: message

34die () {

35 >&2 echo -e "Error: $1"

36 exit 1

37}

39set_benchmarks() {

40 if test -z "$BENCHMARKS"

41 then

42 BENCHMARKS=$DEFAULT_BENCHMARKS

43 elif test "$BENCHMARKS" = "all"

44 then

45 BENCHMARKS=$ALL_BENCHMARKS

46 fi

47 echo "Using benchmark suites [$BENCHMARKS]"

48}

50# $1: benchmark name (linear, nested, base)

51find_report_prog() {

52 local prog_name="chart"

53 hash -r

54 local prog_path=$($WHICH_COMMAND $prog_name)

55 if test -x "$prog_path"

56 then

57 echo $prog_path

58 else

59 return 1

60 fi

61}

63# $1: benchmark name (linear, nested, base)

64build_report_prog() {

65 local prog_name="chart"

66 local prog_path=$($WHICH_COMMAND $prog_name)

68 hash -r

69 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"

70 then

71 echo "Building bench-graph executables"

72 BUILD_ONCE=1

73 $BUILD_CHART_EXE || die "build failed"

74 elif test ! -x "$prog_path"

75 then

76 return 1

77 fi

78 return 0

79}

81build_report_progs() {

82 if test "$RAW" = "0"

83 then

84 build_report_prog || exit 1

85 local prog

86 prog=$(find_report_prog) || \

87 die "Cannot find bench-graph executable"

88 echo "Using bench-graph executable [$prog]"

89 fi

90}

92# We run the benchmarks in isolation in a separate process so that different

93# benchmarks do not interfere with other. To enable that we need to pass the

94# benchmark exe path to gauge as an argument. Unfortunately it cannot find its

95# own path currently.

97# The path is dependent on the architecture and cabal version.

98# Use this command to find the exe if this script fails with an error:

99# find .stack-work/ -type f -name "benchmarks"

100

101stack_bench_prog () {

102 local bench_name=$1

103 local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name

104 if test -x "$bench_prog"

105 then

106 echo $bench_prog

107 else

108 return 1

109 fi

110}

111

112cabal_bench_prog () {

113 local bench_name=$1

114 local bench_prog=`$WHICH_COMMAND $1`

115 if test -x "$bench_prog"

116 then

117 echo $bench_prog

118 else

119 return 1

120 fi

121}

122

123bench_output_file() {

124 local bench_name=$1

125 echo "charts/$bench_name/results.csv"

126}

127

128# --min-duration 0 means exactly one iteration per sample. We use a million

129# iterations in the benchmarking code explicitly and do not use the iterations

130# done by the benchmarking tool.

131#

132# Benchmarking tool by default discards the first iteration to remove

133# aberrations due to initial evaluations etc. We do not discard it because we

134# are anyway doing iterations in the benchmarking code and many of them so that

135# any constant factor gets amortized and anyway it is a cost that we pay in

136# real life.

137#

138# We can pass --min-samples value from the command line as second argument

139# after the benchmark name in case we want to use more than one sample.

140

141run_bench () {

142 local bench_name=$1

143 local output_file=$(bench_output_file $bench_name)

144 local bench_prog

145 bench_prog=$($GET_BENCH_PROG $bench_name) || \

146 die "Cannot find benchmark executable for benchmark $bench_name"

147

148 mkdir -p `dirname $output_file`

149

150 echo "Running benchmark $bench_name ..."

151

152 local QUICK_OPTS="--quick --time-limit 1 --min-duration 0"

153 local SPEED_OPTIONS

154 if test "$LONG" -eq 0

155 then

156 if test "$SLOW" -eq 0

157 then

158 if test "$QUICK" -eq 0

159 then

160 # reasonably quick

161 SPEED_OPTIONS="$QUICK_OPTS --min-samples 10"

162 else

163 # super quick but less accurate

164 SPEED_OPTIONS="$QUICK_OPTS --include-first-iter"

165 fi

166 else

167 SPEED_OPTIONS="--min-duration 0"

168 fi

169 else

170 SPEED_OPTIONS="--stream-size 10000000 $QUICK_OPTS --include-first-iter"

171 fi

172

173 $bench_prog $SPEED_OPTIONS \

174 --csvraw=$output_file \

175 -v 2 \

176 --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"

177}

178

179run_benches() {

180 for i in $1

181 do

182 run_bench $i

183 done

184}

185

186run_benches_comparing() {

187 local bench_list=$1

188

189 if test -z "$CANDIDATE"

190 then

191 CANDIDATE=$(git rev-parse HEAD)

192 fi

193 if test -z "$BASE"

194 then

195 # XXX Should be where the current branch is forked from master

196 BASE="$CANDIDATE^"

197 fi

198 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"

199 echo "Checking out base commit [$BASE] for benchmarking"

200 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"

201

202 $BUILD_BENCH || die "build failed"

203 run_benches "$bench_list"

204

205 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"

206 git checkout "$CANDIDATE" || \

207 die "Checkout of candidate [$CANDIDATE] commit failed"

208

209 $BUILD_BENCH || die "build failed"

210 run_benches "$bench_list"

211 # XXX reset back to the original commit

212}

213

214backup_output_file() {

215 local bench_name=$1

216 local output_file=$(bench_output_file $bench_name)

217

218 if test -e $output_file -a "$APPEND" != 1

219 then

220 mv -f -v $output_file ${output_file}.prev

221 fi

222}

223

224run_measurements() {

225 local bench_list=$1

226

227 for i in $bench_list

228 do

229 backup_output_file $i

230 done

231

232 if test "$COMPARE" = "0"

233 then

234 run_benches "$bench_list"

235 else

236 run_benches_comparing "$bench_list"

237 fi

238}

239

240run_reports() {

241 local prog

242 prog=$(find_report_prog) || \

243 die "Cannot find bench-graph executable"

244 echo

245

246 for i in $1

247 do

248 echo "Generating reports for ${i}..."

249 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

250 $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \

251 --benchmark $i

252 done

253}

254

255#-----------------------------------------------------------------------------

256# Execution starts here

257#-----------------------------------------------------------------------------

258

259DEFAULT_BENCHMARKS="linear"

260GROUP_DIFF=0

261

262COMPARE=0

263BASE=

264CANDIDATE=

265

266APPEND=0

267SLOW=0

268QUICK=0

269LONG=0

270RAW=0

271GRAPH=0

272MEASURE=1

273

274GAUGE_ARGS=

275BUILD_ONCE=0

276USE_STACK=0

277CABAL_BUILD_FLAGS=""

278

279GHC_VERSION=$(ghc --numeric-version)

280

281cabal_which() {

282 find dist-newstyle -type f -path "*${GHC_VERSION}*/$1"

283}

284

285#-----------------------------------------------------------------------------

286# Read command line

287#-----------------------------------------------------------------------------

288

289while test -n "$1"

290do

291 case $1 in

292 -h|--help|help) print_help ;;

293 # options with arguments

294 --benchmarks) shift; BENCHMARKS=$1; shift ;;

295 --base) shift; BASE=$1; shift ;;

296 --candidate) shift; CANDIDATE=$1; shift ;;

297 --cabal-build-flags) shift; CABAL_BUILD_FLAGS=$1; shift ;;

298 # flags

299 --slow) SLOW=1; shift ;;

300 --quick) QUICK=1; shift ;;

301 --compare) COMPARE=1; shift ;;

302 --raw) RAW=1; shift ;;

303 --append) APPEND=1; shift ;;

304 --long) LONG=1; shift ;;

305 --group-diff) GROUP_DIFF=1; shift ;;

306 --graphs) GRAPH=1; shift ;;

307 --no-measure) MEASURE=0; shift ;;

308 --) shift; break ;;

309 -*|--*) print_help ;;

310 *) break ;;

311 esac

312done

313GAUGE_ARGS=$*

314

315echo "Using stack command [$STACK]"

316set_benchmarks

317

318# XXX we can remove these if we pass stack build flags from command line like

319# cabal build flags.

320if echo "$BENCHMARKS" | grep -q base

321then

322 STACK_BUILD_FLAGS="--flag streamly:dev"

323fi

324

325for i in $BENCHMARKS

326do

327 if test "$i" = concurrent

328 then

329 STACK_BUILD_FLAGS="--flag streamly:dev"

330 break

331 fi

332done

333

334if test "$USE_STACK" = "1"

335then

336 WHICH_COMMAND="stack exec which"

337 BUILD_CHART_EXE="stack build --flag streamly:dev"

338 GET_BENCH_PROG=stack_bench_prog

339 BUILD_BENCH="stack build $STACK_BUILD_FLAGS --bench --no-run-benchmarks"

340else

341 # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables

342 #WHICH_COMMAND="cabal v2-exec which"

343 WHICH_COMMAND=cabal_which

344 BUILD_CHART_EXE="cabal v2-build --flags dev chart"

345 GET_BENCH_PROG=cabal_bench_prog

346 BUILD_BENCH="cabal v2-build $CABAL_BUILD_FLAGS --enable-benchmarks"

347fi

348

349#-----------------------------------------------------------------------------

350# Build stuff

351#-----------------------------------------------------------------------------

352

353# We need to build the report progs first at the current (latest) commit before

354# checking out any other commit for benchmarking.

355build_report_progs "$BENCHMARKS"

356

357#-----------------------------------------------------------------------------

358# Run benchmarks

359#-----------------------------------------------------------------------------

360

361if test "$MEASURE" = "1"

362then

363 echo $BUILD_BENCH

364 if test "$USE_STACK" = "1"

365 then

366 $BUILD_BENCH || die "build failed"

367 else

368 $BUILD_BENCH $BENCHMARKS || die "build failed"

369 fi

370 run_measurements "$BENCHMARKS"

371fi

372

373#-----------------------------------------------------------------------------

374# Run reports

375#-----------------------------------------------------------------------------

376

377if test "$RAW" = "0"

378then

379 run_reports "$BENCHMARKS"

380fi