bench.sh · streamly · git.iko.soy

git.iko.soy/streamly

streamly/bench.sh

361 lines 9.3 kB history raw

1#!/bin/bash

3ALL_BENCHMARKS="linear linear-async linear-rate nested nested-concurrent nested-unfold concurrent fileio array base"

5print_help () {

6 echo "Usage: $0 "

8 echo " [--group-diff]"

9 echo " [--graphs]"

10 echo " [--no-measure]"

11 echo " [--append]"

12 echo " [--long]"

13 echo " [--compare] [--base commit] [--candidate commit]"

14 echo " [--slow]"

15 echo " -- <gauge options>"

16 echo

17 echo "Multiple benchmarks can be specified as a space separate list"

18 echo " e.g. --benchmarks \"linear nested\""

19 echo

20 echo "--group-diff is used to compare groups within a single benchmark"

21 echo " e.g. StreamD vs StreamK in base benchmark."

22 echo

23 echo "When using --compare, by default comparative chart of HEAD^ vs HEAD"

24 echo "commit is generated, in the 'charts' directory."

25 echo "Use --base and --candidate to select the commits to compare."

26 echo

27 echo "Any arguments after a '--' are passed directly to gauge"

28 exit

29}

31# $1: message

32die () {

33 >&2 echo -e "Error: $1"

34 exit 1

35}

37set_benchmarks() {

38 if test -z "$BENCHMARKS"

39 then

40 BENCHMARKS=$DEFAULT_BENCHMARKS

41 elif test "$BENCHMARKS" = "all"

42 then

43 BENCHMARKS=$ALL_BENCHMARKS

44 fi

45 echo "Using benchmark suites [$BENCHMARKS]"

46}

48# $1: benchmark name (linear, nested, base)

49find_report_prog() {

50 local prog_name="chart"

51 hash -r

52 local prog_path=$($WHICH_COMMAND $prog_name)

53 if test -x "$prog_path"

54 then

55 echo $prog_path

56 else

57 return 1

58 fi

59}

61# $1: benchmark name (linear, nested, base)

62build_report_prog() {

63 local prog_name="chart"

64 local prog_path=$($WHICH_COMMAND $prog_name)

66 hash -r

67 if test ! -x "$prog_path" -a "$BUILD_ONCE" = "0"

68 then

69 echo "Building bench-graph executables"

70 BUILD_ONCE=1

71 $BUILD_CHART_EXE || die "build failed"

72 elif test ! -x "$prog_path"

73 then

74 return 1

75 fi

76 return 0

77}

79build_report_progs() {

80 if test "$RAW" = "0"

81 then

82 build_report_prog || exit 1

83 local prog

84 prog=$(find_report_prog) || \

85 die "Cannot find bench-graph executable"

86 echo "Using bench-graph executable [$prog]"

87 fi

88}

90# We run the benchmarks in isolation in a separate process so that different

91# benchmarks do not interfere with other. To enable that we need to pass the

92# benchmark exe path to gauge as an argument. Unfortunately it cannot find its

93# own path currently.

95# The path is dependent on the architecture and cabal version.

96# Use this command to find the exe if this script fails with an error:

97# find .stack-work/ -type f -name "benchmarks"

99stack_bench_prog () {

100 local bench_name=$1

101 local bench_prog=`stack path --dist-dir`/build/$bench_name/$bench_name

102 if test -x "$bench_prog"

103 then

104 echo $bench_prog

105 else

106 return 1

107 fi

108}

109

110cabal_bench_prog () {

111 local bench_name=$1

112 local bench_prog=`$WHICH_COMMAND $1`

113 if test -x "$bench_prog"

114 then

115 echo $bench_prog

116 else

117 return 1

118 fi

119}

120

121bench_output_file() {

122 local bench_name=$1

123 echo "charts/$bench_name/results.csv"

124}

125

126# --min-duration 0 means exactly one iteration per sample. We use a million

127# iterations in the benchmarking code explicitly and do not use the iterations

128# done by the benchmarking tool.

129#

130# Benchmarking tool by default discards the first iteration to remove

131# aberrations due to initial evaluations etc. We do not discard it because we

132# are anyway doing iterations in the benchmarking code and many of them so that

133# any constant factor gets amortized and anyway it is a cost that we pay in

134# real life.

135#

136# We can pass --min-samples value from the command line as second argument

137# after the benchmark name in case we want to use more than one sample.

138

139run_bench () {

140 local bench_name=$1

141 local output_file=$(bench_output_file $bench_name)

142 local bench_prog

143 bench_prog=$($GET_BENCH_PROG $bench_name) || \

144 die "Cannot find benchmark executable for benchmark $bench_name"

145

146 mkdir -p `dirname $output_file`

147

148 echo "Running benchmark $bench_name ..."

149

150 local SPEED_OPTIONS

151 if test "$LONG" -eq 0

152 then

153 if test "$SLOW" -eq 0

154 then

155 SPEED_OPTIONS="--quick --min-samples 10 --time-limit 1 --min-duration 0"

156 else

157 SPEED_OPTIONS="--min-duration 0"

158 fi

159 else

160 SPEED_OPTIONS="--stream-size 10000000 -v2 --quick --include-first-iter --time-limit 1 --min-duration 0"

161 fi

162

163 $bench_prog $SPEED_OPTIONS \

164 --csvraw=$output_file \

165 -v 2 \

166 --measure-with $bench_prog $GAUGE_ARGS || die "Benchmarking failed"

167}

168

169run_benches() {

170 for i in $1

171 do

172 run_bench $i

173 done

174}

175

176run_benches_comparing() {

177 local bench_list=$1

178

179 if test -z "$CANDIDATE"

180 then

181 CANDIDATE=$(git rev-parse HEAD)

182 fi

183 if test -z "$BASE"

184 then

185 # XXX Should be where the current branch is forked from master

186 BASE="$CANDIDATE^"

187 fi

188 echo "Comparing baseline commit [$BASE] with candidate [$CANDIDATE]"

189 echo "Checking out base commit [$BASE] for benchmarking"

190 git checkout "$BASE" || die "Checkout of base commit [$BASE] failed"

191

192 $BUILD_BENCH || die "build failed"

193 run_benches "$bench_list"

194

195 echo "Checking out candidate commit [$CANDIDATE] for benchmarking"

196 git checkout "$CANDIDATE" || \

197 die "Checkout of candidate [$CANDIDATE] commit failed"

198

199 $BUILD_BENCH || die "build failed"

200 run_benches "$bench_list"

201 # XXX reset back to the original commit

202}

203

204backup_output_file() {

205 local bench_name=$1

206 local output_file=$(bench_output_file $bench_name)

207

208 if test -e $output_file -a "$APPEND" != 1

209 then

210 mv -f -v $output_file ${output_file}.prev

211 fi

212}

213

214run_measurements() {

215 local bench_list=$1

216

217 for i in $bench_list

218 do

219 backup_output_file $i

220 done

221

222 if test "$COMPARE" = "0"

223 then

224 run_benches "$bench_list"

225 else

226 run_benches_comparing "$bench_list"

227 fi

228}

229

230run_reports() {

231 local prog

232 prog=$(find_report_prog) || \

233 die "Cannot find bench-graph executable"

234 echo

235

236 for i in $1

237 do

238 echo "Generating reports for ${i}..."

239 $prog $(test "$GRAPH" = 1 && echo "--graphs") \

240 $(test "$GROUP_DIFF" = 1 && echo "--group-diff") \

241 --benchmark $i

242 done

243}

244

245#-----------------------------------------------------------------------------

246# Execution starts here

247#-----------------------------------------------------------------------------

248

249DEFAULT_BENCHMARKS="linear"

250GROUP_DIFF=0

251

252COMPARE=0

253BASE=

254CANDIDATE=

255

256APPEND=0

257SLOW=0

258LONG=0

259RAW=0

260GRAPH=0

261MEASURE=1

262

263GAUGE_ARGS=

264BUILD_ONCE=0

265USE_STACK=0

266

267GHC_VERSION=$(ghc --numeric-version)

268

269cabal_which() {

270 find dist-newstyle -type f -path "*${GHC_VERSION}*/$1"

271}

272

273#-----------------------------------------------------------------------------

274# Read command line

275#-----------------------------------------------------------------------------

276

277while test -n "$1"

278do

279 case $1 in

280 -h|--help|help) print_help ;;

281 # options with arguments

282 --slow) SLOW=1; shift ;;

283 --benchmarks) shift; BENCHMARKS=$1; shift ;;

284 --base) shift; BASE=$1; shift ;;

285 --candidate) shift; CANDIDATE=$1; shift ;;

286 # flags

287 --compare) COMPARE=1; shift ;;

288 --raw) RAW=1; shift ;;

289 --append) APPEND=1; shift ;;

290 --long) LONG=1; shift ;;

291 --group-diff) GROUP_DIFF=1; shift ;;

292 --graphs) GRAPH=1; shift ;;

293 --no-measure) MEASURE=0; shift ;;

294 --) shift; break ;;

295 -*|--*) print_help ;;

296 *) break ;;

297 esac

298done

299GAUGE_ARGS=$*

300

301echo "Using stack command [$STACK]"

302set_benchmarks

303

304if echo "$BENCHMARKS" | grep -q base

305then

306 STACK_BUILD_FLAGS="--flag streamly:dev"

307 CABAL_BUILD_FLAGS="--flags dev"

308fi

309

310for i in $BENCHMARKS

311do

312 if test "$i" = concurrent

313 then

314 STACK_BUILD_FLAGS="--flag streamly:dev"

315 CABAL_BUILD_FLAGS="--flags dev"

316 break

317 fi

318done

319

320if test "$USE_STACK" = "1"

321then

322 WHICH_COMMAND="stack exec which"

323 BUILD_CHART_EXE="stack build --flag streamly:dev"

324 GET_BENCH_PROG=stack_bench_prog

325 BUILD_BENCH="stack build $STACK_BUILD_FLAGS --bench --no-run-benchmarks"

326else

327 # XXX cabal issue "cabal v2-exec which" cannot find benchmark/test executables

328 #WHICH_COMMAND="cabal v2-exec which"

329 WHICH_COMMAND=cabal_which

330 BUILD_CHART_EXE="cabal v2-build --flags dev chart"

331 GET_BENCH_PROG=cabal_bench_prog

332 BUILD_BENCH="cabal v2-build $CABAL_BUILD_FLAGS --enable-benchmarks"

333fi

334

335#-----------------------------------------------------------------------------

336# Build stuff

337#-----------------------------------------------------------------------------

338

339# We need to build the report progs first at the current (latest) commit before

340# checking out any other commit for benchmarking.

341build_report_progs "$BENCHMARKS"

342

343#-----------------------------------------------------------------------------

344# Run benchmarks

345#-----------------------------------------------------------------------------

346

347if test "$MEASURE" = "1"

348then

349 echo $BUILD_BENCH

350 $BUILD_BENCH || die "build failed"

351 run_measurements "$BENCHMARKS"

352fi

353

354#-----------------------------------------------------------------------------

355# Run reports

356#-----------------------------------------------------------------------------

357

358if test "$RAW" = "0"

359then

360 run_reports "$BENCHMARKS"

361fi