git.iko.soy/elk
on master help sign in
Change Benchmark API to allow arbitrary nesting of suites
e29deedEvan Czaplicki
bench/src/Benchmark.elm+86−198
compare with master
unifiedsplit
@@ -1,212 +1,100 @@
1module Benchmark
1module Benchmark
2− ( Never
3− , Benchmark
4− , Suite (..)
2+ ( Suite
5− , BenchStats
6− , run
7− , runWithProgress
8 , bench
3 , bench
9− , bench1
4+ , suite
10− , bench2
5+ , run
11− , bench3
6+ , Statistics
12− , bench4
7+ , statisticsToString
13− , bench5
14− , bench6
15− , bench7
16− , bench8
17− , bench9
18 )where
8 )where
19−{-| A wrapper around benchmark.js that allows benchmarking
9+{-| A wrapper around benchmark.js that allows benchmarking of pure functions.
20−of pure functions to be evaluated.
21−
22−# Error and Benchmark types
23−@docs Never, Benchmark, Suite, BenchStats
24
10
25−# Running benchmarks
11+# Creating Benchmarks
26−@docs run, runWithProgress
12+@docs Benchmark, bench, suite
27
13
28−# Creating benchmarks
14+# Run benchmarks
29−@docs bench, bench1, bench2, bench3, bench4, bench5, bench6, bench7, bench8, bench9
15+@docs run, Statistics
30-}
16-}
31
17
32
18
33−import Task
34−import String
35import Native.Benchmark
19import Native.Benchmark
36import Native.BenchmarkJS
20import Native.BenchmarkJS
37−import Signal
21+import String
38−import Time
22+import Task
39−
23+
40−
24+
41−{-|
25+
42−Our error return type: can never be instantiated
26+-- SET UP BENCHMARK SUITE
43−because running benchmarks should not fail
27+
44−|-}
28+
45−type Never =
46− Never Never
47−
48−
49−{-|
50−Opaque type representing a function to be timed
51−|-}
52−type Benchmark =
53− Benchmark
54−
55−
56−{-|
57−The results of running a benchmark
58−|-}
59−type BenchStats =
60− BenchStats
61− { name : String
62− , hz : Float
63− , marginOfError : Float
64− , moePercent : Float
65−-- , numRunsSampled : Int
66− }
67−
68−
69−{-|
70−A single or collection of benchmarks
29+{-| A set of benchmarks. -}
71−that can be run, generating output
30+type Benchmark
72−|-}
31+ = Test String Thunk
73−type Suite =
74− SingleBenchmark Benchmark
75 | Suite String (List Benchmark)
32 | Suite String (List Benchmark)
76
33
77
34
78−{-|
35+type Thunk = Thunk
79−Run a benchmark, generating a list of results for each benchmark.
36+
80−Results contain a pretty-printed string summary, and a list
37+
81−of detailed data for each benchmark.
38+{-| An individual benchmark with a name. It measures how long it takes to
82−|-}
39+evaluate the given function when it is passed a `()` value.
83−run
84− : Suite
85− -> Task.Task Never (String, List BenchStats)
86−run = runWithProgress Nothing
87−
88−
89−{-|
90−Run a benchmark, generating a list of results for each benchmark
91−and updating a String signal with progress as the benchmarks run
92−|-}
93−runWithProgress
94− : Maybe (Signal.Mailbox String)
95− -> Suite
96− -> Task.Task Never (String, List BenchStats)
97−runWithProgress maybeMailbox suite =
98− let
99− ourTask =
100− case maybeMailbox of
101− Nothing -> \_ -> Task.sleep 0
102− Just mailbox ->
103− \s ->
104− (Signal.send mailbox.address s
105− `Task.andThen` \_ -> Task.sleep (5*Time.second))
106− -- `Task.andThen` \_ -> Task.keepGoing
107− in
108− Native.Benchmark.runWithProgress ourTask suite
109−
110−{-|
111−Create a benchmark with the given name
112−running an arbitrary thunk
113−|-}
40+-}
114bench : String -> (() -> result) -> Benchmark
41bench : String -> (() -> result) -> Benchmark
42+bench name thunk =
115−bench = Native.Benchmark.makeBenchmark
43+ Test name (Native.Benchmark.makeThunk thunk)
116−
44+
117−
45+
118−{-|
46+{-| A collection of benchmarks to run together.
119−Functions for creating named benchmarks with 1 to 9 arguments
120−|-}
47+-}
121−bench1 : String -> (a -> result) -> a -> Benchmark
48+suite : String -> List Benchmark -> Benchmark
122−bench1 name f a =
49+suite =
123− Native.Benchmark.makeBenchmark name (\_ -> f a)
50+ Suite
124−
51+
125−
52+
126−{-||-}
53+
127−bench2 : String -> (a -> b -> result) -> a -> b -> Benchmark
54+-- RUN THE BENCHMARKS
128−bench2 name f a b =
55+
56+
57+{-| Statistics -}
58+type Statistics
59+ = StatSuite String (List Statistics)
60+ | Result String
61+ { mean : Float
62+ }
63+
64+
65+{-| Run a benchmark.
66+-}
67+run : Benchmark -> Task.Task x Statistics
68+run =
129− Native.Benchmark.makeBenchmark name (\_ -> f a b)
69+ Native.Benchmark.run
130−
70+
131−
71+
132−{-||-}
72+
133−bench3 : String -> (a -> b -> c -> result) -> a -> b -> c -> Benchmark
73+-- TO STRING
134−bench3 name f a b c =
74+
135− Native.Benchmark.makeBenchmark name (\_ -> f a b c)
75+
136−
76+statisticsToString : Statistics -> String
137−
77+statisticsToString statistics =
138−{-||-}
78+ List.join "\n" (statisticsToStringHelp statistics)
139−bench4 : String -> (a -> b -> c -> d -> result) -> a -> b -> c -> d -> Benchmark
79+
140−bench4 name f a b c d =
80+
141− Native.Benchmark.makeBenchmark name (\_ -> f a b c d)
81+statisticsToStringHelp : Statistics -> (List String, Float)
142−
82+statisticsToStringHelp statistics =
143−
83+ case statistics of
144−{-||-}
84+ Result name {mean} ->
145−bench5 : String -> (a -> b -> c -> d -> e -> result) -> a -> b -> c -> d -> e -> Benchmark
85+ ( [ name ++ " - " ++ toString mean ]
146−bench5 name f a b c d e =
86+ , mean
147− Native.Benchmark.makeBenchmark name (\_ -> f a b c d e)
87+ )
148−
88+
149−
89+ StatSuite name subStats ->
150−{-||-}
90+ let
151−bench6 :
91+ (strings, subMeans) =
152− String
92+ List.unzip (List.map statisticsToStringHelp subStats)
153− -> (a -> b -> c -> d -> e -> f -> result)
93+
154− -> a
94+ total =
155− -> b
95+ List.sum means
156− -> c
96+ in
157− -> d
97+ ( name ++ " - " ++ toString total
158− -> e
98+ :: List.map (String.padLeft 4 ' ') (List.concat strings)
159− -> f
99+ , total
160− -> Benchmark
100+ )
161−bench6 name fn a b c d e f =
162− Native.Benchmark.makeBenchmark name (\_ -> fn a b c d e f)
163−
164−
165−{-||-}
166−bench7 :
167− String
168− -> (a -> b -> c -> d -> e -> f -> g -> result)
169− -> a
170− -> b
171− -> c
172− -> d
173− -> e
174− -> f
175− -> g
176− -> Benchmark
177−bench7 name fn a b c d e f g =
178− Native.Benchmark.makeBenchmark name (\_ -> fn a b c d e f g)
179−
180−
181−{-||-}
182−bench8 :
183− String
184− -> (a -> b -> c -> d -> e -> f -> g -> h -> result)
185− -> a
186− -> b
187− -> c
188− -> d
189− -> e
190− -> f
191− -> g
192− -> h
193− -> Benchmark
194−bench8 name fn a b c d e f g h =
195− Native.Benchmark.makeBenchmark name (\_ -> fn a b c d e f g h)
196−
197−{-||-}
198−bench9 :
199− String
200− -> (a -> b -> c -> d -> e -> f -> g -> h -> i -> result)
201− -> a
202− -> b
203− -> c
204− -> d
205− -> e
206− -> f
207− -> g
208− -> h
209− -> i
210− -> Benchmark
211−bench9 name fn a b c d e f g h i =
212− Native.Benchmark.makeBenchmark name (\_ -> fn a b c d e f g h i)
bench/src/Native/Benchmark.js+59−86
@@ -8,95 +8,68 @@ Elm.Native.Benchmark.make = function(localRuntime) {
8 return localRuntime.Native.Benchmark.values;
8 return localRuntime.Native.Benchmark.values;
9 }
9 }
10
10
11− var List = Elm.Native.List.make(localRuntime);
11+ var List = Elm.List.make(localRuntime);
12 var Task = Elm.Native.Task.make(localRuntime);
12 var Task = Elm.Native.Task.make(localRuntime);
13− var Signal = Elm.Signal.make(localRuntime);
13+
14− var Utils = Elm.Native.Utils.make(localRuntime);
14+
15−
15+ // RUN SOME BENCHMARKS
16− //Nothing special happens here, but we need to use it as a Native function
16+
17− //So that we can accept thunks of any type
17+ function run(benchmark)
18− function makeBenchmark(name, thunk)
19− {
18+ {
20− return {name : name, thunk : thunk};
19+ return Task.asyncFunction(function(callback)
21− }
20+ {
22−
21+ var overallSuite = new Benchmark.Suite;
23− //Generate the task for running a benchmark suite
22+ var statistics = addTests(overallSuite, benchmark);
24− //Possibly updating a given mailbox with a string describing
23+ overallSuite.on('complete', function(event) {
25− //Our progress of running the benchmarks so far
24+ callback(Task.succeed(statistics));
26− function runWithProgress(maybeTaskFn, inSuite)
25+ });
26+ overallSuite.run({ async: true });
27+ });
28+ }
29+
30+
31+ // ADD TESTS TO THE JS BENCHMARK SUITE
32+
33+ function addTests(overallSuite, benchmark)
27 {
34 {
28−
35+ if (benchmark.ctor === 'Suite')
29− return Task.asyncFunction(function(callback) {
36+ {
37+ return {
38+ ctor: 'StatSuite',
39+ _0: benchmark._0,
40+ _1: A2(List.map, curriedAddTests(overallSuite), benchmark._1)
41+ };
42+ }
43+
44+ // WE MUST BE LOOKING AT A TEST
45+ var name = benchmark._0;
46+ var thunk = benchmark._1;
47+
48+ var statistics = {
49+ ctor: 'Result',
50+ _0: name
51+ _1: null
52+ };
53+
30− var bjsSuite = new Benchmark.Suite;
54+ var jsBenchmark = new Benchmark(name, thunk, {
31− var benchArray;
55+ onComplete: function() {
32− var retData = [];
56+ var info = {
33− var finalString = "";
57+ mean: jsBenchmark.stats.mean
34− var numCompleted = 0;
35− var numToRun;
36−
37− switch (inSuite.ctor)
38− {
39− case "Suite":
40− benchArray = List.toArray(inSuite._1);
41− break;
42− case "SingleBenchmark":
43− benchArray = [inSuite._0 ];
44− break;
45− }
58+ };
46−
59+ statistics._1 = info;
47− numToRun = benchArray.length;
60+ }
48− Task.perform(maybeTaskFn("Running benchmark 1 of " + numToRun));
61+ });
49−
62+ overallSuite.add(jsBenchmark);
50− //Initialize each benchmark in the suite
63+
51− for (i = 0; i < benchArray.length; i++)
64+ return statistics;
52− {
53− var ourThunk = function (){
54− //Run the thing we're timing, then mark the asynch benchmark as done
55− benchArray[i].thunk();
56− deferred.resolve();
57− }
58− bjsSuite.add(benchArray[i].name, benchArray[i].thunk );
59− }
60−
61− //Every time a benchmark finishes, we store its results
62− //and update our progress string
63− bjsSuite.on('cycle', function(event) {
64− numCompleted += 1;
65− retData.push(
66− { name : event.target.options.name
67− , hz : event.target.hz
68− , marginOfError : event.target.stats.moe
69− , moePercent : event.target.stats.rme
70− }
65+ }
71− );
66+
72− finalString += String(event.target) + "\n";
67+ var curriedAddTests = F2(addTests);
73− var intermedString =
68+
74− "Running benchmark "
69+
75− + (numCompleted + 1)
76− + " of " + numToRun
77− + "\nLast result: " + String(event.target);
78− Task.perform(maybeTaskFn(intermedString));
79− //retString += String(event.target) + "\n";
80− });
81−
82− //When the last benchmark finishes, we show all results collected
83− bjsSuite.on('complete', function(event) {
84− Task.perform(maybeTaskFn("Final results:\n\n" + finalString) );
85− return callback(Task.succeed(Utils.Tuple2(finalString, List.fromArray(retData))));
86− });
87−
88− //Finally: actually run the suite
89− Task.perform(
90− Task.asyncFunction(function(otherCallback){
91− bjsSuite.run({ 'async': true });
92− }));
93− });
94− }
95
70
96 return localRuntime.Native.Benchmark.values = {
71 return localRuntime.Native.Benchmark.values = {
97− makeBenchmark: F2(makeBenchmark),
72+ makeThunk: function(x) { return x; },
98− runWithProgress: F2(runWithProgress)
73+ run: run
99 };
74 };
100−
101−
102};
75};