2018-01-24 15:08:46 +03:00
(load "Statistics.carp")
2018-01-24 18:08:18 +03:00
(system-include "carp_bench.h")
2017-10-25 14:07:34 +03:00
2017-11-06 20:08:07 +03:00
(defmodule Bench
2018-01-24 15:28:57 +03:00
(def min-runs 50)
2018-03-29 12:36:42 +03:00
(private min-runs)
(hidden min-runs)
2018-01-24 15:28:57 +03:00
2020-01-27 18:56:22 +03:00
(private get-time-elapsed)
(hidden get-time-elapsed)
(register get-time-elapsed (Fn [] Double) "get_time_elapsed")
2019-02-15 16:48:49 +03:00
(doc set-min-runs! "sets the minimum number of runs to `n`.
If your functions takes a large amount of time, experimenting with this might make sense. The default value is `50`, which means that your function runs at least `50` times before even hitting a timeout.")
2018-01-24 15:28:57 +03:00
(defn set-min-runs! [n]
; / 2 because we run it twice per benchmarking run,
; but this is an implementation detail
2018-02-02 09:19:10 +03:00
(set! min-runs (/ n 2)))
2018-01-24 15:28:57 +03:00
2018-03-29 12:36:42 +03:00
(private get-unit)
(hidden get-unit)
2017-10-25 14:07:34 +03:00
(defn get-unit [n]
(cond
2019-05-27 19:05:44 +03:00
(< n 1000.0) (String.append &(Double.str n) "ns")
(< n 1000000.0) (String.append &(Double.str (/ n 1000.0)) "µs")
(< n 1000000000.0) (String.append &(Double.str (/ n 1000000.0)) "ms")
(String.append &(Double.str (/ n 1000000000.0)) "s")))
2017-10-25 14:07:34 +03:00
2018-03-29 12:36:42 +03:00
(private print)
(hidden print)
2017-10-25 14:07:34 +03:00
(defn print [title n]
(let [unit (get-unit n)]
(do
2017-11-06 20:08:07 +03:00
(IO.print title)
2017-11-06 15:16:20 +03:00
(IO.println &unit))))
2017-10-25 14:07:34 +03:00
2018-03-29 12:36:42 +03:00
(private ns-iter-inner)
(hidden ns-iter-inner)
2017-11-06 20:08:07 +03:00
(defn ns-iter-inner [f n]
2017-11-06 13:37:24 +03:00
(let [start (get-time-elapsed)]
(do
2019-02-15 16:48:49 +03:00
(for [i 0 n] (ignore (~f)))
2017-11-06 20:08:07 +03:00
(Double.- (get-time-elapsed) start))))
2017-11-06 15:16:20 +03:00
2018-03-29 12:36:42 +03:00
(private print-bench-results)
(hidden print-bench-results)
2017-11-06 22:47:21 +03:00
(defn print-bench-results [res total]
(do
(print "Total time elapsed: " total)
2018-02-01 19:18:05 +03:00
(print "Best case: " @(Statistics.Summary.min res))
(print "Worst case: " @(Statistics.Summary.max res))
(print "Standard deviation: " @(Statistics.Summary.stdev res))))
2017-11-06 22:47:21 +03:00
2018-03-29 12:36:42 +03:00
(private get-samples)
(hidden get-samples)
2017-11-06 22:47:21 +03:00
(defn get-samples [f n]
(let [zero 0.0
2018-01-24 15:28:57 +03:00
samples (Array.replicate min-runs &zero)]
2017-11-06 22:47:21 +03:00
(do
2018-01-24 15:28:57 +03:00
(for [i 0 min-runs]
2019-02-15 16:48:49 +03:00
(Array.aset! &samples i (Double./ (ns-iter-inner f (Double.to-int n)) n)))
2017-11-06 22:47:21 +03:00
(Statistics.summary &(Statistics.winsorize &samples 5.0)))))
2018-03-29 12:36:42 +03:00
(private min-one)
(hidden min-one)
2017-11-06 22:47:21 +03:00
(defn min-one [n]
(if (> 1.0 n) n 1.0))
; it is actually possible to make this run forever by supplying a _really_
; long-running function, where long-running is everything over 30ms.
2019-02-15 16:48:49 +03:00
(doc bench "benchmarks the function `f` and prints the results to `stdout`.")
2017-11-06 13:37:24 +03:00
(defn bench [f]
2019-02-15 16:48:49 +03:00
(let [ns (ns-iter-inner &f 1)
2017-11-06 13:37:24 +03:00
ns-target-total 1000000.0
2017-11-06 22:47:21 +03:00
_n (Double./ ns-target-total (min-one ns))
n (min-one _n)
2017-11-06 20:08:07 +03:00
total 0.0
done false
2018-03-07 17:20:44 +03:00
res (Statistics.summary &[0.0])]
2017-11-06 20:08:07 +03:00
(do
(while (and (Double.< total 3000000000.0) (not done))
2017-11-06 22:47:21 +03:00
(let [loop-start (get-time-elapsed)
2019-02-15 16:48:49 +03:00
summ (get-samples &f n)
summ5 (get-samples &f n)
2017-11-06 22:47:21 +03:00
loop-run (- (get-time-elapsed) loop-start)]
(if (and
(Double.> loop-run 100000.0)
(and
2018-02-01 19:18:05 +03:00
(Double.< @(Statistics.Summary.median-abs-dev-pct &summ) 1.0)
(Double.< (Double.- @(Statistics.Summary.median &summ)
@(Statistics.Summary.median &summ5))
@(Statistics.Summary.median-abs-dev &summ5))))
2017-11-06 22:47:21 +03:00
(do
2018-02-02 09:19:10 +03:00
(set! total (Double.+ total loop-run))
(set! done true)
2018-03-07 17:20:44 +03:00
(set! res summ5))
2017-11-06 22:47:21 +03:00
(do
2018-02-02 09:19:10 +03:00
(set! total (Double.+ total loop-run))
2017-11-06 22:47:21 +03:00
(if (< (Double.* n 10.0) n)
; abort on overflow
2018-02-02 09:19:10 +03:00
(set! total (Double.+ total 3000000000.0))
(set! n (Double.* n 2.0)))))))
2017-11-06 20:08:07 +03:00
(if done
2018-03-07 17:20:44 +03:00
(print-bench-results &res total)
2017-11-06 22:32:59 +03:00
(IO.println "Could not stabilize benchmark after more than 3 seconds!")))))
2017-11-06 20:08:07 +03:00
)
2017-11-06 13:37:24 +03:00
(defmacro benchn [n form]
2020-01-27 18:56:22 +03:00
(list 'let ['before (Bench.get-time-elapsed)
2017-10-25 14:07:34 +03:00
'times []]
(list 'do
(list 'for ['i 0 n]
2020-01-27 18:56:22 +03:00
(list 'let ['before-once (Bench.get-time-elapsed)]
2017-10-25 14:07:34 +03:00
(list 'do
form
2020-01-27 18:56:22 +03:00
(list 'set! × (Array.push-back (Array.copy ×) (Double.- (Bench.get-time-elapsed) before-once))))))
(list 'let ['total (Double.- (Bench.get-time-elapsed) before)
2017-10-25 14:07:34 +03:00
'per (list 'Double./ 'total (list 'Double.from-int n))]
(do
(Bench.print "Total time elapsed: " total)
(Bench.print "Time elapsed per run (average): " per)
(Bench.print "Best case: " (Statistics.min ×))
(Bench.print "Worst case: " (Statistics.max ×))
(Bench.print "Standard deviation: " (Statistics.stdev ×)))))))