Carp/core/Bench.carp

114 lines
4.2 KiB
Plaintext

(load "Statistics.carp")
(system-include "carp_bench.h")
(defmodule Bench
(def- min-runs 50)
(private get-time-elapsed)
(hidden get-time-elapsed)
(register get-time-elapsed (Fn [] Double) "get_time_elapsed")
(doc set-min-runs! "sets the minimum number of runs to `n`.
If your functions takes a large amount of time, experimenting with this might make sense. The default value is `50`, which means that your function runs at least `50` times before even hitting a timeout.")
(defn set-min-runs! [n]
; / 2 because we run it twice per benchmarking run,
; but this is an implementation detail
(set! min-runs (/ n 2)))
(defn- get-unit [n]
(cond
(< n 1000.0) (String.append &(Double.str n) "ns")
(< n 1000000.0) (String.append &(Double.str (/ n 1000.0)) "µs")
(< n 1000000000.0) (String.append &(Double.str (/ n 1000000.0)) "ms")
(String.append &(Double.str (/ n 1000000000.0)) "s")))
(defn- print [title n]
(let [unit (get-unit n)]
(do
(IO.print title)
(IO.println &unit))))
(defn- ns-iter-inner [f n]
(let [start (get-time-elapsed)]
(do
(for [i 0 n] (ignore (~f)))
(Double.- (get-time-elapsed) start))))
(defn- print-bench-results [res total]
(do
(print "Total time elapsed: " total)
(print "Best case: " @(Statistics.Summary.min res))
(print "Worst case: " @(Statistics.Summary.max res))
(print "Standard deviation: " @(Statistics.Summary.stdev res))))
(defn- get-samples [f n]
(let [zero 0.0
samples (Array.replicate min-runs &zero)]
(do
(for [i 0 min-runs]
(Array.aset! &samples i (Double./ (ns-iter-inner f (Double.to-int n)) n)))
(Statistics.summary &(Statistics.winsorize &samples 5.0)))))
(defn- min-one [n]
(if (> 1.0 n) n 1.0))
; it is actually possible to make this run forever by supplying a _really_
; long-running function, where long-running is everything over 30ms.
(doc bench "benchmarks the function `f` and prints the results to `stdout`.")
(defn bench [f]
(let [ns (ns-iter-inner &f 1)
ns-target-total 1000000.0
_n (Double./ ns-target-total (min-one ns))
n (min-one _n)
total 0.0
done false
res (Statistics.summary &[0.0])]
(do
(while (and (Double.< total 3000000000.0) (not done))
(let [loop-start (get-time-elapsed)
summ (get-samples &f n)
summ5 (get-samples &f n)
loop-run (- (get-time-elapsed) loop-start)]
(if (and
(Double.> loop-run 100000.0)
(and
(Double.< @(Statistics.Summary.median-abs-dev-pct &summ) 1.0)
(Double.< (Double.- @(Statistics.Summary.median &summ)
@(Statistics.Summary.median &summ5))
@(Statistics.Summary.median-abs-dev &summ5))))
(do
(set! total (Double.+ total loop-run))
(set! done true)
(set! res summ5))
(do
(set! total (Double.+ total loop-run))
(if (< (Double.* n 10.0) n)
; abort on overflow
(set! total (Double.+ total 3000000000.0))
(set! n (Double.* n 2.0)))))))
(if done
(print-bench-results &res total)
(IO.println "Could not stabilize benchmark after more than 3 seconds!")))))
)
(defmacro benchn [n form]
`(let [before (Bench.get-time-elapsed)
times []]
(do
(for [i 0 %n]
(let [before-once (Bench.get-time-elapsed)]
(do
%form
(set! &times (Array.push-back (Array.copy &times) (Double.- (Bench.get-time-elapsed) before-once))))))
(let [total (Double.- (Bench.get-time-elapsed) before)
per (Double./ total (Double.from-int %n))]
(do
(Bench.print "Total time elapsed: " total)
(Bench.print "Time elapsed per run (average): " per)
(Bench.print "Best case: " (Statistics.min &times))
(Bench.print "Worst case: " (Statistics.max &times))
(Bench.print "Standard deviation: " (Statistics.stdev &times)))))))