Carp/core/Bench.carp

(load "Statistics.carp")

(system-include "carp_bench.h")

(defmodule Bench
  (def min-runs 50)
  (private min-runs)
  (hidden min-runs)

  (private get-time-elapsed)
  (hidden get-time-elapsed)
  (register get-time-elapsed (Fn [] Double) "get_time_elapsed")

  (doc set-min-runs! "sets the minimum number of runs to `n`.

If your functions takes a large amount of time, experimenting with this might make sense. The default value is `50`, which means that your function runs at least `50` times before even hitting a timeout.")
  (defn set-min-runs! [n]
    ; / 2 because we run it twice per benchmarking run,
    ; but this is an implementation detail
    (set! min-runs (/ n 2)))

  (private get-unit)
  (hidden get-unit)
  (defn get-unit [n]
    (cond
      (< n 1000.0) (String.append &(Double.str n) "ns")
      (< n 1000000.0) (String.append &(Double.str (/ n 1000.0)) "µs")
      (< n 1000000000.0) (String.append &(Double.str (/ n 1000000.0)) "ms")
      (String.append &(Double.str (/ n 1000000000.0)) "s")))

  (private print)
  (hidden print)
  (defn print [title n]
    (let [unit (get-unit n)]
      (do
        (IO.print title)
        (IO.println &unit))))

  (private ns-iter-inner)
  (hidden ns-iter-inner)
  (defn ns-iter-inner [f n]
    (let [start (get-time-elapsed)]
      (do
        (for [i 0 n] (ignore (~f)))
        (Double.- (get-time-elapsed) start))))

  (private print-bench-results)
  (hidden print-bench-results)
  (defn print-bench-results [res total]
    (do
      (print "Total time elapsed: " total)
      (print "Best case: " @(Statistics.Summary.min res))
      (print "Worst case: " @(Statistics.Summary.max res))
      (print "Standard deviation: " @(Statistics.Summary.stdev res))))

  (private get-samples)
  (hidden get-samples)
  (defn get-samples [f n]
    (let [zero 0.0
          samples (Array.replicate min-runs &zero)]
      (do
        (for [i 0 min-runs]
          (Array.aset! &samples i (Double./ (ns-iter-inner f (Double.to-int n)) n)))
        (Statistics.summary &(Statistics.winsorize &samples 5.0)))))

  (private min-one)
  (hidden min-one)
  (defn min-one [n]
    (if (> 1.0 n) n 1.0))

  ; it is actually possible to make this run forever by supplying a _really_
  ; long-running function, where long-running is everything over 30ms.
  (doc bench "benchmarks the function `f` and prints the results to `stdout`.")
  (defn bench [f]
    (let [ns (ns-iter-inner &f 1)
          ns-target-total 1000000.0
          _n (Double./ ns-target-total (min-one ns))
          n (min-one _n)
          total 0.0
          done false
          res (Statistics.summary &[0.0])]
      (do
        (while (and (Double.< total 3000000000.0) (not done))
          (let [loop-start (get-time-elapsed)
                summ (get-samples &f n)
                summ5 (get-samples &f n)
                loop-run (- (get-time-elapsed) loop-start)]
            (if (and
                  (Double.> loop-run 100000.0)
                  (and
                    (Double.< @(Statistics.Summary.median-abs-dev-pct &summ) 1.0)
                    (Double.< (Double.- @(Statistics.Summary.median &summ)
                                        @(Statistics.Summary.median &summ5))
                              @(Statistics.Summary.median-abs-dev &summ5))))
              (do
                (set! total (Double.+ total loop-run))
                (set! done true)
                (set! res summ5))
              (do
                (set! total (Double.+ total loop-run))
                (if (< (Double.* n 10.0) n)
                  ; abort on overflow
                  (set! total (Double.+ total 3000000000.0))
                  (set! n (Double.* n 2.0)))))))
          (if done
            (print-bench-results &res total)
            (IO.println "Could not stabilize benchmark after more than 3 seconds!")))))
)


(defmacro benchn [n form]
  (list 'let ['before (Bench.get-time-elapsed)
              'times []]
    (list 'do
      (list 'for ['i 0 n]
        (list 'let ['before-once (Bench.get-time-elapsed)]
          (list 'do
            form
            (list 'set! &times (Array.push-back (Array.copy &times) (Double.- (Bench.get-time-elapsed) before-once))))))
      (list 'let ['total (Double.- (Bench.get-time-elapsed) before)
                  'per (list 'Double./ 'total (list 'Double.from-int n))]
        (do
          (Bench.print "Total time elapsed: " total)
          (Bench.print "Time elapsed per run (average): " per)
          (Bench.print "Best case: " (Statistics.min &times))
          (Bench.print "Worst case: " (Statistics.max &times))
          (Bench.print "Standard deviation: " (Statistics.stdev &times)))))))