git.iko.soy/typeable/pandoc
on master help sign in
Switch to gauge for now for benchmarks.

tasty-bench is displaying odd behavior, with different timings depending on the `--pattern` specified.

commitff0fcedcb3c33015f9d550acce8fad2d662d49bd authorJohn MacFarlane <jgm@berkeley.edu> dateMon 15 Mar 2021, 22:50 (6 years ago) parentd94f9b9
3 files changed+20−13
Makefile+10−3 benchmark/benchmark-pandoc.hs+7−7 pandoc.cabal+3−3
Makefile+10−3
@@ -51,10 +51,17 @@ ghcid-test:
5151 ghcid -c "stack repl --ghc-options=-XNoImplicitPrelude --flag 'pandoc:embed_data_files' --ghci-options=-fobject-code pandoc:lib pandoc:test-pandoc"
5252
5353bench:
54− stack bench --benchmark-arguments='$(BENCHARGS) $(BASELINE) --csv bench_$(TIMESTAMP).csv' --ghc-options '-Rghc-timing $(GHCOPTS)'
54+ stack bench \
55+ --ghc-options '-Rghc-timing $(GHCOPTS)' \
56+ --benchmark-arguments='--small --time-limit=2 \
57+ --match=pattern $(PATTERN)' 2>&1 | \
58+ tee "bench_latest.txt"
59+ perl -pe 's/\x1b\[[0-9;]*[mGK]//g;s/\r//;' bench_latest.txt > \
60+ "bench_$(TIMESTAMP).txt"
5561
56−weigh:
57− stack build --ghc-options '$(GHCOPTS)' pandoc:weigh-pandoc && stack exec weigh-pandoc
62+
63+# With tasty-bench, we used
64+# --benchmark-arguments='$(BENCHARGS) $(BASELINE) --csv bench_$(TIMESTAMP).csv' --ghc-options '-Rghc-timing $(GHCOPTS)'
5865
5966reformat:
6067 for f in $(SOURCEFILES); do echo $$f; stylish-haskell -i $$f ; done
benchmark/benchmark-pandoc.hs+7−7
@@ -18,12 +18,12 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
1818-}
1919import Text.Pandoc
2020import Text.Pandoc.MIME
21−import Control.Monad (when)
21+import Control.DeepSeq (force)
2222import Control.Monad.Except (throwError)
2323import qualified Text.Pandoc.UTF8 as UTF8
2424import qualified Data.ByteString as B
2525import qualified Data.Text as T
26−import Test.Tasty.Bench
26+import Gauge
2727import qualified Data.ByteString.Lazy as BL
2828import Data.Maybe (mapMaybe)
2929import Data.List (sortOn)
@@ -31,10 +31,10 @@ import Data.List (sortOn)
3131readerBench :: Pandoc
3232 -> T.Text
3333 -> Maybe Benchmark
34+readerBench _ name
35+ | name `elem` ["bibtex", "biblatex", "csljson"] = Nothing
3436readerBench doc name = either (const Nothing) Just $
3537 runPure $ do
36− when (name `elem` ["bibtex", "biblatex", "csljson"]) $
37− throwError $ PandocSomeError $ name <> " not supported for benchmark"
3838 (rdr, rexts) <- getReader name
3939 (wtr, wexts) <- getWriter name
4040 case (rdr, wtr) of
@@ -65,10 +65,10 @@ writerBench :: [(FilePath, MimeType, BL.ByteString)]
6565 -> Pandoc
6666 -> T.Text
6767 -> Maybe Benchmark
68+writerBench _ _ name
69+ | name `elem` ["bibtex", "biblatex", "csljson"] = Nothing
6870writerBench imgs doc name = either (const Nothing) Just $
6971 runPure $ do
70− when (name `elem` ["bibtex", "biblatex", "csljson"]) $
71− throwError $ PandocSomeError $ name <> " not supported for benchmark"
7272 (wtr, wexts) <- getWriter name
7373 case wtr of
7474 TextWriter writerFun ->
@@ -90,7 +90,7 @@ main :: IO ()
9090main = do
9191 inp <- UTF8.toText <$> B.readFile "test/testsuite.txt"
9292 let opts = def
93− let doc = either (error . show) id $ runPure $ readMarkdown opts inp
93+ let doc = either (error . show) force $ runPure $ readMarkdown opts inp
9494 defaultMain
9595 [ env getImages $ \imgs ->
9696 bgroup "writers" $ mapMaybe (writerBench imgs doc . fst)
pandoc.cabal+3−3
@@ -826,8 +826,8 @@ benchmark benchmark-pandoc
826826 hs-source-dirs: benchmark
827827 build-depends: bytestring,
828828 containers,
829− tasty,
830− tasty-bench >= 0.2 && <= 0.3,
829+ gauge >= 0.2 && < 0.3,
831830 mtl >= 2.2 && < 2.3,
832831 text >= 1.1.1.0 && < 1.3,
833− time
832+ time,
833+ deepseq