[add] the Reader benchmark.

This commit is contained in:
Yamada Ryo 2024-10-18 14:36:38 +09:00
parent f96c21c264
commit b07a1a54b6
No known key found for this signature in database
GPG Key ID: AAE3C7A542B02DBF
8 changed files with 320 additions and 12 deletions

View File

@ -0,0 +1,106 @@
<svg xmlns="http://www.w3.org/2000/svg" height="502" width="960.0" font-size="16" font-family="sans-serif" stroke-width="2">
<g transform="translate(10.0 0)">
<text fill="hsl(0, 100%, 40%)" y="22">local.deep.10000.heftia.5+5+0 2.33 ms</text>
<g>
<title>2.33 ms ± 185 μs</title>
<rect y="28" rx="5" height="22" width="245.9585448635316" fill="hsl(0, 100%, 80%)" stroke="hsl(0, 100%, 55%)" />
<g stroke="hsl(0, 100%, 40%)"><line x1="226.41179725842534" x2="265.5052924686379" y1="39" y2="39" />
<line x1="226.41179725842534" x2="226.41179725842534" y1="33" y2="45" />
<line x1="265.5052924686379" x2="265.5052924686379" y1="33" y2="45" />
</g>
</g>
<g fill="hsl(40, 100%, 40%)">
<text y="77">local.deep.10000.heftia.5+4+1</text>
<text y="77" x="307.954676786032" text-anchor="end">2.92 ms</text>
</g>
<g>
<title>2.92 ms ± 175 μs</title>
<rect y="83" rx="5" height="22" width="307.954676786032" fill="hsl(40, 100%, 80%)" stroke="hsl(40, 100%, 55%)" />
<g stroke="hsl(40, 100%, 40%)"><line x1="289.4698636723321" x2="326.4394898997319" y1="94" y2="94" />
<line x1="289.4698636723321" x2="289.4698636723321" y1="88" y2="100" />
<line x1="326.4394898997319" x2="326.4394898997319" y1="88" y2="100" />
</g>
</g>
<g fill="hsl(80, 100%, 40%)">
<text y="132">local.deep.10000.heftia.5+3+2</text>
<text y="132" x="382.2609768376057" text-anchor="end">3.63 ms</text>
</g>
<g>
<title>3.63 ms ± 234 μs</title>
<rect y="138" rx="5" height="22" width="382.2609768376057" fill="hsl(80, 100%, 80%)" stroke="hsl(80, 100%, 55%)" />
<g stroke="hsl(80, 100%, 40%)"><line x1="357.5740428908989" x2="406.9479107843125" y1="149" y2="149" />
<line x1="357.5740428908989" x2="357.5740428908989" y1="143" y2="155" />
<line x1="406.9479107843125" x2="406.9479107843125" y1="143" y2="155" />
</g>
</g>
<g fill="hsl(120, 100%, 40%)">
<text y="187">local.deep.10000.heftia.5+2+3</text>
<text y="187" x="481.8174612465912" text-anchor="end">4.57 ms</text>
</g>
<g>
<title>4.57 ms ± 268 μs</title>
<rect y="193" rx="5" height="22" width="481.8174612465912" fill="hsl(120, 100%, 80%)" stroke="hsl(120, 100%, 55%)" />
<g stroke="hsl(120, 100%, 40%)"><line x1="453.60562275019765" x2="510.0292997429848" y1="204" y2="204" />
<line x1="453.60562275019765" x2="453.60562275019765" y1="198" y2="210" />
<line x1="510.0292997429848" x2="510.0292997429848" y1="198" y2="210" />
</g>
</g>
<g fill="hsl(160, 100%, 40%)">
<text y="242">local.deep.10000.heftia.5+1+4</text>
<text y="242" x="546.5384337712587" text-anchor="end">5.18 ms</text>
</g>
<g>
<title>5.18 ms ± 317 μs</title>
<rect y="248" rx="5" height="22" width="546.5384337712587" fill="hsl(160, 100%, 80%)" stroke="hsl(160, 100%, 55%)" />
<g stroke="hsl(160, 100%, 40%)"><line x1="513.0935880990652" x2="579.9832794434523" y1="259" y2="259" />
<line x1="513.0935880990652" x2="513.0935880990652" y1="253" y2="265" />
<line x1="579.9832794434523" x2="579.9832794434523" y1="253" y2="265" />
</g>
</g>
<g fill="hsl(200, 100%, 40%)">
<text y="297">local.deep.10000.heftia.5+0+5</text>
<text y="297" x="672.2969269780732" text-anchor="end">6.38 ms</text>
</g>
<g>
<title>6.38 ms ± 616 μs</title>
<rect y="303" rx="5" height="22" width="672.2969269780732" fill="hsl(200, 100%, 80%)" stroke="hsl(200, 100%, 55%)" />
<g stroke="hsl(200, 100%, 40%)"><line x1="607.3452151700138" x2="737.2486387861326" y1="314" y2="314" />
<line x1="607.3452151700138" x2="607.3452151700138" y1="308" y2="320" />
<line x1="737.2486387861326" x2="737.2486387861326" y1="308" y2="320" />
</g>
</g>
<g fill="hsl(240, 100%, 40%)">
<text y="352">local.deep.10000.polysemy.5+5</text>
<text y="352" x="874.8905492882927" text-anchor="end">8.30 ms</text>
</g>
<g>
<title>8.30 ms ± 618 μs</title>
<rect y="358" rx="5" height="22" width="874.8905492882927" fill="hsl(240, 100%, 80%)" stroke="hsl(240, 100%, 55%)" />
<g stroke="hsl(240, 100%, 40%)"><line x1="809.7810985765855" x2="940.0" y1="369" y2="369" />
<line x1="809.7810985765855" x2="809.7810985765855" y1="363" y2="375" />
<line x1="940.0" x2="940.0" y1="363" y2="375" />
</g>
</g>
<g fill="hsl(280, 100%, 40%)">
<text y="407">local.deep.10000.fused.5+5</text>
<text y="407" x="791.7405544181797" text-anchor="end">7.51 ms</text>
</g>
<g>
<title>7.51 ms ± 411 μs</title>
<rect y="413" rx="5" height="22" width="791.7405544181797" fill="hsl(280, 100%, 80%)" stroke="hsl(280, 100%, 55%)" />
<g stroke="hsl(280, 100%, 40%)"><line x1="748.3870734133866" x2="835.0940354229729" y1="424" y2="424" />
<line x1="748.3870734133866" x2="748.3870734133866" y1="418" y2="430" />
<line x1="835.0940354229729" x2="835.0940354229729" y1="418" y2="430" />
</g>
</g>
<text fill="hsl(320, 100%, 40%)" y="462">local.deep.10000.effectful.5+5 992 μs</text>
<g>
<title>992 μs ± 96 μs</title>
<rect y="468" rx="5" height="22" width="104.61546838900283" fill="hsl(320, 100%, 80%)" stroke="hsl(320, 100%, 55%)" />
<g stroke="hsl(320, 100%, 40%)"><line x1="94.47911613847512" x2="114.75182063953054" y1="479" y2="479" />
<line x1="94.47911613847512" x2="94.47911613847512" y1="473" y2="485" />
<line x1="114.75182063953054" x2="114.75182063953054" y1="473" y2="485" />
</g>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 5.0 KiB

View File

@ -0,0 +1,49 @@
<svg xmlns="http://www.w3.org/2000/svg" height="227" width="960.0" font-size="16" font-family="sans-serif" stroke-width="2">
<g transform="translate(10.0 0)">
<g fill="hsl(0, 100%, 40%)">
<text y="22">local.shallow.10000.heftia</text>
<text y="22" x="508.74118543228576" text-anchor="end">2.33 ms</text>
</g>
<g>
<title>2.33 ms ± 172 μs</title>
<rect y="28" rx="5" height="22" width="508.74118543228576" fill="hsl(0, 100%, 80%)" stroke="hsl(0, 100%, 55%)" />
<g stroke="hsl(0, 100%, 40%)"><line x1="471.1688210293156" x2="546.3135498352559" y1="39" y2="39" />
<line x1="471.1688210293156" x2="471.1688210293156" y1="33" y2="45" />
<line x1="546.3135498352559" x2="546.3135498352559" y1="33" y2="45" />
</g>
</g>
<g fill="hsl(90, 100%, 40%)">
<text y="77">local.shallow.10000.polysemy</text>
<text y="77" x="892.703792324007" text-anchor="end">4.09 ms</text>
</g>
<g>
<title>4.09 ms ± 217 μs</title>
<rect y="83" rx="5" height="22" width="892.703792324007" fill="hsl(90, 100%, 80%)" stroke="hsl(90, 100%, 55%)" />
<g stroke="hsl(90, 100%, 40%)"><line x1="845.4075846480141" x2="940.0" y1="94" y2="94" />
<line x1="845.4075846480141" x2="845.4075846480141" y1="88" y2="100" />
<line x1="940.0" x2="940.0" y1="88" y2="100" />
</g>
</g>
<g fill="hsl(180, 100%, 40%)">
<text y="132">local.shallow.10000.fused</text>
<text y="132" x="385.2457271160473" text-anchor="end">1.76 ms</text>
</g>
<g>
<title>1.76 ms ± 154 μs</title>
<rect y="138" rx="5" height="22" width="385.2457271160473" fill="hsl(180, 100%, 80%)" stroke="hsl(180, 100%, 55%)" />
<g stroke="hsl(180, 100%, 40%)"><line x1="351.6389956251121" x2="418.8524586069825" y1="149" y2="149" />
<line x1="351.6389956251121" x2="351.6389956251121" y1="143" y2="155" />
<line x1="418.8524586069825" x2="418.8524586069825" y1="143" y2="155" />
</g>
</g>
<text fill="hsl(270, 100%, 40%)" y="187">local.shallow.10000.effectful 956 μs</text>
<g>
<title>956 μs ± 62 μs</title>
<rect y="193" rx="5" height="22" width="208.69904846996374" fill="hsl(270, 100%, 80%)" stroke="hsl(270, 100%, 55%)" />
<g stroke="hsl(270, 100%, 40%)"><line x1="195.05756999348839" x2="222.3405269464391" y1="204" y2="204" />
<line x1="195.05756999348839" x2="195.05756999348839" y1="198" y2="210" />
<line x1="222.3405269464391" x2="222.3405269464391" y1="198" y2="210" />
</g>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.3 KiB

View File

@ -7,5 +7,7 @@ cabal run heftia-bench -- --svg ../benchmark/bench-result/nondet-shallow.svg --p
cabal run heftia-bench -- --svg ../benchmark/bench-result/nondet-deep.svg --pattern '$2 == "nondet.deep"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/catch-shallow.svg --pattern '$2 == "catch.shallow"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/catch-deep.svg --pattern '$2 == "catch.deep"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/local-shallow.svg --pattern '$2 == "local.shallow"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/local-deep.svg --pattern '$2 == "local.deep"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/coroutine-shallow.svg --pattern '$2 == "coroutine.shallow"'
cabal run heftia-bench -- --svg ../benchmark/bench-result/coroutine-deep.svg --pattern '$2 == "coroutine.deep"'

View File

@ -7,6 +7,8 @@ In the State benchmark, it is about 3-6x faster than `polysemy` and about 2-7x s
In the Throw/Catch benchmark, it is slightly faster than `effectful` in the fastest case, and at least 2x as fast as `polysemy` in the other cases.
In the Ask/Local benchmark, it still outperforms `polysemy` and `fused-effects` in deep effect stack cases.
Non-deterministic computations are somewhat slow;
in cases with a shallow effect stack, they are about 5x slower than `fused-effects`.
However, because their performance in terms of stack depth is better than that of `fused-effects`,
@ -34,25 +36,25 @@ Larger values indicate that `heftia` is faster.
| nondet, deep | 0.66x |
## Reproduction
The benchmark code is available at [heftia-effects/bench](https://github.com/sayo-hs/heftia/blob/v0.4.0/heftia-effects/bench).
The benchmark code is available at [heftia-effects/bench](https://github.com/sayo-hs/heftia/blob/v0.5.0/heftia-effects/bench).
To run the benchmarks, move your working directory to the root directory of the `heftia` repository and execute
[`./benchmark/bench.sh`](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench.sh).
[`./benchmark/bench.sh`](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench.sh).
## Benchmark Results
The code was compiled with GHC 9.8.2 and run on a Ryzen 9 3900XT.
* State Effect Benchmark (Shallow Effect Stack):
![countdown.shallow](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/countdown-shallow.svg)
![countdown.shallow](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/countdown-shallow.svg)
* State Effect Benchmark (Deep Effect Stack):
![countdown.deep](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/countdown-deep.svg)
![countdown.deep](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/countdown-deep.svg)
* Throw/Catch Effect Benchmark (Shallow Effect Stack):
![catch.shallow](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/catch-shallow.svg)
![catch.shallow](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/catch-shallow.svg)
* Throw/Catch Effect Benchmark (Deep Effect Stack):
![catch.deep](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/catch-deep.svg)
![catch.deep](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/catch-deep.svg)
Note: Here, the $5 + (5 - N) + N$ corresponds to the interpretation stack of `catchHeftiaDeepN` represented below:
@ -69,14 +71,20 @@ run :: Heftia.Eff eh (Heftia.Ask () ': ef) a -> Heftia.Eff eh ef a
run = Heftia.runAsk ()
```
* Ask/Local Effect Benchmark (Shallow Effect Stack):
![local.shallow](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/local-shallow.svg)
* Ask/Local Effect Benchmark (Deep Effect Stack):
![local.deep](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/local-deep.svg)
* NonDet Effect Benchmark (Shallow Effect Stack):
![nondet.shallow](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/nondet-shallow.svg)
![nondet.shallow](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/nondet-shallow.svg)
* NonDet Effect Benchmark (Deep Effect Stack):
![nondet.deep](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/nondet-deep.svg)
![nondet.deep](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/nondet-deep.svg)
* Coroutine Benchmark (Shallow Effect Stack):
![coroutine.shallow](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/coroutine-shallow.svg)
![coroutine.shallow](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/coroutine-shallow.svg)
* Coroutine Benchmark (Deep Effect Stack):
![coroutine.deep](https://github.com/sayo-hs/heftia/blob/v0.4.0/benchmark/bench-result/coroutine-deep.svg)
![coroutine.deep](https://github.com/sayo-hs/heftia/blob/v0.5.0/benchmark/bench-result/coroutine-deep.svg)

View File

@ -1,7 +1,7 @@
-- SPDX-License-Identifier: BSD-3-Clause
-- (c) 2022 Xy Ren; 2024 Sayo Koyoneda
-- Benchmarking scoped effects #1: Catching errors
-- Benchmarking higher-order effects #1: Catching errors
module BenchCatch where

View File

@ -0,0 +1,114 @@
-- SPDX-License-Identifier: BSD-3-Clause
-- (c) 2022 Xy Ren; 2024 Sayo Koyoneda
-- Benchmarking higher-order effects #2: Local environments
module BenchLocal where
import Control.Carrier.Reader qualified as F
import Control.Monad.Hefty qualified as H
import Control.Monad.Hefty.Reader qualified as H
import Effectful qualified as EL
import Effectful.Reader.Dynamic qualified as EL
import Polysemy qualified as P
import Polysemy.Reader qualified as P
import "eff" Control.Effect qualified as E
programHeftia :: (H.Member (H.Ask Int) ef, H.MemberH (H.Local Int) eh) => Int -> H.Eff eh ef Int
programHeftia = \case
0 -> H.ask
n -> H.local @Int (+ 1) (programHeftia (n - 1))
{-# NOINLINE programHeftia #-}
localHeftia :: Int -> Int
localHeftia n = H.runPure $ H.runAsk @Int 0 $ H.runLocal @Int $ programHeftia n
localHeftiaDeep0, localHeftiaDeep1, localHeftiaDeep2, localHeftiaDeep3, localHeftiaDeep4, localHeftiaDeep5 :: Int -> Int
localHeftiaDeep0 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runLocal @Int $ programHeftia n
localHeftiaDeep1 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ hrun $ hrun $ hrun $ hrun $ H.runLocal @Int $ hrun $ programHeftia n
localHeftiaDeep2 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ hrun $ hrun $ hrun $ H.runLocal @Int $ hrun $ hrun $ programHeftia n
localHeftiaDeep3 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ hrun $ hrun $ H.runLocal @Int $ hrun $ hrun $ hrun $ programHeftia n
localHeftiaDeep4 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ hrun $ H.runLocal @Int $ hrun $ hrun $ hrun $ hrun $ programHeftia n
localHeftiaDeep5 n = H.runPure $ hrun $ hrun $ hrun $ hrun $ hrun $ H.runAsk @Int 0 $ H.runLocal @Int $ hrun $ hrun $ hrun $ hrun $ hrun $ programHeftia n
hrun :: H.Eff eh (H.Ask () ': ef) a -> H.Eff eh ef a
hrun = H.runAsk ()
programSem :: (P.Reader Int `P.Member` es) => Int -> P.Sem es Int
programSem = \case
0 -> P.ask
n -> P.local @Int (+ 1) (programSem (n - 1))
{-# NOINLINE programSem #-}
localSem :: Int -> Int
localSem n = P.run $ P.runReader @Int 0 $ programSem n
localSemDeep :: Int -> Int
localSemDeep n = P.run $ run $ run $ run $ run $ run $ P.runReader @Int 0 $ run $ run $ run $ run $ run $ programSem n
where
run = P.runReader ()
programFused :: (F.Has (F.Reader Int) sig m) => Int -> m Int
programFused = \case
0 -> F.ask
n -> F.local @Int (+ 1) (programFused (n - 1))
{-# NOINLINE programFused #-}
localFused :: Int -> Int
localFused n = F.run $ F.runReader @Int 0 $ programFused n
localFusedDeep :: Int -> Int
localFusedDeep n = F.run $ run $ run $ run $ run $ run $ F.runReader @Int 0 $ run $ run $ run $ run $ run $ programFused n
where
run = F.runReader ()
programEffectful :: (EL.Reader Int EL.:> es) => Int -> EL.Eff es Int
programEffectful = \case
0 -> EL.ask
n -> EL.local @Int (+ 1) (programEffectful (n - 1))
{-# NOINLINE programEffectful #-}
localEffectful :: Int -> Int
localEffectful n = EL.runPureEff $ EL.runReader @Int 0 $ programEffectful n
localEffectfulDeep :: Int -> Int
localEffectfulDeep n =
EL.runPureEff $ run $ run $ run $ run $ run $ EL.runReader @Int 0 $ run $ run $ run $ run $ run $ programEffectful n
where
run = EL.runReader ()
programEff :: (E.Reader Int E.:< es) => Int -> E.Eff es Int
programEff = \case
0 -> E.ask
n -> E.local @Int (+ 1) (programEff (n - 1))
{-# NOINLINE programEff #-}
localEff :: Int -> Int
localEff n = E.run $ E.runReader @Int 0 $ programEff n
localEffDeep :: Int -> Int
localEffDeep n = E.run $ run $ run $ run $ run $ run $ E.runReader @Int 0 $ run $ run $ run $ run $ run $ programEff n
where
run = E.runReader ()
{-
The MTL case is disabled because of conflicting functional dependencies.
When using something other than Reader to build up the stack, it is considered
that the performance degradation caused by the pure stack factor cannot be
measured.
programMtl :: (M.MonadReader Int m) => Int -> m Int
programMtl = \case
0 -> M.ask
n -> M.local (+ 1) (programMtl (n - 1))
{-# NOINLINE programMtl #-}
localMtl :: Int -> Int
localMtl n = M.runReader 0 $ programMtl n
localMtlDeep :: Int -> Int
localMtlDeep n = M.runIdentity $ run $ run $ run $ run $ run $ M.runReader 0 $ run $ run $ run $ run $ run $ programMtl n
where
run = (`M.runReaderT` ())
-}

View File

@ -6,6 +6,7 @@ module Main where
import BenchCatch
import BenchCoroutine
import BenchCountdown
import BenchLocal
import BenchParallel
import BenchPyth
import Data.Functor ((<&>))
@ -68,6 +69,34 @@ main =
, -- , bench "eff.5+5" $ nf catchEffDeep x
bench "mtl.5+5" $ nf catchMtlDeep x
]
, bgroup "local.shallow" $
[10000] <&> \x ->
bgroup
(show x)
[ bench "heftia" $ nf localHeftia x
, bench "polysemy" $ nf localSem x
, bench "fused" $ nf localFused x
, bench "effectful" $ nf localEffectful x
-- , bench "eff" $ nf localEff x
-- `eff` is x500 slow in this case, so it is excluded because it makes the graph hard to read.
-- bench "mtl" $ nf localMtl x
]
, bgroup "local.deep" $
[10000] <&> \x ->
bgroup
(show x)
[ bench "heftia.5+5+0" $ nf localHeftiaDeep0 x
, bench "heftia.5+4+1" $ nf localHeftiaDeep1 x
, bench "heftia.5+3+2" $ nf localHeftiaDeep2 x
, bench "heftia.5+2+3" $ nf localHeftiaDeep3 x
, bench "heftia.5+1+4" $ nf localHeftiaDeep4 x
, bench "heftia.5+0+5" $ nf localHeftiaDeep5 x
, bench "polysemy.5+5" $ nf localSemDeep x
, bench "fused.5+5" $ nf localFusedDeep x
, bench "effectful.5+5" $ nf localEffectfulDeep x
-- bench "eff.5+5" $ nf localEffDeep x
-- bench "mtl.5+5" $ nf localMtlDeep x
]
, bgroup "nondet.shallow" $
[32] <&> \x ->
bgroup

View File

@ -269,7 +269,7 @@ benchmark heftia-bench
other-modules:
BenchCountdown
BenchCatch
-- BenchLocal
BenchLocal
BenchCoroutine
BenchPyth
BenchParallel