on master help sign in
Use less memory for single chunk chunksOf benchmark

So that we can limit the heap to a reasonable value in tests. Also move the chunksOf benchmarks out of the DEVBUILD flag as they do not depend on a well formed text file.

commit4f4964afca4fd656e346ae81d9ef01924dc0f890 authorHarendra Kumar <harendra.kumar@gmail.com> dateFri 14 Feb 2020, 15:54 (7 years ago) parent7e4b43c
2 files changed+19−9
benchmark/FileIO.hs+14−9 benchmark/Streamly/Benchmark/FileIO/Stream.hs+5
benchmark/FileIO.hs+14−9
@@ -30,6 +30,9 @@ blockSize, blockCount :: Int
3030blockSize = 32768
3131blockCount = 3200
3232
33+fileSize :: Int
34+fileSize = blockSize * blockCount
35+
3336#ifdef DEVBUILD
3437-- This is a 500MB text file for text processing benchmarks. We cannot
3538-- have it in the repo, therefore we use it locally with DEVBUILD
@@ -38,9 +41,6 @@ blockCount = 3200
3841infile :: String
3942infile = "benchmark/text-processing/gutenberg-500.txt"
4043
41−fileSize :: Int
42−fileSize = blockSize * blockCount
43−
4444#else
4545infile :: String
4646infile = scratchDir ++ "in-100MB.txt"
@@ -204,21 +204,26 @@ main = do
204204 Handles inh outh <- readIORef href
205205 BFS.copyCodecUtf8Lenient inh outh
206206 ]
207− , bgroup "grouping"
208− [ mkBench "chunksOf (single chunk)" href $ do
207+#endif
208+ , bgroup "grouping-chunks"
209+ [ mkBench "sumChunksOf (single chunk)" href $ do
209210 Handles inh _ <- readIORef href
210− BFS.chunksOf fileSize inh
211+ BFS.chunksOfSum fileSize inh
212+ , mkBench "sumChunksOf 1" href $ do
213+ Handles inh _ <- readIORef href
214+ BFS.chunksOfSum 1 inh
211215
212− , mkBench "chunksOf 1" href $ do
216+ , mkBench "arraysOf 1" href $ do
213217 Handles inh _ <- readIORef href
214218 BFS.chunksOf 1 inh
215− , mkBench "chunksOf 10" href $ do
219+ , mkBench "arraysOf 10" href $ do
216220 Handles inh _ <- readIORef href
217221 BFS.chunksOf 10 inh
218− , mkBench "chunksOf 1000" href $ do
222+ , mkBench "arraysOf 1000" href $ do
219223 Handles inh _ <- readIORef href
220224 BFS.chunksOf 1000 inh
221225 ]
226+#ifdef DEVBUILD
222227 , bgroup "group-ungroup-stream"
223228 [ mkBench "lines-unlines-[Char]" href $ do
224229 Handles inh outh <- readIORef href
benchmark/Streamly/Benchmark/FileIO/Stream.hs+5
@@ -56,6 +56,7 @@ module Streamly.Benchmark.FileIO.Stream
5656 , copyCodecUtf8
5757 , decodeUtf8Lax
5858 , copyCodecUtf8Lenient
59+ , chunksOfSum
5960 , chunksOf
6061 , chunksOfD
6162 , splitOn
@@ -403,6 +404,10 @@ inspect $ hasNoTypeClasses 'copyCodecUtf8Lenient
403404-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''D.ConcatMapUState
404405#endif
405406
407+{-# INLINE chunksOfSum #-}
408+chunksOfSum :: Int -> Handle -> IO Int
409+chunksOfSum n inh = S.length $ S.chunksOf n FL.sum (S.unfold FH.read inh)
410+
406411-- | Slice in chunks of size n and get the count of chunks.
407412{-# INLINE chunksOf #-}
408413chunksOf :: Int -> Handle -> IO Int