git.iko.soy/streamly
on master help sign in
Use less memory for single chunk chunksOf benchmark
4f4964aHarendra Kumar
benchmark/FileIO.hs+14−9
compare with master
unifiedsplit
@@ -30,6 +30,9 @@ blockSize, blockCount :: Int
30blockSize = 32768
30blockSize = 32768
31blockCount = 3200
31blockCount = 3200
32
32
33+fileSize :: Int
34+fileSize = blockSize * blockCount
35+
33#ifdef DEVBUILD
36#ifdef DEVBUILD
34-- This is a 500MB text file for text processing benchmarks. We cannot
37-- This is a 500MB text file for text processing benchmarks. We cannot
35-- have it in the repo, therefore we use it locally with DEVBUILD
38-- have it in the repo, therefore we use it locally with DEVBUILD
@@ -38,9 +41,6 @@ blockCount = 3200
38infile :: String
41infile :: String
39infile = "benchmark/text-processing/gutenberg-500.txt"
42infile = "benchmark/text-processing/gutenberg-500.txt"
40
43
41−fileSize :: Int
42−fileSize = blockSize * blockCount
43−
44#else
44#else
45infile :: String
45infile :: String
46infile = scratchDir ++ "in-100MB.txt"
46infile = scratchDir ++ "in-100MB.txt"
@@ -204,21 +204,26 @@ main = do
204 Handles inh outh <- readIORef href
204 Handles inh outh <- readIORef href
205 BFS.copyCodecUtf8Lenient inh outh
205 BFS.copyCodecUtf8Lenient inh outh
206 ]
206 ]
207+#endif
207− , bgroup "grouping"
208+ , bgroup "grouping-chunks"
208− [ mkBench "chunksOf (single chunk)" href $ do
209+ [ mkBench "sumChunksOf (single chunk)" href $ do
209 Handles inh _ <- readIORef href
210 Handles inh _ <- readIORef href
210− BFS.chunksOf fileSize inh
211+ BFS.chunksOfSum fileSize inh
212+ , mkBench "sumChunksOf 1" href $ do
213+ Handles inh _ <- readIORef href
214+ BFS.chunksOfSum 1 inh
211
215
212− , mkBench "chunksOf 1" href $ do
216+ , mkBench "arraysOf 1" href $ do
213 Handles inh _ <- readIORef href
217 Handles inh _ <- readIORef href
214 BFS.chunksOf 1 inh
218 BFS.chunksOf 1 inh
215− , mkBench "chunksOf 10" href $ do
219+ , mkBench "arraysOf 10" href $ do
216 Handles inh _ <- readIORef href
220 Handles inh _ <- readIORef href
217 BFS.chunksOf 10 inh
221 BFS.chunksOf 10 inh
218− , mkBench "chunksOf 1000" href $ do
222+ , mkBench "arraysOf 1000" href $ do
219 Handles inh _ <- readIORef href
223 Handles inh _ <- readIORef href
220 BFS.chunksOf 1000 inh
224 BFS.chunksOf 1000 inh
221 ]
225 ]
226+#ifdef DEVBUILD
222 , bgroup "group-ungroup-stream"
227 , bgroup "group-ungroup-stream"
223 [ mkBench "lines-unlines-[Char]" href $ do
228 [ mkBench "lines-unlines-[Char]" href $ do
224 Handles inh outh <- readIORef href
229 Handles inh outh <- readIORef href
benchmark/Streamly/Benchmark/FileIO/Stream.hs+5
@@ -56,6 +56,7 @@ module Streamly.Benchmark.FileIO.Stream
56 , copyCodecUtf8
56 , copyCodecUtf8
57 , decodeUtf8Lax
57 , decodeUtf8Lax
58 , copyCodecUtf8Lenient
58 , copyCodecUtf8Lenient
59+ , chunksOfSum
59 , chunksOf
60 , chunksOf
60 , chunksOfD
61 , chunksOfD
61 , splitOn
62 , splitOn
@@ -403,6 +404,10 @@ inspect $ hasNoTypeClasses 'copyCodecUtf8Lenient
403-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''D.ConcatMapUState
404-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''D.ConcatMapUState
404#endif
405#endif
405
406
407+{-# INLINE chunksOfSum #-}
408+chunksOfSum :: Int -> Handle -> IO Int
409+chunksOfSum n inh = S.length $ S.chunksOf n FL.sum (S.unfold FH.read inh)
410+
406-- | Slice in chunks of size n and get the count of chunks.
411-- | Slice in chunks of size n and get the count of chunks.
407{-# INLINE chunksOf #-}
412{-# INLINE chunksOf #-}
408chunksOf :: Int -> Handle -> IO Int
413chunksOf :: Int -> Handle -> IO Int