git.iko.soy/streamly
on master help sign in
Add some tests and benchmarks for decode-utf8
commit9e46ec65fcc96bb997f230d89f035fa79bd9a640 authorHarendra Kumar <harendra.kumar@gmail.com> dateTue 17 Sep 2019, 19:07 (7 years ago) parent3b44db7
4 files changed+113−1
compare with master
unifiedsplit
benchmark/FileIO.hs+21 src/Streamly/Benchmark/FileIO/Array.hs+33 src/Streamly/Benchmark/FileIO/Stream.hs+38−1 test/String.hs+21
benchmark/FileIO.hs+21
@@ -88,6 +88,9 @@ main = do
8888 , mkBench "cat" href $ do
8989 Handles inh _ <- readIORef href
9090 BFA.cat devNull inh
91+ , mkBench "read-utf8" href $ do
92+ Handles inh _ <- readIORef href
93+ BFA.decodeUtf8Lenient inh
9194 ]
9295 , bgroup "readStream"
9396 [ mkBench "last" href $ do
@@ -114,6 +117,15 @@ main = do
114117 , mkBench "catStream" href $ do
115118 Handles inh _ <- readIORef href
116119 BFS.catStreamWrite devNull inh
120+ , mkBench "read-word8" href $ do
121+ Handles inh _ <- readIORef href
122+ BFS.readWord8 inh
123+ , mkBench "read-char8" href $ do
124+ Handles inh _ <- readIORef href
125+ BFS.decodeChar8 inh
126+ , mkBench "read-utf8" href $ do
127+ Handles inh _ <- readIORef href
128+ BFS.decodeUtf8Lenient inh
117129 ]
118130 , bgroup "copyArray"
119131 [ mkBench "copy" href $ do
@@ -132,6 +144,9 @@ main = do
132144 [ mkBench "char8" href $ do
133145 Handles inh outh <- readIORef href
134146 BFS.copyCodecChar8 inh outh
147+ , mkBench "utf8-arrays" href $ do
148+ Handles inh outh <- readIORef href
149+ BFA.copyCodecUtf8Lenient inh outh
135150 , mkBench "utf8" href $ do
136151 Handles inh outh <- readIORef href
137152 BFS.copyCodecUtf8Lenient inh outh
@@ -215,6 +230,9 @@ main = do
215230 , mkBench "splitOnSeq abcdefgh" href $ do
216231 Handles inh _ <- readIORef href
217232 BFS.splitOnSeq "abcdefgh" inh
233+ , mkBench "splitOnSeqUtf8 abcdefgh" href $ do
234+ Handles inh _ <- readIORef href
235+ BFS.splitOnSeqUtf8 "abcdefgh" inh
218236 ]
219237 , bgroup "long-pattern"
220238 [ mkBench "splitOnSeq abcdefghi" href $ do
@@ -229,6 +247,9 @@ main = do
229247 , mkBench "splitOnSuffixSeq abc...xyz" href $ do
230248 Handles inh _ <- readIORef href
231249 BFS.splitOnSuffixSeq "abcdefghijklmnopqrstuvwxyz" inh
250+ , mkBench "splitOnSeqUtf8 abc...xyz" href $ do
251+ Handles inh _ <- readIORef href
252+ BFS.splitOnSeqUtf8 "abcdefghijklmnopqrstuvwxyz" inh
232253 ]
233254 ]
234255#endif
src/Streamly/Benchmark/FileIO/Array.hs+33
@@ -29,6 +29,8 @@ module Streamly.Benchmark.FileIO.Array
2929 , copy
3030 , linesUnlinesCopy
3131 , wordsUnwordsCopy
32+ , decodeUtf8Lenient
33+ , copyCodecUtf8Lenient
3234 )
3335where
3436
@@ -40,6 +42,7 @@ import Prelude hiding (last)
4042import qualified Streamly.FileSystem.Handle as FH
4143import qualified Streamly.Memory.Array as A
4244import qualified Streamly.Prelude as S
45+import qualified Streamly.Data.String as SS
4346import qualified Streamly.Internal as Internal
4447
4548#ifdef INSPECTION
@@ -160,3 +163,33 @@ wordsUnwordsCopy inh outh =
160163inspect $ hasNoTypeClassesExcept 'wordsUnwordsCopy [''Storable]
161164-- inspect $ 'wordsUnwordsCopy `hasNoType` ''Step
162165#endif
166+
167+{-# INLINE decodeUtf8Lenient #-}
168+decodeUtf8Lenient :: Handle -> IO ()
169+decodeUtf8Lenient inh =
170+ S.drain
171+ $ SS.decodeUtf8ArraysLenient
172+ $ FH.readArraysOf (1024*1024) inh
173+
174+#ifdef INSPECTION
175+inspect $ hasNoTypeClasses 'decodeUtf8Lenient
176+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''Step
177+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''AT.FlattenState
178+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''D.ConcatMapUState
179+#endif
180+
181+-- | Copy file
182+{-# INLINE copyCodecUtf8Lenient #-}
183+copyCodecUtf8Lenient :: Handle -> Handle -> IO ()
184+copyCodecUtf8Lenient inh outh =
185+ S.runFold (FH.write outh)
186+ $ SS.encodeUtf8
187+ $ SS.decodeUtf8ArraysLenient
188+ $ FH.readArraysOf (1024*1024) inh
189+
190+#ifdef INSPECTION
191+inspect $ hasNoTypeClasses 'copyCodecUtf8Lenient
192+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''Step
193+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''AT.FlattenState
194+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''D.ConcatMapUState
195+#endif
src/Streamly/Benchmark/FileIO/Stream.hs+38−1
@@ -33,8 +33,11 @@ module Streamly.Benchmark.FileIO.Stream
3333 , linesUnlinesCopy
3434 , wordsUnwordsCopyWord8
3535 , wordsUnwordsCopy
36+ , readWord8
37+ , decodeChar8
3638 , copyCodecChar8
3739 , copyCodecUtf8
40+ , decodeUtf8Lenient
3841 , copyCodecUtf8Lenient
3942 , chunksOf
4043 , chunksOfD
@@ -42,6 +45,7 @@ module Streamly.Benchmark.FileIO.Stream
4245 , splitOnSuffix
4346 , wordsBy
4447 , splitOnSeq
48+ , splitOnSeqUtf8
4549 , splitOnSuffixSeq
4650 )
4751where
@@ -186,6 +190,17 @@ inspect $ 'copy `hasNoType` ''AT.FlattenState
186190inspect $ 'copy `hasNoType` ''D.ConcatMapUState
187191#endif
188192
193+{-# INLINE readWord8 #-}
194+readWord8 :: Handle -> IO ()
195+readWord8 inh = S.drain $ FH.read inh
196+
197+{-# INLINE decodeChar8 #-}
198+decodeChar8 :: Handle -> IO ()
199+decodeChar8 inh =
200+ S.drain
201+ $ SS.decodeChar8
202+ $ FH.read inh
203+
189204-- | Copy file
190205{-# INLINE copyCodecChar8 #-}
191206copyCodecChar8 :: Handle -> Handle -> IO ()
@@ -202,6 +217,20 @@ inspect $ 'copyCodecChar8 `hasNoType` ''AT.FlattenState
202217inspect $ 'copyCodecChar8 `hasNoType` ''D.ConcatMapUState
203218#endif
204219
220+{-# INLINE decodeUtf8Lenient #-}
221+decodeUtf8Lenient :: Handle -> IO ()
222+decodeUtf8Lenient inh =
223+ S.drain
224+ $ SS.decodeUtf8Lenient
225+ $ FH.read inh
226+
227+#ifdef INSPECTION
228+inspect $ hasNoTypeClasses 'decodeUtf8Lenient
229+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''Step
230+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''AT.FlattenState
231+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''D.ConcatMapUState
232+#endif
233+
205234-- | Copy file
206235{-# INLINE copyCodecUtf8 #-}
207236copyCodecUtf8 :: Handle -> Handle -> IO ()
@@ -406,7 +435,7 @@ inspect $ 'wordsBy `hasNoType` ''AT.FlattenState
406435inspect $ 'wordsBy `hasNoType` ''D.ConcatMapUState
407436#endif
408437
409−-- | Split on a character sequence.
438+-- | Split on a word8 sequence.
410439{-# INLINE splitOnSeq #-}
411440splitOnSeq :: String -> Handle -> IO Int
412441splitOnSeq str inh =
@@ -420,6 +449,14 @@ inspect $ hasNoTypeClasses 'splitOnSeq
420449-- inspect $ 'splitOnSeq `hasNoType` ''D.ConcatMapUState
421450#endif
422451
452+-- | Split on a character sequence.
453+{-# INLINE splitOnSeqUtf8 #-}
454+splitOnSeqUtf8 :: String -> Handle -> IO Int
455+splitOnSeqUtf8 str inh =
456+ (S.length $ Internal.splitOnSeq (A.fromList str) FL.drain
457+ $ SS.decodeUtf8ArraysLenient
458+ $ FH.readArrays inh) -- >>= print
459+
423460-- | Split on suffix sequence.
424461{-# INLINE splitOnSuffixSeq #-}
425462splitOnSuffixSeq :: String -> Handle -> IO Int
test/String.hs+21
@@ -41,6 +41,24 @@ propDecodeEncodeId =
4141 chrs <- S.toList $ SS.decodeUtf8 wrds
4242 assert (chrs == list)
4343
44+-- XXX need to use invalid characters
45+propDecodeEncodeIdLenient :: Property
46+propDecodeEncodeIdLenient =
47+ forAll genUnicode $ \list ->
48+ monadicIO $ do
49+ let wrds = SS.encodeUtf8 $ S.fromList list
50+ chrs <- S.toList $ SS.decodeUtf8Lenient wrds
51+ assert (chrs == list)
52+
53+propDecodeEncodeIdArrays :: Property
54+propDecodeEncodeIdArrays =
55+ forAll genUnicode $ \list ->
56+ monadicIO $ do
57+ let wrds = SS.encodeUtf8 $ S.fromList list
58+ chrs <- S.toList $ SS.decodeUtf8ArraysLenient
59+ (S.runFold A.write wrds)
60+ assert (chrs == list)
61+
4462testLines :: Property
4563testLines =
4664 forAll genUnicode $ \list ->
@@ -100,6 +118,9 @@ main = hspec
100118 $ do
101119 describe "UTF8 - Encoding / Decoding" $ do
102120 prop "decodeUtf8 . encodeUtf8 == id" $ propDecodeEncodeId
121+ prop "decodeUtf8Lenient . encodeUtf8 == id" $ propDecodeEncodeIdLenient
122+ prop "decodeUtf8ArraysLenient . encodeUtf8 == id"
123+ $ propDecodeEncodeIdArrays
103124 prop "Streamly.Data.String.lines == Prelude.lines" $ testLines
104125 prop "Arrays Streamly.Data.String.lines == Prelude.lines" $ testLinesArray
105126 prop "Streamly.Data.String.words == Prelude.words" $ testWords