git.iko.soy/streamly
on master help sign in
Add some tests and benchmarks for decode-utf8
9e46ec6Harendra Kumar
benchmark/FileIO.hs+21
compare with master
unifiedsplit
@@ -88,6 +88,9 @@ main = do
88 , mkBench "cat" href $ do
88 , mkBench "cat" href $ do
89 Handles inh _ <- readIORef href
89 Handles inh _ <- readIORef href
90 BFA.cat devNull inh
90 BFA.cat devNull inh
91+ , mkBench "read-utf8" href $ do
92+ Handles inh _ <- readIORef href
93+ BFA.decodeUtf8Lenient inh
91 ]
94 ]
92 , bgroup "readStream"
95 , bgroup "readStream"
93 [ mkBench "last" href $ do
96 [ mkBench "last" href $ do
@@ -114,6 +117,15 @@ main = do
114 , mkBench "catStream" href $ do
117 , mkBench "catStream" href $ do
115 Handles inh _ <- readIORef href
118 Handles inh _ <- readIORef href
116 BFS.catStreamWrite devNull inh
119 BFS.catStreamWrite devNull inh
120+ , mkBench "read-word8" href $ do
121+ Handles inh _ <- readIORef href
122+ BFS.readWord8 inh
123+ , mkBench "read-char8" href $ do
124+ Handles inh _ <- readIORef href
125+ BFS.decodeChar8 inh
126+ , mkBench "read-utf8" href $ do
127+ Handles inh _ <- readIORef href
128+ BFS.decodeUtf8Lenient inh
117 ]
129 ]
118 , bgroup "copyArray"
130 , bgroup "copyArray"
119 [ mkBench "copy" href $ do
131 [ mkBench "copy" href $ do
@@ -132,6 +144,9 @@ main = do
132 [ mkBench "char8" href $ do
144 [ mkBench "char8" href $ do
133 Handles inh outh <- readIORef href
145 Handles inh outh <- readIORef href
134 BFS.copyCodecChar8 inh outh
146 BFS.copyCodecChar8 inh outh
147+ , mkBench "utf8-arrays" href $ do
148+ Handles inh outh <- readIORef href
149+ BFA.copyCodecUtf8Lenient inh outh
135 , mkBench "utf8" href $ do
150 , mkBench "utf8" href $ do
136 Handles inh outh <- readIORef href
151 Handles inh outh <- readIORef href
137 BFS.copyCodecUtf8Lenient inh outh
152 BFS.copyCodecUtf8Lenient inh outh
@@ -215,6 +230,9 @@ main = do
215 , mkBench "splitOnSeq abcdefgh" href $ do
230 , mkBench "splitOnSeq abcdefgh" href $ do
216 Handles inh _ <- readIORef href
231 Handles inh _ <- readIORef href
217 BFS.splitOnSeq "abcdefgh" inh
232 BFS.splitOnSeq "abcdefgh" inh
233+ , mkBench "splitOnSeqUtf8 abcdefgh" href $ do
234+ Handles inh _ <- readIORef href
235+ BFS.splitOnSeqUtf8 "abcdefgh" inh
218 ]
236 ]
219 , bgroup "long-pattern"
237 , bgroup "long-pattern"
220 [ mkBench "splitOnSeq abcdefghi" href $ do
238 [ mkBench "splitOnSeq abcdefghi" href $ do
@@ -229,6 +247,9 @@ main = do
229 , mkBench "splitOnSuffixSeq abc...xyz" href $ do
247 , mkBench "splitOnSuffixSeq abc...xyz" href $ do
230 Handles inh _ <- readIORef href
248 Handles inh _ <- readIORef href
231 BFS.splitOnSuffixSeq "abcdefghijklmnopqrstuvwxyz" inh
249 BFS.splitOnSuffixSeq "abcdefghijklmnopqrstuvwxyz" inh
250+ , mkBench "splitOnSeqUtf8 abc...xyz" href $ do
251+ Handles inh _ <- readIORef href
252+ BFS.splitOnSeqUtf8 "abcdefghijklmnopqrstuvwxyz" inh
232 ]
253 ]
233 ]
254 ]
234#endif
255#endif
src/Streamly/Benchmark/FileIO/Array.hs+33
@@ -29,6 +29,8 @@ module Streamly.Benchmark.FileIO.Array
29 , copy
29 , copy
30 , linesUnlinesCopy
30 , linesUnlinesCopy
31 , wordsUnwordsCopy
31 , wordsUnwordsCopy
32+ , decodeUtf8Lenient
33+ , copyCodecUtf8Lenient
32 )
34 )
33where
35where
34
36
@@ -40,6 +42,7 @@ import Prelude hiding (last)
40import qualified Streamly.FileSystem.Handle as FH
42import qualified Streamly.FileSystem.Handle as FH
41import qualified Streamly.Memory.Array as A
43import qualified Streamly.Memory.Array as A
42import qualified Streamly.Prelude as S
44import qualified Streamly.Prelude as S
45+import qualified Streamly.Data.String as SS
43import qualified Streamly.Internal as Internal
46import qualified Streamly.Internal as Internal
44
47
45#ifdef INSPECTION
48#ifdef INSPECTION
@@ -160,3 +163,33 @@ wordsUnwordsCopy inh outh =
160inspect $ hasNoTypeClassesExcept 'wordsUnwordsCopy [''Storable]
163inspect $ hasNoTypeClassesExcept 'wordsUnwordsCopy [''Storable]
161-- inspect $ 'wordsUnwordsCopy `hasNoType` ''Step
164-- inspect $ 'wordsUnwordsCopy `hasNoType` ''Step
162#endif
165#endif
166+
167+{-# INLINE decodeUtf8Lenient #-}
168+decodeUtf8Lenient :: Handle -> IO ()
169+decodeUtf8Lenient inh =
170+ S.drain
171+ $ SS.decodeUtf8ArraysLenient
172+ $ FH.readArraysOf (1024*1024) inh
173+
174+#ifdef INSPECTION
175+inspect $ hasNoTypeClasses 'decodeUtf8Lenient
176+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''Step
177+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''AT.FlattenState
178+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''D.ConcatMapUState
179+#endif
180+
181+-- | Copy file
182+{-# INLINE copyCodecUtf8Lenient #-}
183+copyCodecUtf8Lenient :: Handle -> Handle -> IO ()
184+copyCodecUtf8Lenient inh outh =
185+ S.runFold (FH.write outh)
186+ $ SS.encodeUtf8
187+ $ SS.decodeUtf8ArraysLenient
188+ $ FH.readArraysOf (1024*1024) inh
189+
190+#ifdef INSPECTION
191+inspect $ hasNoTypeClasses 'copyCodecUtf8Lenient
192+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''Step
193+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''AT.FlattenState
194+-- inspect $ 'copyCodecUtf8Lenient `hasNoType` ''D.ConcatMapUState
195+#endif
src/Streamly/Benchmark/FileIO/Stream.hs+38−1
@@ -33,8 +33,11 @@ module Streamly.Benchmark.FileIO.Stream
33 , linesUnlinesCopy
33 , linesUnlinesCopy
34 , wordsUnwordsCopyWord8
34 , wordsUnwordsCopyWord8
35 , wordsUnwordsCopy
35 , wordsUnwordsCopy
36+ , readWord8
37+ , decodeChar8
36 , copyCodecChar8
38 , copyCodecChar8
37 , copyCodecUtf8
39 , copyCodecUtf8
40+ , decodeUtf8Lenient
38 , copyCodecUtf8Lenient
41 , copyCodecUtf8Lenient
39 , chunksOf
42 , chunksOf
40 , chunksOfD
43 , chunksOfD
@@ -42,6 +45,7 @@ module Streamly.Benchmark.FileIO.Stream
42 , splitOnSuffix
45 , splitOnSuffix
43 , wordsBy
46 , wordsBy
44 , splitOnSeq
47 , splitOnSeq
48+ , splitOnSeqUtf8
45 , splitOnSuffixSeq
49 , splitOnSuffixSeq
46 )
50 )
47where
51where
@@ -186,6 +190,17 @@ inspect $ 'copy `hasNoType` ''AT.FlattenState
186inspect $ 'copy `hasNoType` ''D.ConcatMapUState
190inspect $ 'copy `hasNoType` ''D.ConcatMapUState
187#endif
191#endif
188
192
193+{-# INLINE readWord8 #-}
194+readWord8 :: Handle -> IO ()
195+readWord8 inh = S.drain $ FH.read inh
196+
197+{-# INLINE decodeChar8 #-}
198+decodeChar8 :: Handle -> IO ()
199+decodeChar8 inh =
200+ S.drain
201+ $ SS.decodeChar8
202+ $ FH.read inh
203+
189-- | Copy file
204-- | Copy file
190{-# INLINE copyCodecChar8 #-}
205{-# INLINE copyCodecChar8 #-}
191copyCodecChar8 :: Handle -> Handle -> IO ()
206copyCodecChar8 :: Handle -> Handle -> IO ()
@@ -202,6 +217,20 @@ inspect $ 'copyCodecChar8 `hasNoType` ''AT.FlattenState
202inspect $ 'copyCodecChar8 `hasNoType` ''D.ConcatMapUState
217inspect $ 'copyCodecChar8 `hasNoType` ''D.ConcatMapUState
203#endif
218#endif
204
219
220+{-# INLINE decodeUtf8Lenient #-}
221+decodeUtf8Lenient :: Handle -> IO ()
222+decodeUtf8Lenient inh =
223+ S.drain
224+ $ SS.decodeUtf8Lenient
225+ $ FH.read inh
226+
227+#ifdef INSPECTION
228+inspect $ hasNoTypeClasses 'decodeUtf8Lenient
229+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''Step
230+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''AT.FlattenState
231+-- inspect $ 'decodeUtf8Lenient `hasNoType` ''D.ConcatMapUState
232+#endif
233+
205-- | Copy file
234-- | Copy file
206{-# INLINE copyCodecUtf8 #-}
235{-# INLINE copyCodecUtf8 #-}
207copyCodecUtf8 :: Handle -> Handle -> IO ()
236copyCodecUtf8 :: Handle -> Handle -> IO ()
@@ -406,7 +435,7 @@ inspect $ 'wordsBy `hasNoType` ''AT.FlattenState
406inspect $ 'wordsBy `hasNoType` ''D.ConcatMapUState
435inspect $ 'wordsBy `hasNoType` ''D.ConcatMapUState
407#endif
436#endif
408
437
409−-- | Split on a character sequence.
438+-- | Split on a word8 sequence.
410{-# INLINE splitOnSeq #-}
439{-# INLINE splitOnSeq #-}
411splitOnSeq :: String -> Handle -> IO Int
440splitOnSeq :: String -> Handle -> IO Int
412splitOnSeq str inh =
441splitOnSeq str inh =
@@ -420,6 +449,14 @@ inspect $ hasNoTypeClasses 'splitOnSeq
420-- inspect $ 'splitOnSeq `hasNoType` ''D.ConcatMapUState
449-- inspect $ 'splitOnSeq `hasNoType` ''D.ConcatMapUState
421#endif
450#endif
422
451
452+-- | Split on a character sequence.
453+{-# INLINE splitOnSeqUtf8 #-}
454+splitOnSeqUtf8 :: String -> Handle -> IO Int
455+splitOnSeqUtf8 str inh =
456+ (S.length $ Internal.splitOnSeq (A.fromList str) FL.drain
457+ $ SS.decodeUtf8ArraysLenient
458+ $ FH.readArrays inh) -- >>= print
459+
423-- | Split on suffix sequence.
460-- | Split on suffix sequence.
424{-# INLINE splitOnSuffixSeq #-}
461{-# INLINE splitOnSuffixSeq #-}
425splitOnSuffixSeq :: String -> Handle -> IO Int
462splitOnSuffixSeq :: String -> Handle -> IO Int
test/String.hs+21
@@ -41,6 +41,24 @@ propDecodeEncodeId =
41 chrs <- S.toList $ SS.decodeUtf8 wrds
41 chrs <- S.toList $ SS.decodeUtf8 wrds
42 assert (chrs == list)
42 assert (chrs == list)
43
43
44+-- XXX need to use invalid characters
45+propDecodeEncodeIdLenient :: Property
46+propDecodeEncodeIdLenient =
47+ forAll genUnicode $ \list ->
48+ monadicIO $ do
49+ let wrds = SS.encodeUtf8 $ S.fromList list
50+ chrs <- S.toList $ SS.decodeUtf8Lenient wrds
51+ assert (chrs == list)
52+
53+propDecodeEncodeIdArrays :: Property
54+propDecodeEncodeIdArrays =
55+ forAll genUnicode $ \list ->
56+ monadicIO $ do
57+ let wrds = SS.encodeUtf8 $ S.fromList list
58+ chrs <- S.toList $ SS.decodeUtf8ArraysLenient
59+ (S.runFold A.write wrds)
60+ assert (chrs == list)
61+
44testLines :: Property
62testLines :: Property
45testLines =
63testLines =
46 forAll genUnicode $ \list ->
64 forAll genUnicode $ \list ->
@@ -100,6 +118,9 @@ main = hspec
100 $ do
118 $ do
101 describe "UTF8 - Encoding / Decoding" $ do
119 describe "UTF8 - Encoding / Decoding" $ do
102 prop "decodeUtf8 . encodeUtf8 == id" $ propDecodeEncodeId
120 prop "decodeUtf8 . encodeUtf8 == id" $ propDecodeEncodeId
121+ prop "decodeUtf8Lenient . encodeUtf8 == id" $ propDecodeEncodeIdLenient
122+ prop "decodeUtf8ArraysLenient . encodeUtf8 == id"
123+ $ propDecodeEncodeIdArrays
103 prop "Streamly.Data.String.lines == Prelude.lines" $ testLines
124 prop "Streamly.Data.String.lines == Prelude.lines" $ testLines
104 prop "Arrays Streamly.Data.String.lines == Prelude.lines" $ testLinesArray
125 prop "Arrays Streamly.Data.String.lines == Prelude.lines" $ testLinesArray
105 prop "Streamly.Data.String.words == Prelude.words" $ testWords
126 prop "Streamly.Data.String.words == Prelude.words" $ testWords