@@ -1,4 +1,4 @@
1−-- |
1+--
22-- Module : Streamly.Unicode.Stream
33-- Copyright : (c) 2019 Composewell Technologies
44-- License : BSD-3-Clause
@@ -21,16 +21,14 @@
2121import Prelude hiding (last, length)
2222import System.IO (Handle)
2323
24−import qualified Streamly.Data.Fold as FL
25−import qualified Streamly.Unicode.Stream as SS
26−import qualified Streamly.FileSystem.Handle as FH
27−import qualified Streamly.Internal.Data.Unfold as IUF
28−import qualified Streamly.Internal.Unicode.Stream as IUS
29−import qualified Streamly.Internal.FileSystem.Handle as IFH
30−import qualified Streamly.Internal.Unicode.Array.Char as IUA
31−import qualified Streamly.Internal.Data.Stream.IsStream as IP
32−import qualified Streamly.Data.Array.Foreign as A
33−import qualified Streamly.Prelude as S
24+import qualified Streamly.Data.Array.Foreign as Array
25+import qualified Streamly.Data.Fold as Fold
26+import qualified Streamly.FileSystem.Handle as Handle
27+import qualified Streamly.Internal.Data.Stream.IsStream as Stream
28+import qualified Streamly.Internal.Data.Unfold as Unfold
29+import qualified Streamly.Internal.FileSystem.Handle as Handle
30+import qualified Streamly.Internal.Unicode.Array.Char as UnicodeArr
31+import qualified Streamly.Internal.Unicode.Stream as Unicode
3432
3533import Gauge hiding (env)
3634import Streamly.Benchmark.Common
@@ -41,8 +39,8 @@ import Foreign.Storable (Storable)
4139import Streamly.Internal.Data.Stream.StreamD.Type (Step(..))
4240import qualified Streamly.Internal.Data.Fold.Type as Fold
4341import qualified Streamly.Internal.Data.Tuple.Strict as Strict
44−import qualified Streamly.Internal.Data.Array.Foreign.Type as AT
45−import qualified Streamly.Internal.Data.Array.Foreign.Mut.Type as MA
42+import qualified Streamly.Internal.Data.Array.Foreign.Type as Array
43+import qualified Streamly.Internal.Data.Array.Foreign.Mut.Type as MArray
4644
4745import Test.Inspection
4846#endif
@@ -54,10 +52,10 @@ moduleName = "Unicode.Stream"
5452{-# NOINLINE copyCodecUtf8ArraysLenient #-}
5553copyCodecUtf8ArraysLenient :: Handle -> Handle -> IO ()
5654copyCodecUtf8ArraysLenient inh outh =
57− S.fold (FH.write outh)
58− $ SS.encodeUtf8'
59− $ IUS.decodeUtf8Arrays
60− $ IFH.toChunks inh
55+ Stream.fold (Handle.write outh)
56+ $ Unicode.encodeUtf8'
57+ $ Unicode.decodeUtf8Arrays
58+ $ Handle.toChunks inh
6159
6260#ifdef INSPECTION
6361inspect $ hasNoTypeClasses 'copyCodecUtf8ArraysLenient
@@ -68,7 +66,7 @@ o_1_space_decode_encode_chunked :: BenchEnv -> [Benchmark]
6866o_1_space_decode_encode_chunked env =
6967 [ bgroup "decode-encode/toChunks"
7068 [
71− mkBenchSmall "decodeEncodeUtf8Lenient" env $ \inH outH ->
69+ mkBenchSmall "encodeUtf8' . decodeUtf8Arrays" env $ \inH outH ->
7270 copyCodecUtf8ArraysLenient inH outH
7371 ]
7472 ]
@@ -80,60 +78,60 @@ o_1_space_decode_encode_chunked env =
8078{-# NOINLINE linesUnlinesCopy #-}
8179linesUnlinesCopy :: Handle -> Handle -> IO ()
8280linesUnlinesCopy inh outh =
83− S.fold (FH.write outh)
84− $ SS.encodeLatin1'
85− $ IUS.unlines IUF.fromList
86− $ S.splitOnSuffix (== '\n') FL.toList
87− $ SS.decodeLatin1
88− $ S.unfold FH.read inh
81+ Stream.fold (Handle.write outh)
82+ $ Unicode.encodeLatin1'
83+ $ Unicode.unlines Unfold.fromList
84+ $ Stream.splitOnSuffix (== '\n') Fold.toList
85+ $ Unicode.decodeLatin1
86+ $ Stream.unfold Handle.read inh
8987
9088{-# NOINLINE linesUnlinesArrayWord8Copy #-}
9189linesUnlinesArrayWord8Copy :: Handle -> Handle -> IO ()
9290linesUnlinesArrayWord8Copy inh outh =
93− S.fold (FH.write outh)
94− $ IP.interposeSuffix 10 A.read
95− $ S.splitOnSuffix (== 10) A.write
96− $ S.unfold FH.read inh
91+ Stream.fold (Handle.write outh)
92+ $ Stream.interposeSuffix 10 Array.read
93+ $ Stream.splitOnSuffix (== 10) Array.write
94+ $ Stream.unfold Handle.read inh
9795
9896-- XXX splitSuffixOn requires -funfolding-use-threshold=150 for better fusion
9997-- | Lines and unlines
10098{-# NOINLINE linesUnlinesArrayCharCopy #-}
10199linesUnlinesArrayCharCopy :: Handle -> Handle -> IO ()
102100linesUnlinesArrayCharCopy inh outh =
103− S.fold (FH.write outh)
104− $ SS.encodeLatin1'
105− $ IUA.unlines
106− $ IUA.lines
107− $ SS.decodeLatin1
108− $ S.unfold FH.read inh
101+ Stream.fold (Handle.write outh)
102+ $ Unicode.encodeLatin1'
103+ $ UnicodeArr.unlines
104+ $ UnicodeArr.lines
105+ $ Unicode.decodeLatin1
106+ $ Stream.unfold Handle.read inh
109107
110108#ifdef INSPECTION
111109inspect $ hasNoTypeClassesExcept 'linesUnlinesArrayCharCopy [''Storable]
112110-- inspect $ 'linesUnlinesArrayCharCopy `hasNoType` ''Step
113111#endif
114112
115−-- XXX to write this we need to be able to map decodeUtf8 on the A.read fold.
113+-- XXX to write this we need to be able to map decodeUtf8 on the Array.read fold.
116114-- For that we have to write decodeUtf8 as a Pipe.
117115{-
118116{-# INLINE linesUnlinesArrayUtf8Copy #-}
119117linesUnlinesArrayUtf8Copy :: Handle -> Handle -> IO ()
120118linesUnlinesArrayUtf8Copy inh outh =
121− S.fold (FH.write outh)
122− $ SS.encodeLatin1'
123− $ IP.intercalate (A.fromList [10]) (pipe SS.decodeUtf8P A.read)
124− $ S.splitOnSuffix (== '\n') (IFL.map SS.encodeUtf8' A.write)
125− $ SS.decodeLatin1
126− $ S.unfold FH.read inh
119+ Stream.fold (Handle.write outh)
120+ $ Unicode.encodeLatin1'
121+ $ Stream.intercalate (Array.fromList [10]) (pipe Unicode.decodeUtf8P Array.read)
122+ $ Stream.splitOnSuffix (== '\n') (IFold.map Unicode.encodeUtf8' Array.write)
123+ $ Unicode.decodeLatin1
124+ $ Stream.unfold Handle.read inh
127125-}
128126
129127-- | Word, unwords and copy
130128{-# NOINLINE wordsUnwordsCopyWord8 #-}
131129wordsUnwordsCopyWord8 :: Handle -> Handle -> IO ()
132130wordsUnwordsCopyWord8 inh outh =
133− S.fold (FH.write outh)
134− $ IP.interposeSuffix 32 IUF.fromList
135− $ S.wordsBy isSp FL.toList
136− $ S.unfold FH.read inh
131+ Stream.fold (Handle.write outh)
132+ $ Stream.interposeSuffix 32 Unfold.fromList
133+ $ Stream.wordsBy isSp Fold.toList
134+ $ Stream.unfold Handle.read inh
137135
138136#ifdef INSPECTION
139137inspect $ hasNoTypeClasses 'wordsUnwordsCopyWord8
@@ -144,9 +142,9 @@ inspect $ hasNoTypeClasses 'wordsUnwordsCopyWord8
144142{-# NOINLINE wordsUnwordsCopy #-}
145143wordsUnwordsCopy :: Handle -> Handle -> IO ()
146144wordsUnwordsCopy inh outh =
147− S.fold (FH.write outh)
148− $ SS.encodeLatin1'
149− $ IUS.unwords IUF.fromList
145+ Stream.fold (Handle.write outh)
146+ $ Unicode.encodeLatin1'
147+ $ Unicode.unwords Unfold.fromList
150148 -- XXX This pipeline does not fuse with wordsBy but fuses with splitOn
151149 -- with -funfolding-use-threshold=300. With wordsBy it does not fuse
152150 -- even with high limits for inlining and spec-constr ghc options. With
@@ -154,10 +152,10 @@ wordsUnwordsCopy inh outh =
154152 -- is no evidence in the core that a join point involving Step
155153 -- constructors is not getting inlined. Not being able to fuse at all in
156154 -- this case could be an unknown issue, need more investigation.
157− $ S.wordsBy isSpace FL.toList
158− -- -- $ S.splitOn isSpace FL.toList
159− $ SS.decodeLatin1
160− $ S.unfold FH.read inh
155+ $ Stream.wordsBy isSpace Fold.toList
156+ -- -- $ Stream.splitOn isSpace Fold.toList
157+ $ Unicode.decodeLatin1
158+ $ Stream.unfold Handle.read inh
161159
162160#ifdef INSPECTION
163161-- inspect $ hasNoTypeClasses 'wordsUnwordsCopy
@@ -167,28 +165,28 @@ wordsUnwordsCopy inh outh =
167165{-# NOINLINE wordsUnwordsCharArrayCopy #-}
168166wordsUnwordsCharArrayCopy :: Handle -> Handle -> IO ()
169167wordsUnwordsCharArrayCopy inh outh =
170− S.fold (FH.write outh)
171− $ SS.encodeLatin1'
172− $ IUA.unwords
173− $ IUA.words
174− $ SS.decodeLatin1
175− $ S.unfold FH.read inh
168+ Stream.fold (Handle.write outh)
169+ $ Unicode.encodeLatin1'
170+ $ UnicodeArr.unwords
171+ $ UnicodeArr.words
172+ $ Unicode.decodeLatin1
173+ $ Stream.unfold Handle.read inh
176174
177175o_1_space_copy_read_group_ungroup :: BenchEnv -> [Benchmark]
178176o_1_space_copy_read_group_ungroup env =
179177 [ bgroup "ungroup-group"
180− [ mkBenchSmall "US.unlines . S.splitOnSuffix ([Word8])" env
178+ [ mkBenchSmall "unlines . splitOnSuffix ([Word8])" env
181179 $ \inh outh -> linesUnlinesCopy inh outh
182− , mkBenchSmall "S.interposeSuffix . S.splitOnSuffix(Array Word8)" env
180+ , mkBenchSmall "interposeSuffix . splitOnSuffix (Array Word8)" env
183181 $ \inh outh -> linesUnlinesArrayWord8Copy inh outh
184− , mkBenchSmall "UA.unlines . UA.lines (Array Char)" env
182+ , mkBenchSmall "UnicodeArr.unlines . UnicodeArr.lines (Array Char)" env
185183 $ \inh outh -> linesUnlinesArrayCharCopy inh outh
186184
187− , mkBenchSmall "S.interposeSuffix . S.wordsBy ([Word8])" env
185+ , mkBenchSmall "interposeSuffix . wordsBy ([Word8])" env
188186 $ \inh outh -> wordsUnwordsCopyWord8 inh outh
189− , mkBenchSmall "US.unwords . S.wordsBy ([Char])" env
187+ , mkBenchSmall "unwords . wordsBy ([Char])" env
190188 $ \inh outh -> wordsUnwordsCopy inh outh
191− , mkBenchSmall "UA.unwords . UA.words (Array Char)" env
189+ , mkBenchSmall "UnicodeArr.unwords . UnicodeArr.words (Array Char)" env
192190 $ \inh outh -> wordsUnwordsCharArrayCopy inh outh
193191 ]
194192 ]
@@ -201,55 +199,55 @@ o_1_space_copy_read_group_ungroup env =
201199{-# NOINLINE copyStreamLatin1' #-}
202200copyStreamLatin1' :: Handle -> Handle -> IO ()
203201copyStreamLatin1' inh outh =
204− S.fold (FH.write outh)
205− $ SS.encodeLatin1'
206− $ SS.decodeLatin1
207− $ S.unfold FH.read inh
202+ Stream.fold (Handle.write outh)
203+ $ Unicode.encodeLatin1'
204+ $ Unicode.decodeLatin1
205+ $ Stream.unfold Handle.read inh
208206
209207#ifdef INSPECTION
210208inspect $ hasNoTypeClasses 'copyStreamLatin1'
211209inspect $ 'copyStreamLatin1' `hasNoType` ''Step
212−inspect $ 'copyStreamLatin1' `hasNoType` ''IUF.ConcatState -- FH.read/UF.many
213−inspect $ 'copyStreamLatin1' `hasNoType` ''MA.ReadUState -- FH.read/A.read
210+inspect $ 'copyStreamLatin1' `hasNoType` ''Unfold.ConcatState -- Handle.read/UF.many
211+inspect $ 'copyStreamLatin1' `hasNoType` ''MArray.ReadUState -- Handle.read/Array.read
214212
215213inspect $ 'copyStreamLatin1' `hasNoType` ''Fold.Step
216−inspect $ 'copyStreamLatin1' `hasNoType` ''AT.ArrayUnsafe -- FH.write/writeNUnsafe
217−inspect $ 'copyStreamLatin1' `hasNoType` ''Strict.Tuple3' -- FH.write/chunksOf
214+inspect $ 'copyStreamLatin1' `hasNoType` ''Array.ArrayUnsafe -- Handle.write/writeNUnsafe
215+inspect $ 'copyStreamLatin1' `hasNoType` ''Strict.Tuple3' -- Handle.write/chunksOf
218216#endif
219217
220218-- | Copy file (encodeLatin1)
221219{-# NOINLINE copyStreamLatin1 #-}
222220copyStreamLatin1 :: Handle -> Handle -> IO ()
223221copyStreamLatin1 inh outh =
224− S.fold (FH.write outh)
225− $ SS.encodeLatin1
226− $ SS.decodeLatin1
227− $ S.unfold FH.read inh
222+ Stream.fold (Handle.write outh)
223+ $ Unicode.encodeLatin1
224+ $ Unicode.decodeLatin1
225+ $ Stream.unfold Handle.read inh
228226
229227#ifdef INSPECTION
230228inspect $ hasNoTypeClasses 'copyStreamLatin1
231229inspect $ 'copyStreamLatin1 `hasNoType` ''Step
232−inspect $ 'copyStreamLatin1 `hasNoType` ''IUF.ConcatState -- FH.read/UF.many
233−inspect $ 'copyStreamLatin1 `hasNoType` ''MA.ReadUState -- FH.read/A.read
230+inspect $ 'copyStreamLatin1 `hasNoType` ''Unfold.ConcatState -- Handle.read/UF.many
231+inspect $ 'copyStreamLatin1 `hasNoType` ''MArray.ReadUState -- Handle.read/Array.read
234232
235233inspect $ 'copyStreamLatin1 `hasNoType` ''Fold.ManyState
236234inspect $ 'copyStreamLatin1 `hasNoType` ''Fold.Step
237−inspect $ 'copyStreamLatin1 `hasNoType` ''AT.ArrayUnsafe -- FH.write/writeNUnsafe
238−inspect $ 'copyStreamLatin1 `hasNoType` ''Strict.Tuple3' -- FH.write/chunksOf
235+inspect $ 'copyStreamLatin1 `hasNoType` ''Array.ArrayUnsafe -- Handle.write/writeNUnsafe
236+inspect $ 'copyStreamLatin1 `hasNoType` ''Strict.Tuple3' -- Handle.write/chunksOf
239237#endif
240238
241239-- | Copy file
242240_copyStreamUtf8' :: Handle -> Handle -> IO ()
243241_copyStreamUtf8' inh outh =
244− S.fold (FH.write outh)
245− $ SS.encodeUtf8'
246− $ SS.decodeUtf8'
247− $ S.unfold FH.read inh
242+ Stream.fold (Handle.write outh)
243+ $ Unicode.encodeUtf8'
244+ $ Unicode.decodeUtf8'
245+ $ Stream.unfold Handle.read inh
248246
249247#ifdef INSPECTION
250248inspect $ hasNoTypeClasses '_copyStreamUtf8'
251249-- inspect $ '_copyStreamUtf8 `hasNoType` ''Step
252−-- inspect $ '_copyStreamUtf8 `hasNoType` ''AT.FlattenState
250+-- inspect $ '_copyStreamUtf8 `hasNoType` ''Array.FlattenState
253251-- inspect $ '_copyStreamUtf8 `hasNoType` ''D.ConcatMapUState
254252#endif
255253
@@ -257,15 +255,15 @@ inspect $ hasNoTypeClasses '_copyStreamUtf8'
257255{-# NOINLINE copyStreamUtf8 #-}
258256copyStreamUtf8 :: Handle -> Handle -> IO ()
259257copyStreamUtf8 inh outh =
260− S.fold (FH.write outh)
261− $ SS.encodeUtf8
262− $ SS.decodeUtf8
263− $ S.unfold FH.read inh
258+ Stream.fold (Handle.write outh)
259+ $ Unicode.encodeUtf8
260+ $ Unicode.decodeUtf8
261+ $ Stream.unfold Handle.read inh
264262
265263#ifdef INSPECTION
266264inspect $ hasNoTypeClasses 'copyStreamUtf8
267265-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''Step
268−-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''AT.FlattenState
266+-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''Array.FlattenState
269267-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''D.ConcatMapUState
270268#endif
271269
@@ -274,15 +272,16 @@ o_1_space_decode_encode_read env =
274272 [ bgroup "decode-encode"
275273 [
276274 -- This needs an ascii file, as decode just errors out.
277− mkBench "SS.encodeLatin1' . SS.decodeLatin1" env $ \inh outh ->
275+ mkBench "encodeLatin1' . decodeLatin1" env $ \inh outh ->
278276 copyStreamLatin1' inh outh
279− , mkBench "SS.encodeLatin1 . SS.decodeLatin1" env $ \inh outh ->
277+ , mkBench "encodeLatin1 . decodeLatin1" env $ \inh outh ->
280278 copyStreamLatin1 inh outh
281279#ifdef DEVBUILD
282− , mkBench "copyUtf8" env $ \inh outh ->
280+ -- Requires valid unicode input
281+ , mkBench "encodeUtf8' . decodeUtf8'" env $ \inh outh ->
283282 _copyStreamUtf8' inh outh
284283#endif
285− , mkBenchSmall "SS.encodeUtf8 . SS.decodeUtf8Lax" env $ \inh outh ->
284+ , mkBenchSmall "encodeUtf8 . decodeUtf8" env $ \inh outh ->
286285 copyStreamUtf8 inh outh
287286 ]
288287 ]
@@ -296,7 +295,7 @@ main = do
296295 where
297296
298297 allBenchmarks env =
299− [ bgroup (o_1_space_prefix moduleName) $ Prelude.concat $
298+ [ bgroup (o_1_space_prefix moduleName) $ Prelude.concat
300299 [ o_1_space_copy_read_group_ungroup env
301300 , o_1_space_decode_encode_chunked env
302301 , o_1_space_decode_encode_read env