git.iko.soy/streamly
on master help sign in
hide char array string ops from Unicode stream module
44e3cb3Harendra Kumar
examples/WordCount.hs+12−9
compare with master
unifiedsplit
@@ -36,6 +36,9 @@ import Data.Word (Word8)
36import GHC.Conc (numCapabilities)
36import GHC.Conc (numCapabilities)
37import System.Environment (getArgs)
37import System.Environment (getArgs)
38import System.IO (Handle, openFile, IOMode(..))
38import System.IO (Handle, openFile, IOMode(..))
39+import Streamly.Internal.Data.Unicode.Stream
40+ (DecodeState, DecodeError(..), CodePoint, decodeUtf8Either,
41+ resumeDecodeUtf8Either)
39
42
40import qualified Streamly as S
43import qualified Streamly as S
41import qualified Streamly.Data.Unicode.Stream as S
44import qualified Streamly.Data.Unicode.Stream as S
@@ -239,7 +242,7 @@ resetHeaderOnNewChar counts = do
239-- Manipulating the trailer
242-- Manipulating the trailer
240-------------------------------------------------------------------------------
243-------------------------------------------------------------------------------
241
244
242−setTrailer :: V.IOVector Int -> S.DecodeState -> S.CodePoint -> IO ()
245+setTrailer :: V.IOVector Int -> DecodeState -> CodePoint -> IO ()
243setTrailer counts st cp = do
246setTrailer counts st cp = do
244 writeField counts TrailerState (fromIntegral st)
247 writeField counts TrailerState (fromIntegral st)
245 writeField counts TrailerCodePoint cp
248 writeField counts TrailerCodePoint cp
@@ -257,7 +260,7 @@ resetTrailerOnNewChar counts = do
257-------------------------------------------------------------------------------
260-------------------------------------------------------------------------------
258
261
259{-# INLINE countChar #-}
262{-# INLINE countChar #-}
260−countChar :: V.IOVector Int -> Either S.DecodeError Char -> IO ()
263+countChar :: V.IOVector Int -> Either DecodeError Char -> IO ()
261countChar counts inp =
264countChar counts inp =
262 case inp of
265 case inp of
263 Right ch -> do
266 Right ch -> do
@@ -272,7 +275,7 @@ countChar counts inp =
272 wasSpace <- readField counts WasSpace
275 wasSpace <- readField counts WasSpace
273 when (wasSpace /= 0) $ modifyField counts WordCount (+ 1)
276 when (wasSpace /= 0) $ modifyField counts WordCount (+ 1)
274 accountChar counts False
277 accountChar counts False
275− Left (S.DecodeError st cp) -> do
278+ Left (DecodeError st cp) -> do
276 hdone <- readField counts HeaderDone
279 hdone <- readField counts HeaderDone
277 if hdone == 0
280 if hdone == 0
278 then do
281 then do
@@ -309,7 +312,7 @@ _wc_mwl_parserial :: Handle -> IO (V.IOVector Int)
309_wc_mwl_parserial src = do
312_wc_mwl_parserial src = do
310 counts <- newCounts
313 counts <- newCounts
311 S.mapM_ (countChar counts)
314 S.mapM_ (countChar counts)
312− $ S.decodeUtf8Either
315+ $ decodeUtf8Either
313 $ S.unfold FH.read src
316 $ S.unfold FH.read src
314 return counts
317 return counts
315
318
@@ -348,7 +351,7 @@ _wc_mwl_serial src = print =<< (
348reconstructChar :: Int
351reconstructChar :: Int
349 -> V.IOVector Int
352 -> V.IOVector Int
350 -> V.IOVector Int
353 -> V.IOVector Int
351− -> IO (S.SerialT IO (Either S.DecodeError Char))
354+ -> IO (S.SerialT IO (Either DecodeError Char))
352reconstructChar hdrCnt v1 v2 = do
355reconstructChar hdrCnt v1 v2 = do
353 when (hdrCnt > 3 || hdrCnt < 0) $ error "reconstructChar: hdrCnt > 3"
356 when (hdrCnt > 3 || hdrCnt < 0) $ error "reconstructChar: hdrCnt > 3"
354 stream1 <-
357 stream1 <-
@@ -372,7 +375,7 @@ reconstructChar hdrCnt v1 v2 = do
372
375
373 state <- readField v1 TrailerState
376 state <- readField v1 TrailerState
374 cp <- readField v1 TrailerCodePoint
377 cp <- readField v1 TrailerCodePoint
375− return $ S.resumeDecodeUtf8Either (fromIntegral state) cp stream3
378+ return $ resumeDecodeUtf8Either (fromIntegral state) cp stream3
376
379
377getHdrChar :: V.IOVector Int -> IO (Maybe Int)
380getHdrChar :: V.IOVector Int -> IO (Maybe Int)
378getHdrChar v = do
381getHdrChar v = do
@@ -543,7 +546,7 @@ addCounts v1 v2 = do
543 writeField v1 TrailerState trailerState2
546 writeField v1 TrailerState trailerState2
544 writeField v1 TrailerCodePoint trailerCodePoint2
547 writeField v1 TrailerCodePoint trailerCodePoint2
545 return v1
548 return v1
546− Left (S.DecodeError st cp) -> do
549+ Left (DecodeError st cp) -> do
547 -- if header was incomplete it may result
550 -- if header was incomplete it may result
548 -- in partially decoded char to be written
551 -- in partially decoded char to be written
549 -- as trailer. Check if the last error is
552 -- as trailer. Check if the last error is
@@ -554,7 +557,7 @@ addCounts v1 v2 = do
554 Nothing -> (st, cp)
557 Nothing -> (st, cp)
555 Just lst -> case lst of
558 Just lst -> case lst of
556 Right _ -> error "addCounts: Bug"
559 Right _ -> error "addCounts: Bug"
557− Left (S.DecodeError st1 cp1) -> (st1, cp1)
560+ Left (DecodeError st1 cp1) -> (st1, cp1)
558 if hdone2 == 0 && st' /= 12
561 if hdone2 == 0 && st' /= 12
559 then do
562 then do
560 -- all elements before the last one must be errors
563 -- all elements before the last one must be errors
@@ -593,7 +596,7 @@ countArray :: A.Array Word8 -> IO (V.IOVector Int)
593countArray src = do
596countArray src = do
594 counts <- newCounts
597 counts <- newCounts
595 S.mapM_ (countChar counts)
598 S.mapM_ (countChar counts)
596− $ S.decodeUtf8Either
599+ $ decodeUtf8Either
597 $ S.unfold A.read src
600 $ S.unfold A.read src
598 return counts
601 return counts
599
602
src/Streamly/Benchmark/FileIO/Array.hs+3−2
@@ -46,6 +46,7 @@ import qualified Streamly.FileSystem.Handle as FH
46import qualified Streamly.Memory.Array as A
46import qualified Streamly.Memory.Array as A
47import qualified Streamly.Prelude as S
47import qualified Streamly.Prelude as S
48import qualified Streamly.Data.Unicode.Stream as SS
48import qualified Streamly.Data.Unicode.Stream as SS
49+import qualified Streamly.Internal.Data.Unicode.Stream as IUS
49
50
50import qualified Streamly.Internal.FileSystem.Handle as IFH
51import qualified Streamly.Internal.FileSystem.Handle as IFH
51import qualified Streamly.Internal.Memory.Array as IA
52import qualified Streamly.Internal.Memory.Array as IA
@@ -214,7 +215,7 @@ inspect $ hasNoTypeClassesExcept 'wordsUnwordsCopy [''Storable]
214decodeUtf8Lenient :: Handle -> IO ()
215decodeUtf8Lenient :: Handle -> IO ()
215decodeUtf8Lenient inh =
216decodeUtf8Lenient inh =
216 S.drain
217 S.drain
217− $ SS.decodeUtf8ArraysLenient
218+ $ IUS.decodeUtf8ArraysLenient
218 $ IFH.toStreamArraysOf (1024*1024) inh
219 $ IFH.toStreamArraysOf (1024*1024) inh
219
220
220#ifdef INSPECTION
221#ifdef INSPECTION
@@ -230,7 +231,7 @@ copyCodecUtf8Lenient :: Handle -> Handle -> IO ()
230copyCodecUtf8Lenient inh outh =
231copyCodecUtf8Lenient inh outh =
231 S.fold (FH.write outh)
232 S.fold (FH.write outh)
232 $ SS.encodeUtf8
233 $ SS.encodeUtf8
233− $ SS.decodeUtf8ArraysLenient
234+ $ IUS.decodeUtf8ArraysLenient
234 $ IFH.toStreamArraysOf (1024*1024) inh
235 $ IFH.toStreamArraysOf (1024*1024) inh
235
236
236#ifdef INSPECTION
237#ifdef INSPECTION
src/Streamly/Benchmark/FileIO/Stream.hs+6−5
@@ -78,6 +78,7 @@ import qualified Streamly.Prelude as S
78import qualified Streamly.Data.Fold as FL
78import qualified Streamly.Data.Fold as FL
79-- import qualified Streamly.Internal.Data.Fold as IFL
79-- import qualified Streamly.Internal.Data.Fold as IFL
80import qualified Streamly.Data.Unicode.Stream as SS
80import qualified Streamly.Data.Unicode.Stream as SS
81+import qualified Streamly.Internal.Data.Unicode.Stream as IUS
81import qualified Streamly.Internal.Data.Unfold as IUF
82import qualified Streamly.Internal.Data.Unfold as IUF
82import qualified Streamly.Internal.Prelude as IP
83import qualified Streamly.Internal.Prelude as IP
83import qualified Streamly.Streams.StreamD as D
84import qualified Streamly.Streams.StreamD as D
@@ -431,8 +432,8 @@ linesUnlinesArrayCharCopy :: Handle -> Handle -> IO ()
431linesUnlinesArrayCharCopy inh outh =
432linesUnlinesArrayCharCopy inh outh =
432 S.fold (FH.write outh)
433 S.fold (FH.write outh)
433 $ SS.encodeChar8
434 $ SS.encodeChar8
434− $ SS.unlines
435+ $ IUS.unlines
435− $ SS.lines
436+ $ IUS.lines
436 $ SS.decodeChar8
437 $ SS.decodeChar8
437 $ S.unfold FH.read inh
438 $ S.unfold FH.read inh
438
439
@@ -525,8 +526,8 @@ wordsUnwordsCharArrayCopy :: Handle -> Handle -> IO ()
525wordsUnwordsCharArrayCopy inh outh =
526wordsUnwordsCharArrayCopy inh outh =
526 S.fold (FH.write outh)
527 S.fold (FH.write outh)
527 $ SS.encodeChar8
528 $ SS.encodeChar8
528− $ SS.unwords
529+ $ IUS.unwords
529− $ SS.words
530+ $ IUS.words
530 $ SS.decodeChar8
531 $ SS.decodeChar8
531 $ S.unfold FH.read inh
532 $ S.unfold FH.read inh
532
533
@@ -597,7 +598,7 @@ inspect $ hasNoTypeClasses 'splitOnSeq
597splitOnSeqUtf8 :: String -> Handle -> IO Int
598splitOnSeqUtf8 :: String -> Handle -> IO Int
598splitOnSeqUtf8 str inh =
599splitOnSeqUtf8 str inh =
599 (S.length $ IP.splitOnSeq (A.fromList str) FL.drain
600 (S.length $ IP.splitOnSeq (A.fromList str) FL.drain
600− $ SS.decodeUtf8ArraysLenient
601+ $ IUS.decodeUtf8ArraysLenient
601 $ IFH.toStreamArrays inh) -- >>= print
602 $ IFH.toStreamArrays inh) -- >>= print
602
603
603-- | Split on suffix sequence.
604-- | Split on suffix sequence.
src/Streamly/Data/Unicode/Stream.hs+4−22
@@ -46,40 +46,22 @@ module Streamly.Data.Unicode.Stream
46 decodeChar8
46 decodeChar8
47 , decodeUtf8
47 , decodeUtf8
48 , decodeUtf8Lenient
48 , decodeUtf8Lenient
49− , D.DecodeError(..)
50− , D.DecodeState
51− , D.CodePoint
52− , decodeUtf8Either
53− , resumeDecodeUtf8Either
54− , decodeUtf8Arrays
55− , decodeUtf8ArraysLenient
56
49
57 -- * Elimination (Encoding)
50 -- * Elimination (Encoding)
58 , encodeChar8
51 , encodeChar8
59 , encodeChar8Unchecked
52 , encodeChar8Unchecked
60 , encodeUtf8
53 , encodeUtf8
61−{-
54+ {-
62− -- * Unicode aware operations
63− , toCaseFold
64− , toLower
65− , toUpper
66− , toTitle
67−
68 -- * Operations on character strings
55 -- * Operations on character strings
69 , strip -- (dropAround isSpace)
56 , strip -- (dropAround isSpace)
70− , stripEnd-}
57+ , stripEnd
71− -- * Transformation
72 , stripStart
58 , stripStart
59+ -}
60+ -- * Transformation
73 , foldLines
61 , foldLines
74 , foldWords
62 , foldWords
75 , unfoldLines
63 , unfoldLines
76 , unfoldWords
64 , unfoldWords
77−
78− -- * Streams of Strings
79− , lines
80− , words
81− , unlines
82− , unwords
83 )
65 )
84where
66where
85
67
test/String.hs+8−7
@@ -16,6 +16,7 @@ import qualified Streamly.Memory.Array as A
16import qualified Streamly.Internal.Memory.ArrayStream as AS
16import qualified Streamly.Internal.Memory.ArrayStream as AS
17import qualified Streamly.Prelude as S
17import qualified Streamly.Prelude as S
18import qualified Streamly.Data.Unicode.Stream as SS
18import qualified Streamly.Data.Unicode.Stream as SS
19+import qualified Streamly.Internal.Data.Unicode.Stream as IUS
19
20
20-- Coverage build takes too long with default number of tests
21-- Coverage build takes too long with default number of tests
21{-
22{-
@@ -56,7 +57,7 @@ propDecodeEncodeIdArrays =
56 forAll genUnicode $ \list ->
57 forAll genUnicode $ \list ->
57 monadicIO $ do
58 monadicIO $ do
58 let wrds = SS.encodeUtf8 $ S.fromList list
59 let wrds = SS.encodeUtf8 $ S.fromList list
59− chrs <- S.toList $ SS.decodeUtf8ArraysLenient
60+ chrs <- S.toList $ IUS.decodeUtf8ArraysLenient
60 (S.fold A.write wrds)
61 (S.fold A.write wrds)
61 assert (chrs == list)
62 assert (chrs == list)
62
63
@@ -66,7 +67,7 @@ testLines =
66 monadicIO $ do
67 monadicIO $ do
67 xs <- S.toList
68 xs <- S.toList
68 $ S.map A.toList
69 $ S.map A.toList
69− $ SS.lines
70+ $ IUS.lines
70 $ S.fromList list
71 $ S.fromList list
71 assert (xs == lines list)
72 assert (xs == lines list)
72
73
@@ -87,7 +88,7 @@ testWords =
87 monadicIO $ do
88 monadicIO $ do
88 xs <- S.toList
89 xs <- S.toList
89 $ S.map A.toList
90 $ S.map A.toList
90− $ SS.words
91+ $ IUS.words
91 $ S.fromList list
92 $ S.fromList list
92 assert (xs == words list)
93 assert (xs == words list)
93
94
@@ -96,8 +97,8 @@ testUnlines =
96 forAll genUnicode $ \list ->
97 forAll genUnicode $ \list ->
97 monadicIO $ do
98 monadicIO $ do
98 xs <- S.toList
99 xs <- S.toList
99− $ SS.unlines
100+ $ IUS.unlines
100− $ SS.lines
101+ $ IUS.lines
101 $ S.fromList list
102 $ S.fromList list
102 assert (xs == unlines (lines list))
103 assert (xs == unlines (lines list))
103
104
@@ -107,8 +108,8 @@ testUnwords =
107 monadicIO $ do
108 monadicIO $ do
108 xs <- run
109 xs <- run
109 $ S.toList
110 $ S.toList
110− $ SS.unwords
111+ $ IUS.unwords
111− $ SS.words
112+ $ IUS.words
112 $ S.fromList list
113 $ S.fromList list
113 assert (xs == unwords (words list))
114 assert (xs == unwords (words list))
114
115