@@ -11,9 +11,9 @@
11module Streamly.Internal.Data.Unicode.Stream
11module Streamly.Internal.Data.Unicode.Stream
13 -- * Construction (Decoding)
13 -- * Construction (Decoding)
14− decodeChar8
14+ decodeLatin1
15 , decodeUtf8
15 , decodeUtf8
16− , decodeUtf8Lenient
16+ , decodeUtf8Lax
17 , D.DecodeError(..)
17 , D.DecodeError(..)
18 , D.DecodeState
18 , D.DecodeState
19 , D.CodePoint
19 , D.CodePoint
@@ -23,27 +23,16 @@ module Streamly.Internal.Data.Unicode.Stream
23 , decodeUtf8ArraysLenient
23 , decodeUtf8ArraysLenient
25 -- * Elimination (Encoding)
25 -- * Elimination (Encoding)
26− , encodeChar8
26+ , encodeLatin1
27− , encodeChar8Unchecked
27+ , encodeLatin1Lax
28 , encodeUtf8
28 , encodeUtf8
30− -- * Unicode aware operations
36 -- * Operations on character strings
30 -- * Operations on character strings
37 , strip -- (dropAround isSpace)
31 , strip -- (dropAround isSpace)
38− , stripEnd-}
32+ , stripEnd
39 -- * Transformation
34 -- * Transformation
40 , stripStart
35 , stripStart
46− -- * Streams of Strings
@@ -55,62 +44,63 @@ import Control.Monad.IO.Class (MonadIO)
55import Data.Char (ord)
44import Data.Char (ord)
56import Data.Word (Word8)
45import Data.Word (Word8)
57import GHC.Base (unsafeChr)
46import GHC.Base (unsafeChr)
58−import Streamly (IsStream, MonadAsync)
47+import Streamly (IsStream)
59import Prelude hiding (String, lines, words, unlines, unwords)
48import Prelude hiding (String, lines, words, unlines, unwords)
60import Streamly.Data.Fold (Fold)
49import Streamly.Data.Fold (Fold)
61import Streamly.Memory.Array (Array)
50import Streamly.Memory.Array (Array)
62import Streamly.Internal.Data.Unfold (Unfold)
51import Streamly.Internal.Data.Unfold (Unfold)
64import qualified Streamly.Internal.Prelude as S
53import qualified Streamly.Internal.Prelude as S
65−import qualified Streamly.Memory.Array as A
66import qualified Streamly.Streams.StreamD as D
54import qualified Streamly.Streams.StreamD as D
68−-- type String = List Char
70−-------------------------------------------------------------------------------
71−-- Encoding/Decoding Characters
72−-------------------------------------------------------------------------------
74−-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
75−-- decodeWith = undefined
77-------------------------------------------------------------------------------
56-------------------------------------------------------------------------------
78-- Encoding/Decoding Unicode Characters
57-- Encoding/Decoding Unicode Characters
79-------------------------------------------------------------------------------
58-------------------------------------------------------------------------------
81-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
60-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
82-- corresponding Unicode 'Char' in 0-255 range.
61-- corresponding Unicode 'Char' in 0-255 range.
83−{-# INLINE decodeChar8 #-}
62+--
64+{-# INLINE decodeLatin1 #-}
84−decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
65+decodeLatin1 :: (IsStream t, Monad m) => t m Word8 -> t m Char
85−decodeChar8 = S.map (unsafeChr . fromIntegral)
66+decodeLatin1 = S.map (unsafeChr . fromIntegral)
87-- | Encode a stream of Unicode characters to bytes by mapping each character
68-- | Encode a stream of Unicode characters to bytes by mapping each character
88-- to a byte in 0-255 range. Throws an error if the input stream contains
69-- to a byte in 0-255 range. Throws an error if the input stream contains
89-- characters beyond 255.
70-- characters beyond 255.
90−{-# INLINE encodeChar8 #-}
71+--
73+{-# INLINE encodeLatin1 #-}
91−encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
74+encodeLatin1 :: (IsStream t, Monad m) => t m Char -> t m Word8
92−encodeChar8 = S.map convert
75+encodeLatin1 = S.map convert
94 convert c =
77 convert c =
95 let codepoint = ord c
78 let codepoint = ord c
96 in if codepoint > 255
79 in if codepoint > 255
97− then error $ "Streamly.String.encodeChar8 invalid \
80+ then error $ "Streamly.String.encodeLatin1 invalid \
98 \input char codepoint " ++ show codepoint
81 \input char codepoint " ++ show codepoint
99 else fromIntegral codepoint
82 else fromIntegral codepoint
101−-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
84+-- | Like 'encodeLatin1' but silently truncates and maps input characters beyond
102-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
85-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
103-- when such truncation occurs.
86-- when such truncation occurs.
104−{-# INLINE encodeChar8Unchecked #-}
87+--
89+{-# INLINE encodeLatin1Lax #-}
105−encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
90+encodeLatin1Lax :: (IsStream t, Monad m) => t m Char -> t m Word8
106−encodeChar8Unchecked = S.map (fromIntegral . ord)
91+encodeLatin1Lax = S.map (fromIntegral . ord)
108-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
93-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
109-- The incoming stream is truncated if an invalid codepoint is encountered.
94-- The incoming stream is truncated if an invalid codepoint is encountered.
110{-# INLINE decodeUtf8 #-}
97{-# INLINE decodeUtf8 #-}
111decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
98decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
112decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
99decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
114{-# INLINE decodeUtf8Arrays #-}
104{-# INLINE decodeUtf8Arrays #-}
115decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
105decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
116decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
106decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
@@ -118,15 +108,23 @@ decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
118-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
108-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
119-- Any invalid codepoint encountered is replaced with the unicode replacement
109-- Any invalid codepoint encountered is replaced with the unicode replacement
120-- character.
110-- character.
121−{-# INLINE decodeUtf8Lenient #-}
111+--
113+{-# INLINE decodeUtf8Lax #-}
122−decodeUtf8Lenient :: (Monad m, IsStream t) => t m Word8 -> t m Char
114+decodeUtf8Lax :: (Monad m, IsStream t) => t m Word8 -> t m Char
123−decodeUtf8Lenient = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
115+decodeUtf8Lax = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
125{-# INLINE decodeUtf8Either #-}
120{-# INLINE decodeUtf8Either #-}
126decodeUtf8Either :: (Monad m, IsStream t)
121decodeUtf8Either :: (Monad m, IsStream t)
127 => t m Word8 -> t m (Either D.DecodeError Char)
122 => t m Word8 -> t m (Either D.DecodeError Char)
128decodeUtf8Either = D.fromStreamD . D.decodeUtf8Either . D.toStreamD
123decodeUtf8Either = D.fromStreamD . D.decodeUtf8Either . D.toStreamD
130{-# INLINE resumeDecodeUtf8Either #-}
128{-# INLINE resumeDecodeUtf8Either #-}
131resumeDecodeUtf8Either
129resumeDecodeUtf8Either
132 :: (Monad m, IsStream t)
130 :: (Monad m, IsStream t)
@@ -137,6 +135,9 @@ resumeDecodeUtf8Either
137resumeDecodeUtf8Either st cp =
135resumeDecodeUtf8Either st cp =
138 D.fromStreamD . D.resumeDecodeUtf8Either st cp . D.toStreamD
136 D.fromStreamD . D.resumeDecodeUtf8Either st cp . D.toStreamD
140{-# INLINE decodeUtf8ArraysLenient #-}
141{-# INLINE decodeUtf8ArraysLenient #-}
141decodeUtf8ArraysLenient ::
142decodeUtf8ArraysLenient ::
142 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
143 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
@@ -144,27 +145,13 @@ decodeUtf8ArraysLenient =
144 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
145 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
146-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
147-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
147{-# INLINE encodeUtf8 #-}
150{-# INLINE encodeUtf8 #-}
148encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
151encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
149encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
152encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
152−-------------------------------------------------------------------------------
153−-- Unicode aware operations on strings
154−-------------------------------------------------------------------------------
156−toCaseFold :: IsStream t => t m Char -> t m Char
157−toCaseFold = undefined
159−toLower :: IsStream t => t m Char -> t m Char
162−toUpper :: IsStream t => t m Char -> t m Char
165−toTitle :: IsStream t => t m Char -> t m Char
168-------------------------------------------------------------------------------
155-------------------------------------------------------------------------------
169-- Utility operations on strings
156-- Utility operations on strings
170-------------------------------------------------------------------------------
157-------------------------------------------------------------------------------
@@ -179,6 +166,8 @@ stripEnd = undefined
179-- | Remove leading whitespace from a string.
166-- | Remove leading whitespace from a string.
181-- > stripStart = S.dropWhile isSpace
168-- > stripStart = S.dropWhile isSpace
182{-# INLINE stripStart #-}
171{-# INLINE stripStart #-}
183stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
172stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
184stripStart = S.dropWhile isSpace
173stripStart = S.dropWhile isSpace
@@ -186,26 +175,15 @@ stripStart = S.dropWhile isSpace
186-- | Fold each line of the stream using the supplied 'Fold'
175-- | Fold each line of the stream using the supplied 'Fold'
187-- and stream the result.
176-- and stream the result.
189−-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
178+-- >>> S.toList $ lines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
190-- ["lines", "this", "string", "", ""]
179-- ["lines", "this", "string", "", ""]
192−-- > foldLines = S.splitOnSuffix (== '\n')
194−{-# INLINE foldLines #-}
195−foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
196−foldLines = S.splitOnSuffix (== '\n')
181+-- > lines = S.splitOnSuffix (== '\n')
198−-- | Fold each word of the stream using the supplied 'Fold'
199−-- and stream the result.
201−-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
202−-- ["fold", "these", "words"]
204−-- > foldWords = S.wordsBy isSpace
206−{-# INLINE foldWords #-}
184+{-# INLINE lines #-}
207−foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
185+lines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
208−foldWords = S.wordsBy isSpace
186+lines = S.splitOnSuffix (== '\n')
210foreign import ccall unsafe "u_iswspace"
188foreign import ccall unsafe "u_iswspace"
211 iswspace :: Int -> Int
189 iswspace :: Int -> Int
@@ -219,79 +197,32 @@ isSpace c
220 uc = fromIntegral (ord c) :: Word
198 uc = fromIntegral (ord c) :: Word
222−-- | Break a string up into a stream of strings at newline characters.
200+-- | Fold each word of the stream using the supplied 'Fold'
223−-- The resulting strings do not contain newlines.
201+-- and stream the result.
225−-- > lines = foldLines A.write
227−-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
228−-- ["lines","this","string","",""]
230−-- If you're dealing with lines of massive length, consider using
231−-- 'foldLines' instead to avoid buffering the data in 'Array'.
233−lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
234−lines = foldLines A.write
236−-- | Break a string up into a stream of strings, which were delimited
237−-- by characters representing white space.
239−-- > words = foldWords A.write
203+-- >>> S.toList $ words FL.toList (S.fromList "fold these words")
204+-- ["fold", "these", "words"]
241−-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
206+-- > words = S.wordsBy isSpace
242−-- ["A", "newline", "is", "considered", "white", "space?"]
244−-- If you're dealing with words of massive length, consider using
208+-- /Internal/
245−-- 'foldWords' instead to avoid buffering the data in 'Array'.
246{-# INLINE words #-}
209{-# INLINE words #-}
247−words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
210+words :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
248−words = foldWords A.write
211+words = S.wordsBy isSpace
250-- | Unfold a stream to character streams using the supplied 'Unfold'
213-- | Unfold a stream to character streams using the supplied 'Unfold'
251-- and concat the results suffixing a newline character @\\n@ to each stream.
214-- and concat the results suffixing a newline character @\\n@ to each stream.
253−{-# INLINE unfoldLines #-}
216+-- /Internal/
254−unfoldLines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
255−unfoldLines = S.interposeSuffix '\n'
257−-- | Flattens the stream of @Array Char@, after appending a terminating
258−-- newline to each string.
260−-- 'unlines' is an inverse operation to 'lines'.
262−-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
263−-- "lines\nthis\nstring\n"
265−-- > unlines = unfoldLines A.read
267−-- Note that, in general
269−-- > unlines . lines /= id
270{-# INLINE unlines #-}
217{-# INLINE unlines #-}
271−unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
218+unlines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
272−unlines = unfoldLines A.read
219+unlines = S.interposeSuffix '\n'
274-- | Unfold the elements of a stream to character streams using the supplied
221-- | Unfold the elements of a stream to character streams using the supplied
275-- 'Unfold' and concat the results with a whitespace character infixed between
222-- 'Unfold' and concat the results with a whitespace character infixed between
276-- the streams.
223-- the streams.
278−{-# INLINE unfoldWords #-}
225+-- /Internal/
279−unfoldWords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
280−unfoldWords = S.interpose ' '
282−-- | Flattens the stream of @Array Char@, after appending a separating
283−-- space to each string.
285−-- 'unwords' is an inverse operation to 'words'.
287−-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
288−-- "unwords this string"
290−-- > unwords = unfoldWords A.read
292−-- Note that, in general
294−-- > unwords . words /= id
295{-# INLINE unwords #-}
226{-# INLINE unwords #-}
296−unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
227+unwords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
297−unwords = unfoldWords A.read
228+unwords = S.interpose ' '