@@ -11,9 +11,9 @@
1111module Streamly.Internal.Data.Unicode.Stream
1212 (
1313 -- * Construction (Decoding)
14− decodeChar8
14+ decodeLatin1
1515 , decodeUtf8
16− , decodeUtf8Lenient
16+ , decodeUtf8Lax
1717 , D.DecodeError(..)
1818 , D.DecodeState
1919 , D.CodePoint
@@ -23,27 +23,16 @@ module Streamly.Internal.Data.Unicode.Stream
2323 , decodeUtf8ArraysLenient
2424
2525 -- * Elimination (Encoding)
26− , encodeChar8
27− , encodeChar8Unchecked
26+ , encodeLatin1
27+ , encodeLatin1Lax
2828 , encodeUtf8
29−{-
30− -- * Unicode aware operations
31− , toCaseFold
32− , toLower
33− , toUpper
34− , toTitle
35−
29+ {-
3630 -- * Operations on character strings
3731 , strip -- (dropAround isSpace)
38− , stripEnd-}
32+ , stripEnd
33+ -}
3934 -- * Transformation
4035 , stripStart
41− , foldLines
42− , foldWords
43− , unfoldLines
44− , unfoldWords
45−
46− -- * Streams of Strings
4736 , lines
4837 , words
4938 , unlines
@@ -55,62 +44,63 @@ import Control.Monad.IO.Class (MonadIO)
5544import Data.Char (ord)
5645import Data.Word (Word8)
5746import GHC.Base (unsafeChr)
58−import Streamly (IsStream, MonadAsync)
47+import Streamly (IsStream)
5948import Prelude hiding (String, lines, words, unlines, unwords)
6049import Streamly.Data.Fold (Fold)
6150import Streamly.Memory.Array (Array)
6251import Streamly.Internal.Data.Unfold (Unfold)
6352
6453import qualified Streamly.Internal.Prelude as S
65−import qualified Streamly.Memory.Array as A
6654import qualified Streamly.Streams.StreamD as D
6755
68−-- type String = List Char
69−
70−-------------------------------------------------------------------------------
71−-- Encoding/Decoding Characters
72−-------------------------------------------------------------------------------
73−
74−-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
75−-- decodeWith = undefined
76−
7756-------------------------------------------------------------------------------
7857-- Encoding/Decoding Unicode Characters
7958-------------------------------------------------------------------------------
8059
8160-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
8261-- corresponding Unicode 'Char' in 0-255 range.
83−{-# INLINE decodeChar8 #-}
84−decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
85−decodeChar8 = S.map (unsafeChr . fromIntegral)
62+--
63+-- /Since: 0.7.0/
64+{-# INLINE decodeLatin1 #-}
65+decodeLatin1 :: (IsStream t, Monad m) => t m Word8 -> t m Char
66+decodeLatin1 = S.map (unsafeChr . fromIntegral)
8667
8768-- | Encode a stream of Unicode characters to bytes by mapping each character
8869-- to a byte in 0-255 range. Throws an error if the input stream contains
8970-- characters beyond 255.
90−{-# INLINE encodeChar8 #-}
91−encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
92−encodeChar8 = S.map convert
71+--
72+-- /Since: 0.7.0/
73+{-# INLINE encodeLatin1 #-}
74+encodeLatin1 :: (IsStream t, Monad m) => t m Char -> t m Word8
75+encodeLatin1 = S.map convert
9376 where
9477 convert c =
9578 let codepoint = ord c
9679 in if codepoint > 255
97− then error $ "Streamly.String.encodeChar8 invalid \
80+ then error $ "Streamly.String.encodeLatin1 invalid \
9881 \input char codepoint " ++ show codepoint
9982 else fromIntegral codepoint
10083
101−-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
84+-- | Like 'encodeLatin1' but silently truncates and maps input characters beyond
10285-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
10386-- when such truncation occurs.
104−{-# INLINE encodeChar8Unchecked #-}
105−encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
106−encodeChar8Unchecked = S.map (fromIntegral . ord)
87+--
88+-- /Since: 0.7.0/
89+{-# INLINE encodeLatin1Lax #-}
90+encodeLatin1Lax :: (IsStream t, Monad m) => t m Char -> t m Word8
91+encodeLatin1Lax = S.map (fromIntegral . ord)
10792
10893-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
10994-- The incoming stream is truncated if an invalid codepoint is encountered.
95+--
96+-- /Since: 0.7.0/
11097{-# INLINE decodeUtf8 #-}
11198decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
11299decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
113100
101+-- |
102+--
103+-- /Internal/
114104{-# INLINE decodeUtf8Arrays #-}
115105decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
116106decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
@@ -118,15 +108,23 @@ decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
118108-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
119109-- Any invalid codepoint encountered is replaced with the unicode replacement
120110-- character.
121−{-# INLINE decodeUtf8Lenient #-}
122−decodeUtf8Lenient :: (Monad m, IsStream t) => t m Word8 -> t m Char
123−decodeUtf8Lenient = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
111+--
112+-- /Since: 0.7.0/
113+{-# INLINE decodeUtf8Lax #-}
114+decodeUtf8Lax :: (Monad m, IsStream t) => t m Word8 -> t m Char
115+decodeUtf8Lax = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
124116
117+-- |
118+--
119+-- /Internal/
125120{-# INLINE decodeUtf8Either #-}
126121decodeUtf8Either :: (Monad m, IsStream t)
127122 => t m Word8 -> t m (Either D.DecodeError Char)
128123decodeUtf8Either = D.fromStreamD . D.decodeUtf8Either . D.toStreamD
129124
125+-- |
126+--
127+-- /Internal/
130128{-# INLINE resumeDecodeUtf8Either #-}
131129resumeDecodeUtf8Either
132130 :: (Monad m, IsStream t)
@@ -137,6 +135,9 @@ resumeDecodeUtf8Either
137135resumeDecodeUtf8Either st cp =
138136 D.fromStreamD . D.resumeDecodeUtf8Either st cp . D.toStreamD
139137
138+-- |
139+--
140+-- /Internal/
140141{-# INLINE decodeUtf8ArraysLenient #-}
141142decodeUtf8ArraysLenient ::
142143 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
@@ -144,27 +145,13 @@ decodeUtf8ArraysLenient =
144145 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
145146
146147-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
148+--
149+-- /Since: 0.7.0/
147150{-# INLINE encodeUtf8 #-}
148151encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
149152encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
150153
151154{-
152−-------------------------------------------------------------------------------
153−-- Unicode aware operations on strings
154−-------------------------------------------------------------------------------
155−
156−toCaseFold :: IsStream t => t m Char -> t m Char
157−toCaseFold = undefined
158−
159−toLower :: IsStream t => t m Char -> t m Char
160−toLower = undefined
161−
162−toUpper :: IsStream t => t m Char -> t m Char
163−toUpper = undefined
164−
165−toTitle :: IsStream t => t m Char -> t m Char
166−toTitle = undefined
167−
168155-------------------------------------------------------------------------------
169156-- Utility operations on strings
170157-------------------------------------------------------------------------------
@@ -179,6 +166,8 @@ stripEnd = undefined
179166-- | Remove leading whitespace from a string.
180167--
181168-- > stripStart = S.dropWhile isSpace
169+--
170+-- /Internal/
182171{-# INLINE stripStart #-}
183172stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
184173stripStart = S.dropWhile isSpace
@@ -186,26 +175,15 @@ stripStart = S.dropWhile isSpace
186175-- | Fold each line of the stream using the supplied 'Fold'
187176-- and stream the result.
188177--
189−-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
178+-- >>> S.toList $ lines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
190179-- ["lines", "this", "string", "", ""]
191180--
192−-- > foldLines = S.splitOnSuffix (== '\n')
193−--
194−{-# INLINE foldLines #-}
195−foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
196−foldLines = S.splitOnSuffix (== '\n')
197−
198−-- | Fold each word of the stream using the supplied 'Fold'
199−-- and stream the result.
200−--
201−-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
202−-- ["fold", "these", "words"]
203−--
204−-- > foldWords = S.wordsBy isSpace
181+-- > lines = S.splitOnSuffix (== '\n')
205182--
206−{-# INLINE foldWords #-}
207−foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
208−foldWords = S.wordsBy isSpace
183+-- /Internal/
184+{-# INLINE lines #-}
185+lines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
186+lines = S.splitOnSuffix (== '\n')
209187
210188foreign import ccall unsafe "u_iswspace"
211189 iswspace :: Int -> Int
@@ -219,79 +197,32 @@ isSpace c
219197 where
220198 uc = fromIntegral (ord c) :: Word
221199
222−-- | Break a string up into a stream of strings at newline characters.
223−-- The resulting strings do not contain newlines.
224−--
225−-- > lines = foldLines A.write
226−--
227−-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
228−-- ["lines","this","string","",""]
229−--
230−-- If you're dealing with lines of massive length, consider using
231−-- 'foldLines' instead to avoid buffering the data in 'Array'.
232−{-# INLINE lines #-}
233−lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
234−lines = foldLines A.write
235−
236−-- | Break a string up into a stream of strings, which were delimited
237−-- by characters representing white space.
200+-- | Fold each word of the stream using the supplied 'Fold'
201+-- and stream the result.
238202--
239−-- > words = foldWords A.write
203+-- >>> S.toList $ words FL.toList (S.fromList "fold these words")
204+-- ["fold", "these", "words"]
240205--
241−-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
242−-- ["A", "newline", "is", "considered", "white", "space?"]
206+-- > words = S.wordsBy isSpace
243207--
244−-- If you're dealing with words of massive length, consider using
245−-- 'foldWords' instead to avoid buffering the data in 'Array'.
208+-- /Internal/
246209{-# INLINE words #-}
247−words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
248−words = foldWords A.write
210+words :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
211+words = S.wordsBy isSpace
249212
250213-- | Unfold a stream to character streams using the supplied 'Unfold'
251214-- and concat the results suffixing a newline character @\\n@ to each stream.
252215--
253−{-# INLINE unfoldLines #-}
254−unfoldLines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
255−unfoldLines = S.interposeSuffix '\n'
256−
257−-- | Flattens the stream of @Array Char@, after appending a terminating
258−-- newline to each string.
259−--
260−-- 'unlines' is an inverse operation to 'lines'.
261−--
262−-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
263−-- "lines\nthis\nstring\n"
264−--
265−-- > unlines = unfoldLines A.read
266−--
267−-- Note that, in general
268−--
269−-- > unlines . lines /= id
216+-- /Internal/
270217{-# INLINE unlines #-}
271−unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
272−unlines = unfoldLines A.read
218+unlines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
219+unlines = S.interposeSuffix '\n'
273220
274221-- | Unfold the elements of a stream to character streams using the supplied
275222-- 'Unfold' and concat the results with a whitespace character infixed between
276223-- the streams.
277224--
278−{-# INLINE unfoldWords #-}
279−unfoldWords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
280−unfoldWords = S.interpose ' '
281−
282−-- | Flattens the stream of @Array Char@, after appending a separating
283−-- space to each string.
284−--
285−-- 'unwords' is an inverse operation to 'words'.
286−--
287−-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
288−-- "unwords this string"
289−--
290−-- > unwords = unfoldWords A.read
291−--
292−-- Note that, in general
293−--
294−-- > unwords . words /= id
225+-- /Internal/
295226{-# INLINE unwords #-}
296−unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
297−unwords = unfoldWords A.read
227+unwords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
228+unwords = S.interpose ' '