@@ -8,23 +8,38 @@
8-- Stability : experimental
8-- Stability : experimental
9-- Portability : GHC
9-- Portability : GHC
11−-- Stream of 'Char' are the equivalent of Haskell Strings but without any
11+-- A unicode string can be represented either as a stream of 'Char' e.g.
12−-- performance issues that are usually associated with the 'String' type.
12+-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13−-- Haskell lists are not suitable for storage in GC memory. We use streams for
13+-- can be done efficiently by applying stream operations and folds on the
14−-- processing and arrays for storing. We can use the usual stream processing
14+-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15−-- routines on these 'Char' streams. Stream processing is very efficient due to
15+-- applied where available or the array can be read into a stream and then
16−-- stream fusion. If strings are to be stored or buffered in memory, they can
16+-- processed using stream operations. Use 'Array' 'Char' when you need to store
17−-- be encoded to 'Word8' arrays using the encoding routines provided in the
17+-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18−-- module. Therefore, a separate type for text representation is not required.
18+-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19−-- A @Stream Identity Char@ can be used almost as a drop-in replacement for the
19+-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20−-- standard Haskell @String@, especially when used with @OverloadedStrings@
20+-- convenience.
21−-- extension, with little differences.
23−-- The 'String' type in this module is just a synonym for the type @List Char@.
24−-- It provides better performance compared to the standard Haskell @String@
25−-- type and can be used almost as a drop-in replacement, especially when used
26−-- with @OverloadedStrings@ extension, with little differences.
22+-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23+-- large amounts of text in memory. Also, it is computationally efficient as
24+-- there is no encoding/decoding involved. We recommend using true streaming
25+-- operations to avoid buffering large amounts of data as long as possible e.g.
26+-- use `foldLines` instead of `lines`. However, if for some reason you are
27+-- buffering very large amounts of text in memory and are worried about space
28+-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29+-- encoded 'Array' 'Word8'.
31+-- Please note the following:
33+-- * Case conversion: Some unicode characters translate to more than one code
34+-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35+-- package do not handle such characters. Therefore, operations like @map
36+-- toUpper@ on a character stream or character array may not always perform
37+-- correct conversion.
38+-- * String comparison: In some cases, visually identical strings may have
39+-- different unicode representations, therefore, a character stream or
40+-- character array cannot be directly compared. A normalized comparison may be
41+-- needed to check string equivalence correctly.
28-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
43-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
29-- <src/test/PureStreams.hs> for comprehensive usage examples.
44-- <src/test/PureStreams.hs> for comprehensive usage examples.
@@ -33,15 +48,14 @@
34module Streamly.Data.String
49module Streamly.Data.String
36− -- String
51+ -- * Construction (Decoding)
38− -- * Encoding and Decoding
55+ -- * Elimination (Encoding)
39− encodeChar8
56+ , encodeChar8
40 , encodeChar8Unchecked
57 , encodeChar8Unchecked
43 , encodeUtf8
58 , encodeUtf8
46 -- * Unicode aware operations
60 -- * Unicode aware operations
47 , toCaseFold
61 , toCaseFold
@@ -52,10 +66,12 @@ module Streamly.Data.String
52 -- * Operations on character strings
66 -- * Operations on character strings
53 , strip -- (dropAround isSpace)
67 , strip -- (dropAround isSpace)
54 , stripEnd-}
68 , stripEnd-}
55− -- * Substrings
69+ -- * Transformation
56 , stripStart
70 , stripStart
57 , foldLines
71 , foldLines
58 , foldWords
72 , foldWords
74+ -- * Streams of Strings
@@ -73,7 +89,6 @@ import Streamly.Fold (Fold)
73import Streamly.Memory.Array (Array)
89import Streamly.Memory.Array (Array)
75import qualified Streamly.Prelude as S
91import qualified Streamly.Prelude as S
76−import qualified Streamly.Memory.Array.Types as A (unlines)
77import qualified Streamly.Memory.Array as A
92import qualified Streamly.Memory.Array as A
78import qualified Streamly.Memory.ArrayStream as AS
93import qualified Streamly.Memory.ArrayStream as AS
79import qualified Streamly.Streams.StreamD as D
94import qualified Streamly.Streams.StreamD as D
@@ -157,27 +172,33 @@ stripEnd :: IsStream t => t m Char -> t m Char
157stripEnd = undefined
172stripEnd = undefined
160−-- | Remove leading whitespace from a String.
175+-- | Remove leading whitespace from a string.
162-- > stripStart = S.dropWhile isSpace
177-- > stripStart = S.dropWhile isSpace
163{-# INLINE stripStart #-}
178{-# INLINE stripStart #-}
164stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
179stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
165stripStart = S.dropWhile isSpace
180stripStart = S.dropWhile isSpace
167−-- | Fold each line of the stream using the supplied Fold
182+-- | Fold each line of the stream using the supplied 'Fold'
168-- and stream the result.
183-- and stream the result.
170-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
185-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
171-- ["lines", "this", "string", "", ""]
186-- ["lines", "this", "string", "", ""]
188+-- > foldLines = S.splitOnSuffix (== '\n')
172{-# INLINE foldLines #-}
190{-# INLINE foldLines #-}
173foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
191foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
174foldLines = S.splitOnSuffix (== '\n')
192foldLines = S.splitOnSuffix (== '\n')
176−-- | Fold each word of the stream using the supplied Fold
194+-- | Fold each word of the stream using the supplied 'Fold'
177-- and stream the result.
195-- and stream the result.
179-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
197-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
180-- ["fold", "these", "words"]
198-- ["fold", "these", "words"]
200+-- > foldWords = S.wordsBy isSpace
181{-# INLINE foldWords #-}
202{-# INLINE foldWords #-}
182foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
203foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
183foldWords = S.wordsBy isSpace
204foldWords = S.wordsBy isSpace
@@ -194,29 +215,33 @@ isSpace c
195 uc = fromIntegral (ord c) :: Word
216 uc = fromIntegral (ord c) :: Word
197−-- | Break a string up into a list of strings at newline characters.
218+-- | Break a string up into a stream of strings at newline characters.
198-- The resulting strings do not contain newlines.
219-- The resulting strings do not contain newlines.
221+-- > lines = foldLines A.write
200-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
223-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
201-- ["lines","this","string","",""]
224-- ["lines","this","string","",""]
203-- If you're dealing with lines of massive length, consider using
226-- If you're dealing with lines of massive length, consider using
204−-- 'foldLines' instead.
227+-- 'foldLines' instead to avoid buffering the data in 'Array'.
205{-# INLINE lines #-}
228{-# INLINE lines #-}
206lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
229lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
207−lines = S.splitOnSuffix (== '\n') A.write
230+lines = foldLines A.write
209−-- | Break a string up into a list of strings, which were delimited
232+-- | Break a string up into a stream of strings, which were delimited
210-- by characters representing white space.
233-- by characters representing white space.
235+-- > words = foldWords A.write
212-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
237-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
213-- ["A", "newline", "is", "considered", "white", "space?"]
238-- ["A", "newline", "is", "considered", "white", "space?"]
215-- If you're dealing with words of massive length, consider using
240-- If you're dealing with words of massive length, consider using
216−-- 'foldWords' instead.
241+-- 'foldWords' instead to avoid buffering the data in 'Array'.
217{-# INLINE words #-}
242{-# INLINE words #-}
218words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
243words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
219−words = S.wordsBy isSpace A.write
244+words = foldWords A.write
221-- | Flattens the stream of @Array Char@, after appending a terminating
246-- | Flattens the stream of @Array Char@, after appending a terminating
222-- newline to each string.
247-- newline to each string.
@@ -226,12 +251,14 @@ words = S.wordsBy isSpace A.write
226-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
251-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
227-- "lines\nthis\nstring\n"
252-- "lines\nthis\nstring\n"
254+-- > unlines = AS.unlinesBy '\n'
229-- Note that, in general
256-- Note that, in general
231-- > unlines . lines /= id
258-- > unlines . lines /= id
232{-# INLINE unlines #-}
259{-# INLINE unlines #-}
233unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
260unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
234−unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
261+unlines = AS.unlinesBy '\n'
236-- | Flattens the stream of @Array Char@, after appending a separating
263-- | Flattens the stream of @Array Char@, after appending a separating
237-- space to each string.
264-- space to each string.
@@ -241,6 +268,9 @@ unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
241-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
268-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
242-- "unwords this string"
269-- "unwords this string"
272+-- > unwords = AS.flatten . (S.intersperse (A.fromList " "))
244-- Note that, in general
274-- Note that, in general
246-- > unwords . words /= id
276-- > unwords . words /= id