@@ -8,23 +8,38 @@
88-- Stability : experimental
99-- Portability : GHC
1010--
11−-- Stream of 'Char' are the equivalent of Haskell Strings but without any
12−-- performance issues that are usually associated with the 'String' type.
13−-- Haskell lists are not suitable for storage in GC memory. We use streams for
14−-- processing and arrays for storing. We can use the usual stream processing
15−-- routines on these 'Char' streams. Stream processing is very efficient due to
16−-- stream fusion. If strings are to be stored or buffered in memory, they can
17−-- be encoded to 'Word8' arrays using the encoding routines provided in the
18−-- module. Therefore, a separate type for text representation is not required.
19−-- A @Stream Identity Char@ can be used almost as a drop-in replacement for the
20−-- standard Haskell @String@, especially when used with @OverloadedStrings@
21−-- extension, with little differences.
22−
23−-- The 'String' type in this module is just a synonym for the type @List Char@.
24−-- It provides better performance compared to the standard Haskell @String@
25−-- type and can be used almost as a drop-in replacement, especially when used
26−-- with @OverloadedStrings@ extension, with little differences.
11+-- A unicode string can be represented either as a stream of 'Char' e.g.
12+-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13+-- can be done efficiently by applying stream operations and folds on the
14+-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15+-- applied where available or the array can be read into a stream and then
16+-- processed using stream operations. Use 'Array' 'Char' when you need to store
17+-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18+-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19+-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20+-- convenience.
2721--
22+-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23+-- large amounts of text in memory. Also, it is computationally efficient as
24+-- there is no encoding/decoding involved. We recommend using true streaming
25+-- operations to avoid buffering large amounts of data as long as possible e.g.
26+-- use `foldLines` instead of `lines`. However, if for some reason you are
27+-- buffering very large amounts of text in memory and are worried about space
28+-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29+-- encoded 'Array' 'Word8'.
30+--
31+-- Please note the following:
32+--
33+-- * Case conversion: Some unicode characters translate to more than one code
34+-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35+-- package do not handle such characters. Therefore, operations like @map
36+-- toUpper@ on a character stream or character array may not always perform
37+-- correct conversion.
38+-- * String comparison: In some cases, visually identical strings may have
39+-- different unicode representations, therefore, a character stream or
40+-- character array cannot be directly compared. A normalized comparison may be
41+-- needed to check string equivalence correctly.
42+
2843-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
2944-- <src/test/PureStreams.hs> for comprehensive usage examples.
3045
@@ -33,15 +48,14 @@
3348--
3449module Streamly.Data.String
3550 (
36− -- String
51+ -- * Construction (Decoding)
52+ decodeChar8
53+ , decodeUtf8
3754
38− -- * Encoding and Decoding
39− encodeChar8
55+ -- * Elimination (Encoding)
56+ , encodeChar8
4057 , encodeChar8Unchecked
41− , decodeChar8
42−
4358 , encodeUtf8
44− , decodeUtf8
4559{-
4660 -- * Unicode aware operations
4761 , toCaseFold
@@ -52,10 +66,12 @@ module Streamly.Data.String
5266 -- * Operations on character strings
5367 , strip -- (dropAround isSpace)
5468 , stripEnd-}
55− -- * Substrings
69+ -- * Transformation
5670 , stripStart
5771 , foldLines
5872 , foldWords
73+
74+ -- * Streams of Strings
5975 , lines
6076 , words
6177 , unlines
@@ -73,7 +89,6 @@ import Streamly.Fold (Fold)
7389import Streamly.Memory.Array (Array)
7490
7591import qualified Streamly.Prelude as S
76−import qualified Streamly.Memory.Array.Types as A (unlines)
7792import qualified Streamly.Memory.Array as A
7893import qualified Streamly.Memory.ArrayStream as AS
7994import qualified Streamly.Streams.StreamD as D
@@ -157,27 +172,33 @@ stripEnd :: IsStream t => t m Char -> t m Char
157172stripEnd = undefined
158173-}
159174
160−-- | Remove leading whitespace from a String.
175+-- | Remove leading whitespace from a string.
161176--
162177-- > stripStart = S.dropWhile isSpace
163178{-# INLINE stripStart #-}
164179stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
165180stripStart = S.dropWhile isSpace
166181
167−-- | Fold each line of the stream using the supplied Fold
182+-- | Fold each line of the stream using the supplied 'Fold'
168183-- and stream the result.
169184--
170185-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
171186-- ["lines", "this", "string", "", ""]
187+--
188+-- > foldLines = S.splitOnSuffix (== '\n')
189+--
172190{-# INLINE foldLines #-}
173191foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
174192foldLines = S.splitOnSuffix (== '\n')
175193
176−-- | Fold each word of the stream using the supplied Fold
194+-- | Fold each word of the stream using the supplied 'Fold'
177195-- and stream the result.
178196--
179197-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
180198-- ["fold", "these", "words"]
199+--
200+-- > foldWords = S.wordsBy isSpace
201+--
181202{-# INLINE foldWords #-}
182203foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
183204foldWords = S.wordsBy isSpace
@@ -194,29 +215,33 @@ isSpace c
194215 where
195216 uc = fromIntegral (ord c) :: Word
196217
197−-- | Break a string up into a list of strings at newline characters.
218+-- | Break a string up into a stream of strings at newline characters.
198219-- The resulting strings do not contain newlines.
199220--
221+-- > lines = foldLines A.write
222+--
200223-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
201224-- ["lines","this","string","",""]
202225--
203226-- If you're dealing with lines of massive length, consider using
204−-- 'foldLines' instead.
227+-- 'foldLines' instead to avoid buffering the data in 'Array'.
205228{-# INLINE lines #-}
206229lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
207−lines = S.splitOnSuffix (== '\n') A.write
230+lines = foldLines A.write
208231
209−-- | Break a string up into a list of strings, which were delimited
232+-- | Break a string up into a stream of strings, which were delimited
210233-- by characters representing white space.
211234--
235+-- > words = foldWords A.write
236+--
212237-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
213238-- ["A", "newline", "is", "considered", "white", "space?"]
214239--
215240-- If you're dealing with words of massive length, consider using
216−-- 'foldWords' instead.
241+-- 'foldWords' instead to avoid buffering the data in 'Array'.
217242{-# INLINE words #-}
218243words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
219−words = S.wordsBy isSpace A.write
244+words = foldWords A.write
220245
221246-- | Flattens the stream of @Array Char@, after appending a terminating
222247-- newline to each string.
@@ -226,12 +251,14 @@ words = S.wordsBy isSpace A.write
226251-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
227252-- "lines\nthis\nstring\n"
228253--
254+-- > unlines = AS.unlinesBy '\n'
255+--
229256-- Note that, in general
230257--
231258-- > unlines . lines /= id
232259{-# INLINE unlines #-}
233260unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
234−unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
261+unlines = AS.unlinesBy '\n'
235262
236263-- | Flattens the stream of @Array Char@, after appending a separating
237264-- space to each string.
@@ -241,6 +268,9 @@ unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
241268-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
242269-- "unwords this string"
243270--
271+--
272+-- > unwords = AS.flatten . (S.intersperse (A.fromList " "))
273+--
244274-- Note that, in general
245275--
246276-- > unwords . words /= id