git.iko.soy/streamly
on 3336bb1 help sign in
streamly/src/Streamly/Data/String.hs
278 lines 9.6 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.Data.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- A unicode string can be represented either as a stream of 'Char' e.g.
12-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13-- can be done efficiently by applying stream operations and folds on the
14-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15-- applied where available or the array can be read into a stream and then
16-- processed using stream operations. Use 'Array' 'Char' when you need to store
17-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20-- convenience.
21--
22-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23-- large amounts of text in memory. Also, it is computationally efficient as
24-- there is no encoding/decoding involved. We recommend using true streaming
25-- operations to avoid buffering large amounts of data as long as possible e.g.
26-- use `foldLines` instead of `lines`. However, if for some reason you are
27-- buffering very large amounts of text in memory and are worried about space
28-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29-- encoded 'Array' 'Word8'.
30--
31-- Please note the following:
32--
33-- * Case conversion: Some unicode characters translate to more than one code
34-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35-- package do not handle such characters. Therefore, operations like @map
36-- toUpper@ on a character stream or character array may not always perform
37-- correct conversion.
38-- * String comparison: In some cases, visually identical strings may have
39-- different unicode representations, therefore, a character stream or
40-- character array cannot be directly compared. A normalized comparison may be
41-- needed to check string equivalence correctly.
43-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
44-- <src/test/PureStreams.hs> for comprehensive usage examples.
46-- TBD write a note on performance comparison with text, bytestrings, lists and
47-- vector.
48--
49module Streamly.Data.String
50 (
51 -- * Construction (Decoding)
52 decodeChar8
53 , decodeUtf8
55 -- * Elimination (Encoding)
56 , encodeChar8
57 , encodeChar8Unchecked
58 , encodeUtf8
59{-
60 -- * Unicode aware operations
61 , toCaseFold
62 , toLower
63 , toUpper
64 , toTitle
66 -- * Operations on character strings
67 , strip -- (dropAround isSpace)
68 , stripEnd-}
69 -- * Transformation
70 , stripStart
71 , foldLines
72 , foldWords
74 -- * Streams of Strings
75 , lines
76 , words
77 , unlines
78 , unwords
79 )
80where
82import Control.Monad.IO.Class (MonadIO)
83import Data.Char (ord)
84import Data.Word (Word8)
85import GHC.Base (unsafeChr)
86import Streamly (IsStream, MonadAsync)
87import Prelude hiding (String, lines, words, unlines, unwords)
88import Streamly.Fold (Fold)
89import Streamly.Memory.Array (Array)
91import qualified Streamly.Prelude as S
92import qualified Streamly.Memory.Array as A
93import qualified Streamly.Streams.StreamD as D
95-- type String = List Char
97-------------------------------------------------------------------------------
98-- Encoding/Decoding Characters
99-------------------------------------------------------------------------------
101-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
102-- decodeWith = undefined
104-------------------------------------------------------------------------------
105-- Encoding/Decoding Unicode Characters
106-------------------------------------------------------------------------------
108-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
109-- corresponding Unicode 'Char' in 0-255 range.
110{-# INLINE decodeChar8 #-}
111decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
112decodeChar8 = S.map (unsafeChr . fromIntegral)
114-- | Encode a stream of Unicode characters to bytes by mapping each character
115-- to a byte in 0-255 range. Throws an error if the input stream contains
116-- characters beyond 255.
117{-# INLINE encodeChar8 #-}
118encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
119encodeChar8 = S.map convert
120 where
121 convert c =
122 let codepoint = ord c
123 in if codepoint > 255
124 then error $ "Streamly.String.encodeChar8 invalid \
125 \input char codepoint " ++ show codepoint
126 else fromIntegral codepoint
128-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
129-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
130-- when such truncation occurs.
131{-# INLINE encodeChar8Unchecked #-}
132encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
133encodeChar8Unchecked = S.map (fromIntegral . ord)
135-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
136-- The incoming stream is truncated if an invalid codepoint is encountered.
137{-# INLINE decodeUtf8 #-}
138decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
139decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
141-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
142{-# INLINE encodeUtf8 #-}
143encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
144encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
146{-
147-------------------------------------------------------------------------------
148-- Unicode aware operations on strings
149-------------------------------------------------------------------------------
151toCaseFold :: IsStream t => t m Char -> t m Char
152toCaseFold = undefined
154toLower :: IsStream t => t m Char -> t m Char
155toLower = undefined
157toUpper :: IsStream t => t m Char -> t m Char
158toUpper = undefined
160toTitle :: IsStream t => t m Char -> t m Char
161toTitle = undefined
163-------------------------------------------------------------------------------
164-- Utility operations on strings
165-------------------------------------------------------------------------------
167strip :: IsStream t => t m Char -> t m Char
168strip = undefined
170stripEnd :: IsStream t => t m Char -> t m Char
171stripEnd = undefined
172-}
174-- | Remove leading whitespace from a string.
175--
176-- > stripStart = S.dropWhile isSpace
177{-# INLINE stripStart #-}
178stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
179stripStart = S.dropWhile isSpace
181-- | Fold each line of the stream using the supplied 'Fold'
182-- and stream the result.
183--
184-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
185-- ["lines", "this", "string", "", ""]
186--
187-- > foldLines = S.splitOnSuffix (== '\n')
188--
189{-# INLINE foldLines #-}
190foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
191foldLines = S.splitOnSuffix (== '\n')
193-- | Fold each word of the stream using the supplied 'Fold'
194-- and stream the result.
195--
196-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
197-- ["fold", "these", "words"]
198--
199-- > foldWords = S.wordsBy isSpace
200--
201{-# INLINE foldWords #-}
202foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
203foldWords = S.wordsBy isSpace
205foreign import ccall unsafe "u_iswspace"
206 iswspace :: Int -> Int
208-- | Code copied from base/Data.Char to INLINE it
209{-# INLINE isSpace #-}
210isSpace :: Char -> Bool
211isSpace c
212 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
213 | otherwise = iswspace (ord c) /= 0
214 where
215 uc = fromIntegral (ord c) :: Word
217-- | Break a string up into a stream of strings at newline characters.
218-- The resulting strings do not contain newlines.
219--
220-- > lines = foldLines A.write
221--
222-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
223-- ["lines","this","string","",""]
224--
225-- If you're dealing with lines of massive length, consider using
226-- 'foldLines' instead to avoid buffering the data in 'Array'.
227{-# INLINE lines #-}
228lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
229lines = foldLines A.write
231-- | Break a string up into a stream of strings, which were delimited
232-- by characters representing white space.
233--
234-- > words = foldWords A.write
235--
236-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
237-- ["A", "newline", "is", "considered", "white", "space?"]
238--
239-- If you're dealing with words of massive length, consider using
240-- 'foldWords' instead to avoid buffering the data in 'Array'.
241{-# INLINE words #-}
242words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
243words = foldWords A.write
245-- | Flattens the stream of @Array Char@, after appending a terminating
246-- newline to each string.
247--
248-- 'unlines' is an inverse operation to 'lines'.
249--
250-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
251-- "lines\nthis\nstring\n"
252--
253-- > unlines = A.concatWithSuffix '\n'
254--
255-- Note that, in general
256--
257-- > unlines . lines /= id
258{-# INLINE unlines #-}
259unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
260unlines = A.concatWithSuffix '\n'
262-- | Flattens the stream of @Array Char@, after appending a separating
263-- space to each string.
264--
265-- 'unwords' is an inverse operation to 'words'.
266--
267-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
268-- "unwords this string"
269--
270--
271-- > unwords = A.concat . (S.intersperse (A.fromList " "))
272--
273-- Note that, in general
274--
275-- > unwords . words /= id
276{-# INLINE unwords #-}
277unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
278unwords = A.concat . (S.intersperse (A.fromList " "))