git.iko.soy/streamly
on 48fc6bf help sign in
streamly/src/Streamly/Data/String.hs
311 lines 11 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.Data.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- A unicode string can be represented either as a stream of 'Char' e.g.
12-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13-- can be done efficiently by applying stream operations and folds on the
14-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15-- applied where available or the array can be read into a stream and then
16-- processed using stream operations. Use 'Array' 'Char' when you need to store
17-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20-- convenience.
21--
22-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23-- large amounts of text in memory. Also, it is computationally efficient as
24-- there is no encoding/decoding involved. We recommend using true streaming
25-- operations to avoid buffering large amounts of data as long as possible e.g.
26-- use `foldLines` instead of `lines`. However, if for some reason you are
27-- buffering very large amounts of text in memory and are worried about space
28-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29-- encoded 'Array' 'Word8'.
30--
31-- Please note the following:
32--
33-- * Case conversion: Some unicode characters translate to more than one code
34-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35-- package do not handle such characters. Therefore, operations like @map
36-- toUpper@ on a character stream or character array may not always perform
37-- correct conversion.
38-- * String comparison: In some cases, visually identical strings may have
39-- different unicode representations, therefore, a character stream or
40-- character array cannot be directly compared. A normalized comparison may be
41-- needed to check string equivalence correctly.
43-- See "Streamly.Internal.Data.List", <src/docs/streamly-vs-lists.md> for more details and
44-- <src/test/PureStreams.hs> for comprehensive usage examples.
46-- TBD write a note on performance comparison with text, bytestrings, lists and
47-- vector.
48--
49module Streamly.Data.String
50 (
51 -- * Construction (Decoding)
52 decodeChar8
53 , decodeUtf8
54 , decodeUtf8Lenient
55 , decodeUtf8Arrays
56 , decodeUtf8ArraysLenient
58 -- * Elimination (Encoding)
59 , encodeChar8
60 , encodeChar8Unchecked
61 , encodeUtf8
62{-
63 -- * Unicode aware operations
64 , toCaseFold
65 , toLower
66 , toUpper
67 , toTitle
69 -- * Operations on character strings
70 , strip -- (dropAround isSpace)
71 , stripEnd-}
72 -- * Transformation
73 , stripStart
74 , foldLines
75 , foldWords
76 , unfoldLines
78 -- * Streams of Strings
79 , lines
80 , words
81 , unlines
82 , unwords
83 )
84where
86import Control.Monad.IO.Class (MonadIO)
87import Data.Char (ord)
88import Data.Word (Word8)
89import GHC.Base (unsafeChr)
90import Streamly (IsStream, MonadAsync)
91import Prelude hiding (String, lines, words, unlines, unwords)
92import Streamly.Data.Fold (Fold)
93import Streamly.Memory.Array (Array)
94import Streamly.Internal.Data.Unfold (Unfold)
96import qualified Streamly.Internal.Prelude as S
97import qualified Streamly.Memory.Array as A
98import qualified Streamly.Internal.Memory.ArrayStream as AS
99import qualified Streamly.Streams.StreamD as D
100import qualified Streamly.Internal.Data.Unfold as UF
102-- type String = List Char
104-------------------------------------------------------------------------------
105-- Encoding/Decoding Characters
106-------------------------------------------------------------------------------
108-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
109-- decodeWith = undefined
111-------------------------------------------------------------------------------
112-- Encoding/Decoding Unicode Characters
113-------------------------------------------------------------------------------
115-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
116-- corresponding Unicode 'Char' in 0-255 range.
117{-# INLINE decodeChar8 #-}
118decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
119decodeChar8 = S.map (unsafeChr . fromIntegral)
121-- | Encode a stream of Unicode characters to bytes by mapping each character
122-- to a byte in 0-255 range. Throws an error if the input stream contains
123-- characters beyond 255.
124{-# INLINE encodeChar8 #-}
125encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
126encodeChar8 = S.map convert
127 where
128 convert c =
129 let codepoint = ord c
130 in if codepoint > 255
131 then error $ "Streamly.String.encodeChar8 invalid \
132 \input char codepoint " ++ show codepoint
133 else fromIntegral codepoint
135-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
136-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
137-- when such truncation occurs.
138{-# INLINE encodeChar8Unchecked #-}
139encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
140encodeChar8Unchecked = S.map (fromIntegral . ord)
142-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
143-- The incoming stream is truncated if an invalid codepoint is encountered.
144{-# INLINE decodeUtf8 #-}
145decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
146decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
148{-# INLINE decodeUtf8Arrays #-}
149decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
150decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
152-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
153-- Any invalid codepoint encountered is replaced with the unicode replacement
154-- character.
155{-# INLINE decodeUtf8Lenient #-}
156decodeUtf8Lenient :: (Monad m, IsStream t) => t m Word8 -> t m Char
157decodeUtf8Lenient = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
159{-# INLINE decodeUtf8ArraysLenient #-}
160decodeUtf8ArraysLenient ::
161 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
162decodeUtf8ArraysLenient =
163 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
165-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
166{-# INLINE encodeUtf8 #-}
167encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
168encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
170{-
171-------------------------------------------------------------------------------
172-- Unicode aware operations on strings
173-------------------------------------------------------------------------------
175toCaseFold :: IsStream t => t m Char -> t m Char
176toCaseFold = undefined
178toLower :: IsStream t => t m Char -> t m Char
179toLower = undefined
181toUpper :: IsStream t => t m Char -> t m Char
182toUpper = undefined
184toTitle :: IsStream t => t m Char -> t m Char
185toTitle = undefined
187-------------------------------------------------------------------------------
188-- Utility operations on strings
189-------------------------------------------------------------------------------
191strip :: IsStream t => t m Char -> t m Char
192strip = undefined
194stripEnd :: IsStream t => t m Char -> t m Char
195stripEnd = undefined
196-}
198-- | Remove leading whitespace from a string.
199--
200-- > stripStart = S.dropWhile isSpace
201{-# INLINE stripStart #-}
202stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
203stripStart = S.dropWhile isSpace
205-- | Fold each line of the stream using the supplied 'Fold'
206-- and stream the result.
207--
208-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
209-- ["lines", "this", "string", "", ""]
210--
211-- > foldLines = S.splitOnSuffix (== '\n')
212--
213{-# INLINE foldLines #-}
214foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
215foldLines = S.splitOnSuffix (== '\n')
217-- | Fold each word of the stream using the supplied 'Fold'
218-- and stream the result.
219--
220-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
221-- ["fold", "these", "words"]
222--
223-- > foldWords = S.wordsBy isSpace
224--
225{-# INLINE foldWords #-}
226foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
227foldWords = S.wordsBy isSpace
229foreign import ccall unsafe "u_iswspace"
230 iswspace :: Int -> Int
232-- | Code copied from base/Data.Char to INLINE it
233{-# INLINE isSpace #-}
234isSpace :: Char -> Bool
235isSpace c
236 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
237 | otherwise = iswspace (ord c) /= 0
238 where
239 uc = fromIntegral (ord c) :: Word
241-- | Break a string up into a stream of strings at newline characters.
242-- The resulting strings do not contain newlines.
243--
244-- > lines = foldLines A.write
245--
246-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
247-- ["lines","this","string","",""]
248--
249-- If you're dealing with lines of massive length, consider using
250-- 'foldLines' instead to avoid buffering the data in 'Array'.
251{-# INLINE lines #-}
252lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
253lines = foldLines A.write
255-- | Break a string up into a stream of strings, which were delimited
256-- by characters representing white space.
257--
258-- > words = foldWords A.write
259--
260-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
261-- ["A", "newline", "is", "considered", "white", "space?"]
262--
263-- If you're dealing with words of massive length, consider using
264-- 'foldWords' instead to avoid buffering the data in 'Array'.
265{-# INLINE words #-}
266words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
267words = foldWords A.write
269-- | Unfold a stream to character streams using the supplied 'Unfold'
270-- and concat the results suffixing a newline character @\\n@ to each stream.
271--
272-- > unfoldLines = S.intercalateSuffix UF.identity '\n'
273--
274{-# INLINE unfoldLines #-}
275unfoldLines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
276unfoldLines unf = S.intercalateSuffix UF.identity '\n' unf
278-- | Flattens the stream of @Array Char@, after appending a terminating
279-- newline to each string.
280--
281-- 'unlines' is an inverse operation to 'lines'.
282--
283-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
284-- "lines\nthis\nstring\n"
285--
286-- > unlines = unfoldLines A.read
287--
288-- Note that, in general
289--
290-- > unlines . lines /= id
291{-# INLINE unlines #-}
292unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
293unlines = unfoldLines A.read
295-- | Flattens the stream of @Array Char@, after appending a separating
296-- space to each string.
297--
298-- 'unwords' is an inverse operation to 'words'.
299--
300-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
301-- "unwords this string"
302--
303--
304-- > unwords = A.concat . (S.intersperse (A.fromList " "))
305--
306-- Note that, in general
307--
308-- > unwords . words /= id
309{-# INLINE unwords #-}
310unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
311unwords = AS.concat . (S.intersperse (A.fromList " "))