git.iko.soy/streamly
on bcecc5a help sign in
streamly/src/Streamly/Data/String.hs
312 lines 11 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.Data.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- A unicode string can be represented either as a stream of 'Char' e.g.
12-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13-- can be done efficiently by applying stream operations and folds on the
14-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15-- applied where available or the array can be read into a stream and then
16-- processed using stream operations. Use 'Array' 'Char' when you need to store
17-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
20-- convenience.
21--
22-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23-- large amounts of text in memory. Also, it is computationally efficient as
24-- there is no encoding/decoding involved. We recommend using true streaming
25-- operations to avoid buffering large amounts of data as long as possible e.g.
26-- use `foldLines` instead of `lines`. However, if for some reason you are
27-- buffering very large amounts of text in memory and are worried about space
28-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29-- encoded 'Array' 'Word8'.
30--
31-- Please note the following:
32--
33-- * Case conversion: Some unicode characters translate to more than one code
34-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35-- package do not handle such characters. Therefore, operations like @map
36-- toUpper@ on a character stream or character array may not always perform
37-- correct conversion.
38-- * String comparison: In some cases, visually identical strings may have
39-- different unicode representations, therefore, a character stream or
40-- character array cannot be directly compared. A normalized comparison may be
41-- needed to check string equivalence correctly.
43-- See "Streamly.Internal.Data.List", <src/docs/streamly-vs-lists.md> for more details and
44-- <src/test/PureStreams.hs> for comprehensive usage examples.
46-- TBD write a note on performance comparison with text, bytestrings, lists and
47-- vector.
48--
49module Streamly.Data.String
50 (
51 -- * Construction (Decoding)
52 decodeChar8
53 , decodeUtf8
54 , decodeUtf8Lenient
55 , decodeUtf8Arrays
56 , decodeUtf8ArraysLenient
58 -- * Elimination (Encoding)
59 , encodeChar8
60 , encodeChar8Unchecked
61 , encodeUtf8
62{-
63 -- * Unicode aware operations
64 , toCaseFold
65 , toLower
66 , toUpper
67 , toTitle
69 -- * Operations on character strings
70 , strip -- (dropAround isSpace)
71 , stripEnd-}
72 -- * Transformation
73 , stripStart
74 , foldLines
75 , foldWords
76 , unfoldLines
78 -- * Streams of Strings
79 , lines
80 , words
81 , unlines
82 , unwords
83 )
84where
86import Control.Monad.IO.Class (MonadIO)
87import Data.Char (ord)
88import Data.Word (Word8)
89import GHC.Base (unsafeChr)
90import Streamly (IsStream, MonadAsync)
91import Prelude hiding (String, lines, words, unlines, unwords)
92import Streamly.Data.Fold (Fold)
93import Streamly.Memory.Array (Array)
94import Streamly.Internal.Data.Unfold (Unfold)
96import qualified Streamly.Internal.Prelude as S
97import qualified Streamly.Memory.Array as A
98import qualified Streamly.Internal.Memory.Array.Types as A
99import qualified Streamly.Internal.Memory.ArrayStream as AS
100import qualified Streamly.Streams.StreamD as D
101import qualified Streamly.Internal.Data.Unfold as UF
103-- type String = List Char
105-------------------------------------------------------------------------------
106-- Encoding/Decoding Characters
107-------------------------------------------------------------------------------
109-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
110-- decodeWith = undefined
112-------------------------------------------------------------------------------
113-- Encoding/Decoding Unicode Characters
114-------------------------------------------------------------------------------
116-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
117-- corresponding Unicode 'Char' in 0-255 range.
118{-# INLINE decodeChar8 #-}
119decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
120decodeChar8 = S.map (unsafeChr . fromIntegral)
122-- | Encode a stream of Unicode characters to bytes by mapping each character
123-- to a byte in 0-255 range. Throws an error if the input stream contains
124-- characters beyond 255.
125{-# INLINE encodeChar8 #-}
126encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
127encodeChar8 = S.map convert
128 where
129 convert c =
130 let codepoint = ord c
131 in if codepoint > 255
132 then error $ "Streamly.String.encodeChar8 invalid \
133 \input char codepoint " ++ show codepoint
134 else fromIntegral codepoint
136-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
137-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
138-- when such truncation occurs.
139{-# INLINE encodeChar8Unchecked #-}
140encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
141encodeChar8Unchecked = S.map (fromIntegral . ord)
143-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
144-- The incoming stream is truncated if an invalid codepoint is encountered.
145{-# INLINE decodeUtf8 #-}
146decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
147decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
149{-# INLINE decodeUtf8Arrays #-}
150decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
151decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
153-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
154-- Any invalid codepoint encountered is replaced with the unicode replacement
155-- character.
156{-# INLINE decodeUtf8Lenient #-}
157decodeUtf8Lenient :: (Monad m, IsStream t) => t m Word8 -> t m Char
158decodeUtf8Lenient = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
160{-# INLINE decodeUtf8ArraysLenient #-}
161decodeUtf8ArraysLenient ::
162 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
163decodeUtf8ArraysLenient =
164 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
166-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
167{-# INLINE encodeUtf8 #-}
168encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
169encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
171{-
172-------------------------------------------------------------------------------
173-- Unicode aware operations on strings
174-------------------------------------------------------------------------------
176toCaseFold :: IsStream t => t m Char -> t m Char
177toCaseFold = undefined
179toLower :: IsStream t => t m Char -> t m Char
180toLower = undefined
182toUpper :: IsStream t => t m Char -> t m Char
183toUpper = undefined
185toTitle :: IsStream t => t m Char -> t m Char
186toTitle = undefined
188-------------------------------------------------------------------------------
189-- Utility operations on strings
190-------------------------------------------------------------------------------
192strip :: IsStream t => t m Char -> t m Char
193strip = undefined
195stripEnd :: IsStream t => t m Char -> t m Char
196stripEnd = undefined
197-}
199-- | Remove leading whitespace from a string.
200--
201-- > stripStart = S.dropWhile isSpace
202{-# INLINE stripStart #-}
203stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
204stripStart = S.dropWhile isSpace
206-- | Fold each line of the stream using the supplied 'Fold'
207-- and stream the result.
208--
209-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
210-- ["lines", "this", "string", "", ""]
211--
212-- > foldLines = S.splitOnSuffix (== '\n')
213--
214{-# INLINE foldLines #-}
215foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
216foldLines = S.splitOnSuffix (== '\n')
218-- | Fold each word of the stream using the supplied 'Fold'
219-- and stream the result.
220--
221-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
222-- ["fold", "these", "words"]
223--
224-- > foldWords = S.wordsBy isSpace
225--
226{-# INLINE foldWords #-}
227foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
228foldWords = S.wordsBy isSpace
230foreign import ccall unsafe "u_iswspace"
231 iswspace :: Int -> Int
233-- | Code copied from base/Data.Char to INLINE it
234{-# INLINE isSpace #-}
235isSpace :: Char -> Bool
236isSpace c
237 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
238 | otherwise = iswspace (ord c) /= 0
239 where
240 uc = fromIntegral (ord c) :: Word
242-- | Break a string up into a stream of strings at newline characters.
243-- The resulting strings do not contain newlines.
244--
245-- > lines = foldLines A.write
246--
247-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
248-- ["lines","this","string","",""]
249--
250-- If you're dealing with lines of massive length, consider using
251-- 'foldLines' instead to avoid buffering the data in 'Array'.
252{-# INLINE lines #-}
253lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
254lines = foldLines A.write
256-- | Break a string up into a stream of strings, which were delimited
257-- by characters representing white space.
258--
259-- > words = foldWords A.write
260--
261-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
262-- ["A", "newline", "is", "considered", "white", "space?"]
263--
264-- If you're dealing with words of massive length, consider using
265-- 'foldWords' instead to avoid buffering the data in 'Array'.
266{-# INLINE words #-}
267words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
268words = foldWords A.write
270-- | Unfold a stream to character streams using the supplied 'Unfold'
271-- and concat the results suffixing a newline character @\\n@ to each stream.
272--
273-- > unfoldLines = S.intercalateSuffix UF.singleton '\n'
274--
275{-# INLINE unfoldLines #-}
276unfoldLines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
277unfoldLines unf = S.intercalateSuffix UF.singleton '\n' unf
279-- | Flattens the stream of @Array Char@, after appending a terminating
280-- newline to each string.
281--
282-- 'unlines' is an inverse operation to 'lines'.
283--
284-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
285-- "lines\nthis\nstring\n"
286--
287-- > unlines = unfoldLines A.readU
288--
289-- Note that, in general
290--
291-- > unlines . lines /= id
292{-# INLINE unlines #-}
293unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
294unlines = unfoldLines A.readU
296-- | Flattens the stream of @Array Char@, after appending a separating
297-- space to each string.
298--
299-- 'unwords' is an inverse operation to 'words'.
300--
301-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
302-- "unwords this string"
303--
304--
305-- > unwords = A.concat . (S.intersperse (A.fromList " "))
306--
307-- Note that, in general
308--
309-- > unwords . words /= id
310{-# INLINE unwords #-}
311unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
312unwords = AS.concat . (S.intersperse (A.fromList " "))