1{-# LANGUAGE FlexibleContexts #-}
3-- Module : Streamly.Data.String
4-- Copyright : (c) 2018 Composewell Technologies
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
11-- A unicode string can be represented either as a stream of 'Char' e.g.
12-- 'SerialT' 'Identity' 'Char' or as 'Array' 'Char'. Unicode text processing
13-- can be done efficiently by applying stream operations and folds on the
14-- stream of 'Char'. When using 'Array' 'Char' direct array operations can be
15-- applied where available or the array can be read into a stream and then
16-- processed using stream operations. Use 'Array' 'Char' when you need to store
17-- or buffer strings temporarily in memory. Streams in 'Identity' monad and
18-- 'Array' 'Char' are instances of 'IsString' and 'IsList', therefore,
19-- 'OverloadedStrings' and 'OverloadedLists' extensions can be used for
22-- 'Array' 'Char' is usually perfectly fine to buffer short to medium or even
23-- large amounts of text in memory. Also, it is computationally efficient as
24-- there is no encoding/decoding involved. We recommend using true streaming
25-- operations to avoid buffering large amounts of data as long as possible e.g.
26-- use `foldLines` instead of `lines`. However, if for some reason you are
27-- buffering very large amounts of text in memory and are worried about space
28-- efficiency you can use 'encodeUtf8' on the stream to convert it to a utf8
29-- encoded 'Array' 'Word8'.
31-- Please note the following:
33-- * Case conversion: Some unicode characters translate to more than one code
34-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@
35-- package do not handle such characters. Therefore, operations like @map
36-- toUpper@ on a character stream or character array may not always perform
38-- * String comparison: In some cases, visually identical strings may have
39-- different unicode representations, therefore, a character stream or
40-- character array cannot be directly compared. A normalized comparison may be
41-- needed to check string equivalence correctly.
43-- See "Streamly.Internal.Data.List", <src/docs/streamly-vs-lists.md> for more details and
44-- <src/test/PureStreams.hs> for comprehensive usage examples.
46-- TBD write a note on performance comparison with text, bytestrings, lists and
49module Streamly.Data.String
51 -- * Construction (Decoding)
59 , resumeDecodeUtf8Either
61 , decodeUtf8ArraysLenient
63 -- * Elimination (Encoding)
65 , encodeChar8Unchecked
68 -- * Unicode aware operations
74 -- * Operations on character strings
75 , strip -- (dropAround isSpace)
84 -- * Streams of Strings
92import Control.Monad.IO.Class (MonadIO)
94import Data.Word (Word8)
95import GHC.Base (unsafeChr)
96import Streamly (IsStream, MonadAsync)
97import Prelude hiding (String, lines, words, unlines, unwords)
98import Streamly.Data.Fold (Fold)
99import Streamly.Memory.Array (Array)
100import Streamly.Internal.Data.Unfold (Unfold)
102import qualified Streamly.Internal.Prelude as S
103import qualified Streamly.Memory.Array as A
104import qualified Streamly.Streams.StreamD as D
106-- type String = List Char
108-------------------------------------------------------------------------------
109-- Encoding/Decoding Characters
110-------------------------------------------------------------------------------
112-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
113-- decodeWith = undefined
115-------------------------------------------------------------------------------
116-- Encoding/Decoding Unicode Characters
117-------------------------------------------------------------------------------
119-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
120-- corresponding Unicode 'Char' in 0-255 range.
121{-# INLINE decodeChar8 #-}
122decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
123decodeChar8 = S.map (unsafeChr . fromIntegral)
125-- | Encode a stream of Unicode characters to bytes by mapping each character
126-- to a byte in 0-255 range. Throws an error if the input stream contains
127-- characters beyond 255.
128{-# INLINE encodeChar8 #-}
129encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
130encodeChar8 = S.map convert
133 let codepoint = ord c
134 in if codepoint > 255
135 then error $ "Streamly.String.encodeChar8 invalid \
136 \input char codepoint " ++ show codepoint
137 else fromIntegral codepoint
139-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
140-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
141-- when such truncation occurs.
142{-# INLINE encodeChar8Unchecked #-}
143encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
144encodeChar8Unchecked = S.map (fromIntegral . ord)
146-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
147-- The incoming stream is truncated if an invalid codepoint is encountered.
148{-# INLINE decodeUtf8 #-}
149decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
150decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
152{-# INLINE decodeUtf8Arrays #-}
153decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
154decodeUtf8Arrays = D.fromStreamD . D.decodeUtf8Arrays . D.toStreamD
156-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
157-- Any invalid codepoint encountered is replaced with the unicode replacement
159{-# INLINE decodeUtf8Lenient #-}
160decodeUtf8Lenient :: (Monad m, IsStream t) => t m Word8 -> t m Char
161decodeUtf8Lenient = D.fromStreamD . D.decodeUtf8Lenient . D.toStreamD
163{-# INLINE decodeUtf8Either #-}
164decodeUtf8Either :: (Monad m, IsStream t)
165 => t m Word8 -> t m (Either D.DecodeError Char)
166decodeUtf8Either = D.fromStreamD . D.decodeUtf8Either . D.toStreamD
168{-# INLINE resumeDecodeUtf8Either #-}
169resumeDecodeUtf8Either
170 :: (Monad m, IsStream t)
174 -> t m (Either D.DecodeError Char)
175resumeDecodeUtf8Either st cp =
176 D.fromStreamD . D.resumeDecodeUtf8Either st cp . D.toStreamD
178{-# INLINE decodeUtf8ArraysLenient #-}
179decodeUtf8ArraysLenient ::
180 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char
181decodeUtf8ArraysLenient =
182 D.fromStreamD . D.decodeUtf8ArraysLenient . D.toStreamD
184-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
185{-# INLINE encodeUtf8 #-}
186encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
187encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
190-------------------------------------------------------------------------------
191-- Unicode aware operations on strings
192-------------------------------------------------------------------------------
194toCaseFold :: IsStream t => t m Char -> t m Char
195toCaseFold = undefined
197toLower :: IsStream t => t m Char -> t m Char
200toUpper :: IsStream t => t m Char -> t m Char
203toTitle :: IsStream t => t m Char -> t m Char
206-------------------------------------------------------------------------------
207-- Utility operations on strings
208-------------------------------------------------------------------------------
210strip :: IsStream t => t m Char -> t m Char
213stripEnd :: IsStream t => t m Char -> t m Char
217-- | Remove leading whitespace from a string.
219-- > stripStart = S.dropWhile isSpace
220{-# INLINE stripStart #-}
221stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
222stripStart = S.dropWhile isSpace
224-- | Fold each line of the stream using the supplied 'Fold'
225-- and stream the result.
227-- >>> S.toList $ foldLines FL.toList (S.fromList "lines\nthis\nstring\n\n\n")
228-- ["lines", "this", "string", "", ""]
230-- > foldLines = S.splitOnSuffix (== '\n')
232{-# INLINE foldLines #-}
233foldLines :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
234foldLines = S.splitOnSuffix (== '\n')
236-- | Fold each word of the stream using the supplied 'Fold'
237-- and stream the result.
239-- >>> S.toList $ foldWords FL.toList (S.fromList "fold these words")
240-- ["fold", "these", "words"]
242-- > foldWords = S.wordsBy isSpace
244{-# INLINE foldWords #-}
245foldWords :: (Monad m, IsStream t) => Fold m Char b -> t m Char -> t m b
246foldWords = S.wordsBy isSpace
248foreign import ccall unsafe "u_iswspace"
249 iswspace :: Int -> Int
251-- | Code copied from base/Data.Char to INLINE it
252{-# INLINE isSpace #-}
253isSpace :: Char -> Bool
255 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
256 | otherwise = iswspace (ord c) /= 0
258 uc = fromIntegral (ord c) :: Word
260-- | Break a string up into a stream of strings at newline characters.
261-- The resulting strings do not contain newlines.
263-- > lines = foldLines A.write
265-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
266-- ["lines","this","string","",""]
268-- If you're dealing with lines of massive length, consider using
269-- 'foldLines' instead to avoid buffering the data in 'Array'.
271lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
272lines = foldLines A.write
274-- | Break a string up into a stream of strings, which were delimited
275-- by characters representing white space.
277-- > words = foldWords A.write
279-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
280-- ["A", "newline", "is", "considered", "white", "space?"]
282-- If you're dealing with words of massive length, consider using
283-- 'foldWords' instead to avoid buffering the data in 'Array'.
285words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
286words = foldWords A.write
288-- | Unfold a stream to character streams using the supplied 'Unfold'
289-- and concat the results suffixing a newline character @\\n@ to each stream.
291{-# INLINE unfoldLines #-}
292unfoldLines :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
293unfoldLines = S.interposeSuffix '\n'
295-- | Flattens the stream of @Array Char@, after appending a terminating
296-- newline to each string.
298-- 'unlines' is an inverse operation to 'lines'.
300-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
301-- "lines\nthis\nstring\n"
303-- > unlines = unfoldLines A.read
305-- Note that, in general
307-- > unlines . lines /= id
308{-# INLINE unlines #-}
309unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
310unlines = unfoldLines A.read
312-- | Unfold the elements of a stream to character streams using the supplied
313-- 'Unfold' and concat the results with a whitespace character infixed between
316{-# INLINE unfoldWords #-}
317unfoldWords :: (MonadIO m, IsStream t) => Unfold m a Char -> t m a -> t m Char
318unfoldWords = S.interpose ' '
320-- | Flattens the stream of @Array Char@, after appending a separating
321-- space to each string.
323-- 'unwords' is an inverse operation to 'words'.
325-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
326-- "unwords this string"
328-- > unwords = unfoldWords A.read
330-- Note that, in general
332-- > unwords . words /= id
333{-# INLINE unwords #-}
334unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
335unwords = unfoldWords A.read
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list