git.iko.soy/streamly
on 20b204c help sign in
streamly/src/Streamly/String.hs
246 lines 8.3 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- Stream of 'Char' are the equivalent of Haskell Strings but without any
12-- performance issues that are usually associated with the 'String' type.
13-- Haskell lists are not suitable for storage in GC memory. We use streams for
14-- processing and arrays for storing. We can use the usual stream processing
15-- routines on these 'Char' streams. Stream processing is very efficient due to
16-- stream fusion. If strings are to be stored or buffered in memory, they can
17-- be encoded to 'Word8' arrays using the encoding routines provided in the
18-- module. Therefore, a separate type for text representation is not required.
20-- The 'String' type in this module is just a synonym for the type @List Char@.
21-- It provides better performance compared to the standard Haskell @String@
22-- type and can be used almost as a drop-in replacement, especially when used
23-- with @OverloadedStrings@ extension, with little differences.
24--
25-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
26-- <src/test/PureStreams.hs> for comprehensive usage examples.
28-- TBD write a note on performance comparison with text, bytestrings, lists and
29-- vector.
30--
31module Streamly.String
32 (
33 -- String
35 -- * Encoding and Decoding
36 encodeChar8
37 , encodeChar8Unchecked
38 , decodeChar8
40 , encodeUtf8
41 , decodeUtf8
42{-
43 -- * Unicode aware operations
44 , toCaseFold
45 , toLower
46 , toUpper
47 , toTitle
49 -- * Operations on character strings
50 , strip -- (dropAround isSpace)
51 , stripEnd-}
52 -- * Substrings
53 , stripStart
54 , foldLines
55 , foldWords
56 , lines
57 , words
58 , unlines
59 , unwords
60 )
61where
63import Control.Monad.IO.Class (MonadIO)
64import Data.Char (ord)
65import Data.Word (Word8)
66import GHC.Base (unsafeChr)
67import Streamly (IsStream, MonadAsync)
68import Prelude hiding (String, lines, words, unlines, unwords)
69import Streamly.Fold (Fold)
70import Streamly.Mem.Array (Array, toArray)
72import qualified Streamly.Prelude as S
73import qualified Streamly.Fold as FL
74import qualified Streamly.Mem.Array.Types as A (unlines)
75import qualified Streamly.Mem.Array as A
76import qualified Streamly.Streams.StreamD as D
78-- type String = List Char
80-------------------------------------------------------------------------------
81-- Encoding/Decoding Characters
82-------------------------------------------------------------------------------
84-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
85-- decodeWith = undefined
87-------------------------------------------------------------------------------
88-- Encoding/Decoding Unicode Characters
89-------------------------------------------------------------------------------
91-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
92-- corresponding Unicode 'Char' in 0-255 range.
93{-# INLINE decodeChar8 #-}
94decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
95decodeChar8 = S.map (unsafeChr . fromIntegral)
97-- | Encode a stream of Unicode characters to bytes by mapping each character
98-- to a byte in 0-255 range. Throws an error if the input stream contains
99-- characters beyond 255.
100{-# INLINE encodeChar8 #-}
101encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
102encodeChar8 = S.map convert
103 where
104 convert c =
105 let codepoint = ord c
106 in if codepoint > 255
107 then error $ "Streamly.String.encodeChar8 invalid \
108 \input char codepoint " ++ show codepoint
109 else fromIntegral codepoint
111-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
112-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
113-- when such truncation occurs.
114{-# INLINE encodeChar8Unchecked #-}
115encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
116encodeChar8Unchecked = S.map (fromIntegral . ord)
118-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
119-- The incoming stream is truncated if an invalid codepoint is encountered.
120{-# INLINE decodeUtf8 #-}
121decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
122decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
124-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
125{-# INLINE encodeUtf8 #-}
126encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
127encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
129{-
130-------------------------------------------------------------------------------
131-- Unicode aware operations on strings
132-------------------------------------------------------------------------------
134toCaseFold :: IsStream t => t m Char -> t m Char
135toCaseFold = undefined
137toLower :: IsStream t => t m Char -> t m Char
138toLower = undefined
140toUpper :: IsStream t => t m Char -> t m Char
141toUpper = undefined
143toTitle :: IsStream t => t m Char -> t m Char
144toTitle = undefined
146-------------------------------------------------------------------------------
147-- Utility operations on strings
148-------------------------------------------------------------------------------
150strip :: IsStream t => t m Char -> t m Char
151strip = undefined
153stripEnd :: IsStream t => t m Char -> t m Char
154stripEnd = undefined
155-}
157-- | Remove leading whitespace from a String.
158--
159-- > stripStart = S.dropWhile isSpace
160{-# INLINE stripStart #-}
161stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
162stripStart = S.dropWhile isSpace
164-- | Fold each line of the stream using the supplied Fold
165-- and stream the result.
166--
167-- >>> foldLines "lines\nthis\nstring\n\n\n" FL.toList
168-- ["lines", "this", "string"]
169{-# INLINE foldLines #-}
170foldLines :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
171foldLines = flip (FL.splitSuffixBy (== '\n'))
173-- | Fold each word of the stream using the supplied Fold
174-- and stream the result.
175--
176-- >>> foldWords "fold these words" FL.toList
177-- ["fold", "these", "words"]
178{-# INLINE foldWords #-}
179foldWords :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
180foldWords = flip (FL.wordsBy isSpace)
182foreign import ccall unsafe "u_iswspace"
183 iswspace :: Int -> Int
185-- | Code copied from base/Data.Char to INLINE it
186{-# INLINE isSpace #-}
187isSpace :: Char -> Bool
188isSpace c
189 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
190 | otherwise = iswspace (ord c) /= 0
191 where
192 uc = fromIntegral (ord c) :: Word
194-- | Break a string up into a list of strings at newline characters.
195-- The resulting strings do not contain newlines.
196--
197-- >>> lines "lines\nthis\nstring\n\n\n"
198-- ["lines","this","string"]
199--
200-- If you're dealing with lines of massive length, consider using
201-- 'foldLines' instead.
202{-# INLINE lines #-}
203lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
204lines = FL.splitSuffixBy (== '\n') toArray
206-- | Break a string up into a list of strings, which were delimited
207-- by characters representing white space.
208--
209-- >>> words "A newline\nis considered white space?"
210-- ["A", "newline", "is", "considered", "white", "space?"]
211--
212-- If you're dealing with words of massive length, consider using
213-- 'foldWords' instead.
214{-# INLINE words #-}
215words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
216words = FL.wordsBy isSpace toArray
218-- | Flattens the stream of 'Array' 'Char', after appending a terminating
219-- newline to each string.
220--
221-- 'unlines' is an inverse operation to 'lines'.
222--
223-- >>> unlines ["lines", "this", "string"]
224-- "lines\nthis\nstring\n"
225--
226-- Note that, in general
227--
228-- > unlines . lines /= id
229{-# INLINE unlines #-}
230unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
231unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
233-- | Flattens the stream of 'Array' 'Char', after appending a separating
234-- space to each string.
235--
236-- 'unwords' is an inverse operation to 'words'.
237--
238-- >>> unwords ["unwords", "this", "string"]
239-- "unwords this string"
240--
241-- Note that, in general
242--
243-- > unwords . words /= id
244{-# INLINE unwords #-}
245unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
246unwords = A.flattenArrays . (S.intersperse (A.fromList " "))