git.iko.soy/streamly
on 3cdecba help sign in
streamly/src/Streamly/String.hs
250 lines 8.6 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- Stream of 'Char' are the equivalent of Haskell Strings but without any
12-- performance issues that are usually associated with the 'String' type.
13-- Haskell lists are not suitable for storage in GC memory. We use streams for
14-- processing and arrays for storing. We can use the usual stream processing
15-- routines on these 'Char' streams. Stream processing is very efficient due to
16-- stream fusion. If strings are to be stored or buffered in memory, they can
17-- be encoded to 'Word8' arrays using the encoding routines provided in the
18-- module. Therefore, a separate type for text representation is not required.
19-- A @Stream Identity Char@ can be used almost as a drop-in replacement for the
20-- standard Haskell @String@, especially when used with @OverloadedStrings@
21-- extension, with little differences.
23-- The 'String' type in this module is just a synonym for the type @List Char@.
24-- It provides better performance compared to the standard Haskell @String@
25-- type and can be used almost as a drop-in replacement, especially when used
26-- with @OverloadedStrings@ extension, with little differences.
27--
28-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
29-- <src/test/PureStreams.hs> for comprehensive usage examples.
31-- TBD write a note on performance comparison with text, bytestrings, lists and
32-- vector.
33--
34module Streamly.String
35 (
36 -- String
38 -- * Encoding and Decoding
39 encodeChar8
40 , encodeChar8Unchecked
41 , decodeChar8
43 , encodeUtf8
44 , decodeUtf8
45{-
46 -- * Unicode aware operations
47 , toCaseFold
48 , toLower
49 , toUpper
50 , toTitle
52 -- * Operations on character strings
53 , strip -- (dropAround isSpace)
54 , stripEnd-}
55 -- * Substrings
56 , stripStart
57 , foldLines
58 , foldWords
59 , lines
60 , words
61 , unlines
62 , unwords
63 )
64where
66import Control.Monad.IO.Class (MonadIO)
67import Data.Char (ord)
68import Data.Word (Word8)
69import GHC.Base (unsafeChr)
70import Streamly (IsStream, MonadAsync)
71import Prelude hiding (String, lines, words, unlines, unwords)
72import Streamly.Fold (Fold)
73import Streamly.Mem.Array (Array, toArray)
75import qualified Streamly.Prelude as S
76import qualified Streamly.Fold as FL
77import qualified Streamly.Mem.Array.Types as A (unlines)
78import qualified Streamly.Mem.Array as A
79import qualified Streamly.Mem.Array.Stream as AS
80import qualified Streamly.Streams.StreamD as D
82-- type String = List Char
84-------------------------------------------------------------------------------
85-- Encoding/Decoding Characters
86-------------------------------------------------------------------------------
88-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
89-- decodeWith = undefined
91-------------------------------------------------------------------------------
92-- Encoding/Decoding Unicode Characters
93-------------------------------------------------------------------------------
95-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
96-- corresponding Unicode 'Char' in 0-255 range.
97{-# INLINE decodeChar8 #-}
98decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
99decodeChar8 = S.map (unsafeChr . fromIntegral)
101-- | Encode a stream of Unicode characters to bytes by mapping each character
102-- to a byte in 0-255 range. Throws an error if the input stream contains
103-- characters beyond 255.
104{-# INLINE encodeChar8 #-}
105encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
106encodeChar8 = S.map convert
107 where
108 convert c =
109 let codepoint = ord c
110 in if codepoint > 255
111 then error $ "Streamly.String.encodeChar8 invalid \
112 \input char codepoint " ++ show codepoint
113 else fromIntegral codepoint
115-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
116-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
117-- when such truncation occurs.
118{-# INLINE encodeChar8Unchecked #-}
119encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
120encodeChar8Unchecked = S.map (fromIntegral . ord)
122-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
123-- The incoming stream is truncated if an invalid codepoint is encountered.
124{-# INLINE decodeUtf8 #-}
125decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
126decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
128-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
129{-# INLINE encodeUtf8 #-}
130encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
131encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
133{-
134-------------------------------------------------------------------------------
135-- Unicode aware operations on strings
136-------------------------------------------------------------------------------
138toCaseFold :: IsStream t => t m Char -> t m Char
139toCaseFold = undefined
141toLower :: IsStream t => t m Char -> t m Char
142toLower = undefined
144toUpper :: IsStream t => t m Char -> t m Char
145toUpper = undefined
147toTitle :: IsStream t => t m Char -> t m Char
148toTitle = undefined
150-------------------------------------------------------------------------------
151-- Utility operations on strings
152-------------------------------------------------------------------------------
154strip :: IsStream t => t m Char -> t m Char
155strip = undefined
157stripEnd :: IsStream t => t m Char -> t m Char
158stripEnd = undefined
159-}
161-- | Remove leading whitespace from a String.
162--
163-- > stripStart = S.dropWhile isSpace
164{-# INLINE stripStart #-}
165stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
166stripStart = S.dropWhile isSpace
168-- | Fold each line of the stream using the supplied Fold
169-- and stream the result.
170--
171-- >>> S.toList $ foldLines (S.fromList "lines\nthis\nstring\n\n\n") FL.toList
172-- ["lines", "this", "string", "", ""]
173{-# INLINE foldLines #-}
174foldLines :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
175foldLines = flip (FL.splitOnSuffix (== '\n'))
177-- | Fold each word of the stream using the supplied Fold
178-- and stream the result.
179--
180-- >>> S.toList $ foldWords (S.fromList "fold these words") FL.toList
181-- ["fold", "these", "words"]
182{-# INLINE foldWords #-}
183foldWords :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
184foldWords = flip (FL.wordsBy isSpace)
186foreign import ccall unsafe "u_iswspace"
187 iswspace :: Int -> Int
189-- | Code copied from base/Data.Char to INLINE it
190{-# INLINE isSpace #-}
191isSpace :: Char -> Bool
192isSpace c
193 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
194 | otherwise = iswspace (ord c) /= 0
195 where
196 uc = fromIntegral (ord c) :: Word
198-- | Break a string up into a list of strings at newline characters.
199-- The resulting strings do not contain newlines.
200--
201-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
202-- ["lines","this","string","",""]
203--
204-- If you're dealing with lines of massive length, consider using
205-- 'foldLines' instead.
206{-# INLINE lines #-}
207lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
208lines = FL.splitOnSuffix (== '\n') toArray
210-- | Break a string up into a list of strings, which were delimited
211-- by characters representing white space.
212--
213-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
214-- ["A", "newline", "is", "considered", "white", "space?"]
215--
216-- If you're dealing with words of massive length, consider using
217-- 'foldWords' instead.
218{-# INLINE words #-}
219words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
220words = FL.wordsBy isSpace toArray
222-- | Flattens the stream of @Array Char@, after appending a terminating
223-- newline to each string.
224--
225-- 'unlines' is an inverse operation to 'lines'.
226--
227-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
228-- "lines\nthis\nstring\n"
229--
230-- Note that, in general
231--
232-- > unlines . lines /= id
233{-# INLINE unlines #-}
234unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
235unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
237-- | Flattens the stream of @Array Char@, after appending a separating
238-- space to each string.
239--
240-- 'unwords' is an inverse operation to 'words'.
241--
242-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
243-- "unwords this string"
244--
245-- Note that, in general
246--
247-- > unwords . words /= id
248{-# INLINE unwords #-}
249unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
250unwords = AS.flatten . (S.intersperse (A.fromList " "))