git.iko.soy/streamly
on 3fcfe4a help sign in
streamly/src/Streamly/String.hs
249 lines 8.6 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- Stream of 'Char' are the equivalent of Haskell Strings but without any
12-- performance issues that are usually associated with the 'String' type.
13-- Haskell lists are not suitable for storage in GC memory. We use streams for
14-- processing and arrays for storing. We can use the usual stream processing
15-- routines on these 'Char' streams. Stream processing is very efficient due to
16-- stream fusion. If strings are to be stored or buffered in memory, they can
17-- be encoded to 'Word8' arrays using the encoding routines provided in the
18-- module. Therefore, a separate type for text representation is not required.
19-- A @Stream Identity Char@ can be used almost as a drop-in replacement for the
20-- standard Haskell @String@, especially when used with @OverloadedStrings@
21-- extension, with little differences.
23-- The 'String' type in this module is just a synonym for the type @List Char@.
24-- It provides better performance compared to the standard Haskell @String@
25-- type and can be used almost as a drop-in replacement, especially when used
26-- with @OverloadedStrings@ extension, with little differences.
27--
28-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
29-- <src/test/PureStreams.hs> for comprehensive usage examples.
31-- TBD write a note on performance comparison with text, bytestrings, lists and
32-- vector.
33--
34module Streamly.String
35 (
36 -- String
38 -- * Encoding and Decoding
39 encodeChar8
40 , encodeChar8Unchecked
41 , decodeChar8
43 , encodeUtf8
44 , decodeUtf8
45{-
46 -- * Unicode aware operations
47 , toCaseFold
48 , toLower
49 , toUpper
50 , toTitle
52 -- * Operations on character strings
53 , strip -- (dropAround isSpace)
54 , stripEnd-}
55 -- * Substrings
56 , stripStart
57 , foldLines
58 , foldWords
59 , lines
60 , words
61 , unlines
62 , unwords
63 )
64where
66import Control.Monad.IO.Class (MonadIO)
67import Data.Char (ord)
68import Data.Word (Word8)
69import GHC.Base (unsafeChr)
70import Streamly (IsStream, MonadAsync)
71import Prelude hiding (String, lines, words, unlines, unwords)
72import Streamly.Fold (Fold)
73import Streamly.Mem.Array (Array, toArray)
75import qualified Streamly.Prelude as S
76import qualified Streamly.Fold as FL
77import qualified Streamly.Mem.Array.Types as A (unlines)
78import qualified Streamly.Mem.Array as A
79import qualified Streamly.Streams.StreamD as D
81-- type String = List Char
83-------------------------------------------------------------------------------
84-- Encoding/Decoding Characters
85-------------------------------------------------------------------------------
87-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
88-- decodeWith = undefined
90-------------------------------------------------------------------------------
91-- Encoding/Decoding Unicode Characters
92-------------------------------------------------------------------------------
94-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
95-- corresponding Unicode 'Char' in 0-255 range.
96{-# INLINE decodeChar8 #-}
97decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
98decodeChar8 = S.map (unsafeChr . fromIntegral)
100-- | Encode a stream of Unicode characters to bytes by mapping each character
101-- to a byte in 0-255 range. Throws an error if the input stream contains
102-- characters beyond 255.
103{-# INLINE encodeChar8 #-}
104encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
105encodeChar8 = S.map convert
106 where
107 convert c =
108 let codepoint = ord c
109 in if codepoint > 255
110 then error $ "Streamly.String.encodeChar8 invalid \
111 \input char codepoint " ++ show codepoint
112 else fromIntegral codepoint
114-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
115-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
116-- when such truncation occurs.
117{-# INLINE encodeChar8Unchecked #-}
118encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
119encodeChar8Unchecked = S.map (fromIntegral . ord)
121-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
122-- The incoming stream is truncated if an invalid codepoint is encountered.
123{-# INLINE decodeUtf8 #-}
124decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
125decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
127-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
128{-# INLINE encodeUtf8 #-}
129encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
130encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
132{-
133-------------------------------------------------------------------------------
134-- Unicode aware operations on strings
135-------------------------------------------------------------------------------
137toCaseFold :: IsStream t => t m Char -> t m Char
138toCaseFold = undefined
140toLower :: IsStream t => t m Char -> t m Char
141toLower = undefined
143toUpper :: IsStream t => t m Char -> t m Char
144toUpper = undefined
146toTitle :: IsStream t => t m Char -> t m Char
147toTitle = undefined
149-------------------------------------------------------------------------------
150-- Utility operations on strings
151-------------------------------------------------------------------------------
153strip :: IsStream t => t m Char -> t m Char
154strip = undefined
156stripEnd :: IsStream t => t m Char -> t m Char
157stripEnd = undefined
158-}
160-- | Remove leading whitespace from a String.
161--
162-- > stripStart = S.dropWhile isSpace
163{-# INLINE stripStart #-}
164stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
165stripStart = S.dropWhile isSpace
167-- | Fold each line of the stream using the supplied Fold
168-- and stream the result.
169--
170-- >>> S.toList $ foldLines (S.fromList "lines\nthis\nstring\n\n\n") FL.toList
171-- ["lines", "this", "string", "", ""]
172{-# INLINE foldLines #-}
173foldLines :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
174foldLines = flip (FL.splitSuffixBy (== '\n'))
176-- | Fold each word of the stream using the supplied Fold
177-- and stream the result.
178--
179-- >>> S.toList $ foldWords (S.fromList "fold these words") FL.toList
180-- ["fold", "these", "words"]
181{-# INLINE foldWords #-}
182foldWords :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
183foldWords = flip (FL.wordsBy isSpace)
185foreign import ccall unsafe "u_iswspace"
186 iswspace :: Int -> Int
188-- | Code copied from base/Data.Char to INLINE it
189{-# INLINE isSpace #-}
190isSpace :: Char -> Bool
191isSpace c
192 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
193 | otherwise = iswspace (ord c) /= 0
194 where
195 uc = fromIntegral (ord c) :: Word
197-- | Break a string up into a list of strings at newline characters.
198-- The resulting strings do not contain newlines.
199--
200-- >>> S.toList $ lines $ S.fromList "lines\nthis\nstring\n\n\n"
201-- ["lines","this","string","",""]
202--
203-- If you're dealing with lines of massive length, consider using
204-- 'foldLines' instead.
205{-# INLINE lines #-}
206lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
207lines = FL.splitSuffixBy (== '\n') toArray
209-- | Break a string up into a list of strings, which were delimited
210-- by characters representing white space.
211--
212-- >>> S.toList $ words $ S.fromList "A newline\nis considered white space?"
213-- ["A", "newline", "is", "considered", "white", "space?"]
214--
215-- If you're dealing with words of massive length, consider using
216-- 'foldWords' instead.
217{-# INLINE words #-}
218words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
219words = FL.wordsBy isSpace toArray
221-- | Flattens the stream of @Array Char@, after appending a terminating
222-- newline to each string.
223--
224-- 'unlines' is an inverse operation to 'lines'.
225--
226-- >>> S.toList $ unlines $ S.fromList ["lines", "this", "string"]
227-- "lines\nthis\nstring\n"
228--
229-- Note that, in general
230--
231-- > unlines . lines /= id
232{-# INLINE unlines #-}
233unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
234unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
236-- | Flattens the stream of @Array Char@, after appending a separating
237-- space to each string.
238--
239-- 'unwords' is an inverse operation to 'words'.
240--
241-- >>> S.toList $ unwords $ S.fromList ["unwords", "this", "string"]
242-- "unwords this string"
243--
244-- Note that, in general
245--
246-- > unwords . words /= id
247{-# INLINE unwords #-}
248unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
249unwords = A.flattenArrays . (S.intersperse (A.fromList " "))