git.iko.soy/streamly
on 54947ea help sign in
streamly/src/Streamly/String.hs
194 lines 6.6 kB history raw
1{-# LANGUAGE FlexibleContexts #-}
2-- |
3-- Module : Streamly.String
4-- Copyright : (c) 2018 Composewell Technologies
5--
6-- License : BSD3
7-- Maintainer : harendra.kumar@gmail.com
8-- Stability : experimental
9-- Portability : GHC
10--
11-- Stream of 'Char' are the equivalent of Haskell Strings but without any
12-- performance issues that are usually associated with the 'String' type.
13-- Haskell lists are not suitable for storage in GC memory. We use streams for
14-- processing and arrays for storing. We can use the usual stream processing
15-- routines on these 'Char' streams. Stream processing is very efficient due to
16-- stream fusion. If strings are to be stored or buffered in memory, they can
17-- be encoded to 'Word8' arrays using the encoding routines provided in the
18-- module. Therefore, a separate type for text representation is not required.
20-- The 'String' type in this module is just a synonym for the type @List Char@.
21-- It provides better performance compared to the standard Haskell @String@
22-- type and can be used almost as a drop-in replacement, especially when used
23-- with @OverloadedStrings@ extension, with little differences.
24--
25-- See "Streamly.List", <src/docs/streamly-vs-lists.md> for more details and
26-- <src/test/PureStreams.hs> for comprehensive usage examples.
28-- TBD write a note on performance comparison with text, bytestrings, lists and
29-- vector.
30--
31module Streamly.String
32 (
33 -- String
35 -- * Encoding and Decoding
36 encodeChar8
37 , encodeChar8Unchecked
38 , decodeChar8
40 , encodeUtf8
41 , decodeUtf8
42{-
43 -- * Unicode aware operations
44 , toCaseFold
45 , toLower
46 , toUpper
47 , toTitle
49 -- * Operations on character strings
50 , strip -- (dropAround isSpace)
51 , stripEnd-}
52 , stripStart
53 , foldLines
54 , foldWords
55 , lines
56 , words
57 , unlines
58 , unwords
59 )
60where
62import Control.Monad.IO.Class (MonadIO)
63import Data.Char (ord)
64import Data.Word (Word8)
65import GHC.Base (unsafeChr)
66import Streamly (IsStream, MonadAsync)
67import Prelude hiding (String, lines, words, unlines, unwords)
68import Streamly.Fold (Fold)
69import Streamly.Mem.Array (Array, toArray)
71import qualified Streamly.Prelude as S
72import qualified Streamly.Fold as FL
73import qualified Streamly.Mem.Array.Types as A (unlines)
74import qualified Streamly.Mem.Array as A
75import qualified Streamly.Streams.StreamD as D
77-- type String = List Char
79-------------------------------------------------------------------------------
80-- Encoding/Decoding Characters
81-------------------------------------------------------------------------------
83-- decodeWith :: TextEncoding -> t m Word8 -> t m Char
84-- decodeWith = undefined
86-------------------------------------------------------------------------------
87-- Encoding/Decoding Unicode Characters
88-------------------------------------------------------------------------------
90-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
91-- corresponding Unicode 'Char' in 0-255 range.
92{-# INLINE decodeChar8 #-}
93decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
94decodeChar8 = S.map (unsafeChr . fromIntegral)
96-- | Encode a stream of Unicode characters to bytes by mapping each character
97-- to a byte in 0-255 range. Throws an error if the input stream contains
98-- characters beyond 255.
99{-# INLINE encodeChar8 #-}
100encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
101encodeChar8 = S.map convert
102 where
103 convert c =
104 let codepoint = ord c
105 in if codepoint > 255
106 then error $ "Streamly.String.encodeChar8 invalid \
107 \input char codepoint " ++ show codepoint
108 else fromIntegral codepoint
110-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
111-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
112-- when such truncation occurs.
113{-# INLINE encodeChar8Unchecked #-}
114encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
115encodeChar8Unchecked = S.map (fromIntegral . ord)
117-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
118-- The incoming stream is truncated if an invalid codepoint is encountered.
119{-# INLINE decodeUtf8 #-}
120decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char
121decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD
123-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.
124{-# INLINE encodeUtf8 #-}
125encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8
126encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD
128{-
129-------------------------------------------------------------------------------
130-- Unicode aware operations on strings
131-------------------------------------------------------------------------------
133toCaseFold :: IsStream t => t m Char -> t m Char
134toCaseFold = undefined
136toLower :: IsStream t => t m Char -> t m Char
137toLower = undefined
139toUpper :: IsStream t => t m Char -> t m Char
140toUpper = undefined
142toTitle :: IsStream t => t m Char -> t m Char
143toTitle = undefined
145-------------------------------------------------------------------------------
146-- Utility operations on strings
147-------------------------------------------------------------------------------
149strip :: IsStream t => t m Char -> t m Char
150strip = undefined
152stripEnd :: IsStream t => t m Char -> t m Char
153stripEnd = undefined
154-}
156{-# INLINE stripStart #-}
157stripStart :: (Monad m, IsStream t) => t m Char -> t m Char
158stripStart = S.dropWhile isSpace
160{-# INLINE foldLines #-}
161foldLines :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
162foldLines = flip (FL.splitSuffixBy (== '\n'))
164{-# INLINE foldWords #-}
165foldWords :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b
166foldWords = flip (FL.wordsBy isSpace)
168foreign import ccall unsafe "u_iswspace"
169 iswspace :: Int -> Int
171-- Code copied from base/Data.Char to INLINE it
172{-# INLINE isSpace #-}
173isSpace :: Char -> Bool
174isSpace c
175 | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0
176 | otherwise = iswspace (ord c) /= 0
177 where
178 uc = fromIntegral (ord c) :: Word
180{-# INLINE lines #-}
181lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
182lines = FL.splitSuffixBy (== '\n') toArray
184{-# INLINE words #-}
185words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)
186words = FL.wordsBy isSpace toArray
188{-# INLINE unlines #-}
189unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char
190unlines = D.fromStreamD . A.unlines '\n' . D.toStreamD
192{-# INLINE unwords #-}
193unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char
194unwords = A.flattenArrays . (S.intersperse (A.fromList " "))