@@ -7,13 +7,15 @@
77-- Stability : experimental
88-- Portability : GHC
99--
10−-- Strings are simply character streams. Stream of 'Char' are the equivalent of
11−-- Haskell Strings but without any performance issues that are usually
12−-- associated with the 'String' type. We can use the usual stream processing
13−-- routines in these 'Char' streams. The processing of streams is very
14−-- efficient in terms of CPU and space usage. Streamly employs stream fusion
15−-- which makes all the stream operations extremely fast.
16−--
10+-- Stream of 'Char' are the equivalent of Haskell Strings but without any
11+-- performance issues that are usually associated with the 'String' type.
12+-- Haskell lists are not suitable for storage in GC memory. We use streams for
13+-- processing and arrays for storing. We can use the usual stream processing
14+-- routines on these 'Char' streams. Stream processing is very efficient due to
15+-- stream fusion. If strings are to be stored or buffered in memory, they can
16+-- be encoded to 'Word8' arrays using the encoding routines provided in the
17+-- module. Therefore, a separate type for text representation is not required.
18+
1719-- The 'String' type in this module is just a synonym for the type @List Char@.
1820-- It provides better performance compared to the standard Haskell @String@
1921-- type and can be used almost as a drop-in replacement, especially when used
@@ -26,13 +28,13 @@
2628-- vector.
2729--
2830module Streamly.String
29− ( String
30− -- * Encoding and Decoding
31− -- | Streams of 'Word8' read from a file handle can be decoded into a
32− -- stream of unicode characters, similalrly before we write unicode text to
33− -- a file we can use an encoding routine to convert it to a stream of bytes
34− -- before writing.
31+ (
32+ -- String
3533
34+ -- * Encoding and Decoding
35+ encodeChar8
36+ , encodeChar8Unchecked
37+ , decodeChar8
3638{-
3739 , encodeUtf8
3840 , decodeUtf8
@@ -57,14 +59,18 @@ module Streamly.String
5759 )
5860where
5961
60−-- import Data.Word (Word8)
61−-- import Streamly (IsStream)
62−import Streamly.List (List)
62+import Data.Char (ord)
63+import Data.Word (Word8)
64+import GHC.Base (unsafeChr)
65+import Streamly (IsStream)
66+-- import Streamly.List (List)
6367import Prelude hiding (String, lines, words, unlines, unwords)
6468-- import Streamly.Fold (Fold)
6569-- import Streamly.Array (Array)
6670
67−type String = List Char
71+import qualified Streamly.Prelude as S
72+
73+-- type String = List Char
6874
6975-------------------------------------------------------------------------------
7076-- Encoding/Decoding Characters
@@ -77,6 +83,30 @@ type String = List Char
7783-- Encoding/Decoding Unicode Characters
7884-------------------------------------------------------------------------------
7985
86+-- | Decode a stream of bytes to Unicode characters by mapping each byte to a
87+-- corresponding Unicode 'Char' in 0-255 range.
88+decodeChar8 :: (IsStream t, Monad m) => t m Word8 -> t m Char
89+decodeChar8 = S.map (unsafeChr . fromIntegral)
90+
91+-- | Encode a stream of Unicode characters to bytes by mapping each character
92+-- to a byte in 0-255 range. Throws an error if the input stream contains
93+-- characters beyond 255.
94+encodeChar8 :: (IsStream t, Monad m) => t m Char -> t m Word8
95+encodeChar8 = S.map convert
96+ where
97+ convert c =
98+ let codepoint = ord c
99+ in if codepoint > 255
100+ then error $ "Streamly.String.encodeChar8 invalid \
101+ \input char codepoint " ++ show codepoint
102+ else fromIntegral codepoint
103+
104+-- | Like 'encodeChar8' but silently truncates and maps input characters beyond
105+-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown
106+-- when such truncation occurs.
107+encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8
108+encodeChar8Unchecked = S.map (fromIntegral . ord)
109+
80110{-
81111-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.
82112decodeUtf8 :: IsStream t => t m Word8 -> t m Char