git.iko.soy/text
on master help sign in
Decode lazy UTF-8 ByteStrings
0c32368Bryan O'Sullivan
Data/Text/Lazy/Encoding.hs+9−1
compare with master
unifiedsplit
@@ -20,7 +20,7 @@ module Data.Text.Lazy.Encoding
20 (
20 (
21 -- * Decoding ByteStrings to Text
21 -- * Decoding ByteStrings to Text
22 -- decodeASCII
22 -- decodeASCII
23− --, decodeUtf8
23+ decodeUtf8
24 --, decodeUtf16LE
24 --, decodeUtf16LE
25 --, decodeUtf16BE
25 --, decodeUtf16BE
26 --, decodeUtf32LE
26 --, decodeUtf32LE
@@ -35,3 +35,11 @@ module Data.Text.Lazy.Encoding
35 ) where
35 ) where
36
36
37import Data.ByteString.Lazy (ByteString)
37import Data.ByteString.Lazy (ByteString)
38+import Data.Text.Lazy (Text)
39+import qualified Data.Text.Lazy.Fusion as F
40+import qualified Data.Text.Lazy.Encoding.Fusion as E
41+
42+-- | Decode a 'ByteString' containing UTF-8 encoded text.
43+decodeUtf8 :: ByteString -> Text
44+decodeUtf8 bs = F.unstream (E.streamUtf8 bs)
45+{-# INLINE decodeUtf8 #-}
Data/Text/Lazy/Encoding/Fusion.hs+39−2
@@ -15,7 +15,7 @@ module Data.Text.Lazy.Encoding.Fusion
15 (
15 (
16 -- * Streaming
16 -- * Streaming
17 -- streamASCII
17 -- streamASCII
18− --, streamUtf8
18+ streamUtf8
19 --, streamUtf16LE
19 --, streamUtf16LE
20 --, streamUtf16BE
20 --, streamUtf16BE
21 --, streamUtf32LE
21 --, streamUtf32LE
@@ -24,7 +24,44 @@ module Data.Text.Lazy.Encoding.Fusion
24 -- * Unstreaming
24 -- * Unstreaming
25 --, unstream
25 --, unstream
26
26
27− module Data.Text.Encoding.Fusion.Common
27+ , module Data.Text.Encoding.Fusion.Common
28 ) where
28 ) where
29
29
30+import Data.ByteString.Lazy.Internal (ByteString(..))
31+import qualified Data.ByteString as B
32+import qualified Data.ByteString.Unsafe as B
30import Data.Text.Encoding.Fusion.Common
33import Data.Text.Encoding.Fusion.Common
34+import Data.Text.Fusion (Step(..), Stream(..))
35+import Data.Text.Fusion.Internal (PairS(..))
36+import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)
37+import qualified Data.Text.Encoding.Utf8 as U8
38+
39+unknownLength :: Int
40+unknownLength = 4
41+
42+-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8
43+-- encoding.
44+streamUtf8 :: ByteString -> Stream Char
45+streamUtf8 bs0 = Stream next (bs0 :!: 0) unknownLength
46+ where
47+ {-# INLINE next #-}
48+ next (c@(Chunk bs rest) :!: i)
49+ | i >= l = next (rest :!: 0)
50+ | U8.validate1 x1 = Yield (unsafeChr8 x1) (c :!: i+1)
51+ | i+1 < l && U8.validate2 x1 x2 = Yield (U8.chr2 x1 x2) (c :!: i+2)
52+ | i+2 < l && U8.validate3 x1 x2 x3 = Yield (U8.chr3 x1 x2 x3) (c :!: i+3)
53+ | i+3 < l && U8.validate4 x1 x2 x3 x4 = Yield (U8.chr4 x1 x2 x3 x4) (c :!: i+4)
54+ | otherwise = encodingError "UTF-8"
55+ where
56+ x1 = idx i
57+ x2 = idx (i + 1)
58+ x3 = idx (i + 2)
59+ x4 = idx (i + 3)
60+ idx = B.unsafeIndex bs
61+ l = B.length bs
62+ next (Empty :!: _) = Done
63+{-# INLINE [0] streamUtf8 #-}
64+
65+encodingError :: String -> a
66+encodingError encoding =
67+ error $ "Data.Text.Lazy.Encoding.Fusion: Bad " ++ encoding ++ " stream"