git.iko.soy/text
on master help sign in
Try a different (apparently unsuccessful) tack with UTF-8 decoding
2469a55Bryan O'Sullivan
Data/Text/Encoding/Fusion.hs+41−4
compare with master
unifiedsplit
@@ -1,4 +1,4 @@
1−{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}
1+{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, Rank2Types #-}
2
2
3-- |
3-- |
4-- Module : Data.Text.Encoding.Fusion
4-- Module : Data.Text.Encoding.Fusion
@@ -34,16 +34,20 @@ module Data.Text.Encoding.Fusion
34#if defined(ASSERTS)
34#if defined(ASSERTS)
35import Control.Exception (assert)
35import Control.Exception (assert)
36#endif
36#endif
37+import Data.Bits ((.&.), (.|.))
37import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)
38import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)
38import Data.Text.Fusion (Step(..), Stream(..))
39import Data.Text.Fusion (Step(..), Stream(..))
39import Data.Text.Fusion.Size
40import Data.Text.Fusion.Size
40import Data.Text.Encoding.Error
41import Data.Text.Encoding.Error
41import Data.Text.Encoding.Fusion.Common
42import Data.Text.Encoding.Fusion.Common
43+import Data.Text.Unsafe (inlinePerformIO)
42import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)
44import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)
43import Data.Text.UnsafeShift (shiftL, shiftR)
45import Data.Text.UnsafeShift (shiftL, shiftR)
44import Data.Word (Word8, Word16, Word32)
46import Data.Word (Word8, Word16, Word32)
45import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)
47import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)
48+import Foreign.Ptr (Ptr, plusPtr)
49+import GHC.Ptr (Ptr(..))
46−import Foreign.Storable (pokeByteOff)
50+import Foreign.Storable (peek, pokeByteOff)
47import System.IO.Unsafe (unsafePerformIO)
51import System.IO.Unsafe (unsafePerformIO)
48import qualified Data.ByteString as B
52import qualified Data.ByteString as B
49import qualified Data.ByteString.Unsafe as B
53import qualified Data.ByteString.Unsafe as B
@@ -65,8 +69,8 @@ streamASCII bs = Stream next 0 (maxSize l)
65
69
66-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8
70-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8
67-- encoding.
71-- encoding.
68−streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
72+streamUtf8' :: OnDecodeError -> ByteString -> Stream Char
69−streamUtf8 onErr bs = Stream next 0 (maxSize l)
73+streamUtf8' onErr bs = Stream next 0 (maxSize l)
70 where
74 where
71 l = B.length bs
75 l = B.length bs
72 next i
76 next i
@@ -84,6 +88,33 @@ streamUtf8 onErr bs = Stream next 0 (maxSize l)
84 idx = B.unsafeIndex bs
88 idx = B.unsafeIndex bs
85{-# INLINE [0] streamUtf8 #-}
89{-# INLINE [0] streamUtf8 #-}
86
90
91+accept, reject :: Word32
92+accept = 0
93+reject = 12
94+
95+data S8 = S8 {-# UNPACK #-} !Word32 {-# UNPACK #-} !Int {-# UNPACK #-} !Word32
96+
97+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
98+streamUtf8 onErr bs = Stream next (S8 accept 0 0) (maxSize l)
99+ where
100+ l = B.length bs
101+ next (S8 state i code)
102+ | i >= l = Done
103+ | state' == accept = Yield (unsafeChr32 code') (s' accept)
104+ | state' /= reject = Skip (s' state')
105+ | otherwise = decodeError "streamUtf8" "UTF-8" onErr
106+ (Just byte) (s' accept)
107+ where s' s = S8 s (i+1) code'
108+ byte = B.unsafeIndex bs i
109+ word = fromIntegral byte
110+ peeku :: Int -> Word32
111+ peeku n = fromIntegral (inlinePerformIO (peek (utf8d `plusPtr` n) :: IO Word8))
112+ !kind = peeku (fromIntegral word)
113+ !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)
114+ | otherwise = (0xff `shiftR` fromIntegral kind) .&. word
115+ !state' = peeku (256 + fromIntegral (state + kind))
116+
117+
87-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
118-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
88-- endian UTF-16 encoding.
119-- endian UTF-16 encoding.
89streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
120streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
@@ -202,3 +233,9 @@ decodeError func kind onErr mb i =
202 Just c -> Yield c i
233 Just c -> Yield c i
203 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++
234 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++
204 kind ++ " stream"
235 kind ++ " stream"
236+
237+utf8d :: Ptr Word8
238+utf8d@(Ptr _) = hs_utf8d
239+{-# NOINLINE utf8d #-}
240+
241+foreign import ccall unsafe "__hs_utf8d" hs_utf8d :: Ptr Word8
cbits/utf8d.cnew file+25
@@ -0,0 +1,25 @@
1+static const unsigned char utf8d[364] = {
2+ // The first part of the table maps bytes to character classes that
3+ // to reduce the size of the transition table and create bitmasks.
4+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
5+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
6+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
7+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
8+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
9+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
10+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
11+ 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
12+
13+ // The second part is a transition table that maps a combination
14+ // of a state of the automaton and a character class to a state.
15+ 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
16+ 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
17+ 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
18+ 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
19+ 12,36,12,12,12,12,12,12,12,12,12,12,
20+};
21+
22+const unsigned char *__hs_utf8d()
23+{
24+ return utf8d;
25+}
text.cabal+1
@@ -33,6 +33,7 @@ flag developer
33 default: False
33 default: False
34
34
35library
35library
36+ c-sources: cbits/utf8d.c
36 exposed-modules:
37 exposed-modules:
37 Data.Text
38 Data.Text
38 Data.Text.Encoding
39 Data.Text.Encoding