git.iko.soy/text
on master help sign in
Try a different (apparently unsuccessful) tack with UTF-8 decoding

--HG-- extra : convert_revision : 57472c58c34c921818dc0f9f21e58586bcbf886e

commit2469a55517b4174cfc04395cf9a7f840738781c9 authorBryan O'Sullivan <bos@serpentine.com> dateThu 19 Aug 2010, 19:33 (16 years ago) parente7872c2
3 files changed+67−4
compare with master
unifiedsplit
Data/Text/Encoding/Fusion.hs+41−4 cbits/utf8d.c+25 text.cabal+1
Data/Text/Encoding/Fusion.hs+41−4
@@ -1,4 +1,4 @@
1−{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}
1+{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, Rank2Types #-}
22
33-- |
44-- Module : Data.Text.Encoding.Fusion
@@ -34,16 +34,20 @@ module Data.Text.Encoding.Fusion
3434#if defined(ASSERTS)
3535import Control.Exception (assert)
3636#endif
37+import Data.Bits ((.&.), (.|.))
3738import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)
3839import Data.Text.Fusion (Step(..), Stream(..))
3940import Data.Text.Fusion.Size
4041import Data.Text.Encoding.Error
4142import Data.Text.Encoding.Fusion.Common
43+import Data.Text.Unsafe (inlinePerformIO)
4244import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)
4345import Data.Text.UnsafeShift (shiftL, shiftR)
4446import Data.Word (Word8, Word16, Word32)
4547import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)
46−import Foreign.Storable (pokeByteOff)
48+import Foreign.Ptr (Ptr, plusPtr)
49+import GHC.Ptr (Ptr(..))
50+import Foreign.Storable (peek, pokeByteOff)
4751import System.IO.Unsafe (unsafePerformIO)
4852import qualified Data.ByteString as B
4953import qualified Data.ByteString.Unsafe as B
@@ -65,8 +69,8 @@ streamASCII bs = Stream next 0 (maxSize l)
6569
6670-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using UTF-8
6771-- encoding.
68−streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
69−streamUtf8 onErr bs = Stream next 0 (maxSize l)
72+streamUtf8' :: OnDecodeError -> ByteString -> Stream Char
73+streamUtf8' onErr bs = Stream next 0 (maxSize l)
7074 where
7175 l = B.length bs
7276 next i
@@ -84,6 +88,33 @@ streamUtf8 onErr bs = Stream next 0 (maxSize l)
8488 idx = B.unsafeIndex bs
8589{-# INLINE [0] streamUtf8 #-}
8690
91+accept, reject :: Word32
92+accept = 0
93+reject = 12
94+
95+data S8 = S8 {-# UNPACK #-} !Word32 {-# UNPACK #-} !Int {-# UNPACK #-} !Word32
96+
97+streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
98+streamUtf8 onErr bs = Stream next (S8 accept 0 0) (maxSize l)
99+ where
100+ l = B.length bs
101+ next (S8 state i code)
102+ | i >= l = Done
103+ | state' == accept = Yield (unsafeChr32 code') (s' accept)
104+ | state' /= reject = Skip (s' state')
105+ | otherwise = decodeError "streamUtf8" "UTF-8" onErr
106+ (Just byte) (s' accept)
107+ where s' s = S8 s (i+1) code'
108+ byte = B.unsafeIndex bs i
109+ word = fromIntegral byte
110+ peeku :: Int -> Word32
111+ peeku n = fromIntegral (inlinePerformIO (peek (utf8d `plusPtr` n) :: IO Word8))
112+ !kind = peeku (fromIntegral word)
113+ !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)
114+ | otherwise = (0xff `shiftR` fromIntegral kind) .&. word
115+ !state' = peeku (256 + fromIntegral (state + kind))
116+
117+
87118-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
88119-- endian UTF-16 encoding.
89120streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
@@ -202,3 +233,9 @@ decodeError func kind onErr mb i =
202233 Just c -> Yield c i
203234 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++
204235 kind ++ " stream"
236+
237+utf8d :: Ptr Word8
238+utf8d@(Ptr _) = hs_utf8d
239+{-# NOINLINE utf8d #-}
240+
241+foreign import ccall unsafe "__hs_utf8d" hs_utf8d :: Ptr Word8
cbits/utf8d.cnew file+25
@@ -0,0 +1,25 @@
1+static const unsigned char utf8d[364] = {
2+ // The first part of the table maps bytes to character classes that
3+ // to reduce the size of the transition table and create bitmasks.
4+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
5+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
6+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
7+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
8+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
9+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
10+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
11+ 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
12+
13+ // The second part is a transition table that maps a combination
14+ // of a state of the automaton and a character class to a state.
15+ 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
16+ 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
17+ 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
18+ 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
19+ 12,36,12,12,12,12,12,12,12,12,12,12,
20+};
21+
22+const unsigned char *__hs_utf8d()
23+{
24+ return utf8d;
25+}
text.cabal+1
@@ -33,6 +33,7 @@ flag developer
3333 default: False
3434
3535library
36+ c-sources: cbits/utf8d.c
3637 exposed-modules:
3738 Data.Text
3839 Data.Text.Encoding