aeade62 · text · git.iko.soy

git.iko.soy/text

on master help sign in

Move the DFA LUT into Haskell

aeade62Bryan O'Sullivan16 years ago

Data/Text/Array.hs+22−3

compare with master

unifiedsplit

@@ -39,8 +39,10 @@ module Data.Text.Array

39 , toList

40 , unsafeFreeze

41 , unsafeIndex

42+ , unsafeIndexWord8

42 , unsafeNew

43 , unsafeNew

43 , unsafeWrite

44 , unsafeWrite

45+ , unsafeWriteWord8

44 ) where

46 ) where

46#if defined(ASSERTS)

48#if defined(ASSERTS)

@@ -60,11 +62,11 @@ import Control.Exception (assert)

60import Data.Bits ((.&.))

62import Data.Bits ((.&.))

61import Data.Text.UnsafeShift (shiftL, shiftR)

63import Data.Text.UnsafeShift (shiftL, shiftR)

62import GHC.Base (ByteArray#, MutableByteArray#, Int(..),

64import GHC.Base (ByteArray#, MutableByteArray#, Int(..),

63− indexWord16Array#, indexWordArray#, newByteArray#,

65+ indexWord16Array#, indexWord8Array#, indexWordArray#, newByteArray#,

64 readWord16Array#, readWordArray#, unsafeCoerce#,

66 readWord16Array#, readWordArray#, unsafeCoerce#,

65− writeWord16Array#, writeWordArray#)

67+ writeWord16Array#, writeWord8Array#, writeWordArray#)

66import GHC.ST (ST(..), runST)

68import GHC.ST (ST(..), runST)

67−import GHC.Word (Word16(..), Word(..))

69+import GHC.Word (Word16(..), Word8(..), Word(..))

68import Prelude hiding (length, read)

70import Prelude hiding (length, read)

70-- | Immutable array type.

72-- | Immutable array type.

@@ -134,6 +136,14 @@ unsafeIndexWord (Array len ba#) i@(I# i#) =

134 case indexWordArray# ba# i# of r# -> (W# r#)

136 case indexWordArray# ba# i# of r# -> (W# r#)

135{-# INLINE unsafeIndexWord #-}

137{-# INLINE unsafeIndexWord #-}

136

138

139+-- | Unchecked read of an immutable array. May return garbage or

140+-- crash on an out-of-bounds access.

141+unsafeIndexWord8 :: Array -> Int -> Word8

142+unsafeIndexWord8 (Array len ba#) i@(I# i#) =

143+ CHECK_BOUNDS("unsafeIndexWord8",len,i)

144+ case indexWord8Array# ba# i# of r# -> (W8# r#)

145+{-# INLINE unsafeIndexWord8 #-}

146+

137-- | Unchecked read of a mutable array. May return garbage or

147-- | Unchecked read of a mutable array. May return garbage or

138-- crash on an out-of-bounds access.

148-- crash on an out-of-bounds access.

139unsafeRead :: MArray s -> Int -> ST s Word16

149unsafeRead :: MArray s -> Int -> ST s Word16

@@ -170,6 +180,15 @@ unsafeWriteWord (MArray len marr#) i@(I# i#) (W# e#) = ST $ \s1# ->

170 s2# -> (# s2#, () #)

180 s2# -> (# s2#, () #)

171{-# INLINE unsafeWriteWord #-}

181{-# INLINE unsafeWriteWord #-}

172

182

183+-- | Unchecked write of a mutable array. May return garbage or crash

184+-- on an out-of-bounds access.

185+unsafeWriteWord8 :: MArray s -> Int -> Word8 -> ST s ()

186+unsafeWriteWord8 (MArray len marr#) i@(I# i#) (W8# e#) = ST $ \s1# ->

187+ CHECK_BOUNDS("unsafeWriteWord8",len,i)

188+ case writeWord8Array# marr# i# e# s1# of

189+ s2# -> (# s2#, () #)

190+{-# INLINE unsafeWriteWord8 #-}

191+

173-- | Convert an immutable array to a list.

192-- | Convert an immutable array to a list.

174toList :: Array -> [Word16]

193toList :: Array -> [Word16]

175toList a = loop 0

194toList a = loop 0

Data/Text/Encoding/Fusion.hs+18−10

@@ -1,4 +1,4 @@

1−{-# LANGUAGE BangPatterns, CPP, ForeignFunctionInterface, Rank2Types #-}

1+{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}

3-- |

4-- Module : Data.Text.Encoding.Fusion

@@ -36,18 +36,17 @@ import Control.Exception (assert)

36#endif

37import Data.Bits ((.&.), (.|.))

38import Data.ByteString.Internal (ByteString(..), mallocByteString, memcpy)

39+import Control.Monad.ST

40+import Data.Text.Array

39import Data.Text.Fusion (Step(..), Stream(..))

41import Data.Text.Fusion (Step(..), Stream(..))

40import Data.Text.Fusion.Size

42import Data.Text.Fusion.Size

41import Data.Text.Encoding.Error

43import Data.Text.Encoding.Error

42import Data.Text.Encoding.Fusion.Common

44import Data.Text.Encoding.Fusion.Common

43−import Data.Text.Unsafe (inlinePerformIO)

44import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)

45import Data.Text.UnsafeChar (unsafeChr, unsafeChr8, unsafeChr32)

45import Data.Text.UnsafeShift (shiftL, shiftR)

46import Data.Text.UnsafeShift (shiftL, shiftR)

46import Data.Word (Word8, Word16, Word32)

47import Data.Word (Word8, Word16, Word32)

47import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)

48import Foreign.ForeignPtr (withForeignPtr, ForeignPtr)

48−import Foreign.Ptr (Ptr, plusPtr)

49−import GHC.Ptr (Ptr(..))

50−import Foreign.Storable (peek, pokeByteOff)

49+import Foreign.Storable (pokeByteOff)

51import System.IO.Unsafe (unsafePerformIO)

50import System.IO.Unsafe (unsafePerformIO)

52import qualified Data.ByteString as B

51import qualified Data.ByteString as B

53import qualified Data.ByteString.Unsafe as B

52import qualified Data.ByteString.Unsafe as B

@@ -108,7 +107,7 @@ streamUtf8 onErr bs = Stream next (S8 accept 0 0) (maxSize l)

108 byte = B.unsafeIndex bs i

107 byte = B.unsafeIndex bs i

109 word = fromIntegral byte

108 word = fromIntegral byte

110 peeku :: Int -> Word32

109 peeku :: Int -> Word32

111− peeku n = fromIntegral (inlinePerformIO (peek (utf8d `plusPtr` n) :: IO Word8))

110+ peeku n = fromIntegral (unsafeIndexWord8 utf8d n)

112 !kind = peeku (fromIntegral word)

111 !kind = peeku (fromIntegral word)

113 !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)

112 !code' | state /= accept = (word .&. 0x3f) .|. (code `shiftL` 6)

114 | otherwise = (0xff `shiftR` fromIntegral kind) .&. word

113 | otherwise = (0xff `shiftR` fromIntegral kind) .&. word

@@ -234,8 +233,17 @@ decodeError func kind onErr mb i =

234 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++

233 where desc = "Data.Text.Encoding.Fusion." ++ func ++ ": Invalid " ++

235 kind ++ " stream"

234 kind ++ " stream"

236

235

237−utf8d :: Ptr Word8

236+utf8d :: Array

238−utf8d@(Ptr _) = hs_utf8d

239{-# NOINLINE utf8d #-}

237{-# NOINLINE utf8d #-}

240−

238+utf8d = runST fill where

241−foreign import ccall unsafe "__hs_utf8d" hs_utf8d :: Ptr Word8

239+ fill = do

240+ ary <- unsafeNew . (`div` 2) . sum . map fst $ xs

241+ mapM_ (uncurry (unsafeWriteWord8 ary))

242+ (zip [0..] (concatMap (uncurry replicate) xs))

243+ unsafeFreeze ary

244+ xs = [(128,0),(16,1),(16,9),(32,7),(2,8),(30,2),(1,10),(12,3),(1,4),(2,3),

245+ (1,11),(3,6),(1,5),(11,8),(1,0),(1,12),(1,24),(1,36),(1,60),(1,96),

246+ (1,84),(3,12),(1,48),(1,72),(13,12),(1,0),(5,12),(1,0),(1,12),(1,0),

247+ (3,12),(1,24),(5,12),(1,24),(1,12),(1,24),(9,12),(1,24),(5,12),(1,24),

248+ (7,12),(1,24),(9,12),(1,36),(1,12),(1,36),(3,12),(1,36),(5,12),(1,36),

249+ (1,12),(1,36),(3,12),(1,36),(10,12)]

cbits/utf8d.cdeleted−25

@@ -1,25 +0,0 @@

1−static const unsigned char utf8d[364] = {

2− // The first part of the table maps bytes to character classes that

3− // to reduce the size of the transition table and create bitmasks.

4− 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

5− 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

6− 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

7− 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,

8− 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,

9− 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,

10− 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,

11− 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,

12−

13− // The second part is a transition table that maps a combination

14− // of a state of the automaton and a character class to a state.

15− 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,

16− 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,

17− 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,

18− 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,

19− 12,36,12,12,12,12,12,12,12,12,12,12,

20−};

21−

22−const unsigned char *__hs_utf8d()

23−{

24− return utf8d;

25−}

text.cabal−1

@@ -33,7 +33,6 @@ flag developer

33 default: False

35library

36− c-sources: cbits/utf8d.c

37 exposed-modules:

36 exposed-modules:

38 Data.Text

37 Data.Text

39 Data.Text.Encoding

38 Data.Text.Encoding