ef77903 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

pass the utf-8 lookup table instead of using it as global

ef77903Harendra Kumar7 years ago

src/Streamly/Streams/StreamD.hs+32−16

compare with master

unifiedsplit

@@ -3036,15 +3036,26 @@ utf8d = A.fromList [

3036 12,36,12,12,12,12,12,12,12,12,12,12

3037 ]

3038

3039+-- | Return element at the specified index without checking the bounds.

3040+-- and without touching the foreign ptr.

3041+{-# INLINE_NORMAL unsafePeekElemOff #-}

3042+unsafePeekElemOff :: forall a. Storable a => Ptr a -> Int -> a

3043+unsafePeekElemOff p i = let !x = A.unsafeInlineIO $ peekElemOff p i in x

3044+

3039{-# INLINE decode #-}

3045{-# INLINE decode #-}

3046+decode

3047+ :: Ptr Word8

3048+ -> DecoderState

3049+ -> CodePoint

3050+ -> Word8

3040−decode :: DecoderState -> CodePoint -> Word8 -> Tuple' DecoderState CodePoint

3051+ -> Tuple' DecoderState CodePoint

3041−decode state codep byte =

3052+decode table state codep byte =

3042− let t = utf8d `A.unsafeIndex` fromIntegral byte

3053+ let !t = table `unsafePeekElemOff` fromIntegral byte

3043 codep' =

3054 codep' =

3044 if state /= 0

3055 if state /= 0

3045 then (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)

3056 then (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)

3046 else (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte

3057 else (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte

3047− state' = utf8d `A.unsafeIndex`

3058+ state' = table `unsafePeekElemOff`

3048 (256 + fromIntegral state + fromIntegral t)

3059 (256 + fromIntegral state + fromIntegral t)

3049 in (Tuple' state' codep')

3060 in (Tuple' state' codep')

3050

3061

@@ -3056,7 +3067,10 @@ data FreshPoint s

3056-- XXX Add proper error messages

3067-- XXX Add proper error messages

3057{-# INLINE_NORMAL decodeUtf8With #-}

3068{-# INLINE_NORMAL decodeUtf8With #-}

3058decodeUtf8With :: Monad m => CodingFailureMode -> Stream m Word8 -> Stream m Char

3069decodeUtf8With :: Monad m => CodingFailureMode -> Stream m Word8 -> Stream m Char

3059−decodeUtf8With cfm (Stream step state) = Stream step' (FreshPoint 0 0 state)

3070+decodeUtf8With cfm (Stream step state) =

3071+ let Array p _ _ = utf8d

3072+ !ptr = (unsafeForeignPtrToPtr p)

3073+ in Stream (step' ptr) (FreshPoint 0 0 state)

3060 where

3074 where

3061 {-# INLINE transliterateOrError #-}

3075 {-# INLINE transliterateOrError #-}

3062 transliterateOrError e s =

3076 transliterateOrError e s =

@@ -3070,7 +3084,7 @@ decodeUtf8With cfm (Stream step state) = Stream step' (FreshPoint 0 0 state)

3070 error "Streamly.Streams.StreamD.decodeUtf8With: Input Underflow"

3084 error "Streamly.Streams.StreamD.decodeUtf8With: Input Underflow"

3071 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

3085 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

3072 {-# INLINE_LATE step' #-}

3086 {-# INLINE_LATE step' #-}

3073− step' gst (FreshPoint codepointPtr statePtr st) = do

3087+ step' table gst (FreshPoint codepointPtr statePtr st) = do

3074 r <- step (adaptState gst) st

3088 r <- step (adaptState gst) st

3075 return $

3089 return $

3076 case r of

3090 case r of

@@ -3083,7 +3097,7 @@ decodeUtf8With cfm (Stream step state) = Stream step' (FreshPoint 0 0 state)

3083 (FreshPoint 0 0 s)

3097 (FreshPoint 0 0 s)

3084 else

3098 else

3085 let (Tuple' sv cp) =

3099 let (Tuple' sv cp) =

3086− decode statePtr codepointPtr x

3100+ decode table statePtr codepointPtr x

3087 in case sv of

3101 in case sv of

3088 12 ->

3102 12 ->

3089 Skip $

3103 Skip $

@@ -3101,8 +3115,8 @@ decodeUtf8With cfm (Stream step state) = Stream step' (FreshPoint 0 0 state)

3101 if statePtr /= 0

3115 if statePtr /= 0

3102 then Skip inputUnderflow

3116 then Skip inputUnderflow

3103 else Skip Done

3117 else Skip Done

3104− step' _ (YieldAndContinue c s) = return $ Yield c s

3118+ step' _ _ (YieldAndContinue c s) = return $ Yield c s

3105− step' _ Done = return Stop

3119+ step' _ _ Done = return Stop

3106

3120

3107{-# INLINE decodeUtf8 #-}

3121{-# INLINE decodeUtf8 #-}

3108decodeUtf8 :: Monad m => Stream m Word8 -> Stream m Char

3122decodeUtf8 :: Monad m => Stream m Word8 -> Stream m Char

@@ -3134,7 +3148,9 @@ decodeUtf8ArraysWith ::

3134 -> Stream m (A.Array Word8)

3148 -> Stream m (A.Array Word8)

3135 -> Stream m Char

3149 -> Stream m Char

3136decodeUtf8ArraysWith cfm (Stream step state) =

3150decodeUtf8ArraysWith cfm (Stream step state) =

3151+ let Array p _ _ = utf8d

3152+ !ptr = (unsafeForeignPtrToPtr p)

3137− Stream step' (OuterLoop 0 0 state)

3153+ in Stream (step' ptr) (OuterLoop 0 0 state)

3138 where

3154 where

3139 {-# INLINE transliterateOrError #-}

3155 {-# INLINE transliterateOrError #-}

3140 transliterateOrError e s =

3156 transliterateOrError e s =

@@ -3149,7 +3165,7 @@ decodeUtf8ArraysWith cfm (Stream step state) =

3149 "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Input Underflow"

3165 "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Input Underflow"

3150 TransliterateCodingFailure -> YAndC replacementChar D

3166 TransliterateCodingFailure -> YAndC replacementChar D

3151 {-# INLINE_LATE step' #-}

3167 {-# INLINE_LATE step' #-}

3152− step' gst (OuterLoop cp ds st) = do

3168+ step' _ gst (OuterLoop cp ds st) = do

3153 r <- step (adaptState gst) st

3169 r <- step (adaptState gst) st

3154 return $

3170 return $

3155 case r of

3171 case r of

@@ -3161,9 +3177,9 @@ decodeUtf8ArraysWith cfm (Stream step state) =

3161 if ds /= 0

3177 if ds /= 0

3162 then Skip inputUnderflow

3178 then Skip inputUnderflow

3163 else Skip D

3179 else Skip D

3164− step' _ (InnerLoop cp ds st _ p end)

3180+ step' _ _ (InnerLoop cp ds st _ p end)

3165 | p == end = return $ Skip $ OuterLoop cp ds st

3181 | p == end = return $ Skip $ OuterLoop cp ds st

3166− step' _ (InnerLoop codepointPtr statePtr st startf p end) = do

3182+ step' table _ (InnerLoop codepointPtr statePtr st startf p end) = do

3167 x <-

3183 x <-

3168 liftIO $ do

3184 liftIO $ do

3169 r <- peek p

3185 r <- peek p

@@ -3175,7 +3191,7 @@ decodeUtf8ArraysWith cfm (Stream step state) =

3175 (unsafeChr (fromIntegral x))

3191 (unsafeChr (fromIntegral x))

3176 (InnerLoop 0 0 st startf (p `plusPtr` 1) end)

3192 (InnerLoop 0 0 st startf (p `plusPtr` 1) end)

3177 else do

3193 else do

3178− let (Tuple' sv cp) = decode statePtr codepointPtr x

3194+ let (Tuple' sv cp) = decode table statePtr codepointPtr x

3179 return $

3195 return $

3180 case sv of

3196 case sv of

3181 12 ->

3197 12 ->

@@ -3189,8 +3205,8 @@ decodeUtf8ArraysWith cfm (Stream step state) =

3189 (unsafeChr (fromIntegral cp))

3205 (unsafeChr (fromIntegral cp))

3190 (InnerLoop cp sv st startf (p `plusPtr` 1) end)

3206 (InnerLoop cp sv st startf (p `plusPtr` 1) end)

3191 _ -> Skip (InnerLoop cp sv st startf (p `plusPtr` 1) end)

3207 _ -> Skip (InnerLoop cp sv st startf (p `plusPtr` 1) end)

3192− step' _ (YAndC c s) = return $ Yield c s

3208+ step' _ _ (YAndC c s) = return $ Yield c s

3193− step' _ D = return Stop

3209+ step' _ _ D = return Stop

3194

3210

3195{-# INLINE decodeUtf8Arrays #-}

3211{-# INLINE decodeUtf8Arrays #-}

3196decodeUtf8Arrays ::

3212decodeUtf8Arrays ::