@@ -3009,7 +3009,8 @@ data CodingFailureMode
30093009replacementChar :: Char
30103010replacementChar = '\xFFFD'
30113011
3012−type CodePoint = Word32
3012+-- Int helps in cheaper conversion from Int to Char
3013+type CodePoint = Int
30133014type DecoderState = Word8
30143015
30153016-- See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
@@ -3050,12 +3051,14 @@ decode
30503051 -> Word8
30513052 -> Tuple' DecoderState CodePoint
30523053decode table state codep byte =
3054+ -- Remember codep is Int type!
3055+ -- Can it be unsafe to convert the resulting to Char?
30533056 let !t = table `unsafePeekElemOff` fromIntegral byte
3054− codep' =
3057+ !codep' =
30553058 if state /= 0
30563059 then (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
30573060 else (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
3058− state' = table `unsafePeekElemOff`
3061+ !state' = table `unsafePeekElemOff`
30593062 (256 + fromIntegral state + fromIntegral t)
30603063 in (Tuple' state' codep')
30613064
@@ -3107,7 +3110,7 @@ decodeUtf8With cfm (Stream step state) =
31073110 0 ->
31083111 Skip $
31093112 YieldAndContinue
3110− (unsafeChr (fromIntegral cp))
3113+ (unsafeChr cp)
31113114 (FreshPoint cp sv s)
31123115 _ -> Skip (FreshPoint cp sv s)
31133116 Skip s -> Skip (FreshPoint codepointPtr statePtr s)
@@ -3127,8 +3130,8 @@ decodeUtf8Lenient :: Monad m => Stream m Word8 -> Stream m Char
31273130decodeUtf8Lenient = decodeUtf8With TransliterateCodingFailure
31283131
31293132data FlattenState s a
3130− = OuterLoop !CodePoint !DecoderState s
3131− | InnerLoop !CodePoint !DecoderState s (ForeignPtr a) (Ptr a) (Ptr a)
3133+ = OuterLoop s !CodePoint !DecoderState
3134+ | InnerLoop s (ForeignPtr a) !CodePoint !DecoderState !(Ptr a) !(Ptr a)
31323135 | YAndC !Char (FlattenState s a) -- These constructors can be
31333136 -- encoded in the FreshPoint
31343137 -- type, I prefer to keep these
@@ -3150,7 +3153,7 @@ decodeUtf8ArraysWith ::
31503153decodeUtf8ArraysWith cfm (Stream step state) =
31513154 let Array p _ _ = utf8d
31523155 !ptr = (unsafeForeignPtrToPtr p)
3153− in Stream (step' ptr) (OuterLoop 0 0 state)
3156+ in Stream (step' ptr) (OuterLoop state 0 0)
31543157 where
31553158 {-# INLINE transliterateOrError #-}
31563159 transliterateOrError e s =
@@ -3165,31 +3168,29 @@ decodeUtf8ArraysWith cfm (Stream step state) =
31653168 "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Input Underflow"
31663169 TransliterateCodingFailure -> YAndC replacementChar D
31673170 {-# INLINE_LATE step' #-}
3168− step' _ gst (OuterLoop cp ds st) = do
3171+ step' _ gst (OuterLoop st cp ds) = do
31693172 r <- step (adaptState gst) st
31703173 return $
31713174 case r of
31723175 Yield A.Array {..} s ->
31733176 let p = unsafeForeignPtrToPtr aStart
3174− in Skip (InnerLoop cp ds s aStart p aEnd)
3175− Skip s -> Skip (OuterLoop cp ds s)
3177+ in Skip (InnerLoop s aStart cp ds p aEnd)
3178+ Skip s -> Skip (OuterLoop s cp ds)
31763179 Stop ->
31773180 if ds /= 0
31783181 then Skip inputUnderflow
31793182 else Skip D
3180− step' _ _ (InnerLoop cp ds st _ p end)
3181− | p == end = return $ Skip $ OuterLoop cp ds st
3182− step' table _ (InnerLoop codepointPtr statePtr st startf p end) = do
3183− x <-
3184− liftIO $ do
3185− r <- peek p
3186− touchForeignPtr startf
3187− return r
3183+ step' _ _ (InnerLoop st startf cp ds p end)
3184+ | p == end = do
3185+ liftIO $ touchForeignPtr startf
3186+ return $ Skip $ OuterLoop st cp ds
3187+ step' table _ (InnerLoop st startf codepointPtr statePtr p end) = do
3188+ x <- liftIO $ peek p
31883189 if statePtr == 0 && x <= 0x7f
31893190 then
31903191 return $ Skip $ YAndC
31913192 (unsafeChr (fromIntegral x))
3192− (InnerLoop 0 0 st startf (p `plusPtr` 1) end)
3193+ (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
31933194 else do
31943195 let (Tuple' sv cp) = decode table statePtr codepointPtr x
31953196 return $
@@ -3198,13 +3199,13 @@ decodeUtf8ArraysWith cfm (Stream step state) =
31983199 Skip $
31993200 transliterateOrError
32003201 "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Invalid UTF8 codepoint encountered"
3201− (InnerLoop 0 0 st startf (p `plusPtr` 1) end)
3202+ (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
32023203 0 ->
32033204 Skip $
32043205 YAndC
3205− (unsafeChr (fromIntegral cp))
3206− (InnerLoop cp sv st startf (p `plusPtr` 1) end)
3207− _ -> Skip (InnerLoop cp sv st startf (p `plusPtr` 1) end)
3206+ (unsafeChr cp)
3207+ (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
3208+ _ -> Skip (InnerLoop st startf cp sv (p `plusPtr` 1) end)
32083209 step' _ _ (YAndC c s) = return $ Yield c s
32093210 step' _ _ D = return Stop
32103211