@@ -3049,27 +3049,41 @@ utf8d = unsafePerformIO $ runFold (A.writeAligned 64) $ fromList [
30493049unsafePeekElemOff :: forall a. Storable a => Ptr a -> Int -> a
30503050unsafePeekElemOff p i = let !x = A.unsafeInlineIO $ peekElemOff p i in x
30513051
3052−{-# INLINE decode #-}
3053−decode
3052+-- decode is split into two separate cases to avoid branching instructions.
3053+-- From the higher level flow we already know which case we are in so we can
3054+-- call the appropriate decode function.
3055+--
3056+-- When the state is 0
3057+{-# INLINE decode0 #-}
3058+decode0 :: Ptr Word8 -> Word8 -> Tuple' DecoderState CodePoint
3059+decode0 table byte =
3060+ let !t = table `unsafePeekElemOff` fromIntegral byte
3061+ !codep' = (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
3062+ !state' = table `unsafePeekElemOff` (256 + fromIntegral t)
3063+ in (Tuple' state' codep')
3064+
3065+-- When the state is not 0
3066+{-# INLINE decode1 #-}
3067+decode1
30543068 :: Ptr Word8
30553069 -> DecoderState
30563070 -> CodePoint
30573071 -> Word8
30583072 -> Tuple' DecoderState CodePoint
3059−decode table state codep byte =
3073+decode1 table state codep byte =
30603074 -- Remember codep is Int type!
30613075 -- Can it be unsafe to convert the resulting to Char?
30623076 let !t = table `unsafePeekElemOff` fromIntegral byte
3063− !codep' =
3064− if state /= 0
3065− then (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
3066− else (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
3077+ !codep' = (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
30673078 !state' = table `unsafePeekElemOff`
30683079 (256 + fromIntegral state + fromIntegral t)
30693080 in (Tuple' state' codep')
30703081
3082+
30713083data FreshPoint s
3072− = FreshPoint !CodePoint !DecoderState s
3084+ = FreshPointDecodeInit s
3085+ | FreshPointDecodeFirst s Word8
3086+ | FreshPointDecoding s !DecoderState !CodePoint
30733087 | YieldAndContinue !Char (FreshPoint s)
30743088 | Done
30753089
@@ -3079,7 +3093,7 @@ decodeUtf8With :: Monad m => CodingFailureMode -> Stream m Word8 -> Stream m Cha
30793093decodeUtf8With cfm (Stream step state) =
30803094 let Array p _ _ = utf8d
30813095 !ptr = (unsafeForeignPtrToPtr p)
3082− in Stream (step' ptr) (FreshPoint 0 0 state)
3096+ in Stream (step' ptr) (FreshPointDecodeInit state)
30833097 where
30843098 {-# INLINE transliterateOrError #-}
30853099 transliterateOrError e s =
@@ -3093,37 +3107,57 @@ decodeUtf8With cfm (Stream step state) =
30933107 error "Streamly.Streams.StreamD.decodeUtf8With: Input Underflow"
30943108 TransliterateCodingFailure -> YieldAndContinue replacementChar Done
30953109 {-# INLINE_LATE step' #-}
3096− step' table gst (FreshPoint codepointPtr statePtr st) = do
3110+ step' _ gst (FreshPointDecodeInit st) = do
30973111 r <- step (adaptState gst) st
3098− return $
3099− case r of
3100− Yield x s ->
3101− if statePtr == 0 && x <= 0x7f
3102− then
3103− Skip $
3104− YieldAndContinue
3112+ case r of
3113+ Yield x s ->
3114+ -- Note: It is important to use a ">" instead of a "<=" test
3115+ -- here for GHC to generate code layout for default branch
3116+ -- prediction for the common case. This is fragile and might
3117+ -- change with the compiler versions, we need a more reliable
3118+ -- "likely" primitive to control branch predication.
3119+ case x > 0x7f of
3120+ False ->
3121+ return $ Skip $ YieldAndContinue
31053122 (unsafeChr (fromIntegral x))
3106− (FreshPoint 0 0 s)
3107− else
3108− let (Tuple' sv cp) =
3109− decode table statePtr codepointPtr x
3110− in case sv of
3111− 12 ->
3112− Skip $
3113− transliterateOrError
3114− "Streamly.Streams.StreamD.decodeUtf8With: Invalid UTF8 codepoint encountered"
3115− (FreshPoint 0 0 s)
3116− 0 ->
3117− Skip $
3118− YieldAndContinue
3119− (unsafeChr cp)
3120− (FreshPoint cp sv s)
3121− _ -> Skip (FreshPoint cp sv s)
3122− Skip s -> Skip (FreshPoint codepointPtr statePtr s)
3123− Stop ->
3124− if statePtr /= 0
3125− then Skip inputUnderflow
3126− else Skip Done
3123+ (FreshPointDecodeInit s)
3124+ -- Using a separate state here generates a jump to a
3125+ -- separate code block in the core which seems to perform
3126+ -- slightly better for the non-ascii case.
3127+ True -> return $ Skip $ FreshPointDecodeFirst s x
3128+ Skip s -> return $ Skip (FreshPointDecodeInit s)
3129+ Stop -> return $ Skip Done
3130+
3131+ step' table _ (FreshPointDecodeFirst st x) = do
3132+ let (Tuple' sv cp) = decode0 table x
3133+ return $
3134+ case sv of
3135+ 12 ->
3136+ Skip $
3137+ transliterateOrError
3138+ "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Invalid UTF8 codepoint encountered"
3139+ (FreshPointDecodeInit st)
3140+ 0 -> error "unreachable state"
3141+ _ -> Skip (FreshPointDecoding st sv cp)
3142+
3143+ step' table gst (FreshPointDecoding st statePtr codepointPtr) = do
3144+ r <- step (adaptState gst) st
3145+ case r of
3146+ Yield x s -> do
3147+ let (Tuple' sv cp) = decode1 table statePtr codepointPtr x
3148+ return $
3149+ case sv of
3150+ 0 -> Skip $ YieldAndContinue (unsafeChr cp)
3151+ (FreshPointDecodeInit s)
3152+ 12 ->
3153+ Skip $
3154+ transliterateOrError
3155+ "Streamly.Streams.StreamD.decodeUtf8With: Invalid UTF8 codepoint encountered"
3156+ (FreshPointDecodeInit s)
3157+ _ -> Skip (FreshPointDecoding s sv cp)
3158+ Skip s -> return $ Skip (FreshPointDecoding s statePtr codepointPtr)
3159+ Stop -> return $ Skip inputUnderflow
3160+
31273161 step' _ _ (YieldAndContinue c s) = return $ Yield c s
31283162 step' _ _ Done = return Stop
31293163
@@ -3136,8 +3170,11 @@ decodeUtf8Lenient :: Monad m => Stream m Word8 -> Stream m Char
31363170decodeUtf8Lenient = decodeUtf8With TransliterateCodingFailure
31373171
31383172data FlattenState s a
3139− = OuterLoop s !CodePoint !DecoderState
3140− | InnerLoop s (ForeignPtr a) !CodePoint !DecoderState !(Ptr a) !(Ptr a)
3173+ = OuterLoop s !(Maybe (DecoderState, CodePoint))
3174+ | InnerLoopDecodeInit s (ForeignPtr a) !(Ptr a) !(Ptr a)
3175+ | InnerLoopDecodeFirst s (ForeignPtr a) !(Ptr a) !(Ptr a) Word8
3176+ | InnerLoopDecoding s (ForeignPtr a) !(Ptr a) !(Ptr a)
3177+ !DecoderState !CodePoint
31413178 | YAndC !Char (FlattenState s a) -- These constructors can be
31423179 -- encoded in the FreshPoint
31433180 -- type, I prefer to keep these
@@ -3159,7 +3196,7 @@ decodeUtf8ArraysWith ::
31593196decodeUtf8ArraysWith cfm (Stream step state) =
31603197 let Array p _ _ = utf8d
31613198 !ptr = (unsafeForeignPtrToPtr p)
3162− in Stream (step' ptr) (OuterLoop state 0 0)
3199+ in Stream (step' ptr) (OuterLoop state Nothing)
31633200 where
31643201 {-# INLINE transliterateOrError #-}
31653202 transliterateOrError e s =
@@ -3174,44 +3211,76 @@ decodeUtf8ArraysWith cfm (Stream step state) =
31743211 "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Input Underflow"
31753212 TransliterateCodingFailure -> YAndC replacementChar D
31763213 {-# INLINE_LATE step' #-}
3177− step' _ gst (OuterLoop st cp ds) = do
3214+ step' _ gst (OuterLoop st Nothing) = do
31783215 r <- step (adaptState gst) st
31793216 return $
31803217 case r of
31813218 Yield A.Array {..} s ->
31823219 let p = unsafeForeignPtrToPtr aStart
3183− in Skip (InnerLoop s aStart cp ds p aEnd)
3184− Skip s -> Skip (OuterLoop s cp ds)
3185− Stop ->
3186− if ds /= 0
3187− then Skip inputUnderflow
3188− else Skip D
3189− step' _ _ (InnerLoop st startf cp ds p end)
3220+ in Skip (InnerLoopDecodeInit s aStart p aEnd)
3221+ Skip s -> Skip (OuterLoop s Nothing)
3222+ Stop -> Skip D
3223+ step' _ gst (OuterLoop st dst@(Just (ds, cp))) = do
3224+ r <- step (adaptState gst) st
3225+ return $
3226+ case r of
3227+ Yield A.Array {..} s ->
3228+ let p = unsafeForeignPtrToPtr aStart
3229+ in Skip (InnerLoopDecoding s aStart p aEnd ds cp)
3230+ Skip s -> Skip (OuterLoop s dst)
3231+ Stop -> Skip inputUnderflow
3232+ step' _ _ (InnerLoopDecodeInit st startf p end)
3233+ | p == end = do
3234+ liftIO $ touchForeignPtr startf
3235+ return $ Skip $ OuterLoop st Nothing
3236+ step' _ _ (InnerLoopDecodeInit st startf p end) = do
3237+ x <- liftIO $ peek p
3238+ -- Note: It is important to use a ">" instead of a "<=" test here for
3239+ -- GHC to generate code layout for default branch prediction for the
3240+ -- common case. This is fragile and might change with the compiler
3241+ -- versions, we need a more reliable "likely" primitive to control
3242+ -- branch predication.
3243+ case x > 0x7f of
3244+ False ->
3245+ return $ Skip $ YAndC
3246+ (unsafeChr (fromIntegral x))
3247+ (InnerLoopDecodeInit st startf (p `plusPtr` 1) end)
3248+ -- Using a separate state here generates a jump to a separate code
3249+ -- block in the core which seems to perform slightly better for the
3250+ -- non-ascii case.
3251+ True -> return $ Skip $ InnerLoopDecodeFirst st startf p end x
3252+
3253+ step' table _ (InnerLoopDecodeFirst st startf p end x) = do
3254+ let (Tuple' sv cp) = decode0 table x
3255+ return $
3256+ case sv of
3257+ 12 ->
3258+ Skip $
3259+ transliterateOrError
3260+ "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Invalid UTF8 codepoint encountered"
3261+ (InnerLoopDecodeInit st startf (p `plusPtr` 1) end)
3262+ 0 -> error "unreachable state"
3263+ _ -> Skip (InnerLoopDecoding st startf (p `plusPtr` 1) end sv cp)
3264+ step' _ _ (InnerLoopDecoding st startf p end sv cp)
31903265 | p == end = do
31913266 liftIO $ touchForeignPtr startf
3192− return $ Skip $ OuterLoop st cp ds
3193− step' table _ (InnerLoop st startf codepointPtr statePtr p end) = do
3267+ return $ Skip $ OuterLoop st (Just (sv, cp))
3268+ step' table _ (InnerLoopDecoding st startf p end statePtr codepointPtr) = do
31943269 x <- liftIO $ peek p
3195− if statePtr == 0 && x <= 0x7f
3196− then
3197− return $ Skip $ YAndC
3198− (unsafeChr (fromIntegral x))
3199− (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
3200− else do
3201− let (Tuple' sv cp) = decode table statePtr codepointPtr x
3202− return $
3203− case sv of
3204− 12 ->
3205− Skip $
3206− transliterateOrError
3207− "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Invalid UTF8 codepoint encountered"
3208− (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
3209− 0 ->
3210− Skip $
3211− YAndC
3212− (unsafeChr cp)
3213− (InnerLoop st startf 0 0 (p `plusPtr` 1) end)
3214− _ -> Skip (InnerLoop st startf cp sv (p `plusPtr` 1) end)
3270+ let (Tuple' sv cp) = decode1 table statePtr codepointPtr x
3271+ return $
3272+ case sv of
3273+ 0 ->
3274+ Skip $
3275+ YAndC
3276+ (unsafeChr cp)
3277+ (InnerLoopDecodeInit st startf (p `plusPtr` 1) end)
3278+ 12 ->
3279+ Skip $
3280+ transliterateOrError
3281+ "Streamly.Streams.StreamD.decodeUtf8ArraysWith: Invalid UTF8 codepoint encountered"
3282+ (InnerLoopDecodeInit st startf (p `plusPtr` 1) end)
3283+ _ -> Skip (InnerLoopDecoding st startf (p `plusPtr` 1) end sv cp)
32153284 step' _ _ (YAndC c s) = return $ Yield c s
32163285 step' _ _ D = return Stop
32173286