@@ -299,6 +299,8 @@ import Foreign.Ptr (Ptr)
299299import Foreign.Storable (Storable(..))
300300import GHC.Base (assert, Char(..), unsafeChr, ord)
301301import GHC.IO.Encoding.Failure (isSurrogate)
302+import GHC.ForeignPtr (ForeignPtr (..))
303+import GHC.Ptr (Ptr (..))
302304import GHC.Types (SPEC(..))
303305import GHC.Word (Word8(..))
304306import System.IO.Unsafe (unsafePerformIO)
@@ -327,7 +329,7 @@ import qualified Streamly.Streams.StreamK as K
327329
328330import Foreign.Ptr (plusPtr)
329331import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
330−import Foreign.ForeignPtr (ForeignPtr, touchForeignPtr)
332+import Foreign.ForeignPtr (touchForeignPtr)
331333
332334------------------------------------------------------------------------------
333335-- Construction
@@ -3484,13 +3486,17 @@ decode0 table byte =
34843486 !codep' = (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
34853487 !state' = table `unsafePeekElemOff` (256 + fromIntegral t)
34863488 in assert ((byte > 0x7f || error showByte)
3487− && (state' /= 0 || error (showByte ++ showState state')))
3489+ && (state' /= 0 || error (showByte ++ showTable)))
34883490 (Tuple' state' codep')
34893491
34903492 where
34913493
3492− showByte = "decode0: Invalid byte: " ++ show byte
3493− showState st = " state: " ++ show st ++ " table: " ++ show table
3494+ utf8table =
3495+ let !(Ptr addr) = table
3496+ end = table `plusPtr` 364
3497+ in A.Array (ForeignPtr addr undefined) end end :: A.Array Word8
3498+ showByte = "Streamly: decode0: byte: " ++ show byte
3499+ showTable = " table: " ++ show utf8table
34943500
34953501-- When the state is not 0
34963502{-# INLINE decode1 #-}
@@ -3502,12 +3508,25 @@ decode1
35023508 -> Tuple' DecodeState CodePoint
35033509decode1 table state codep byte =
35043510 -- Remember codep is Int type!
3505− -- Can it be unsafe to convert the resulting to Char?
3511+ -- Can it be unsafe to convert the resulting Int to Char?
35063512 let !t = table `unsafePeekElemOff` fromIntegral byte
35073513 !codep' = (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
35083514 !state' = table `unsafePeekElemOff`
35093515 (256 + fromIntegral state + fromIntegral t)
3510− in (Tuple' state' codep')
3516+ in assert (codep' <= 0x10FFFF
3517+ || error (showByte ++ showState state codep))
3518+ (Tuple' state' codep')
3519+ where
3520+
3521+ utf8table =
3522+ let !(Ptr addr) = table
3523+ end = table `plusPtr` 364
3524+ in A.Array (ForeignPtr addr undefined) end end :: A.Array Word8
3525+ showByte = "Streamly: decode1: byte: " ++ show byte
3526+ showState st cp =
3527+ " state: " ++ show st ++
3528+ " codepoint: " ++ show cp ++
3529+ " table: " ++ show utf8table
35113530
35123531-- We can divide the errors in three general categories:
35133532-- * A non-starter was encountered in a begin state