@@ -299,6 +299,8 @@ import Foreign.Ptr (Ptr)
299import Foreign.Storable (Storable(..))
299import Foreign.Storable (Storable(..))
300import GHC.Base (assert, Char(..), unsafeChr, ord)
300import GHC.Base (assert, Char(..), unsafeChr, ord)
301import GHC.IO.Encoding.Failure (isSurrogate)
301import GHC.IO.Encoding.Failure (isSurrogate)
302+import GHC.ForeignPtr (ForeignPtr (..))
303+import GHC.Ptr (Ptr (..))
302import GHC.Types (SPEC(..))
304import GHC.Types (SPEC(..))
303import GHC.Word (Word8(..))
305import GHC.Word (Word8(..))
304import System.IO.Unsafe (unsafePerformIO)
306import System.IO.Unsafe (unsafePerformIO)
@@ -327,7 +329,7 @@ import qualified Streamly.Streams.StreamK as K
328import Foreign.Ptr (plusPtr)
330import Foreign.Ptr (plusPtr)
329import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
331import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
330−import Foreign.ForeignPtr (ForeignPtr, touchForeignPtr)
332+import Foreign.ForeignPtr (touchForeignPtr)
332------------------------------------------------------------------------------
334------------------------------------------------------------------------------
333-- Construction
335-- Construction
@@ -3484,13 +3486,17 @@ decode0 table byte =
3484 !codep' = (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
3486 !codep' = (0xff `shiftR` (fromIntegral t)) .&. fromIntegral byte
3485 !state' = table `unsafePeekElemOff` (256 + fromIntegral t)
3487 !state' = table `unsafePeekElemOff` (256 + fromIntegral t)
3486 in assert ((byte > 0x7f || error showByte)
3488 in assert ((byte > 0x7f || error showByte)
3487− && (state' /= 0 || error (showByte ++ showState state')))
3489+ && (state' /= 0 || error (showByte ++ showTable)))
3488 (Tuple' state' codep')
3490 (Tuple' state' codep')
3495+ let !(Ptr addr) = table
3496+ end = table `plusPtr` 364
3497+ in A.Array (ForeignPtr addr undefined) end end :: A.Array Word8
3492− showByte = "decode0: Invalid byte: " ++ show byte
3498+ showByte = "Streamly: decode0: byte: " ++ show byte
3493− showState st = " state: " ++ show st ++ " table: " ++ show table
3499+ showTable = " table: " ++ show utf8table
3495-- When the state is not 0
3501-- When the state is not 0
3496{-# INLINE decode1 #-}
3502{-# INLINE decode1 #-}
@@ -3502,12 +3508,25 @@ decode1
3502 -> Tuple' DecodeState CodePoint
3508 -> Tuple' DecodeState CodePoint
3503decode1 table state codep byte =
3509decode1 table state codep byte =
3504 -- Remember codep is Int type!
3510 -- Remember codep is Int type!
3505− -- Can it be unsafe to convert the resulting to Char?
3511+ -- Can it be unsafe to convert the resulting Int to Char?
3506 let !t = table `unsafePeekElemOff` fromIntegral byte
3512 let !t = table `unsafePeekElemOff` fromIntegral byte
3507 !codep' = (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
3513 !codep' = (fromIntegral byte .&. 0x3f) .|. (codep `shiftL` 6)
3508 !state' = table `unsafePeekElemOff`
3514 !state' = table `unsafePeekElemOff`
3509 (256 + fromIntegral state + fromIntegral t)
3515 (256 + fromIntegral state + fromIntegral t)
3516+ in assert (codep' <= 0x10FFFF
3517+ || error (showByte ++ showState state codep))
3510− in (Tuple' state' codep')
3518+ (Tuple' state' codep')
3522+ let !(Ptr addr) = table
3523+ end = table `plusPtr` 364
3524+ in A.Array (ForeignPtr addr undefined) end end :: A.Array Word8
3525+ showByte = "Streamly: decode1: byte: " ++ show byte
3527+ " state: " ++ show st ++
3528+ " codepoint: " ++ show cp ++
3529+ " table: " ++ show utf8table
3512-- We can divide the errors in three general categories:
3531-- We can divide the errors in three general categories:
3513-- * A non-starter was encountered in a begin state
3532-- * A non-starter was encountered in a begin state