8307181 · streamly · git.iko.soy

227-- * A starter was encountered without completing a codepoint

228-- * The last codepoint was not complete (input underflow)

229--

230+-- Need to separate resumable and non-resumable error. In case of non-resumable

231+-- error we can also provide the failing byte. In case of resumable error the

232+-- state can be opaque.

233+--

230data DecodeError = DecodeError !DecodeState !CodePoint deriving Show

234data DecodeError = DecodeError !DecodeState !CodePoint deriving Show

231

235

232data FreshPoint s a

236data FreshPoint s a

237 | YieldAndContinue a (FreshPoint s a)

241 | YieldAndContinue a (FreshPoint s a)

238 | Done

242 | Done

239

243

240−-- XXX Add proper error messages

244+-- XXX write it as a parser and use parseMany to decode a stream, need to check

245+-- if that preserves the same performance. Or we can use a resumable parser

246+-- that parses a chunk at a time.

247+--

241−-- XXX Implement this in terms of decodeUtf8Either

248+-- XXX Implement this in terms of decodeUtf8Either. Need to make sure that

249+-- decodeUtf8Either preserves the performance characterstics.

250+--

242{-# INLINE_NORMAL decodeUtf8WithD #-}

251{-# INLINE_NORMAL decodeUtf8WithD #-}

252+decodeUtf8WithD :: Monad m

243−decodeUtf8WithD :: Monad m => CodingFailureMode -> Stream m Word8 -> Stream m Char

253+ => CodingFailureMode -> Stream m Word8 -> Stream m Char

244decodeUtf8WithD cfm (Stream step state) =

254decodeUtf8WithD cfm (Stream step state) =

245 let A.Array p _ = utf8d

255 let A.Array p _ = utf8d

246 !ptr = unsafeForeignPtrToPtr p

256 !ptr = unsafeForeignPtrToPtr p

247 in Stream (step' ptr) (FreshPointDecodeInit state)

257 in Stream (step' ptr) (FreshPointDecodeInit state)

258+

248− where

259+ where

249− {-# INLINE transliterateOrError #-}

260+

250− transliterateOrError e s =

261+ prefix = "Streamly.Internal.Data.Stream.StreamD.decodeUtf8With: "

262+

263+ {-# INLINE handleError #-}

264+ handleError e s =

251 case cfm of

265 case cfm of

252 ErrorOnCodingFailure -> error e

266 ErrorOnCodingFailure -> error e

253 TransliterateCodingFailure -> YieldAndContinue replacementChar s

267 TransliterateCodingFailure -> YieldAndContinue replacementChar s

254− {-# INLINE inputUnderflow #-}

268+

255− inputUnderflow =

269+ {-# INLINE handleUnderflow #-}

270+ handleUnderflow =

256 case cfm of

271 case cfm of

257− ErrorOnCodingFailure ->

272+ ErrorOnCodingFailure -> error $ prefix ++ "Not enough input"

258− error "Streamly.Internal.Data.Stream.StreamD.decodeUtf8With: Input Underflow"

259 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

273 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

274+

260 {-# INLINE_LATE step' #-}

275 {-# INLINE_LATE step' #-}

261 step' _ gst (FreshPointDecodeInit st) = do

276 step' _ gst (FreshPointDecodeInit st) = do

262 r <- step (adaptState gst) st

277 r <- step (adaptState gst) st

287 return $

302 return $

288 case sv of

303 case sv of

289 12 ->

304 12 ->

290− Skip $

305+ let msg = prefix ++ "Invalid first UTF8 byte " ++ show x

291− transliterateOrError

292− "Streamly.Internal.Data.Stream.StreamD.decodeUtf8With: Invalid UTF8 codepoint encountered"

293− (FreshPointDecodeInit st)

306+ in Skip $ handleError msg (FreshPointDecodeInit st)

294 0 -> error "unreachable state"

307 0 -> error "unreachable state"

295 _ -> Skip (FreshPointDecoding st sv cp)

308 _ -> Skip (FreshPointDecoding st sv cp)

296

309

297− -- We recover by trying the new byte x a starter of a new codepoint.

310+ -- We recover by trying the new byte x as a starter of a new codepoint.

298 -- XXX need to use the same recovery in array decoding routine as well

311 -- XXX need to use the same recovery in array decoding routine as well

299 step' table gst (FreshPointDecoding st statePtr codepointPtr) = do

312 step' table gst (FreshPointDecoding st statePtr codepointPtr) = do

300 r <- step (adaptState gst) st

313 r <- step (adaptState gst) st

301 case r of

314 case r of

302 Yield x s -> do

315 Yield x s -> do

303 let (Tuple' sv cp) = decode1 table statePtr codepointPtr x

316 let (Tuple' sv cp) = decode1 table statePtr codepointPtr x

304− return $

305− case sv of

317+ return $ case sv of

306− 0 -> Skip $ YieldAndContinue (unsafeChr cp)

318+ 0 -> Skip $ YieldAndContinue

307− (FreshPointDecodeInit s)

319+ (unsafeChr cp) (FreshPointDecodeInit s)

308− 12 ->

320+ 12 ->

309− Skip $

321+ let msg = prefix

310− transliterateOrError

322+ ++ "Invalid subsequent UTF8 byte "

311− "Streamly.Internal.Data.Stream.StreamD.decodeUtf8With: Invalid UTF8 codepoint encountered"

323+ ++ show x

324+ ++ " in state "

325+ ++ show statePtr

326+ ++ " accumulated value "

327+ ++ show codepointPtr

312− (FreshPointDecodeInit1 s x)

328+ in Skip $ handleError msg (FreshPointDecodeInit1 s x)

313− _ -> Skip (FreshPointDecoding s sv cp)

329+ _ -> Skip (FreshPointDecoding s sv cp)

330+ Skip s -> return $

314− Skip s -> return $ Skip (FreshPointDecoding s statePtr codepointPtr)

331+ Skip (FreshPointDecoding s statePtr codepointPtr)

315− Stop -> return $ Skip inputUnderflow

332+ Stop -> return $ Skip handleUnderflow

316

333

317 step' _ _ (YieldAndContinue c s) = return $ Yield c s

334 step' _ _ (YieldAndContinue c s) = return $ Yield c s

318 step' _ _ Done = return Stop

335 step' _ _ Done = return Stop

377 _ -> Skip (FreshPointDecoding st sv cp)

394 _ -> Skip (FreshPointDecoding st sv cp)

378

395

379 -- We recover by trying the new byte x a starter of a new codepoint.

396 -- We recover by trying the new byte x a starter of a new codepoint.

397+ -- XXX on error need to report the next byte "x" as well.

380 -- XXX need to use the same recovery in array decoding routine as well

398 -- XXX need to use the same recovery in array decoding routine as well

381 step' table gst (FreshPointDecoding st statePtr codepointPtr) = do

399 step' table gst (FreshPointDecoding st statePtr codepointPtr) = do

382 r <- step (adaptState gst) st

400 r <- step (adaptState gst) st