53e6fc4 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Refactor parseDToK for better optimization

53e6fc4Harendra Kumar4 years ago

core/src/Streamly/Internal/Data/Parser/ParserK/Type.hs+75−85

compare with master

unifiedsplit

@@ -358,71 +358,6 @@ instance MonadTrans (ParserK a) where

358-- Convert ParserD to ParserK

359-------------------------------------------------------------------------------

360

361−data ChunkResult s b =

362− ChunkDone !Int !b

363− | ChunkPartial !Int !s

364− | ChunkContinue !Int !s

365− | ChunkError !Int String

366−

367−-- This is very similar to fromParserD in the Array/Unboxed/Fold module.

368−{-# INLINE parseChunk #-}

369−parseChunk

370− :: forall m a s b. (Monad m, Unbox a)

371− => (s -> a -> m (ParserD.Step s b))

372− -> s

373− -> Array a

374− -> Int

375− -> m (ChunkResult s b)

376−parseChunk pstep !state (Array contents start end) !offset = do

377− if offset >= 0

378− then go SPEC (start + offset * SIZE_OF(a)) state

379− else return $ ChunkContinue offset state

380−

381− where

382−

383− {-# INLINE onBack #-}

384− onBack offset1 elemSize constr pst = do

385− let pos = offset1 - start

386− in if pos >= 0

387− then go SPEC offset1 pst

388− else return $ constr (pos `div` elemSize) pst

389−

390− -- Note: div may be expensive but the alternative is to maintain an element

391− -- offset in addition to a byte offset or just the element offset and use

392− -- multiplication to get the byte offset every time, both these options

393− -- turned out to be more expensive than using div.

394− go !_ !cur !pst | cur >= end =

395− return $ ChunkContinue ((end - start) `div` SIZE_OF(a)) pst

396− go !_ !cur !pst = do

397− let !x = unsafeInlineIO $ peekWith contents cur

398− pRes <- pstep pst x

399− let elemSize = SIZE_OF(a)

400− next = INDEX_NEXT(cur,a)

401− back n = next - n * elemSize

402− curOff = (cur - start) `div` elemSize

403− nextOff = (next - start) `div` elemSize

404− case pRes of

405− ParserD.Done 0 b ->

406− return $ ChunkDone nextOff b

407− ParserD.Done 1 b ->

408− return $ ChunkDone curOff b

409− ParserD.Done n b ->

410− return $ ChunkDone ((back n - start) `div` elemSize) b

411− ParserD.Partial 0 pst1 ->

412− go SPEC next pst1

413− ParserD.Partial 1 pst1 ->

414− go SPEC cur pst1

415− ParserD.Partial n pst1 ->

416− onBack (back n) elemSize ChunkPartial pst1

417− ParserD.Continue 0 pst1 ->

418− go SPEC next pst1

419− ParserD.Continue 1 pst1 ->

420− go SPEC cur pst1

421− ParserD.Continue n pst1 ->

422− onBack (back n) elemSize ChunkContinue pst1

423− ParserD.Error err ->

424− return $ ChunkError curOff err

425−

426{-# INLINE parseDToK #-}

361{-# INLINE parseDToK #-}

427parseDToK

362parseDToK

428 :: forall m a s b r. (Monad m, Unbox a)

363 :: forall m a s b r. (Monad m, Unbox a)

@@ -434,37 +369,92 @@ parseDToK

434 -> Int

369 -> Int

435 -> Input a

370 -> Input a

436 -> m (Step a m r)

371 -> m (Step a m r)

437−parseDToK pstep initial extract cont !offset !usedCount !input = do

372+parseDToK pstep initial extract cont !offset0 !usedCount !input = do

438 res <- initial

373 res <- initial

439 case res of

374 case res of

440 ParserD.IPartial pst -> do

375 ParserD.IPartial pst -> do

441 case input of

376 case input of

442− Chunk arr -> parseContChunk usedCount offset pst arr

377+ Chunk arr -> parseContChunk usedCount offset0 pst arr

443 None -> parseContNothing usedCount pst

378 None -> parseContNothing usedCount pst

444− ParserD.IDone b -> cont (Success offset b) usedCount input

379+ ParserD.IDone b -> cont (Success offset0 b) usedCount input

445− ParserD.IError err -> cont (Failure offset err) usedCount input

380+ ParserD.IError err -> cont (Failure offset0 err) usedCount input

446

381

447 where

382 where

448

383

449 -- XXX We can maintain an absolute position instead of relative that will

384 -- XXX We can maintain an absolute position instead of relative that will

450 -- help in reporting of error location in the stream.

385 -- help in reporting of error location in the stream.

451 {-# NOINLINE parseContChunk #-}

386 {-# NOINLINE parseContChunk #-}

452− parseContChunk !count !off !pst !arr = do

387+ parseContChunk !count !offset !state arr@(Array contents start end) = do

388+ if offset >= 0

389+ then go SPEC (start + offset * SIZE_OF(a)) state

390+ else return $ Continue offset (parseCont count state)

391+

392+ where

393+

394+ {-# INLINE onDone #-}

395+ onDone n b =

396+ assert (n <= Array.length arr)

397+ (cont (Success n b) (count + n - offset) (Chunk arr))

398+

399+ {-# INLINE callParseCont #-}

400+ callParseCont constr n pst1 =

401+ assert (n < 0 || n >= Array.length arr)

402+ (return $ constr n (parseCont (count + n - offset) pst1))

403+

404+ {-# INLINE onPartial #-}

405+ onPartial = callParseCont Partial

406+

407+ {-# INLINE onContinue #-}

408+ onContinue = callParseCont Continue

409+

410+ {-# INLINE onError #-}

411+ onError n err =

412+ cont (Failure n err) (count + n - offset) (Chunk arr)

413+

414+ {-# INLINE onBack #-}

415+ onBack offset1 elemSize constr pst = do

416+ let pos = offset1 - start

417+ in if pos >= 0

418+ then go SPEC offset1 pst

419+ else constr (pos `div` elemSize) pst

420+

421+ -- Note: div may be expensive but the alternative is to maintain an element

422+ -- offset in addition to a byte offset or just the element offset and use

423+ -- multiplication to get the byte offset every time, both these options

424+ -- turned out to be more expensive than using div.

425+ go !_ !cur !pst | cur >= end =

426+ onContinue ((end - start) `div` SIZE_OF(a)) pst

427+ go !_ !cur !pst = do

428+ let !x = unsafeInlineIO $ peekWith contents cur

453− pRes <- parseChunk pstep pst arr off

429+ pRes <- pstep pst x

430+ let elemSize = SIZE_OF(a)

431+ next = INDEX_NEXT(cur,a)

432+ back n = next - n * elemSize

433+ curOff = (cur - start) `div` elemSize

434+ nextOff = (next - start) `div` elemSize

454− -- The "n" here is stream position index wrt the array start, and not

435+ -- The "n" here is stream position index wrt the array start, and

455− -- the backtrack count as returned by byte stream parsers.

436+ -- not the backtrack count as returned by byte stream parsers.

456− case pRes of

437+ case pRes of

457− ChunkDone n b ->

438+ ParserD.Done 0 b ->

458− assert (n <= Array.length arr)

439+ onDone nextOff b

459− (cont (Success n b) (count + n - off) (Chunk arr))

440+ ParserD.Done 1 b ->

460− ChunkPartial n pst1 ->

441+ onDone curOff b

461− assert (n < 0 || n >= Array.length arr)

442+ ParserD.Done n b ->

462− (return $ Partial n (parseCont (count + n - off) pst1))

443+ onDone ((back n - start) `div` elemSize) b

463− ChunkContinue n pst1 ->

444+ ParserD.Partial 0 pst1 ->

464− assert (n < 0 || n >= Array.length arr)

445+ go SPEC next pst1

465− (return $ Continue n (parseCont (count + n - off) pst1))

446+ ParserD.Partial 1 pst1 ->

466− ChunkError n err ->

447+ go SPEC cur pst1

467− cont (Failure n err) (count + n - off) (Chunk arr)

448+ ParserD.Partial n pst1 ->

449+ onBack (back n) elemSize onPartial pst1

450+ ParserD.Continue 0 pst1 ->

451+ go SPEC next pst1

452+ ParserD.Continue 1 pst1 ->

453+ go SPEC cur pst1

454+ ParserD.Continue n pst1 ->

455+ onBack (back n) elemSize onContinue pst1

456+ ParserD.Error err ->

457+ onError curOff err

468

458

469 {-# NOINLINE parseContNothing #-}

459 {-# NOINLINE parseContNothing #-}

470 parseContNothing !count !pst = do

460 parseContNothing !count !pst = do

core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+7

@@ -183,6 +183,13 @@ parseD parser strm = do

183parse :: Monad m => PR.Parser a m b -> Stream m a -> m (Either ParseError b)

184parse = parseD

185

186+-- XXX It may be a good idea to use constant sized chunks for backtracking. We

187+-- can take a byte stream but when we have to backtrack we create constant

188+-- sized chunks. We maintain one forward list and one backward list of constant

189+-- sized chunks, and a last backtracking offset. That way we just need lists of

190+-- contents and no need to maintain start/end pointers for individual arrays,

191+-- reducing bookkeeping work.

192+

186-- | Run a 'Parse' over a stream and return rest of the Stream.

193-- | Run a 'Parse' over a stream and return rest of the Stream.

187{-# INLINE_NORMAL parseBreakD #-}

194{-# INLINE_NORMAL parseBreakD #-}

188parseBreakD

195parseBreakD