f763943 · streamly · git.iko.soy

git.iko.soy/streamly

Fix parseBreak losing backtracking Buffer on Error (#2453)

Co-authored-by: Harendra Kumar <harendra@composewell.com>

commitf763943923b23b55fb709bbc3be3e08039ef968c authorRanjeet Ranjan <ranjeetr@composewell.com> dateMon 24 Jul 2023, 17:37 (3 years ago) parent7432dda

4 files changed+73−27

compare with master

unifiedsplit

core/src/Streamly/Internal/Data/Producer/Source.hs+19−6 core/src/Streamly/Internal/Data/Stream/Chunked.hs+23−9 core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+16−7 core/src/Streamly/Internal/Data/StreamK.hs+15−5

core/src/Streamly/Internal/Data/Producer/Source.hs+19−6

@@ -160,7 +160,11 @@ parse

160160 return (Right b, unread src s1)

161161 Error err -> do

162162 s1 <- uextract s

163− return (Left (ParseError err), unread [x] s1)

163+ let src = Prelude.reverse (getList buf)

164+ return

165+ ( Left (ParseError err)

166+ , unread (src ++ [x]) s1

167+ )

164168 Skip s -> go SPEC s buf pst

165169 Stop -> goStop buf pst

166170

@@ -190,7 +194,11 @@ parse

190194 return (Right b, unread src s1)

191195 Error err -> do

192196 s1 <- uextract s

193− return (Left (ParseError err), unread (x:xs) s1)

197+ let src = Prelude.reverse (getList buf)

198+ return

199+ ( Left (ParseError err)

200+ , unread (src ++ (x:xs)) s1

201+ )

194202

195203 -- This is a simplified gobuf

196204 goExtract !_ buf (List []) !pst = goStop buf pst

@@ -216,8 +224,12 @@ parse

216224 let src0 = Prelude.take n (x:getList buf)

217225 src = Prelude.reverse src0

218226 return (Right b, unread src (source Nothing))

219− Error err ->

220− return (Left (ParseError err), unread (x:xs) (source Nothing))

227+ Error err -> do

228+ let src = Prelude.reverse (getList buf)

229+ return

230+ ( Left (ParseError err)

231+ , unread (src ++ (x:xs)) (source Nothing)

232+ )

221233

222234 -- This is a simplified goExtract

223235 {-# INLINE goStop #-}

@@ -238,8 +250,9 @@ parse

238250 let src0 = Prelude.take n (getList buf)

239251 src = Prelude.reverse src0

240252 return (Right b, unread src (source Nothing))

241− Error err ->

242− return (Left (ParseError err), source Nothing)

253+ Error err -> do

254+ let src = Prelude.reverse (getList buf)

255+ return (Left (ParseError err), unread src (source Nothing))

243256

244257{-

245258-- | Parse a buffered source using a parser, returning the parsed value and the

core/src/Streamly/Internal/Data/Stream/Chunked.hs+23−9

@@ -704,7 +704,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

704704 str = K.cons arr0 (K.cons arr1 st)

705705 return (Right b, str)

706706 PR.Error err -> do

707− let str = K.cons (Array contents cur end) stream

707+ let n = Prelude.length backBuf

708+ arr0 = A.fromListN n (Prelude.reverse backBuf)

709+ arr1 = Array contents cur end

710+ str = K.cons arr0 (K.cons arr1 stream)

708711 return (Left (ParseError err), str)

709712

710713 -- This is a simplified goArray

@@ -746,7 +749,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

746749 str = K.cons arr0 (K.fromPure arr1)

747750 return (Right b, str)

748751 PR.Error err -> do

749− let str = K.fromPure (Array contents cur end)

752+ let n = Prelude.length backBuf

753+ arr0 = A.fromListN n (Prelude.reverse backBuf)

754+ arr1 = Array contents cur end

755+ str = K.cons arr0 (K.cons arr1 stream)

750756 return (Left (ParseError err), str)

751757

752758 -- This is a simplified goExtract

@@ -771,8 +777,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

771777 -- arr0 = A.fromListRevN n src0

772778 arr0 = A.fromListN n (Prelude.reverse src0)

773779 return (Right b, K.fromPure arr0)

774− PR.Error err ->

775− return (Left (ParseError err), K.nil)

780+ PR.Error err -> do

781+ let n = Prelude.length backBuf

782+ arr0 = A.fromListN n (Prelude.reverse backBuf)

783+ return (Left (ParseError err), K.fromPure arr0)

776784

777785-- | Parse an array stream using the supplied 'Parser'. Returns the parse

778786-- result and the unconsumed stream. Throws 'ParseError' if the parse fails.

@@ -866,7 +874,9 @@ runArrayParserDBreak

866874 src = Prelude.reverse src0 ++ xs

867875 return (Right b, D.append (D.fromList src) (D.Stream step s))

868876 PR.Error err -> do

869− let strm = D.append (D.fromList (x:xs)) (D.Stream step s)

877+ let src0 = x:getList backBuf

878+ src = Prelude.reverse src0 ++ x:xs

879+ strm = D.append (D.fromList src) (D.Stream step s)

870880 return (Left (ParseError err), strm)

871881

872882 -- This is a simplified gobuf

@@ -901,8 +911,10 @@ runArrayParserDBreak

901911 let src0 = takeArrayListRev n (x:getList backBuf)

902912 src = Prelude.reverse src0 ++ xs

903913 return (Right b, D.fromList src)

904− PR.Error err ->

905− return (Left (ParseError err), D.fromList (x:xs))

914+ PR.Error err -> do

915+ let src0 = getList backBuf

916+ src = Prelude.reverse src0 ++ x:xs

917+ return (Left (ParseError err), D.fromList src)

906918

907919 -- This is a simplified goExtract

908920 {-# INLINE goStop #-}

@@ -927,8 +939,10 @@ runArrayParserDBreak

927939 let src0 = takeArrayListRev n (getList backBuf)

928940 src = Prelude.reverse src0

929941 return (Right b, D.fromList src)

930− PR.Error err ->

931− return (Left (ParseError err), D.nil)

942+ PR.Error err -> do

943+ let src0 = getList backBuf

944+ src = Prelude.reverse src0

945+ return (Left (ParseError err), D.fromList src)

932946

933947{-

934948-- | Parse an array stream using the supplied 'Parser'. Returns the parse

core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+16−7

@@ -238,8 +238,13 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

238238 return

239239 ( Right b,

240240 Nesting.append (fromList src) (Stream step s))

241− PR.Error err ->

242− return (Left (ParseError err), Stream step s)

241+ PR.Error err -> do

242+ let src = Prelude.reverse $ x:getList buf

243+ return

244+ ( Left (ParseError err)

245+ , Nesting.append (fromList src) (Stream step s)

246+ )

247+

243248 Skip s -> go SPEC s buf pst

244249 Stop -> goStop SPEC buf pst

245250

@@ -295,10 +300,11 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

295300 let src0 = Prelude.take n (x:getList buf)

296301 src = Prelude.reverse src0

297302 return (Right b, Nesting.append (fromList src) (Stream step s))

298− PR.Error err ->

303+ PR.Error err -> do

304+ let src = (Prelude.reverse $ getList buf) ++ x:xs

299305 return

300306 ( Left (ParseError err)

301− , Nesting.append (fromList (x:xs)) (Stream step s)

307+ , Nesting.append (fromList src) (Stream step s)

302308 )

303309

304310 -- This is simplified gobuf

@@ -327,7 +333,9 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

327333 let src0 = Prelude.take n (x:getList buf)

328334 src = Prelude.reverse src0

329335 return (Right b, fromList src)

330− PR.Error err -> return (Left (ParseError err), fromList (x:xs))

336+ PR.Error err -> do

337+ let src = (Prelude.reverse $ getList buf) ++ x:xs

338+ return (Left (ParseError err), fromList src)

331339

332340 -- This is simplified goExtract

333341 -- XXX Use SPEC?

@@ -348,8 +356,9 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

348356 let src0 = Prelude.take n (getList buf)

349357 src = Prelude.reverse src0

350358 return (Right b, fromList src)

351− PR.Error err ->

352− return (Left (ParseError err), StreamD.nil)

359+ PR.Error err -> do

360+ let src = Prelude.reverse $ getList buf

361+ return (Left (ParseError err), fromList src)

353362

354363-- | Parse a stream using the supplied 'Parser'.

355364--

core/src/Streamly/Internal/Data/StreamK.hs+15−5

@@ -1171,7 +1171,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

11711171 let stop = do

11721172 r <- extract pst

11731173 case r of

1174− PR.Error err -> return (Left (ParseError err), nil)

1174+ PR.Error err -> do

1175+ let src = Prelude.reverse buf

1176+ return (Left (ParseError err), fromList src)

11751177 PR.Done n b -> do

11761178 assertM(n <= length buf)

11771179 let src0 = Prelude.take n buf

@@ -1206,7 +1208,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

12061208 let src0 = Prelude.take n (x:buf)

12071209 src = Prelude.reverse src0

12081210 return (Right b, append (fromList src) r)

1209− PR.Error err -> return (Left (ParseError err), r)

1211+ PR.Error err -> do

1212+ let src = Prelude.reverse (x:buf)

1213+ return (Left (ParseError err), append (fromList src) r)

12101214 in foldStream defState yieldk single stop st

12111215

12121216 goBuf st buf [] !pst = goStream st buf pst

@@ -1230,7 +1234,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

12301234 let src0 = Prelude.take n (x:buf)

12311235 src = Prelude.reverse src0

12321236 return (Right b, append (fromList src) st)

1233− PR.Error err -> return (Left (ParseError err), nil)

1237+ PR.Error err -> do

1238+ let src = Prelude.reverse buf ++ x:xs

1239+ return (Left (ParseError err), append (fromList src) st)

12341240

12351241-- Using ParserD or ParserK on StreamK may not make much difference. We should

12361242-- perhaps use only chunked parsing on StreamK. We can always convert a stream

@@ -1324,7 +1330,9 @@ parseBreakChunks parser input = do

13241330 assertM(n1 >= 0 && n1 <= sum (Prelude.map Array.length backBuf))

13251331 let (s1, _) = backTrack n1 backBuf nil

13261332 in return (Right b, s1)

1327− ParserK.Error _ err -> return (Left (ParseError err), nil)

1333+ ParserK.Error _ err -> do

1334+ let (s1, _) = backTrack maxBound backBuf nil

1335+ return (Left (ParseError err), s1)

13281336

13291337 seekErr n len =

13301338 error $ "parseBreak: Partial: forward seek not implemented n = "

@@ -1367,7 +1375,9 @@ parseBreakChunks parser input = do

13671375 assertM(n1 <= sum (Prelude.map Array.length (arr:backBuf)))

13681376 let (s1, _) = backTrack n1 (arr:backBuf) stream

13691377 in return (Right b, s1)

1370− ParserK.Error _ err -> return (Left (ParseError err), nil)

1378+ ParserK.Error _ err -> do

1379+ let (s1, _) = backTrack maxBound (arr:backBuf) stream

1380+ return (Left (ParseError err), s1)

13711381

13721382 go backBuf parserk stream = do

13731383 let stop = goStop backBuf parserk