f763943 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Fix parseBreak losing backtracking Buffer on Error (#2453)

f763943Ranjeet Ranjan3 years ago

core/src/Streamly/Internal/Data/Producer/Source.hs+19−6

compare with master

unifiedsplit

@@ -160,7 +160,11 @@ parse

160 return (Right b, unread src s1)

161 Error err -> do

162 s1 <- uextract s

163+ let src = Prelude.reverse (getList buf)

164+ return

163− return (Left (ParseError err), unread [x] s1)

165+ ( Left (ParseError err)

166+ , unread (src ++ [x]) s1

167+ )

164 Skip s -> go SPEC s buf pst

168 Skip s -> go SPEC s buf pst

165 Stop -> goStop buf pst

169 Stop -> goStop buf pst

166

170

@@ -190,7 +194,11 @@ parse

190 return (Right b, unread src s1)

194 return (Right b, unread src s1)

191 Error err -> do

195 Error err -> do

192 s1 <- uextract s

196 s1 <- uextract s

197+ let src = Prelude.reverse (getList buf)

198+ return

193− return (Left (ParseError err), unread (x:xs) s1)

199+ ( Left (ParseError err)

200+ , unread (src ++ (x:xs)) s1

201+ )

194

202

195 -- This is a simplified gobuf

203 -- This is a simplified gobuf

196 goExtract !_ buf (List []) !pst = goStop buf pst

204 goExtract !_ buf (List []) !pst = goStop buf pst

@@ -216,8 +224,12 @@ parse

216 let src0 = Prelude.take n (x:getList buf)

224 let src0 = Prelude.take n (x:getList buf)

217 src = Prelude.reverse src0

225 src = Prelude.reverse src0

218 return (Right b, unread src (source Nothing))

226 return (Right b, unread src (source Nothing))

219− Error err ->

227+ Error err -> do

228+ let src = Prelude.reverse (getList buf)

229+ return

230+ ( Left (ParseError err)

220− return (Left (ParseError err), unread (x:xs) (source Nothing))

231+ , unread (src ++ (x:xs)) (source Nothing)

232+ )

221

233

222 -- This is a simplified goExtract

234 -- This is a simplified goExtract

223 {-# INLINE goStop #-}

235 {-# INLINE goStop #-}

@@ -238,8 +250,9 @@ parse

238 let src0 = Prelude.take n (getList buf)

250 let src0 = Prelude.take n (getList buf)

239 src = Prelude.reverse src0

251 src = Prelude.reverse src0

240 return (Right b, unread src (source Nothing))

252 return (Right b, unread src (source Nothing))

241− Error err ->

253+ Error err -> do

254+ let src = Prelude.reverse (getList buf)

242− return (Left (ParseError err), source Nothing)

255+ return (Left (ParseError err), unread src (source Nothing))

243

256

244{-

257{-

245-- | Parse a buffered source using a parser, returning the parsed value and the

258-- | Parse a buffered source using a parser, returning the parsed value and the

core/src/Streamly/Internal/Data/Stream/Chunked.hs+23−9

@@ -704,7 +704,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

704 str = K.cons arr0 (K.cons arr1 st)

705 return (Right b, str)

706 PR.Error err -> do

707+ let n = Prelude.length backBuf

708+ arr0 = A.fromListN n (Prelude.reverse backBuf)

707− let str = K.cons (Array contents cur end) stream

709+ arr1 = Array contents cur end

710+ str = K.cons arr0 (K.cons arr1 stream)

708 return (Left (ParseError err), str)

711 return (Left (ParseError err), str)

709

712

710 -- This is a simplified goArray

713 -- This is a simplified goArray

@@ -746,7 +749,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

746 str = K.cons arr0 (K.fromPure arr1)

749 str = K.cons arr0 (K.fromPure arr1)

747 return (Right b, str)

750 return (Right b, str)

748 PR.Error err -> do

751 PR.Error err -> do

752+ let n = Prelude.length backBuf

753+ arr0 = A.fromListN n (Prelude.reverse backBuf)

749− let str = K.fromPure (Array contents cur end)

754+ arr1 = Array contents cur end

755+ str = K.cons arr0 (K.cons arr1 stream)

750 return (Left (ParseError err), str)

756 return (Left (ParseError err), str)

751

757

752 -- This is a simplified goExtract

758 -- This is a simplified goExtract

@@ -771,8 +777,10 @@ parseBreakK (PRD.Parser pstep initial extract) stream = do

771 -- arr0 = A.fromListRevN n src0

777 -- arr0 = A.fromListRevN n src0

772 arr0 = A.fromListN n (Prelude.reverse src0)

778 arr0 = A.fromListN n (Prelude.reverse src0)

773 return (Right b, K.fromPure arr0)

779 return (Right b, K.fromPure arr0)

774− PR.Error err ->

780+ PR.Error err -> do

781+ let n = Prelude.length backBuf

782+ arr0 = A.fromListN n (Prelude.reverse backBuf)

775− return (Left (ParseError err), K.nil)

783+ return (Left (ParseError err), K.fromPure arr0)

776

784

777-- | Parse an array stream using the supplied 'Parser'. Returns the parse

785-- | Parse an array stream using the supplied 'Parser'. Returns the parse

778-- result and the unconsumed stream. Throws 'ParseError' if the parse fails.

786-- result and the unconsumed stream. Throws 'ParseError' if the parse fails.

@@ -866,7 +874,9 @@ runArrayParserDBreak

866 src = Prelude.reverse src0 ++ xs

874 src = Prelude.reverse src0 ++ xs

867 return (Right b, D.append (D.fromList src) (D.Stream step s))

875 return (Right b, D.append (D.fromList src) (D.Stream step s))

868 PR.Error err -> do

876 PR.Error err -> do

877+ let src0 = x:getList backBuf

878+ src = Prelude.reverse src0 ++ x:xs

869− let strm = D.append (D.fromList (x:xs)) (D.Stream step s)

879+ strm = D.append (D.fromList src) (D.Stream step s)

870 return (Left (ParseError err), strm)

880 return (Left (ParseError err), strm)

871

881

872 -- This is a simplified gobuf

882 -- This is a simplified gobuf

@@ -901,8 +911,10 @@ runArrayParserDBreak

901 let src0 = takeArrayListRev n (x:getList backBuf)

911 let src0 = takeArrayListRev n (x:getList backBuf)

902 src = Prelude.reverse src0 ++ xs

912 src = Prelude.reverse src0 ++ xs

903 return (Right b, D.fromList src)

913 return (Right b, D.fromList src)

904− PR.Error err ->

914+ PR.Error err -> do

915+ let src0 = getList backBuf

916+ src = Prelude.reverse src0 ++ x:xs

905− return (Left (ParseError err), D.fromList (x:xs))

917+ return (Left (ParseError err), D.fromList src)

906

918

907 -- This is a simplified goExtract

919 -- This is a simplified goExtract

908 {-# INLINE goStop #-}

920 {-# INLINE goStop #-}

@@ -927,8 +939,10 @@ runArrayParserDBreak

927 let src0 = takeArrayListRev n (getList backBuf)

939 let src0 = takeArrayListRev n (getList backBuf)

928 src = Prelude.reverse src0

940 src = Prelude.reverse src0

929 return (Right b, D.fromList src)

941 return (Right b, D.fromList src)

930− PR.Error err ->

942+ PR.Error err -> do

943+ let src0 = getList backBuf

944+ src = Prelude.reverse src0

931− return (Left (ParseError err), D.nil)

945+ return (Left (ParseError err), D.fromList src)

932

946

933{-

947{-

934-- | Parse an array stream using the supplied 'Parser'. Returns the parse

948-- | Parse an array stream using the supplied 'Parser'. Returns the parse

core/src/Streamly/Internal/Data/Stream/StreamD/Eliminate.hs+16−7

@@ -238,8 +238,13 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

238 return

239 ( Right b,

240 Nesting.append (fromList src) (Stream step s))

241− PR.Error err ->

241+ PR.Error err -> do

242+ let src = Prelude.reverse $ x:getList buf

243+ return

242− return (Left (ParseError err), Stream step s)

244+ ( Left (ParseError err)

245+ , Nesting.append (fromList src) (Stream step s)

246+ )

247+

243 Skip s -> go SPEC s buf pst

248 Skip s -> go SPEC s buf pst

244 Stop -> goStop SPEC buf pst

249 Stop -> goStop SPEC buf pst

245

250

@@ -295,10 +300,11 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

295 let src0 = Prelude.take n (x:getList buf)

300 let src0 = Prelude.take n (x:getList buf)

296 src = Prelude.reverse src0

301 src = Prelude.reverse src0

297 return (Right b, Nesting.append (fromList src) (Stream step s))

302 return (Right b, Nesting.append (fromList src) (Stream step s))

298− PR.Error err ->

303+ PR.Error err -> do

304+ let src = (Prelude.reverse $ getList buf) ++ x:xs

299 return

305 return

300 ( Left (ParseError err)

306 ( Left (ParseError err)

301− , Nesting.append (fromList (x:xs)) (Stream step s)

307+ , Nesting.append (fromList src) (Stream step s)

302 )

308 )

303

309

304 -- This is simplified gobuf

310 -- This is simplified gobuf

@@ -327,7 +333,9 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

327 let src0 = Prelude.take n (x:getList buf)

333 let src0 = Prelude.take n (x:getList buf)

328 src = Prelude.reverse src0

334 src = Prelude.reverse src0

329 return (Right b, fromList src)

335 return (Right b, fromList src)

336+ PR.Error err -> do

337+ let src = (Prelude.reverse $ getList buf) ++ x:xs

330− PR.Error err -> return (Left (ParseError err), fromList (x:xs))

338+ return (Left (ParseError err), fromList src)

331

339

332 -- This is simplified goExtract

340 -- This is simplified goExtract

333 -- XXX Use SPEC?

341 -- XXX Use SPEC?

@@ -348,8 +356,9 @@ parseBreakD (PRD.Parser pstep initial extract) stream@(Stream step state) = do

348 let src0 = Prelude.take n (getList buf)

356 let src0 = Prelude.take n (getList buf)

349 src = Prelude.reverse src0

357 src = Prelude.reverse src0

350 return (Right b, fromList src)

358 return (Right b, fromList src)

351− PR.Error err ->

359+ PR.Error err -> do

360+ let src = Prelude.reverse $ getList buf

352− return (Left (ParseError err), StreamD.nil)

361+ return (Left (ParseError err), fromList src)

353

362

354-- | Parse a stream using the supplied 'Parser'.

363-- | Parse a stream using the supplied 'Parser'.

355--

364--

core/src/Streamly/Internal/Data/StreamK.hs+15−5

@@ -1171,7 +1171,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

1171 let stop = do

1172 r <- extract pst

1173 case r of

1174+ PR.Error err -> do

1175+ let src = Prelude.reverse buf

1174− PR.Error err -> return (Left (ParseError err), nil)

1176+ return (Left (ParseError err), fromList src)

1175 PR.Done n b -> do

1177 PR.Done n b -> do

1176 assertM(n <= length buf)

1178 assertM(n <= length buf)

1177 let src0 = Prelude.take n buf

1179 let src0 = Prelude.take n buf

@@ -1206,7 +1208,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

1206 let src0 = Prelude.take n (x:buf)

1208 let src0 = Prelude.take n (x:buf)

1207 src = Prelude.reverse src0

1209 src = Prelude.reverse src0

1208 return (Right b, append (fromList src) r)

1210 return (Right b, append (fromList src) r)

1211+ PR.Error err -> do

1212+ let src = Prelude.reverse (x:buf)

1209− PR.Error err -> return (Left (ParseError err), r)

1213+ return (Left (ParseError err), append (fromList src) r)

1210 in foldStream defState yieldk single stop st

1214 in foldStream defState yieldk single stop st

1211

1215

1212 goBuf st buf [] !pst = goStream st buf pst

1216 goBuf st buf [] !pst = goStream st buf pst

@@ -1230,7 +1234,9 @@ parseDBreak (PR.Parser pstep initial extract) stream = do

1230 let src0 = Prelude.take n (x:buf)

1234 let src0 = Prelude.take n (x:buf)

1231 src = Prelude.reverse src0

1235 src = Prelude.reverse src0

1232 return (Right b, append (fromList src) st)

1236 return (Right b, append (fromList src) st)

1237+ PR.Error err -> do

1238+ let src = Prelude.reverse buf ++ x:xs

1233− PR.Error err -> return (Left (ParseError err), nil)

1239+ return (Left (ParseError err), append (fromList src) st)

1234

1240

1235-- Using ParserD or ParserK on StreamK may not make much difference. We should

1241-- Using ParserD or ParserK on StreamK may not make much difference. We should

1236-- perhaps use only chunked parsing on StreamK. We can always convert a stream

1242-- perhaps use only chunked parsing on StreamK. We can always convert a stream

@@ -1324,7 +1330,9 @@ parseBreakChunks parser input = do

1324 assertM(n1 >= 0 && n1 <= sum (Prelude.map Array.length backBuf))

1330 assertM(n1 >= 0 && n1 <= sum (Prelude.map Array.length backBuf))

1325 let (s1, _) = backTrack n1 backBuf nil

1331 let (s1, _) = backTrack n1 backBuf nil

1326 in return (Right b, s1)

1332 in return (Right b, s1)

1333+ ParserK.Error _ err -> do

1334+ let (s1, _) = backTrack maxBound backBuf nil

1327− ParserK.Error _ err -> return (Left (ParseError err), nil)

1335+ return (Left (ParseError err), s1)

1328

1336

1329 seekErr n len =

1337 seekErr n len =

1330 error $ "parseBreak: Partial: forward seek not implemented n = "

1338 error $ "parseBreak: Partial: forward seek not implemented n = "

@@ -1367,7 +1375,9 @@ parseBreakChunks parser input = do

1367 assertM(n1 <= sum (Prelude.map Array.length (arr:backBuf)))

1375 assertM(n1 <= sum (Prelude.map Array.length (arr:backBuf)))

1368 let (s1, _) = backTrack n1 (arr:backBuf) stream

1376 let (s1, _) = backTrack n1 (arr:backBuf) stream

1369 in return (Right b, s1)

1377 in return (Right b, s1)

1378+ ParserK.Error _ err -> do

1379+ let (s1, _) = backTrack maxBound (arr:backBuf) stream

1370− ParserK.Error _ err -> return (Left (ParseError err), nil)

1380+ return (Left (ParseError err), s1)

1371

1381

1372 go backBuf parserk stream = do

1382 go backBuf parserk stream = do

1373 let stop = goStop backBuf parserk

1383 let stop = goStop backBuf parserk