f3b07c2 · streamly · git.iko.soy

git.iko.soy/streamly

Add an option to preserve quotes in wordQuotedBy

commitf3b07c2922e73e9e1f9222d438385a13ee9d56f1 authorAdithya Kumar <adithya@composewell.com> dateTue 7 Jun 2022, 21:53 (4 years ago) parentc5e0ee5

3 files changed+88−10

compare with master

unifiedsplit

core/src/Streamly/Internal/Data/Parser.hs+10−5 core/src/Streamly/Internal/Data/Parser/ParserD.hs+14−5 test/Streamly/Test/Data/Parser.hs+64

core/src/Streamly/Internal/Data/Parser.hs+10−5

@@ -909,21 +909,26 @@ wordFramedBy isEsc isBegin isEnd isSpc =

909909-- as it encounters any of the closing quotes.

910910--

911911-- >>> q = (`elem` ['"', '\''])

912−-- >>> p = Parser.wordQuotedBy (== '\\') q q id isSpace Fold.toList

913−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f"

912+-- >>> p kQ = Parser.wordQuotedBy kQ (== '\\') q q id isSpace Fold.toList

913+--

914+-- >>> Stream.parse (p False) $ Stream.fromList "a\"b'c\";'d\"e'f ghi"

914915-- "ab'c;d\"ef"

915916--

917+-- >>> Stream.parse (p True) $ Stream.fromList "a\"b'c\";'d\"e'f ghi"

918+-- "a\"b'c\";'d\"e'f"

919+--

916920{-# INLINE wordQuotedBy #-}

917921wordQuotedBy :: (MonadCatch m, Eq a) =>

918− (a -> Bool) -- ^ Escape

922+ Bool -- ^ keep the quotes in the output

923+ -> (a -> Bool) -- ^ Escape

919924 -> (a -> Bool) -- ^ left quote

920925 -> (a -> Bool) -- ^ right quote

921926 -> (a -> a) -- ^ get right quote from left quote

922927 -> (a -> Bool) -- ^ word seperator

923928 -> Fold m a b

924929 -> Parser m a b

925−wordQuotedBy isEsc isBegin isEnd toRight isSpc =

926− D.toParserK . D.wordQuotedBy isEsc isBegin isEnd toRight isSpc

930+wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSpc =

931+ D.toParserK . D.wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSpc

927932

928933-- | Given an input stream @[a,b,c,...]@ and a comparison function @cmp@, the

929934-- parser assigns the element @a@ to the first group, then if @a \`cmp` b@ is

core/src/Streamly/Internal/Data/Parser/ParserD.hs+14−5

@@ -1182,14 +1182,15 @@ data WordQuotedState s b a =

11821182

11831183{-# INLINE wordQuotedBy #-}

11841184wordQuotedBy :: (MonadCatch m, Eq a) =>

1185− (a -> Bool) -- ^ Escape

1185+ Bool -- ^ keep the quotes in the output

1186+ -> (a -> Bool) -- ^ Escape

11861187 -> (a -> Bool) -- ^ left quote

11871188 -> (a -> Bool) -- ^ right quote

11881189 -> (a -> a) -- ^ get right quote from the left quote

11891190 -> (a -> Bool) -- ^ word seperator

11901191 -> Fold m a b

11911192 -> Parser m a b

1192−wordQuotedBy isEsc isBegin isEnd toRight isSep

1193+wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSep

11931194 (Fold fstep finitial fextract) =

11941195 Parser step initial extract

11951196

@@ -1222,7 +1223,10 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

12221223 step (WordQuotedSkipPre s) a

12231224 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s

12241225 | isSep a = return $ Partial 0 $ WordQuotedSkipPre s

1225− | isBegin a = return $ Continue 0 $ WordQuotedWord s 1 a

1226+ | isBegin a =

1227+ if keepQuotes

1228+ then process s a 1 a

1229+ else return $ Continue 0 $ WordQuotedWord s 1 a

12261230 | isEnd a =

12271231 return $ Error "wordQuotedBy: missing frame start"

12281232 | otherwise = processUnquoted s a

@@ -1233,12 +1237,15 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

12331237 return $ Partial 0 $ WordQuotedSkipPost b

12341238 | otherwise = do

12351239 if isBegin a

1236− then return $ Continue 0 $ WordQuotedWord s 1 a

1240+ then if keepQuotes

1241+ then process s a 1 a

1242+ else return $ Continue 0 $ WordQuotedWord s 1 a

12371243 else if isEnd a

12381244 then return $ Error "wordQuotedBy: missing frame start"

12391245 else processUnquoted s a

12401246 step (WordQuotedWord s n q) a

12411247 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n q

1248+ -- XXX Will this ever occur? Will n ever be 0?

12421249 | n == 0 && isSep a = do

12431250 b <- fextract s

12441251 return $ Partial 0 $ WordQuotedSkipPost b

@@ -1246,7 +1253,9 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

12461253 if a == toRight q

12471254 then

12481255 if n == 1

1249− then return $ Continue 0 $ WordUnquotedWord s

1256+ then if keepQuotes

1257+ then processUnquoted s a

1258+ else return $ Continue 0 $ WordUnquotedWord s

12501259 else process s a (n - 1) q

12511260 else if a == q

12521261 then process s a (n + 1) q

test/Streamly/Test/Data/Parser.hs+64

@@ -1,6 +1,7 @@

11module Main (main) where

33import Control.Exception (SomeException(..), displayException)

4+import Data.Foldable (for_)

45import Data.Word (Word8, Word32, Word64)

56import Streamly.Test.Common (listEquals, checkListEqual, chooseInt)

67import Test.Hspec (Spec, hspec, describe)

@@ -467,6 +468,68 @@ wordBy =

467468 let wrds = words lst

468469 in if wrds == [] && length lst > 0 then [""] else wrds

469470

471+parseManyWordQuotedBy :: H.SpecWith ()

472+parseManyWordQuotedBy =

473+ describe "parseMany wordQuotedBy"

474+ $ for_ testCases

475+ $ \c@(kQ, lQ, rQ, fromLQ, input, expected) -> do

476+ let inpStrm = S.fromList input

477+

478+ esc '\\' = True

479+ esc _ = False

480+

481+ spc ' ' = True

482+ spc _ = False

483+

484+ parser = P.wordQuotedBy kQ esc lQ rQ fromLQ spc FL.toList

485+ result <- H.runIO $ S.toList $ S.parseMany parser inpStrm

486+ H.it (showCase c) $ result `H.shouldBe` expected

487+

488+ where

489+

490+ showCase (kQ, _, _, _, input, expected) =

491+ show kQ ++ ", " ++ input ++ " -> " ++ show expected

492+

493+ testCases =

494+ [ ( True

495+ , (== '\'')

496+ , (== '\'')

497+ , id

498+ , "The quick brown fox"

499+ , ["The", "quick", "brown", "fox"])

500+ , ( True

501+ , (== '\'')

502+ , (== '\'')

503+ , id

504+ , "The' quick brown' fox"

505+ , ["The' quick brown'", "fox"])

506+ , ( False

507+ , (== '\'')

508+ , (== '\'')

509+ , id

510+ , "The' quick brown' fox"

511+ , ["The quick brown", "fox"])

512+ , ( True

513+ , (== '[')

514+ , (== '[')

515+ , \x -> if x == '[' then ']' else error "Not an opening quote."

516+ , "The[ quick brown] fox"

517+ , ["The[ quick brown]", "fox"])

518+ , ( True

519+ , (== '[')

520+ , (== '[')

521+ , \x -> if x == '[' then ']' else error "Not an opening quote."

522+ , "The[ qui[ck] brown] \\ f[ ox]"

523+ , ["The[ qui[ck] brown]", " f[ ox]"])

524+ , ( False

525+ , (== '[')

526+ , (== '[')

527+ , \x -> if x == '[' then ']' else error "Not an opening quote."

528+ , "The[ qui[ck] brown] fox"

529+ , ["The qui[ck] brown", "fox"])

530+ ]

531+

532+

470533-- splitWithPass :: Property

471534-- splitWithPass =

472535-- forAll (listOf (chooseInt (0, 1))) $ \ls ->

@@ -1106,6 +1169,7 @@ main =

11061169 prop ("P.takeP = Prelude.take") takeP

11071170 prop "P.groupBy = Prelude.head . Prelude.groupBy" groupBy

11081171 prop "many (P.wordBy ' ') = words'" wordBy

1172+ parseManyWordQuotedBy

11091173 prop "choice" choice

11101174 -- prop "" splitWithPass

11111175 -- prop "" splitWithFailLeft