f3b07c2 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Add an option to preserve quotes in wordQuotedBy

f3b07c2Adithya Kumar4 years ago

core/src/Streamly/Internal/Data/Parser.hs+10−5

compare with master

unifiedsplit

@@ -909,21 +909,26 @@ wordFramedBy isEsc isBegin isEnd isSpc =

909-- as it encounters any of the closing quotes.

910--

911-- >>> q = (`elem` ['"', '\''])

912−-- >>> p = Parser.wordQuotedBy (== '\\') q q id isSpace Fold.toList

912+-- >>> p kQ = Parser.wordQuotedBy kQ (== '\\') q q id isSpace Fold.toList

913+--

913−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f"

914+-- >>> Stream.parse (p False) $ Stream.fromList "a\"b'c\";'d\"e'f ghi"

914-- "ab'c;d\"ef"

915-- "ab'c;d\"ef"

915--

916--

917+-- >>> Stream.parse (p True) $ Stream.fromList "a\"b'c\";'d\"e'f ghi"

918+-- "a\"b'c\";'d\"e'f"

919+--

916{-# INLINE wordQuotedBy #-}

920{-# INLINE wordQuotedBy #-}

917wordQuotedBy :: (MonadCatch m, Eq a) =>

921wordQuotedBy :: (MonadCatch m, Eq a) =>

922+ Bool -- ^ keep the quotes in the output

918− (a -> Bool) -- ^ Escape

923+ -> (a -> Bool) -- ^ Escape

919 -> (a -> Bool) -- ^ left quote

924 -> (a -> Bool) -- ^ left quote

920 -> (a -> Bool) -- ^ right quote

925 -> (a -> Bool) -- ^ right quote

921 -> (a -> a) -- ^ get right quote from left quote

926 -> (a -> a) -- ^ get right quote from left quote

922 -> (a -> Bool) -- ^ word seperator

927 -> (a -> Bool) -- ^ word seperator

923 -> Fold m a b

928 -> Fold m a b

924 -> Parser m a b

929 -> Parser m a b

925−wordQuotedBy isEsc isBegin isEnd toRight isSpc =

930+wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSpc =

926− D.toParserK . D.wordQuotedBy isEsc isBegin isEnd toRight isSpc

931+ D.toParserK . D.wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSpc

927

932

928-- | Given an input stream @[a,b,c,...]@ and a comparison function @cmp@, the

933-- | Given an input stream @[a,b,c,...]@ and a comparison function @cmp@, the

929-- parser assigns the element @a@ to the first group, then if @a \`cmp` b@ is

934-- parser assigns the element @a@ to the first group, then if @a \`cmp` b@ is

core/src/Streamly/Internal/Data/Parser/ParserD.hs+14−5

@@ -1182,14 +1182,15 @@ data WordQuotedState s b a =

1182

1183{-# INLINE wordQuotedBy #-}

1184wordQuotedBy :: (MonadCatch m, Eq a) =>

1185+ Bool -- ^ keep the quotes in the output

1185− (a -> Bool) -- ^ Escape

1186+ -> (a -> Bool) -- ^ Escape

1186 -> (a -> Bool) -- ^ left quote

1187 -> (a -> Bool) -- ^ left quote

1187 -> (a -> Bool) -- ^ right quote

1188 -> (a -> Bool) -- ^ right quote

1188 -> (a -> a) -- ^ get right quote from the left quote

1189 -> (a -> a) -- ^ get right quote from the left quote

1189 -> (a -> Bool) -- ^ word seperator

1190 -> (a -> Bool) -- ^ word seperator

1190 -> Fold m a b

1191 -> Fold m a b

1191 -> Parser m a b

1192 -> Parser m a b

1192−wordQuotedBy isEsc isBegin isEnd toRight isSep

1193+wordQuotedBy keepQuotes isEsc isBegin isEnd toRight isSep

1193 (Fold fstep finitial fextract) =

1194 (Fold fstep finitial fextract) =

1194 Parser step initial extract

1195 Parser step initial extract

1195

1196

@@ -1222,7 +1223,10 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

1222 step (WordQuotedSkipPre s) a

1223 step (WordQuotedSkipPre s) a

1223 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s

1224 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s

1224 | isSep a = return $ Partial 0 $ WordQuotedSkipPre s

1225 | isSep a = return $ Partial 0 $ WordQuotedSkipPre s

1226+ | isBegin a =

1227+ if keepQuotes

1228+ then process s a 1 a

1225− | isBegin a = return $ Continue 0 $ WordQuotedWord s 1 a

1229+ else return $ Continue 0 $ WordQuotedWord s 1 a

1226 | isEnd a =

1230 | isEnd a =

1227 return $ Error "wordQuotedBy: missing frame start"

1231 return $ Error "wordQuotedBy: missing frame start"

1228 | otherwise = processUnquoted s a

1232 | otherwise = processUnquoted s a

@@ -1233,12 +1237,15 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

1233 return $ Partial 0 $ WordQuotedSkipPost b

1237 return $ Partial 0 $ WordQuotedSkipPost b

1234 | otherwise = do

1238 | otherwise = do

1235 if isBegin a

1239 if isBegin a

1240+ then if keepQuotes

1241+ then process s a 1 a

1236− then return $ Continue 0 $ WordQuotedWord s 1 a

1242+ else return $ Continue 0 $ WordQuotedWord s 1 a

1237 else if isEnd a

1243 else if isEnd a

1238 then return $ Error "wordQuotedBy: missing frame start"

1244 then return $ Error "wordQuotedBy: missing frame start"

1239 else processUnquoted s a

1245 else processUnquoted s a

1240 step (WordQuotedWord s n q) a

1246 step (WordQuotedWord s n q) a

1241 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n q

1247 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n q

1248+ -- XXX Will this ever occur? Will n ever be 0?

1242 | n == 0 && isSep a = do

1249 | n == 0 && isSep a = do

1243 b <- fextract s

1250 b <- fextract s

1244 return $ Partial 0 $ WordQuotedSkipPost b

1251 return $ Partial 0 $ WordQuotedSkipPost b

@@ -1246,7 +1253,9 @@ wordQuotedBy isEsc isBegin isEnd toRight isSep

1246 if a == toRight q

1253 if a == toRight q

1247 then

1254 then

1248 if n == 1

1255 if n == 1

1256+ then if keepQuotes

1257+ then processUnquoted s a

1249− then return $ Continue 0 $ WordUnquotedWord s

1258+ else return $ Continue 0 $ WordUnquotedWord s

1250 else process s a (n - 1) q

1259 else process s a (n - 1) q

1251 else if a == q

1260 else if a == q

1252 then process s a (n + 1) q

1261 then process s a (n + 1) q

test/Streamly/Test/Data/Parser.hs+64

@@ -1,6 +1,7 @@

1module Main (main) where

3import Control.Exception (SomeException(..), displayException)

4+import Data.Foldable (for_)

4import Data.Word (Word8, Word32, Word64)

5import Data.Word (Word8, Word32, Word64)

5import Streamly.Test.Common (listEquals, checkListEqual, chooseInt)

6import Streamly.Test.Common (listEquals, checkListEqual, chooseInt)

6import Test.Hspec (Spec, hspec, describe)

7import Test.Hspec (Spec, hspec, describe)

@@ -467,6 +468,68 @@ wordBy =

467 let wrds = words lst

468 let wrds = words lst

468 in if wrds == [] && length lst > 0 then [""] else wrds

469 in if wrds == [] && length lst > 0 then [""] else wrds

469

470

471+parseManyWordQuotedBy :: H.SpecWith ()

472+parseManyWordQuotedBy =

473+ describe "parseMany wordQuotedBy"

474+ $ for_ testCases

475+ $ \c@(kQ, lQ, rQ, fromLQ, input, expected) -> do

476+ let inpStrm = S.fromList input

477+

478+ esc '\\' = True

479+ esc _ = False

480+

481+ spc ' ' = True

482+ spc _ = False

483+

484+ parser = P.wordQuotedBy kQ esc lQ rQ fromLQ spc FL.toList

485+ result <- H.runIO $ S.toList $ S.parseMany parser inpStrm

486+ H.it (showCase c) $ result `H.shouldBe` expected

487+

488+ where

489+

490+ showCase (kQ, _, _, _, input, expected) =

491+ show kQ ++ ", " ++ input ++ " -> " ++ show expected

492+

493+ testCases =

494+ [ ( True

495+ , (== '\'')

496+ , (== '\'')

497+ , id

498+ , "The quick brown fox"

499+ , ["The", "quick", "brown", "fox"])

500+ , ( True

501+ , (== '\'')

502+ , (== '\'')

503+ , id

504+ , "The' quick brown' fox"

505+ , ["The' quick brown'", "fox"])

506+ , ( False

507+ , (== '\'')

508+ , (== '\'')

509+ , id

510+ , "The' quick brown' fox"

511+ , ["The quick brown", "fox"])

512+ , ( True

513+ , (== '[')

514+ , (== '[')

515+ , \x -> if x == '[' then ']' else error "Not an opening quote."

516+ , "The[ quick brown] fox"

517+ , ["The[ quick brown]", "fox"])

518+ , ( True

519+ , (== '[')

520+ , (== '[')

521+ , \x -> if x == '[' then ']' else error "Not an opening quote."

522+ , "The[ qui[ck] brown] \\ f[ ox]"

523+ , ["The[ qui[ck] brown]", " f[ ox]"])

524+ , ( False

525+ , (== '[')

526+ , (== '[')

527+ , \x -> if x == '[' then ']' else error "Not an opening quote."

528+ , "The[ qui[ck] brown] fox"

529+ , ["The qui[ck] brown", "fox"])

530+ ]

531+

532+

470-- splitWithPass :: Property

533-- splitWithPass :: Property

471-- splitWithPass =

534-- splitWithPass =

472-- forAll (listOf (chooseInt (0, 1))) $ \ls ->

535-- forAll (listOf (chooseInt (0, 1))) $ \ls ->

@@ -1106,6 +1169,7 @@ main =

1106 prop ("P.takeP = Prelude.take") takeP

1169 prop ("P.takeP = Prelude.take") takeP

1107 prop "P.groupBy = Prelude.head . Prelude.groupBy" groupBy

1170 prop "P.groupBy = Prelude.head . Prelude.groupBy" groupBy

1108 prop "many (P.wordBy ' ') = words'" wordBy

1171 prop "many (P.wordBy ' ') = words'" wordBy

1172+ parseManyWordQuotedBy

1109 prop "choice" choice

1173 prop "choice" choice

1110 -- prop "" splitWithPass

1174 -- prop "" splitWithPass

1111 -- prop "" splitWithFailLeft

1175 -- prop "" splitWithFailLeft