@@ -146,7 +146,8 @@ module Streamly.Internal.Data.Parser.ParserD
146146 -- spaces or quotes. No nesting.
147147 , wordFramedBy -- XXX Remove this? Covered by wordWithQuotes?
148148 , wordWithQuotes
149− , wordStripQuotes
149+ , wordKeepQuotes
150+ , wordProcessQuotes
150151
151152 -- Framed by separate start and end characters, potentially nested.
152153 -- blockWithQuotes allows quotes inside a block. However,
@@ -1690,33 +1691,100 @@ data WordQuotedState s b a =
16901691 | WordQuotedEsc !s !Int !a !a
16911692 | WordQuotedSkipPost !b
16921693
1693−-- The invalid unquoted char predicate can be used to generate parse error for
1694−-- a bracket end without a bracket begin.
1695−{-# INLINE wordWithQuotesWith #-}
1696−wordWithQuotesWith :: (Monad m, Eq a) =>
1697− (a -> Bool) -- ^ Matches an invalid unquoted char
1698− -> Bool -- ^ Retain the quotes and escape chars in the output
1699− -> (a -> Bool) -- ^ Matches an escape elem?
1694+-- | Quote and bracket aware word splitting with escaping. Like 'wordBy' but
1695+-- word separators within specified quotes or brackets are ignored. Quotes and
1696+-- escape characters can be processed. If the end quote is different from the
1697+-- start quote it is called a bracket. The following quoting rules apply:
1698+--
1699+-- * In an unquoted string a character may be preceded by an escape character.
1700+-- The escape character is removed and the character following it is treated
1701+-- literally with no special meaning e.g. e.g. h\ e\ l\ l\ o is a single word,
1702+-- \n is same as n.
1703+-- * Any part of the word can be placed within quotes. Inside quotes all
1704+-- characters are treated literally with no special meaning. Quoting character
1705+-- itself cannot be used within quotes unless escape processing within quotes
1706+-- is applied to allow it.
1707+-- * Optionally escape processing for quoted part can be specified. Escape
1708+-- character has no special meaning inside quotes unless it is followed by a
1709+-- character that has a escape translation specified, in that case the escape
1710+-- character is removed, and the specified translation is applied to the
1711+-- character following it. This can be used to escape the quoting character
1712+-- itself within quotes.
1713+-- * There can be multiple quoting characters, when a quote starts, all other
1714+-- quoting characters within that quote lose any special meaning until the
1715+-- quote is closed.
1716+-- * A starting quote char without an ending char generates a parse error. An
1717+-- ending bracket char without a corresponding bracket begin is ignored.
1718+-- * Brackets can be nested.
1719+--
1720+-- We should note that unquoted and quoted escape processing are different. In
1721+-- unquoted part escape character is always removed. In quoted part it is
1722+-- removed only if followed by a special meaning character. This is consistent
1723+-- with how shell performs escape processing.
1724+
1725+-- Examples of quotes - "double quotes", 'single quotes', (parens), {braces},
1726+-- ((nested) brackets).
1727+--
1728+-- Example:
1729+--
1730+-- >>> :{
1731+-- >>> q x =
1732+-- >>> case x of
1733+-- >>> '"' -> Just x
1734+-- >>> '\'' -> Just x
1735+-- >>> _ -> Nothing
1736+-- >>> :}
1737+--
1738+-- >>> p = Parser.wordKeepQuotes (== '\\') q isSpace Fold.toList
1739+-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1740+-- Right "a\"b'c\";'d\"e'f"
1741+--
1742+-- Note that outer quotes and backslashes from the input string are consumed by
1743+-- Haskell, therefore, the actual input string passed to the parser is:
1744+-- a"b'c";'d"e'f ghi
1745+--
1746+-- Similarly, when printing, double quotes are escaped by Haskell.
1747+--
1748+-- Limitations:
1749+--
1750+-- Shell like quote processing can be performed by using quote char specific
1751+-- escape processing, single quotes with no escapes, and double quotes with
1752+-- escapes.
1753+--
1754+-- JSON string processing can also be achieved except the "\uXXXX" style
1755+-- escaping for Unicode characters.
1756+--
1757+{-# INLINE wordWithQuotes #-}
1758+wordWithQuotes :: (Monad m, Eq a) =>
1759+ Bool -- ^ Retain the quotes and escape chars in the output
1760+ -> (a -> a -> Maybe a) -- ^ quote char -> escaped char -> translated char
1761+ -> a -- ^ Matches an escape elem?
17001762 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
17011763 -> (a -> Bool) -- ^ Matches a word separator?
17021764 -> Fold m a b
17031765 -> Parser a m b
1704−wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1766+wordWithQuotes keepQuotes tr escChar toRight isSep
17051767 (Fold fstep finitial fextract) =
17061768 Parser step initial extract
17071769
17081770 where
17091771
1772+ -- Can be used to generate parse error for a bracket end without a bracket
1773+ -- begin.
1774+ isInvalid = const False
1775+
1776+ isEsc = (== escChar)
1777+
17101778 initial = do
17111779 res <- finitial
17121780 return $
17131781 case res of
17141782 FL.Partial s -> IPartial (WordQuotedSkipPre s)
17151783 FL.Done _ ->
1716− error "wordWithQuotes: fold done without input"
1784+ error "wordKeepQuotes: fold done without input"
17171785
1718− {-# INLINE process #-}
1719− process s a n ql qr = do
1786+ {-# INLINE processQuoted #-}
1787+ processQuoted s a n ql qr = do
17201788 res <- fstep s a
17211789 return
17221790 $ case res of
@@ -1738,11 +1806,11 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
17381806 case toRight a of
17391807 Just qr ->
17401808 if keepQuotes
1741− then process s a 1 a qr
1809+ then processQuoted s a 1 a qr
17421810 else return $ Continue 0 $ WordQuotedWord s 1 a qr
17431811 Nothing
17441812 | isInvalid a ->
1745− return $ Error "wordWithQuotes: invalid unquoted char"
1813+ return $ Error "wordKeepQuotes: invalid unquoted char"
17461814 | otherwise -> processUnquoted s a
17471815 step (WordUnquotedWord s) a
17481816 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s
@@ -1753,11 +1821,11 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
17531821 case toRight a of
17541822 Just qr ->
17551823 if keepQuotes
1756− then process s a 1 a qr
1824+ then processQuoted s a 1 a qr
17571825 else return $ Continue 0 $ WordQuotedWord s 1 a qr
17581826 Nothing ->
17591827 if isInvalid a
1760− then return $ Error "wordWithQuotes: invalid unquoted char"
1828+ then return $ Error "wordKeepQuotes: invalid unquoted char"
17611829 else processUnquoted s a
17621830 step (WordQuotedWord s n ql qr) a
17631831 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n ql qr
@@ -1774,12 +1842,19 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
17741842 then if keepQuotes
17751843 then processUnquoted s a
17761844 else return $ Continue 0 $ WordUnquotedWord s
1777− else process s a (n - 1) ql qr
1845+ else processQuoted s a (n - 1) ql qr
17781846 else if a == ql
1779− then process s a (n + 1) ql qr
1780− else process s a n ql qr
1847+ then processQuoted s a (n + 1) ql qr
1848+ else processQuoted s a n ql qr
17811849 step (WordUnquotedEsc s) a = processUnquoted s a
1782− step (WordQuotedEsc s n ql qr) a = process s a n ql qr
1850+ step (WordQuotedEsc s n ql qr) a =
1851+ case tr ql a of
1852+ Nothing -> do
1853+ res <- fstep s escChar
1854+ case res of
1855+ FL.Partial s1 -> processQuoted s1 a n ql qr
1856+ FL.Done b -> return $ Done 0 b
1857+ Just x -> processQuoted s x n ql qr
17831858 step (WordQuotedSkipPost b) a =
17841859 return
17851860 $ if not (isSep a)
@@ -1800,86 +1875,42 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
18001875 err "wordWithQuotes: trailing escape"
18011876 extract (WordQuotedSkipPost b) = return (Done 0 b)
18021877
1803−-- | Quote and bracket aware word splitting. Like 'wordBy' but word separators
1804−-- within specified quotes or brackets are ignored. If the end quote is
1805−-- different from the start quote it is called a bracket. The following quoting
1806−-- rules apply:
1807−--
1808−-- * A quoting or bracket character can be used within the same quote or
1809−-- bracket if escaped using the supplied escape char.
1810−-- * A quoting character other than the current quote can be used inside quotes
1811−-- without escaping e.g. "a single quote ' inside a double quote".
1812−-- * A starting quote or bracket char without an ending char generates a parse
1813−-- error.
1814−-- * An ending bracket char without a corresponding begin quote is ignored e.g.
1815−-- this parenthesis ) is ignored.
1816−-- * Brackets can be nested.
1878+-- | 'wordWithQuotes' without processing the quotes and escape function
1879+-- supplied to escape the quote char within a quote. Can be used to parse words
1880+-- keeping the quotes and escapes intact.
18171881--
1818−-- Examples of quotes - "double quotes", 'single quotes', (parens), {braces},
1819−-- ((nested) brackets).
1882+-- >>> wordKeepQuotes = Parser.wordWithQuotes True (\_ _ -> Nothing)
18201883--
1821−-- Example:
1822−--
1823−-- >>> :{
1824−-- >>> q x =
1825−-- >>> case x of
1826−-- >>> '"' -> Just x
1827−-- >>> '\'' -> Just x
1828−-- >>> _ -> Nothing
1829−-- >>> :}
1830−--
1831−-- >>> p = Parser.wordWithQuotes (== '\\') q isSpace Fold.toList
1832−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1833−-- Right "a\"b'c\";'d\"e'f"
1834−--
1835−-- Note that outer quotes and backslashes from the input string are consumed by
1836−-- Haskell, therefore, the actual input string passed to the parser is:
1837−-- a"b'c";'d"e'f ghi
1838−--
1839−-- Similarly, when printing, double quotes are escaped by Haskell.
1840−--
1841−{-# INLINE wordWithQuotes #-}
1842−wordWithQuotes :: (Monad m, Eq a) =>
1843− (a -> Bool) -- ^ Matches an escape elem?
1884+{-# INLINE wordKeepQuotes #-}
1885+wordKeepQuotes :: (Monad m, Eq a) =>
1886+ a -- ^ Escape char
18441887 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
18451888 -> (a -> Bool) -- ^ Matches a word separator?
18461889 -> Fold m a b
18471890 -> Parser a m b
1848−wordWithQuotes = wordWithQuotesWith (const False) True
1891+wordKeepQuotes =
1892+ -- Escape the quote char itself
1893+ wordWithQuotes True (\q x -> if q == x then Just x else Nothing)
18491894
1850−-- | Like 'wordWithQuotes' but strips the quotes and escape chars.
1851−--
1852−-- Example:
1853−--
1854−-- >>> :{
1855−-- >>> q x =
1856−-- >>> case x of
1857−-- >>> '"' -> Just x
1858−-- >>> '\'' -> Just x
1859−-- >>> _ -> Nothing
1860−-- >>> :}
1861−--
1862−-- >>> p = Parser.wordStripQuotes (== '\\') q isSpace Fold.toList
1863−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1864−-- Right "ab'c;d\"ef"
1865−--
1866−-- Note that outer quotes and backslashes from the input string are consumed by
1867−-- Haskell, therefore, the actual input string passed to the parser is:
1868−-- a"b'c";'d"e'f ghi
1869−--
1870−-- From this the outermost quotes are stripped by the parser, double quotes inside
1871−-- single quotes or vice-versa are kept.
1895+-- See the "Quoting Rules" section in the "bash" manual page for a primer on
1896+-- how quotes are used by shells.
1897+
1898+-- | 'wordWithQuotes' with quote processing applied and escape function
1899+-- supplied to escape the quote char within a quote. Can be ysed to parse words
1900+-- and processing the quoting and escaping at the same time.
18721901--
1873−-- Similarly, when printing, double quotes are escaped by Haskell.
1902+-- >>> wordProcessQuotes = Parser.wordWithQuotes False (\_ _ -> Nothing)
18741903--
1875−{-# INLINE wordStripQuotes #-}
1876−wordStripQuotes :: (Monad m, Eq a) =>
1877− (a -> Bool) -- ^ Matches an escape elem?
1904+{-# INLINE wordProcessQuotes #-}
1905+wordProcessQuotes :: (Monad m, Eq a) =>
1906+ a -- ^ Escape char
18781907 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
18791908 -> (a -> Bool) -- ^ Matches a word separator?
18801909 -> Fold m a b
18811910 -> Parser a m b
1882−wordStripQuotes = wordWithQuotesWith (const False) False
1911+wordProcessQuotes =
1912+ -- Escape the quote char itself
1913+ wordWithQuotes False (\q x -> if q == x then Just x else Nothing)
18831914
18841915{-# ANN type GroupByState Fuse #-}
18851916data GroupByState a s