@@ -146,7 +146,8 @@ module Streamly.Internal.Data.Parser.ParserD
146 -- spaces or quotes. No nesting.
146 -- spaces or quotes. No nesting.
147 , wordFramedBy -- XXX Remove this? Covered by wordWithQuotes?
147 , wordFramedBy -- XXX Remove this? Covered by wordWithQuotes?
148 , wordWithQuotes
148 , wordWithQuotes
149− , wordStripQuotes
149+ , wordKeepQuotes
151 -- Framed by separate start and end characters, potentially nested.
152 -- Framed by separate start and end characters, potentially nested.
152 -- blockWithQuotes allows quotes inside a block. However,
153 -- blockWithQuotes allows quotes inside a block. However,
@@ -1690,33 +1691,100 @@ data WordQuotedState s b a =
1690 | WordQuotedEsc !s !Int !a !a
1691 | WordQuotedEsc !s !Int !a !a
1691 | WordQuotedSkipPost !b
1692 | WordQuotedSkipPost !b
1693−-- The invalid unquoted char predicate can be used to generate parse error for
1694+-- | Quote and bracket aware word splitting with escaping. Like 'wordBy' but
1695+-- word separators within specified quotes or brackets are ignored. Quotes and
1696+-- escape characters can be processed. If the end quote is different from the
1697+-- start quote it is called a bracket. The following quoting rules apply:
1699+-- * In an unquoted string a character may be preceded by an escape character.
1700+-- The escape character is removed and the character following it is treated
1701+-- literally with no special meaning e.g. e.g. h\ e\ l\ l\ o is a single word,
1703+-- * Any part of the word can be placed within quotes. Inside quotes all
1704+-- characters are treated literally with no special meaning. Quoting character
1705+-- itself cannot be used within quotes unless escape processing within quotes
1706+-- is applied to allow it.
1707+-- * Optionally escape processing for quoted part can be specified. Escape
1708+-- character has no special meaning inside quotes unless it is followed by a
1709+-- character that has a escape translation specified, in that case the escape
1710+-- character is removed, and the specified translation is applied to the
1711+-- character following it. This can be used to escape the quoting character
1712+-- itself within quotes.
1713+-- * There can be multiple quoting characters, when a quote starts, all other
1714+-- quoting characters within that quote lose any special meaning until the
1716+-- * A starting quote char without an ending char generates a parse error. An
1694−-- a bracket end without a bracket begin.
1717+-- ending bracket char without a corresponding bracket begin is ignored.
1695−{-# INLINE wordWithQuotesWith #-}
1718+-- * Brackets can be nested.
1696−wordWithQuotesWith :: (Monad m, Eq a) =>
1719+--
1697− (a -> Bool) -- ^ Matches an invalid unquoted char
1720+-- We should note that unquoted and quoted escape processing are different. In
1721+-- unquoted part escape character is always removed. In quoted part it is
1722+-- removed only if followed by a special meaning character. This is consistent
1723+-- with how shell performs escape processing.
1725+-- Examples of quotes - "double quotes", 'single quotes', (parens), {braces},
1726+-- ((nested) brackets).
1733+-- >>> '"' -> Just x
1734+-- >>> '\'' -> Just x
1738+-- >>> p = Parser.wordKeepQuotes (== '\\') q isSpace Fold.toList
1739+-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1740+-- Right "a\"b'c\";'d\"e'f"
1742+-- Note that outer quotes and backslashes from the input string are consumed by
1743+-- Haskell, therefore, the actual input string passed to the parser is:
1744+-- a"b'c";'d"e'f ghi
1746+-- Similarly, when printing, double quotes are escaped by Haskell.
1750+-- Shell like quote processing can be performed by using quote char specific
1751+-- escape processing, single quotes with no escapes, and double quotes with
1754+-- JSON string processing can also be achieved except the "\uXXXX" style
1755+-- escaping for Unicode characters.
1757+{-# INLINE wordWithQuotes #-}
1758+wordWithQuotes :: (Monad m, Eq a) =>
1698− -> Bool -- ^ Retain the quotes and escape chars in the output
1759+ Bool -- ^ Retain the quotes and escape chars in the output
1760+ -> (a -> a -> Maybe a) -- ^ quote char -> escaped char -> translated char
1699− -> (a -> Bool) -- ^ Matches an escape elem?
1761+ -> a -- ^ Matches an escape elem?
1700 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1762 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1701 -> (a -> Bool) -- ^ Matches a word separator?
1763 -> (a -> Bool) -- ^ Matches a word separator?
1702 -> Fold m a b
1764 -> Fold m a b
1703 -> Parser a m b
1765 -> Parser a m b
1704−wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1766+wordWithQuotes keepQuotes tr escChar toRight isSep
1705 (Fold fstep finitial fextract) =
1767 (Fold fstep finitial fextract) =
1706 Parser step initial extract
1768 Parser step initial extract
1772+ -- Can be used to generate parse error for a bracket end without a bracket
1774+ isInvalid = const False
1776+ isEsc = (== escChar)
1710 initial = do
1778 initial = do
1711 res <- finitial
1779 res <- finitial
1712 return $
1780 return $
1713 case res of
1781 case res of
1714 FL.Partial s -> IPartial (WordQuotedSkipPre s)
1782 FL.Partial s -> IPartial (WordQuotedSkipPre s)
1715 FL.Done _ ->
1783 FL.Done _ ->
1716− error "wordWithQuotes: fold done without input"
1784+ error "wordKeepQuotes: fold done without input"
1718− {-# INLINE process #-}
1786+ {-# INLINE processQuoted #-}
1719− process s a n ql qr = do
1787+ processQuoted s a n ql qr = do
1720 res <- fstep s a
1788 res <- fstep s a
1722 $ case res of
1790 $ case res of
@@ -1738,11 +1806,11 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1738 case toRight a of
1806 case toRight a of
1739 Just qr ->
1807 Just qr ->
1740 if keepQuotes
1808 if keepQuotes
1741− then process s a 1 a qr
1809+ then processQuoted s a 1 a qr
1742 else return $ Continue 0 $ WordQuotedWord s 1 a qr
1810 else return $ Continue 0 $ WordQuotedWord s 1 a qr
1744 | isInvalid a ->
1812 | isInvalid a ->
1745− return $ Error "wordWithQuotes: invalid unquoted char"
1813+ return $ Error "wordKeepQuotes: invalid unquoted char"
1746 | otherwise -> processUnquoted s a
1814 | otherwise -> processUnquoted s a
1747 step (WordUnquotedWord s) a
1815 step (WordUnquotedWord s) a
1748 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s
1816 | isEsc a = return $ Continue 0 $ WordUnquotedEsc s
@@ -1753,11 +1821,11 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1753 case toRight a of
1821 case toRight a of
1754 Just qr ->
1822 Just qr ->
1755 if keepQuotes
1823 if keepQuotes
1756− then process s a 1 a qr
1824+ then processQuoted s a 1 a qr
1757 else return $ Continue 0 $ WordQuotedWord s 1 a qr
1825 else return $ Continue 0 $ WordQuotedWord s 1 a qr
1758 Nothing ->
1826 Nothing ->
1759 if isInvalid a
1827 if isInvalid a
1760− then return $ Error "wordWithQuotes: invalid unquoted char"
1828+ then return $ Error "wordKeepQuotes: invalid unquoted char"
1761 else processUnquoted s a
1829 else processUnquoted s a
1762 step (WordQuotedWord s n ql qr) a
1830 step (WordQuotedWord s n ql qr) a
1763 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n ql qr
1831 | isEsc a = return $ Continue 0 $ WordQuotedEsc s n ql qr
@@ -1774,12 +1842,19 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1774 then if keepQuotes
1842 then if keepQuotes
1775 then processUnquoted s a
1843 then processUnquoted s a
1776 else return $ Continue 0 $ WordUnquotedWord s
1844 else return $ Continue 0 $ WordUnquotedWord s
1777− else process s a (n - 1) ql qr
1845+ else processQuoted s a (n - 1) ql qr
1778 else if a == ql
1846 else if a == ql
1779− then process s a (n + 1) ql qr
1847+ then processQuoted s a (n + 1) ql qr
1780− else process s a n ql qr
1848+ else processQuoted s a n ql qr
1781 step (WordUnquotedEsc s) a = processUnquoted s a
1849 step (WordUnquotedEsc s) a = processUnquoted s a
1782− step (WordQuotedEsc s n ql qr) a = process s a n ql qr
1850+ step (WordQuotedEsc s n ql qr) a =
1853+ res <- fstep s escChar
1855+ FL.Partial s1 -> processQuoted s1 a n ql qr
1856+ FL.Done b -> return $ Done 0 b
1857+ Just x -> processQuoted s x n ql qr
1783 step (WordQuotedSkipPost b) a =
1858 step (WordQuotedSkipPost b) a =
1785 $ if not (isSep a)
1860 $ if not (isSep a)
@@ -1800,86 +1875,42 @@ wordWithQuotesWith isInvalid keepQuotes isEsc toRight isSep
1800 err "wordWithQuotes: trailing escape"
1875 err "wordWithQuotes: trailing escape"
1801 extract (WordQuotedSkipPost b) = return (Done 0 b)
1876 extract (WordQuotedSkipPost b) = return (Done 0 b)
1803−-- | Quote and bracket aware word splitting. Like 'wordBy' but word separators
1878+-- | 'wordWithQuotes' without processing the quotes and escape function
1804−-- within specified quotes or brackets are ignored. If the end quote is
1879+-- supplied to escape the quote char within a quote. Can be used to parse words
1805−-- different from the start quote it is called a bracket. The following quoting
1880+-- keeping the quotes and escapes intact.
1808−-- * A quoting or bracket character can be used within the same quote or
1809−-- bracket if escaped using the supplied escape char.
1810−-- * A quoting character other than the current quote can be used inside quotes
1811−-- without escaping e.g. "a single quote ' inside a double quote".
1812−-- * A starting quote or bracket char without an ending char generates a parse
1814−-- * An ending bracket char without a corresponding begin quote is ignored e.g.
1815−-- this parenthesis ) is ignored.
1816−-- * Brackets can be nested.
1818−-- Examples of quotes - "double quotes", 'single quotes', (parens), {braces},
1882+-- >>> wordKeepQuotes = Parser.wordWithQuotes True (\_ _ -> Nothing)
1819−-- ((nested) brackets).
1821−-- Example:
1884+{-# INLINE wordKeepQuotes #-}
1822−--
1885+wordKeepQuotes :: (Monad m, Eq a) =>
1823−-- >>> :{
1886+ a -- ^ Escape char
1826−-- >>> '"' -> Just x
1827−-- >>> '\'' -> Just x
1831−-- >>> p = Parser.wordWithQuotes (== '\\') q isSpace Fold.toList
1832−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1833−-- Right "a\"b'c\";'d\"e'f"
1835−-- Note that outer quotes and backslashes from the input string are consumed by
1836−-- Haskell, therefore, the actual input string passed to the parser is:
1837−-- a"b'c";'d"e'f ghi
1839−-- Similarly, when printing, double quotes are escaped by Haskell.
1841−{-# INLINE wordWithQuotes #-}
1842−wordWithQuotes :: (Monad m, Eq a) =>
1843− (a -> Bool) -- ^ Matches an escape elem?
1844 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1887 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1845 -> (a -> Bool) -- ^ Matches a word separator?
1888 -> (a -> Bool) -- ^ Matches a word separator?
1846 -> Fold m a b
1889 -> Fold m a b
1847 -> Parser a m b
1890 -> Parser a m b
1848−wordWithQuotes = wordWithQuotesWith (const False) True
1891+wordKeepQuotes =
1892+ -- Escape the quote char itself
1893+ wordWithQuotes True (\q x -> if q == x then Just x else Nothing)
1895+-- See the "Quoting Rules" section in the "bash" manual page for a primer on
1896+-- how quotes are used by shells.
1850−-- | Like 'wordWithQuotes' but strips the quotes and escape chars.
1898+-- | 'wordWithQuotes' with quote processing applied and escape function
1851−--
1899+-- supplied to escape the quote char within a quote. Can be ysed to parse words
1852−-- Example:
1900+-- and processing the quoting and escaping at the same time.
1857−-- >>> '"' -> Just x
1858−-- >>> '\'' -> Just x
1862−-- >>> p = Parser.wordStripQuotes (== '\\') q isSpace Fold.toList
1863−-- >>> Stream.parse p $ Stream.fromList "a\"b'c\";'d\"e'f ghi"
1864−-- Right "ab'c;d\"ef"
1866−-- Note that outer quotes and backslashes from the input string are consumed by
1867−-- Haskell, therefore, the actual input string passed to the parser is:
1868−-- a"b'c";'d"e'f ghi
1870−-- From this the outermost quotes are stripped by the parser, double quotes inside
1871−-- single quotes or vice-versa are kept.
1873−-- Similarly, when printing, double quotes are escaped by Haskell.
1902+-- >>> wordProcessQuotes = Parser.wordWithQuotes False (\_ _ -> Nothing)
1875−{-# INLINE wordStripQuotes #-}
1904+{-# INLINE wordProcessQuotes #-}
1876−wordStripQuotes :: (Monad m, Eq a) =>
1905+wordProcessQuotes :: (Monad m, Eq a) =>
1877− (a -> Bool) -- ^ Matches an escape elem?
1906+ a -- ^ Escape char
1878 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1907 -> (a -> Maybe a) -- ^ If left quote, return right quote, else Nothing.
1879 -> (a -> Bool) -- ^ Matches a word separator?
1908 -> (a -> Bool) -- ^ Matches a word separator?
1880 -> Fold m a b
1909 -> Fold m a b
1881 -> Parser a m b
1910 -> Parser a m b
1882−wordStripQuotes = wordWithQuotesWith (const False) False
1911+wordProcessQuotes =
1912+ -- Escape the quote char itself
1913+ wordWithQuotes False (\q x -> if q == x then Just x else Nothing)
1884{-# ANN type GroupByState Fuse #-}
1915{-# ANN type GroupByState Fuse #-}
1885data GroupByState a s
1916data GroupByState a s