581a351 · typeable/pandoc

git.iko.soy/typeable/pandoc

RST reader: improve parsing of inline interpreted text roles.

* Use a Span with class "title-reference" for the default title-reference role. * Use B.text to split up contents into Spaces, SoftBreaks, and Strs for title-reference. * Use Code with class "interpreted-text" instead of Span and Str for unknown roles. (The RST writer has also been modified to round-trip this properly.) * Disallow blank lines in interpreted text. * Backslash-escape now works in interpreted text. * Backticks followed by alphanumerics no longer end interpreted text. Closes #4811.

commit581a3514ca266e20e70f05d9ffe314515a0a7bb9 authorJohn MacFarlane <jgm@berkeley.edu> dateSun 5 Aug 2018, 09:15 (8 years ago) parentf7dc3e7

6 files changed+86−19

compare with master

unifiedsplit

src/Text/Pandoc/Readers/RST.hs+28−12 src/Text/Pandoc/Writers/RST.hs+6−3 test/Tests/Readers/RST.hs+1−1 test/command/3407.md+2−2 test/command/4811.md+48 test/rst-reader.native+1−1

src/Text/Pandoc/Readers/RST.hs+28−12

@@ -37,7 +37,7 @@ import Control.Arrow (second)

3737import Control.Monad (forM_, guard, liftM, mplus, mzero, when)

3838import Control.Monad.Except (throwError)

3939import Control.Monad.Identity (Identity (..))

40−import Data.Char (isHexDigit, isSpace, toLower, toUpper)

40+import Data.Char (isHexDigit, isSpace, toLower, toUpper, isAlphaNum)

4141import Data.List (deleteFirstsBy, elemIndex, intercalate, isInfixOf, isSuffixOf,

4242 nub, sort, transpose, union)

4343import qualified Data.Map as M

@@ -1385,7 +1385,6 @@ strong = B.strong . trimInlines . mconcat <$>

13851385--

13861386-- TODO:

13871387-- - Classes are silently discarded in addNewRole

1388−-- - Lacks sensible implementation for title-reference (which is the default)

13891388-- - Allows direct use of the :raw: role, rST only allows inherited use.

13901389interpretedRole :: PandocMonad m => RSTParser m Inlines

13911390interpretedRole = try $ do

@@ -1395,12 +1394,12 @@ interpretedRole = try $ do

13951394renderRole :: PandocMonad m

13961395 => String -> Maybe String -> String -> Attr -> RSTParser m Inlines

13971396renderRole contents fmt role attr = case role of

1398− "sup" -> return $ B.superscript $ B.str contents

1399− "superscript" -> return $ B.superscript $ B.str contents

1400− "sub" -> return $ B.subscript $ B.str contents

1401− "subscript" -> return $ B.subscript $ B.str contents

1402− "emphasis" -> return $ B.emph $ B.str contents

1403− "strong" -> return $ B.strong $ B.str contents

1397+ "sup" -> return $ B.superscript $ treatAsText contents

1398+ "superscript" -> return $ B.superscript $ treatAsText contents

1399+ "sub" -> return $ B.subscript $ treatAsText contents

1400+ "subscript" -> return $ B.subscript $ treatAsText contents

1401+ "emphasis" -> return $ B.emph $ treatAsText contents

1402+ "strong" -> return $ B.strong $ treatAsText contents

14041403 "rfc-reference" -> return $ rfcLink contents

14051404 "RFC" -> return $ rfcLink contents

14061405 "pep-reference" -> return $ pepLink contents

@@ -1411,7 +1410,7 @@ renderRole contents fmt role attr = case role of

14111410 "title" -> titleRef contents

14121411 "t" -> titleRef contents

14131412 "code" -> return $ B.codeWith (addClass "sourceCode" attr) contents

1414− "span" -> return $ B.spanWith attr $ B.str contents

1413+ "span" -> return $ B.spanWith attr $ treatAsText contents

14151414 "raw" -> return $ B.rawInline (fromMaybe "" fmt) contents

14161415 custom -> do

14171416 customRoles <- stateRstCustomRoles <$> getState

@@ -1419,14 +1418,20 @@ renderRole contents fmt role attr = case role of

14191418 Just (newRole, newFmt, newAttr) ->

14201419 renderRole contents newFmt newRole newAttr

14211420 Nothing -> -- undefined role

1422− return $ B.spanWith ("",[],[("role",role)]) (B.str contents)

1421+ return $ B.codeWith ("",["interpreted-text"],[("role",role)])

1422+ contents

14231423 where

1424− titleRef ref = return $ B.str ref -- FIXME: Not a sensible behaviour

1424+ titleRef ref = return $ B.spanWith ("",["title-ref"],[]) $ treatAsText ref

14251425 rfcLink rfcNo = B.link rfcUrl ("RFC " ++ rfcNo) $ B.str ("RFC " ++ rfcNo)

14261426 where rfcUrl = "http://www.faqs.org/rfcs/rfc" ++ rfcNo ++ ".html"

14271427 pepLink pepNo = B.link pepUrl ("PEP " ++ pepNo) $ B.str ("PEP " ++ pepNo)

14281428 where padNo = replicate (4 - length pepNo) '0' ++ pepNo

14291429 pepUrl = "http://www.python.org/dev/peps/pep-" ++ padNo ++ "/"

1430+ treatAsText = B.text . handleEscapes

1431+ handleEscapes [] = []

1432+ handleEscapes ('\\':' ':cs) = handleEscapes cs

1433+ handleEscapes ('\\':c:cs) = c : handleEscapes cs

1434+ handleEscapes (c:cs) = c : handleEscapes cs

14301435

14311436addClass :: String -> Attr -> Attr

14321437addClass c (ident, classes, keyValues) = (ident, classes `union` [c], keyValues)

@@ -1450,7 +1455,18 @@ roleAfter = try $ do

14501455 return (role,contents)

14511456

14521457unmarkedInterpretedText :: PandocMonad m => RSTParser m [Char]

1453−unmarkedInterpretedText = enclosed (atStart $ char '`') (char '`') anyChar

1458+unmarkedInterpretedText = try $ do

1459+ atStart (char '`')

1460+ contents <- mconcat <$> (many1

1461+ ( many1 (noneOf "`\\\n")

1462+ <|> (char '\\' >> ((\c -> ['\\',c]) <$> noneOf "\n"))

1463+ <|> (string "\n" <* notFollowedBy blankline)

1464+ <|> try (string "`" <*

1465+ notFollowedBy (() <$ roleMarker) <*

1466+ lookAhead (satisfy isAlphaNum))

1467+ ))

1468+ char '`'

1469+ return contents

14541470

14551471whitespace :: PandocMonad m => RSTParser m Inlines

14561472whitespace = B.space <$ skipMany1 spaceChar <?> "whitespace"

src/Text/Pandoc/Writers/RST.hs+6−3

@@ -574,15 +574,18 @@ inlineToRST (Quoted DoubleQuote lst) = do

574574 else return $ "“" <> contents <> "”"

575575inlineToRST (Cite _ lst) =

576576 writeInlines lst

577+inlineToRST (Code (_,["interpreted-text"],[("role",role)]) str) = do

578+ return $ ":" <> text role <> ":`" <> text str <> "`"

577579inlineToRST (Code _ str) = do

578580 opts <- gets stOptions

579581 -- we trim the string because the delimiters must adjoin a

580582 -- non-space character; see #3496

581583 -- we use :literal: when the code contains backticks, since

582584 -- :literal: allows backslash-escapes; see #3974

583− return $ if '`' `elem` str

584− then ":literal:`" <> text (escapeString opts (trim str)) <> "`"

585− else "``" <> text (trim str) <> "``"

585+ return $

586+ if '`' `elem` str

587+ then ":literal:`" <> text (escapeString opts (trim str)) <> "`"

588+ else "``" <> text (trim str) <> "``"

586589inlineToRST (Str str) = do

587590 opts <- gets stOptions

588591 return $ text $

test/Tests/Readers/RST.hs+1−1

@@ -177,7 +177,7 @@ tests = [ "line block with blank line" =:

177177 =: ".. role:: haskell(code)\n.. role:: lhs(haskell)\n\n:lhs:`text`"

178178 =?> para (codeWith ("", ["lhs", "haskell", "sourceCode"], []) "text")

179179 , "unknown role" =: ":unknown:`text`" =?>

180− para (spanWith ("",[],[("role","unknown")]) (str "text"))

180+ para (codeWith ("",["interpreted-text"],[("role","unknown")]) "text")

181181 ]

182182 , testGroup "footnotes"

183183 [ "remove space before note" =: T.unlines

test/command/3407.md+2−2

@@ -1,6 +1,6 @@

11```

22% pandoc -f native -t rst

3−[Para [Span ("",[],[("role","foo")]) [Str "text"]]]

3+[Para [Code ("",["interpreted-text"],[("role","foo")]) "text"]]

44^D

55:foo:`text`

66```

@@ -9,5 +9,5 @@

99% pandoc -f rst -t native

1010:foo:`text`

1111^D

12−[Para [Span ("",[],[("role","foo")]) [Str "text"]]]

12+[Para [Code ("",["interpreted-text"],[("role","foo")]) "text"]]

1313```

test/command/4811.mdnew file+48

@@ -0,0 +1,48 @@

1+No blank lines in inline interpreted roles:

3+```

4+% pandoc -f rst -t native

5+`no

7+blank`:myrole:

8+^D

9+[Para [Str "`no"]

10+,Para [Str "blank`:myrole:"]]

11+```

12+

13+Backslash escape behaves properly in interpreted roles:

14+

15+```

16+% pandoc -f rst -t native

17+`hi\ there`:sup:

18+

19+`hi\ there`:code:

20+^D

21+[Para [Superscript [Str "hithere"]]

22+,Para [Code ("",["sourceCode"],[]) "hi\\ there"]]

23+```

24+

25+Backtick followed by alphanumeric doesn't end the span:

26+```

27+% pandoc -f rst -t native

28+`hi`there`:myrole:

29+^D

30+[Para [Code ("",["interpreted-text"],[("role","myrole")]) "hi`there"]]

31+```

32+

33+Newline is okay, as long as not blank:

34+```

35+% pandoc -f rst -t native

36+`hi

37+there`:myrole:

38+^D

39+[Para [Code ("",["interpreted-text"],[("role","myrole")]) "hi\nthere"]]

40+```

41+

42+Use span for title-reference:

43+```

44+% pandoc -f rst -t native

45+`default`

46+^D

47+[Para [Span ("",["title-ref"],[]) [Str "default"]]]

48+```

test/rst-reader.native+1−1

@@ -326,7 +326,7 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "John",Spa

326326,Para [Math DisplayMath "\\alpha = beta",Math DisplayMath "E = mc^2"]

327327,Para [Str "Some",Space,Superscript [Str "of"],Space,Str "these",Space,Superscript [Str "words"],Space,Str "are",Space,Str "in",Space,Superscript [Str "superscript"],Str "."]

328328,Para [Str "Reset",Space,Str "default-role",Space,Str "to",Space,Str "the",Space,Str "default",Space,Str "default."]

329−,Para [Str "And",Space,Str "now",Space,Str "some-invalid-string-3231231",Space,Str "is",Space,Str "nonsense."]

329+,Para [Str "And",Space,Str "now",Space,Span ("",["title-ref"],[]) [Str "some-invalid-string-3231231"],Space,Str "is",Space,Str "nonsense."]

330330,Para [Str "And",Space,Str "now",Space,Str "with",Space,RawInline (Format "html") "<b>inline</b> <span id=\"test\">HTML</span>",Str "."]

331331,Para [Str "And",Space,Str "some",Space,Str "inline",Space,Str "haskell",Space,Code ("",["haskell","sourceCode"],[]) "fmap id [1,2..10]",Str "."]

332332,Para [Str "Indirect",Space,Str "python",Space,Str "role",Space,Code ("",["py","python","indirect","sourceCode"],[]) "[x*x for x in [1,2,3,4,5]]",Str "."]