3f09f53 · typeable/pandoc

git.iko.soy/typeable/pandoc

Implement curly-brace syntax for Markdown citation keys.

The change provides a way to use citation keys that contain special characters not usable with the standard citation key syntax. Example: `@{foo_bar{x}'}` for the key `foo_bar{x}`. Closes #6026. The change requires adding a new parameter to the `citeKey` parser from Text.Pandoc.Parsing [API change]. Markdown reader: recognize @{..} syntax for citatinos. Markdown writer: use @{..} syntax for citations when needed. Update manual with curly-brace syntax for citations. Closes #6026.

commit3f09f53459b877f53072efbf57dec21fa37280b5 authorJohn MacFarlane <jgm@berkeley.edu> dateThu 13 May 2021, 21:11 (5 years ago) parentc46482b

6 files changed+56−19

compare with master

unifiedsplit

MANUAL.txt+10−5 src/Text/Pandoc/Parsing.hs+13−5 src/Text/Pandoc/Readers/Markdown.hs+3−3 src/Text/Pandoc/Readers/Org/Inlines.hs+4−4 src/Text/Pandoc/Writers/Markdown/Inline.hs+7−2 test/command/6026.md+19

MANUAL.txt+10−5

@@ -4949,12 +4949,15 @@ Inline and regular footnotes may be mixed freely.

49494949#### Extension: `citations` ####

49504950

49514951Markdown citations go inside square brackets and are separated

4952−by semicolons. Each citation must have a key, composed of '@' +

4953−the citation identifier from the database, and may optionally

4954−have a prefix, a locator, and a suffix. The citation key must

4955−begin with a letter, digit, or `_`, and may contain

4952+by semicolons. Each citation must have a key and may optionally

4953+have a prefix, a locator, and a suffix. The citation key

4954+consists of `@` plus the citation identifier, possibly

4955+enclosed in curly braces. If the identifier starts

4956+with a letter, digit, or `_`, followed by zero or more

49564957alphanumerics, `_`, and internal punctuation characters

4957−(`:.#$%&-+?<>~/`). Here are some examples:

4958+(`:.#$%&-+?<>~/`), then the curly braces may be omitted.

4959+Identifiers may not contain whitespace characters or unbalanced

4960+curly braces. Here are some examples:

49584961

49594962 Blah blah [see @doe99, pp. 33-35; also @smith04, chap. 1].

49604963

@@ -4962,6 +4965,8 @@ alphanumerics, `_`, and internal punctuation characters

49624965

49634966 Blah blah [@smith04; @doe99].

49644967

4968+ Blah blah [@{https://example.com/bib?name=foobar&date=2000}, p. 33].

4969+

49654970`pandoc` detects locator terms in the [CSL locale files].

49664971Either abbreviated or unabbreviated forms are accepted. In the `en-US`

49674972locale, locator terms can be written in either singular or plural forms,

src/Text/Pandoc/Parsing.hs+13−5

@@ -1605,19 +1605,27 @@ nested p = do

16051605 return res

16061606

16071607citeKey :: (Stream s m Char, UpdateSourcePos s Char, HasLastStrPosition st)

1608− => ParserT s st m (Bool, Text)

1609−citeKey = try $ do

1608+ => Bool -- ^ If True, allow expanded @{..} syntax.

1609+ -> ParserT s st m (Bool, Text)

1610+citeKey allowBraced = try $ do

16101611 guard =<< notAfterString

16111612 suppress_author <- option False (True <$ char '-')

16121613 char '@'

1614+ key <- simpleCiteIdentifier

1615+ <|> if allowBraced

1616+ then charsInBalanced '{' '}' (satisfy (not . isSpace))

1617+ else mzero

1618+ return (suppress_author, key)

1619+

1620+simpleCiteIdentifier :: (Stream s m Char, UpdateSourcePos s Char)

1621+ => ParserT s st m Text

1622+simpleCiteIdentifier = do

16131623 firstChar <- alphaNum <|> char '_' <|> char '*' -- @* for wildcard in nocite

16141624 let regchar = satisfy (\c -> isAlphaNum c || c == '_')

16151625 let internal p = try $ p <* lookAhead regchar

16161626 rest <- many $ regchar <|> internal (oneOf ":.#$%&-+?<>~/") <|>

16171627 try (oneOf ":/" <* lookAhead (char '/'))

1618− let key = firstChar:rest

1619− return (suppress_author, T.pack key)

1620−

1628+ return $ T.pack $ firstChar:rest

16211629

16221630token :: (Stream s m t)

16231631 => (t -> Text)

src/Text/Pandoc/Readers/Markdown.hs+3−3

@@ -2094,7 +2094,7 @@ cite = do

20942094

20952095textualCite :: PandocMonad m => MarkdownParser m (F Inlines)

20962096textualCite = try $ do

2097− (suppressAuthor, key) <- citeKey

2097+ (suppressAuthor, key) <- citeKey True

20982098 -- If this is a reference to an earlier example list item,

20992099 -- then don't parse it as a citation. If the example list

21002100 -- item comes later, we'll parse it here and figure out in

@@ -2174,7 +2174,7 @@ prefix = trimInlinesF . mconcat <$>

21742174 manyTill inline (char ']'

21752175 <|> lookAhead

21762176 (try $ do optional (try (char ';' >> spnl))

2177− citeKey

2177+ citeKey True

21782178 return ']'))

21792179

21802180citeList :: PandocMonad m => MarkdownParser m (F [Citation])

@@ -2183,7 +2183,7 @@ citeList = fmap sequence $ sepBy1 citation (try $ char ';' >> spnl)

21832183citation :: PandocMonad m => MarkdownParser m (F Citation)

21842184citation = try $ do

21852185 pref <- prefix

2186− (suppress_author, key) <- citeKey

2186+ (suppress_author, key) <- citeKey True

21872187 suff <- suffix

21882188 noteNum <- stateNoteNumber <$> getState

21892189 return $ do

src/Text/Pandoc/Readers/Org/Inlines.hs+4−4

@@ -263,7 +263,7 @@ berkeleyCitationList = try $ do

263263 where

264264 citationListPart :: PandocMonad m => OrgParser m (F Inlines)

265265 citationListPart = fmap (trimInlinesF . mconcat) . try . many1 $ do

266− notFollowedBy' citeKey

266+ notFollowedBy' $ citeKey False

267267 notFollowedBy (oneOf ";]")

268268 inline

269269

@@ -278,7 +278,7 @@ berkeleyBareTag' = try $ void (string "cite")

278278

279279berkeleyTextualCite :: PandocMonad m => OrgParser m (F [Citation])

280280berkeleyTextualCite = try $ do

281− (suppressAuthor, key) <- citeKey

281+ (suppressAuthor, key) <- citeKey False

282282 returnF . return $ Citation

283283 { citationId = key

284284 , citationPrefix = mempty

@@ -351,7 +351,7 @@ citeList = sequence <$> sepEndBy1 citation (try $ char ';' *> skipSpaces)

351351citation :: PandocMonad m => OrgParser m (F Citation)

352352citation = try $ do

353353 pref <- prefix

354− (suppress_author, key) <- citeKey

354+ (suppress_author, key) <- citeKey False

355355 suff <- suffix

356356 return $ do

357357 x <- pref

@@ -368,7 +368,7 @@ citation = try $ do

368368 }

369369 where

370370 prefix = trimInlinesF . mconcat <$>

371− manyTill inline (char ']' <|> (']' <$ lookAhead citeKey))

371+ manyTill inline (char ']' <|> (']' <$ lookAhead (citeKey False)))

372372 suffix = try $ do

373373 hasSpace <- option False (notFollowedBy nonspaceChar >> return True)

374374 skipSpaces

src/Text/Pandoc/Writers/Markdown/Inline.hs+7−2

@@ -491,11 +491,16 @@ inlineToMarkdown opts (Cite (c:cs) lst)

491491 rest <- mapM convertOne cs

492492 let inbr = suffs <+> joincits rest

493493 br = if isEmpty inbr then empty else char '[' <> inbr <> char ']'

494− return $ literal ("@" <> citationId c) <+> br

494+ return $ literal ("@" <> maybeInBraces (citationId c)) <+> br

495495 else do

496496 cits <- mapM convertOne (c:cs)

497497 return $ literal "[" <> joincits cits <> literal "]"

498498 where

499+ maybeInBraces key =

500+ case readWith (citeKey False >> spaces >> eof)

501+ defaultParserState ("@" <> key) of

502+ Left _ -> "{" <> key <> "}"

503+ Right _ -> key

499504 joincits = hcat . intersperse (literal "; ") . filter (not . isEmpty)

500505 convertOne Citation { citationId = k

501506 , citationPrefix = pinlines

@@ -504,7 +509,7 @@ inlineToMarkdown opts (Cite (c:cs) lst)

504509 = do

505510 pdoc <- inlineListToMarkdown opts pinlines

506511 sdoc <- inlineListToMarkdown opts sinlines

507− let k' = literal (modekey m <> "@" <> k)

512+ let k' = literal (modekey m <> "@" <> maybeInBraces k)

508513 r = case sinlines of

509514 Str (T.uncons -> Just (y,_)):_ | y `elem` (",;]@" :: String) -> k' <> sdoc

510515 _ -> k' <+> sdoc

test/command/6026.mdnew file+19

@@ -0,0 +1,19 @@

1+```

2+% pandoc -t native

3+@{https://openreview.net/forum?id=HkwoSDPgg}

5+@https://openreview.net/forum?id=HkwoSDPgg

6+^D

7+[Para [Cite [Citation {citationId = "https://openreview.net/forum?id=HkwoSDPgg", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 1, citationHash = 0}] [Str "@https://openreview.net/forum?id=HkwoSDPgg"]]

8+,Para [Cite [Citation {citationId = "https://openreview.net/forum?id", citationPrefix = [], citationSuffix = [], citationMode = AuthorInText, citationNoteNum = 2, citationHash = 0}] [Str "@https://openreview.net/forum?id"],Str "=HkwoSDPgg"]]

9+```

10+```

11+% pandoc -t markdown

12+@{https://openreview.net/forum?id=HkwoSDPgg}

13+

14+@https://openreview.net/forum?id=HkwoSDPgg

15+^D

16+@{https://openreview.net/forum?id=HkwoSDPgg}

17+

18+@https://openreview.net/forum?id=HkwoSDPgg

19+```