git.iko.soy/typeable/pandoc
on master help sign in
Merge pull request #2000 from bosmacs/org-smart

Org reader: add support for smart punctuation

commit2e1e76394a671a127919cd5805bcf61a6cda92e8 authorJohn MacFarlane <jgm@berkeley.edu> dateMon 9 Mar 2015, 09:03 (12 years ago) parentc7c4591513221f
2 files changed+74−8
src/Text/Pandoc/Readers/Org.hs+50−8 tests/Tests/Readers/Org.hs+24
src/Text/Pandoc/Readers/Org.hs+50−8
@@ -1,6 +1,6 @@
11{-# LANGUAGE OverloadedStrings #-}
22{-# LANGUAGE GeneralizedNewtypeDeriving #-}
3−{-# LANGUAGE FlexibleContexts #-}
3+{-# LANGUAGE MultiParamTypeClasses, FlexibleContexts, FlexibleInstances #-}
44{-
55Copyright (C) 2014 Albert Krewinkel <tarleb@moltkeplatz.de>
66
@@ -49,7 +49,7 @@ import Control.Applicative ( Applicative, pure
4949 , (<$>), (<$), (<*>), (<*), (*>) )
5050import Control.Arrow (first)
5151import Control.Monad (foldM, guard, liftM, liftM2, mplus, mzero, when)
52−import Control.Monad.Reader (Reader, runReader, ask, asks)
52+import Control.Monad.Reader (Reader, runReader, ask, asks, local)
5353import Data.Char (isAlphaNum, toLower)
5454import Data.Default
5555import Data.List (intersperse, isPrefixOf, isSuffixOf)
@@ -62,9 +62,11 @@ import Network.HTTP (urlEncode)
6262readOrg :: ReaderOptions -- ^ Reader options
6363 -> String -- ^ String to parse (assuming @'\n'@ line endings)
6464 -> Pandoc
65−readOrg opts s = readWith parseOrg def{ orgStateOptions = opts } (s ++ "\n\n")
65+readOrg opts s = flip runReader def $ readWithM parseOrg def{ orgStateOptions = opts } (s ++ "\n\n")
6666
67−type OrgParser = Parser [Char] OrgParserState
67+data OrgParserLocal = OrgParserLocal { orgLocalQuoteContext :: QuoteContext }
68+
69+type OrgParser = ParserT [Char] OrgParserState (Reader OrgParserLocal)
6870
6971parseOrg :: OrgParser Pandoc
7072parseOrg = do
@@ -125,6 +127,9 @@ data OrgParserState = OrgParserState
125127 , orgStateNotes' :: OrgNoteTable
126128 }
127129
130+instance Default OrgParserLocal where
131+ def = OrgParserLocal NoQuote
132+
128133instance HasReaderOptions OrgParserState where
129134 extractReaderOptions = orgStateOptions
130135
@@ -138,6 +143,10 @@ instance HasLastStrPosition OrgParserState where
138143 getLastStrPos = orgStateLastStrPos
139144 setLastStrPos pos st = st{ orgStateLastStrPos = Just pos }
140145
146+instance HasQuoteContext st (Reader OrgParserLocal) where
147+ getQuoteContext = asks orgLocalQuoteContext
148+ withQuoteContext q = local (\s -> s{orgLocalQuoteContext = q})
149+
141150instance Default OrgParserState where
142151 def = defaultOrgParserState
143152
@@ -964,6 +973,7 @@ inline =
964973 , subscript
965974 , superscript
966975 , inlineLaTeX
976+ , smart
967977 , symbol
968978 ] <* (guard =<< newlinesCountWithinLimits)
969979 <?> "inline"
@@ -1270,12 +1280,16 @@ displayMath :: OrgParser (F Inlines)
12701280displayMath = return . B.displayMath <$> choice [ rawMathBetween "\\[" "\\]"
12711281 , rawMathBetween "$$" "$$"
12721282 ]
1283+
1284+updatePositions :: Char
1285+ -> OrgParser (Char)
1286+updatePositions c = do
1287+ when (c `elem` emphasisPreChars) updateLastPreCharPos
1288+ when (c `elem` emphasisForbiddenBorderChars) updateLastForbiddenCharPos
1289+ return c
1290+
12731291symbol :: OrgParser (F Inlines)
12741292symbol = return . B.str . (: "") <$> (oneOf specialChars >>= updatePositions)
1275− where updatePositions c = do
1276− when (c `elem` emphasisPreChars) updateLastPreCharPos
1277− when (c `elem` emphasisForbiddenBorderChars) updateLastForbiddenCharPos
1278− return c
12791293
12801294emphasisBetween :: Char
12811295 -> OrgParser (F Inlines)
@@ -1486,3 +1500,31 @@ inlineLaTeXCommand = try $ do
14861500 count len anyChar
14871501 return cs
14881502 _ -> mzero
1503+
1504+smart :: OrgParser (F Inlines)
1505+smart = do
1506+ getOption readerSmart >>= guard
1507+ doubleQuoted <|> singleQuoted <|>
1508+ choice (map (return <$>) [orgApostrophe, dash, ellipses])
1509+ where orgApostrophe =
1510+ (char '\'' <|> char '\8217') <* updateLastPreCharPos
1511+ <* updateLastForbiddenCharPos
1512+ *> return (B.str "\x2019")
1513+
1514+singleQuoted :: OrgParser (F Inlines)
1515+singleQuoted = try $ do
1516+ singleQuoteStart
1517+ withQuoteContext InSingleQuote $
1518+ fmap B.singleQuoted . trimInlinesF . mconcat <$>
1519+ many1Till inline singleQuoteEnd
1520+
1521+-- doubleQuoted will handle regular double-quoted sections, as well
1522+-- as dialogues with an open double-quote without a close double-quote
1523+-- in the same paragraph.
1524+doubleQuoted :: OrgParser (F Inlines)
1525+doubleQuoted = try $ do
1526+ doubleQuoteStart
1527+ contents <- mconcat <$> many (try $ notFollowedBy doubleQuoteEnd >> inline)
1528+ (withQuoteContext InDoubleQuote $ (doubleQuoteEnd <* updateLastForbiddenCharPos) >> return
1529+ (fmap B.doubleQuoted . trimInlinesF $ contents))
1530+ <|> (return $ return (B.str "\8220") <> contents)
tests/Tests/Readers/Org.hs+24
@@ -12,6 +12,9 @@ import Data.Monoid (mempty, mappend, mconcat)
1212org :: String -> Pandoc
1313org = readOrg def
1414
15+orgSmart :: String -> Pandoc
16+orgSmart = readOrg def { readerSmart = True }
17+
1518infix 4 =:
1619(=:) :: ToString c
1720 => String -> (String, c) -> Test
@@ -1152,4 +1155,25 @@ tests =
11521155 ]
11531156 in codeBlockWith ( "", classes, params) "code body\n"
11541157 ]
1158+ , testGroup "Smart punctuation"
1159+ [ test orgSmart "quote before ellipses"
1160+ ("'...hi'"
1161+ =?> para (singleQuoted "…hi"))
1162+
1163+ , test orgSmart "apostrophe before emph"
1164+ ("D'oh! A l'/aide/!"
1165+ =?> para ("D’oh! A l’" <> emph "aide" <> "!"))
1166+
1167+ , test orgSmart "apostrophe in French"
1168+ ("À l'arrivée de la guerre, le thème de l'«impossibilité du socialisme»"
1169+ =?> para "À l’arrivée de la guerre, le thème de l’«impossibilité du socialisme»")
1170+
1171+ , test orgSmart "Quotes cannot occur at the end of emphasized text"
1172+ ("/say \"yes\"/" =?>
1173+ para ("/say" <> space <> doubleQuoted "yes" <> "/"))
1174+
1175+ , test orgSmart "Dashes are allowed at the borders of emphasis'"
1176+ ("/foo---/" =?>
1177+ para (emph "foo—"))
1178+ ]
11551179 ]