1db5856 · typeable/pandoc

git.iko.soy/typeable/pandoc

LaTeX reader: tokenize before pulling tokens,

rather than after. This has some performance penalty but is more reliable. Closes #4408.

commit1db585689ad6de5c829851a1e7b3aa0192e4e808 authorJohn MacFarlane <jgm@berkeley.edu> dateMon 15 Oct 2018, 14:52 (8 years ago) parent7e9e24b

2 files changed+16−15

compare with master

unifiedsplit

src/Text/Pandoc/Readers/LaTeX.hs+2−4 src/Text/Pandoc/Readers/LaTeX/Parsing.hs+14−11

src/Text/Pandoc/Readers/LaTeX.hs+2−4

@@ -1480,7 +1480,7 @@ authors = try $ do

14801480

14811481macroDef :: (Monoid a, PandocMonad m) => LP m a

14821482macroDef =

1483− mempty <$ ((commandDef <|> environmentDef) <* doMacros)

1483+ mempty <$ (commandDef <|> environmentDef)

14841484 where commandDef = do

14851485 (name, macro') <- newcommand <|> letmacro <|> defmacro

14861486 guardDisabled Ext_latex_macros <|>

@@ -1501,7 +1501,7 @@ letmacro :: PandocMonad m => LP m (Text, Macro)

15011501letmacro = do

15021502 controlSeq "let"

15031503 (name, contents) <- withVerbatimMode $ do

1504− Tok _ (CtrlSeq name) _ <- withVerbatimMode anyControlSeq

1504+ Tok _ (CtrlSeq name) _ <- anyControlSeq

15051505 optional $ symbol '='

15061506 spaces

15071507 -- we first parse in verbatim mode, and then expand macros,

@@ -1521,7 +1521,6 @@ defmacro = try $

15211521 Tok _ (CtrlSeq name) _ <- anyControlSeq

15221522 argspecs <- many (argspecArg <|> argspecPattern)

15231523 contents <- bracedOrToken

1524− doMacros -- after all this verbatim mode

15251524 return (name, Macro ExpandWhenUsed argspecs Nothing contents)

15261525

15271526argspecArg :: PandocMonad m => LP m ArgSpec

@@ -1559,7 +1558,6 @@ newcommand = do

15591558 case M.lookup name macros of

15601559 Just _ -> report $ MacroAlreadyDefined (T.unpack txt) pos

15611560 Nothing -> return ()

1562− doMacros -- after all this verbatim mode

15631561 return (name, Macro ExpandWhenUsed argspecs optarg contents)

15641562

15651563newenvironment :: PandocMonad m => LP m (Text, Macro, Macro)

src/Text/Pandoc/Readers/LaTeX/Parsing.hs+14−11

@@ -143,6 +143,7 @@ data LaTeXState = LaTeXState{ sOptions :: ReaderOptions

143143 , sLabels :: M.Map String [Inline]

144144 , sHasChapters :: Bool

145145 , sToggles :: M.Map String Bool

146+ , sExpanded :: Bool

146147 }

147148 deriving Show

148149

@@ -164,6 +165,7 @@ defaultLaTeXState = LaTeXState{ sOptions = def

164165 , sLabels = M.empty

165166 , sHasChapters = False

166167 , sToggles = M.empty

168+ , sExpanded = False

167169 }

168170

169171instance PandocMonad m => HasQuoteContext LaTeXState m where

@@ -249,8 +251,7 @@ rawLaTeXParser retokenize parser valParser = do

249251applyMacros :: (PandocMonad m, HasMacros s, HasReaderOptions s)

250252 => String -> ParserT String s m String

251253applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

252− do let retokenize = doMacros *>

253− (toksToString <$> many (satisfyTok (const True)))

254+ do let retokenize = toksToString <$> many (satisfyTok (const True))

254255 pstate <- getState

255256 let lstate = def{ sOptions = extractReaderOptions pstate

256257 , sMacros = extractMacros pstate }

@@ -258,6 +259,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

258259 case res of

259260 Left e -> fail (show e)

260261 Right s' -> return s'

262+

261263tokenize :: SourceName -> Text -> [Tok]

262264tokenize sourcename = totoks (initialPos sourcename)

263265

@@ -371,11 +373,10 @@ toksToString :: [Tok] -> String

371373toksToString = T.unpack . untokenize

372374

373375satisfyTok :: PandocMonad m => (Tok -> Bool) -> LP m Tok

374−satisfyTok f =

375− try $ do

376− res <- tokenPrim (T.unpack . untoken) updatePos matcher

376+satisfyTok f = do

377377 doMacros -- apply macros on remaining input stream

378− return res

378+ updateState $ \st -> st{ sExpanded = False }

379+ tokenPrim (T.unpack . untoken) updatePos matcher

379380 where matcher t | f t = Just t

380381 | otherwise = Nothing

381382 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

@@ -384,12 +385,14 @@ satisfyTok f =

384385

385386doMacros :: PandocMonad m => LP m ()

386387doMacros = do

388+ expanded <- sExpanded <$> getState

387389 verbatimMode <- sVerbatimMode <$> getState

388− unless verbatimMode $ do

389− mbNewInp <- getInput >>= doMacros' 1

390− case mbNewInp of

391− Nothing -> return ()

392− Just inp -> setInput inp

390+ unless (expanded || verbatimMode) $ do

391+ mbNewInp <- getInput >>= doMacros' 1

392+ case mbNewInp of

393+ Nothing -> return ()

394+ Just inp -> setInput inp

395+ updateState $ \st -> st{ sExpanded = True }

393396

394397doMacros' :: PandocMonad m => Int -> [Tok] -> LP m (Maybe [Tok])

395398doMacros' n inp = do