1db5856 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

LaTeX reader: tokenize before pulling tokens,

1db5856John MacFarlane8 years ago

src/Text/Pandoc/Readers/LaTeX.hs+2−4

compare with master

unifiedsplit

@@ -1480,7 +1480,7 @@ authors = try $ do

1480

1481macroDef :: (Monoid a, PandocMonad m) => LP m a

1482macroDef =

1483− mempty <$ ((commandDef <|> environmentDef) <* doMacros)

1483+ mempty <$ (commandDef <|> environmentDef)

1484 where commandDef = do

1485 (name, macro') <- newcommand <|> letmacro <|> defmacro

1486 guardDisabled Ext_latex_macros <|>

@@ -1501,7 +1501,7 @@ letmacro :: PandocMonad m => LP m (Text, Macro)

1501letmacro = do

1502 controlSeq "let"

1503 (name, contents) <- withVerbatimMode $ do

1504− Tok _ (CtrlSeq name) _ <- withVerbatimMode anyControlSeq

1504+ Tok _ (CtrlSeq name) _ <- anyControlSeq

1505 optional $ symbol '='

1506 spaces

1507 -- we first parse in verbatim mode, and then expand macros,

@@ -1521,7 +1521,6 @@ defmacro = try $

1521 Tok _ (CtrlSeq name) _ <- anyControlSeq

1522 argspecs <- many (argspecArg <|> argspecPattern)

1523 contents <- bracedOrToken

1524− doMacros -- after all this verbatim mode

1525 return (name, Macro ExpandWhenUsed argspecs Nothing contents)

1524 return (name, Macro ExpandWhenUsed argspecs Nothing contents)

1526

1525

1527argspecArg :: PandocMonad m => LP m ArgSpec

1526argspecArg :: PandocMonad m => LP m ArgSpec

@@ -1559,7 +1558,6 @@ newcommand = do

1559 case M.lookup name macros of

1558 case M.lookup name macros of

1560 Just _ -> report $ MacroAlreadyDefined (T.unpack txt) pos

1559 Just _ -> report $ MacroAlreadyDefined (T.unpack txt) pos

1561 Nothing -> return ()

1560 Nothing -> return ()

1562− doMacros -- after all this verbatim mode

1563 return (name, Macro ExpandWhenUsed argspecs optarg contents)

1561 return (name, Macro ExpandWhenUsed argspecs optarg contents)

1564

1562

1565newenvironment :: PandocMonad m => LP m (Text, Macro, Macro)

1563newenvironment :: PandocMonad m => LP m (Text, Macro, Macro)

src/Text/Pandoc/Readers/LaTeX/Parsing.hs+14−11

@@ -143,6 +143,7 @@ data LaTeXState = LaTeXState{ sOptions :: ReaderOptions

143 , sLabels :: M.Map String [Inline]

144 , sHasChapters :: Bool

145 , sToggles :: M.Map String Bool

146+ , sExpanded :: Bool

146 }

147 }

147 deriving Show

148 deriving Show

148

149

@@ -164,6 +165,7 @@ defaultLaTeXState = LaTeXState{ sOptions = def

164 , sLabels = M.empty

165 , sLabels = M.empty

165 , sHasChapters = False

166 , sHasChapters = False

166 , sToggles = M.empty

167 , sToggles = M.empty

168+ , sExpanded = False

167 }

169 }

168

170

169instance PandocMonad m => HasQuoteContext LaTeXState m where

171instance PandocMonad m => HasQuoteContext LaTeXState m where

@@ -249,8 +251,7 @@ rawLaTeXParser retokenize parser valParser = do

249applyMacros :: (PandocMonad m, HasMacros s, HasReaderOptions s)

251applyMacros :: (PandocMonad m, HasMacros s, HasReaderOptions s)

250 => String -> ParserT String s m String

252 => String -> ParserT String s m String

251applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

253applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

252− do let retokenize = doMacros *>

253− (toksToString <$> many (satisfyTok (const True)))

254+ do let retokenize = toksToString <$> many (satisfyTok (const True))

254 pstate <- getState

255 pstate <- getState

255 let lstate = def{ sOptions = extractReaderOptions pstate

256 let lstate = def{ sOptions = extractReaderOptions pstate

256 , sMacros = extractMacros pstate }

257 , sMacros = extractMacros pstate }

@@ -258,6 +259,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

258 case res of

259 case res of

259 Left e -> fail (show e)

260 Left e -> fail (show e)

260 Right s' -> return s'

261 Right s' -> return s'

262+

261tokenize :: SourceName -> Text -> [Tok]

263tokenize :: SourceName -> Text -> [Tok]

262tokenize sourcename = totoks (initialPos sourcename)

264tokenize sourcename = totoks (initialPos sourcename)

263

265

@@ -371,11 +373,10 @@ toksToString :: [Tok] -> String

371toksToString = T.unpack . untokenize

373toksToString = T.unpack . untokenize

372

374

373satisfyTok :: PandocMonad m => (Tok -> Bool) -> LP m Tok

375satisfyTok :: PandocMonad m => (Tok -> Bool) -> LP m Tok

374−satisfyTok f =

376+satisfyTok f = do

375− try $ do

376− res <- tokenPrim (T.unpack . untoken) updatePos matcher

377 doMacros -- apply macros on remaining input stream

378− return res

378+ updateState $ \st -> st{ sExpanded = False }

379+ tokenPrim (T.unpack . untoken) updatePos matcher

379 where matcher t | f t = Just t

380 where matcher t | f t = Just t

380 | otherwise = Nothing

381 | otherwise = Nothing

381 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

382 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

@@ -384,12 +385,14 @@ satisfyTok f =

384

385

385doMacros :: PandocMonad m => LP m ()

386doMacros :: PandocMonad m => LP m ()

386doMacros = do

387doMacros = do

388+ expanded <- sExpanded <$> getState

387 verbatimMode <- sVerbatimMode <$> getState

389 verbatimMode <- sVerbatimMode <$> getState

388− unless verbatimMode $ do

390+ unless (expanded || verbatimMode) $ do

389− mbNewInp <- getInput >>= doMacros' 1

391+ mbNewInp <- getInput >>= doMacros' 1

390− case mbNewInp of

392+ case mbNewInp of

391− Nothing -> return ()

393+ Nothing -> return ()

392− Just inp -> setInput inp

394+ Just inp -> setInput inp

395+ updateState $ \st -> st{ sExpanded = True }

393

396

394doMacros' :: PandocMonad m => Int -> [Tok] -> LP m (Maybe [Tok])

397doMacros' :: PandocMonad m => Int -> [Tok] -> LP m (Maybe [Tok])

395doMacros' n inp = do

398doMacros' n inp = do