c40857b · typeable/pandoc

git.iko.soy/typeable/pandoc

Improved handling of include files in LaTeX reader.

Previously `\include` wouldn't work if the included file contained, e.g., a begin without a matching end. We've changed the Tok type so that it stores a full SourcePos, rather than just a line and column. So tokens keeep track of the file they came from. This allows us to use a simpler method for includes, which doesn't require parsing the included document as a whole. Closes #3971.

commitc40857b38905b9ea298a777354eb4cb0da2213c6 authorJohn MacFarlane <jgm@berkeley.edu> dateMon 16 Oct 2017, 22:03 (9 years ago) parent9cf9a64

5 files changed+93−61

compare with master

unifiedsplit

pandoc.cabal+1 src/Text/Pandoc/Readers/LaTeX.hs+77−57 src/Text/Pandoc/Readers/LaTeX/Types.hs+4−4 test/command/3971.md+9 test/command/3971b.tex+2

pandoc.cabal+1

@@ -148,6 +148,7 @@ extra-source-files:

148148 test/command/3510-subdoc.org

149149 test/command/3510-export.latex

150150 test/command/3510-src.hs

151+ test/command/3971b.tex

151152 test/docbook-reader.docbook

152153 test/docbook-xref.docbook

153154 test/html-reader.html

src/Text/Pandoc/Readers/LaTeX.hs+77−57

@@ -71,7 +71,9 @@ import Text.Pandoc.Shared

7171import Text.Pandoc.Readers.LaTeX.Types (Macro(..), ExpansionPoint(..), Tok(..),

7272 TokType(..))

7373import Text.Pandoc.Walk

74−import Text.Pandoc.Error (PandocError(PandocParsecError, PandocMacroLoop))

74+import Text.Pandoc.Error

75+ (PandocError(PandocParsecError, PandocParseError, PandocMacroLoop))

76+import Text.Parsec.Pos

7577

7678-- for debugging:

7779-- import Text.Pandoc.Extensions (getDefaultExtensions)

@@ -85,7 +87,7 @@ readLaTeX :: PandocMonad m

8587 -> m Pandoc

8688readLaTeX opts ltx = do

8789 parsed <- runParserT parseLaTeX def{ sOptions = opts } "source"

88− (tokenize (crFilter ltx))

90+ (tokenize "source" (crFilter ltx))

8991 case parsed of

9092 Right result -> return result

9193 Left e -> throwError $ PandocParsecError (T.unpack ltx) e

@@ -127,7 +129,7 @@ resolveRefs _ x = x

127129-- res <- runIOorExplode (runParserT p defaultLaTeXState{

128130-- sOptions = def{ readerExtensions =

129131-- enableExtension Ext_raw_tex $

130−-- getDefaultExtensions "latex" }} "source" (tokenize t))

132+-- getDefaultExtensions "latex" }} "source" (tokenize "source" t))

131133-- case res of

132134-- Left e -> error (show e)

133135-- Right r -> return r

@@ -238,7 +240,7 @@ rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s)

238240 => LP m a -> ParserT String s m String

239241rawLaTeXParser parser = do

240242 inp <- getInput

241− let toks = tokenize $ T.pack inp

243+ let toks = tokenize "source" $ T.pack inp

242244 pstate <- getState

243245 let lstate = def{ sOptions = extractReaderOptions pstate }

244246 res <- lift $ runParserT ((,) <$> try (snd <$> withRaw parser) <*> getState)

@@ -257,7 +259,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

257259 pstate <- getState

258260 let lstate = def{ sOptions = extractReaderOptions pstate

259261 , sMacros = extractMacros pstate }

260− res <- runParserT retokenize lstate "math" (tokenize (T.pack s))

262+ res <- runParserT retokenize lstate "math" (tokenize "math" (T.pack s))

261263 case res of

262264 Left e -> fail (show e)

263265 Right s' -> return s'

@@ -278,7 +280,7 @@ inlineCommand :: PandocMonad m => ParserT String ParserState m Inlines

278280inlineCommand = do

279281 lookAhead (try (char '\\' >> letter) <|> char '$')

280282 inp <- getInput

281− let toks = tokenize $ T.pack inp

283+ let toks = tokenize "chunk" $ T.pack inp

282284 let rawinline = do

283285 (il, raw) <- try $ withRaw (inlineEnvironment <|> inlineCommand')

284286 st <- getState

@@ -294,32 +296,33 @@ inlineCommand = do

294296 takeP (T.length (untokenize raw))

295297 return il

296298

297−tokenize :: Text -> [Tok]

298−tokenize = totoks (1, 1)

299+tokenize :: SourceName -> Text -> [Tok]

300+tokenize sourcename = totoks (initialPos sourcename)

299301

300−totoks :: (Line, Column) -> Text -> [Tok]

301−totoks (lin,col) t =

302+totoks :: SourcePos -> Text -> [Tok]

303+totoks pos t =

302304 case T.uncons t of

303305 Nothing -> []

304306 Just (c, rest)

305307 | c == '\n' ->

306− Tok (lin, col) Newline "\n"

307− : totoks (lin + 1,1) rest

308+ Tok pos Newline "\n"

309+ : totoks (setSourceColumn (incSourceLine pos 1) 1) rest

308310 | isSpaceOrTab c ->

309311 let (sps, rest') = T.span isSpaceOrTab t

310− in Tok (lin, col) Spaces sps

311− : totoks (lin, col + T.length sps) rest'

312+ in Tok pos Spaces sps

313+ : totoks (incSourceColumn pos (T.length sps))

314+ rest'

312315 | isAlphaNum c ->

313316 let (ws, rest') = T.span isAlphaNum t

314− in Tok (lin, col) Word ws

315− : totoks (lin, col + T.length ws) rest'

317+ in Tok pos Word ws

318+ : totoks (incSourceColumn pos (T.length ws)) rest'

316319 | c == '%' ->

317320 let (cs, rest') = T.break (== '\n') rest

318− in Tok (lin, col) Comment ("%" <> cs)

319− : totoks (lin, col + 1 + T.length cs) rest'

321+ in Tok pos Comment ("%" <> cs)

322+ : totoks (incSourceColumn pos (1 + T.length cs)) rest'

320323 | c == '\\' ->

321324 case T.uncons rest of

322− Nothing -> [Tok (lin, col) Symbol (T.singleton c)]

325+ Nothing -> [Tok pos Symbol (T.singleton c)]

323326 Just (d, rest')

324327 | isLetterOrAt d ->

325328 -- \makeatletter is common in macro defs;

@@ -328,24 +331,24 @@ totoks (lin,col) t =

328331 -- probably best for now

329332 let (ws, rest'') = T.span isLetterOrAt rest

330333 (ss, rest''') = T.span isSpaceOrTab rest''

331− in Tok (lin, col) (CtrlSeq ws) ("\\" <> ws <> ss)

332− : totoks (lin,

333− col + 1 + T.length ws + T.length ss) rest'''

334+ in Tok pos (CtrlSeq ws) ("\\" <> ws <> ss)

335+ : totoks (incSourceColumn pos

336+ (1 + T.length ws + T.length ss)) rest'''

334337 | d == '\t' || d == '\n' ->

335− Tok (lin, col) Symbol ("\\")

336− : totoks (lin, col + 1) rest

338+ Tok pos Symbol ("\\")

339+ : totoks (incSourceColumn pos 1) rest

337340 | otherwise ->

338− Tok (lin, col) (CtrlSeq (T.singleton d)) (T.pack [c,d])

339− : totoks (lin, col + 2) rest'

341+ Tok pos (CtrlSeq (T.singleton d)) (T.pack [c,d])

342+ : totoks (incSourceColumn pos 2) rest'

340343 | c == '#' ->

341344 let (t1, t2) = T.span (\d -> d >= '0' && d <= '9') rest

342345 in case safeRead (T.unpack t1) of

343346 Just i ->

344− Tok (lin, col) (Arg i) ("#" <> t1)

345− : totoks (lin, col + 1 + T.length t1) t2

347+ Tok pos (Arg i) ("#" <> t1)

348+ : totoks (incSourceColumn pos (1 + T.length t1)) t2

346349 Nothing ->

347− Tok (lin, col) Symbol ("#")

348− : totoks (lin, col + 1) t2

350+ Tok pos Symbol ("#")

351+ : totoks (incSourceColumn pos 1) t2

349352 | c == '^' ->

350353 case T.uncons rest of

351354 Just ('^', rest') ->

@@ -354,20 +357,20 @@ totoks (lin,col) t =

354357 | isLowerHex d ->

355358 case T.uncons rest'' of

356359 Just (e, rest''') | isLowerHex e ->

357− Tok (lin, col) Esc2 (T.pack ['^','^',d,e])

358− : totoks (lin, col + 4) rest'''

360+ Tok pos Esc2 (T.pack ['^','^',d,e])

361+ : totoks (incSourceColumn pos 4) rest'''

359362 _ ->

360− Tok (lin, col) Esc1 (T.pack ['^','^',d])

361− : totoks (lin, col + 3) rest''

363+ Tok pos Esc1 (T.pack ['^','^',d])

364+ : totoks (incSourceColumn pos 3) rest''

362365 | d < '\128' ->

363− Tok (lin, col) Esc1 (T.pack ['^','^',d])

364− : totoks (lin, col + 3) rest''

365− _ -> [Tok (lin, col) Symbol ("^"),

366− Tok (lin, col + 1) Symbol ("^")]

367− _ -> Tok (lin, col) Symbol ("^")

368− : totoks (lin, col + 1) rest

366+ Tok pos Esc1 (T.pack ['^','^',d])

367+ : totoks (incSourceColumn pos 3) rest''

368+ _ -> [Tok pos Symbol ("^"),

369+ Tok (incSourceColumn pos 1) Symbol ("^")]

370+ _ -> Tok pos Symbol ("^")

371+ : totoks (incSourceColumn pos 1) rest

369372 | otherwise ->

370− Tok (lin, col) Symbol (T.singleton c) : totoks (lin, col + 1) rest

373+ Tok pos Symbol (T.singleton c) : totoks (incSourceColumn pos 1) rest

371374

372375 where isSpaceOrTab ' ' = True

373376 isSpaceOrTab '\t' = True

@@ -393,8 +396,7 @@ satisfyTok f =

393396 where matcher t | f t = Just t

394397 | otherwise = Nothing

395398 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

396− updatePos spos _ (Tok (lin,col) _ _ : _) =

397− setSourceColumn (setSourceLine spos lin) col

399+ updatePos _spos _ (Tok pos _ _ : _) = pos

398400 updatePos spos _ [] = spos

399401

400402doMacros :: PandocMonad m => Int -> LP m ()

@@ -437,7 +439,7 @@ doMacros n = do

437439 else doMacros (n + 1)

438440 ExpandWhenDefined -> return ()

439441

440−setpos :: (Line, Column) -> Tok -> Tok

442+setpos :: SourcePos -> Tok -> Tok

441443setpos spos (Tok _ tt txt) = Tok spos tt txt

442444

443445anyControlSeq :: PandocMonad m => LP m Tok

@@ -728,15 +730,15 @@ doverb = do

728730

729731verbTok :: PandocMonad m => Char -> LP m Tok

730732verbTok stopchar = do

731− t@(Tok (lin, col) toktype txt) <- satisfyTok (not . isNewlineTok)

733+ t@(Tok pos toktype txt) <- satisfyTok (not . isNewlineTok)

732734 case T.findIndex (== stopchar) txt of

733735 Nothing -> return t

734736 Just i -> do

735737 let (t1, t2) = T.splitAt i txt

736738 inp <- getInput

737− setInput $ Tok (lin, col + i) Symbol (T.singleton stopchar)

738− : (totoks (lin, col + i + 1) (T.drop 1 t2)) ++ inp

739− return $ Tok (lin, col) toktype t1

739+ setInput $ Tok (incSourceColumn pos i) Symbol (T.singleton stopchar)

740+ : (totoks (incSourceColumn pos (i + 1)) (T.drop 1 t2)) ++ inp

741+ return $ Tok pos toktype t1

740742

741743dolstinline :: PandocMonad m => LP m Inlines

742744dolstinline = do

@@ -1117,16 +1119,16 @@ tok = grouped inline <|> inlineCommand' <|> singleChar'

11171119

11181120singleChar :: PandocMonad m => LP m Tok

11191121singleChar = try $ do

1120− Tok (lin,col) toktype t <- satisfyTok (tokTypeIn [Word, Symbol])

1122+ Tok pos toktype t <- satisfyTok (tokTypeIn [Word, Symbol])

11211123 guard $ not $ toktype == Symbol &&

11221124 T.any (`Set.member` specialChars) t

11231125 if T.length t > 1

11241126 then do

11251127 let (t1, t2) = (T.take 1 t, T.drop 1 t)

11261128 inp <- getInput

1127− setInput $ (Tok (lin, col + 1) toktype t2) : inp

1128− return $ Tok (lin,col) toktype t1

1129− else return $ Tok (lin,col) toktype t

1129+ setInput $ (Tok (incSourceColumn pos 1) toktype t2) : inp

1130+ return $ Tok pos toktype t1

1131+ else return $ Tok pos toktype t

11301132

11311133opt :: PandocMonad m => LP m Inlines

11321134opt = bracketed inline

@@ -1159,7 +1161,7 @@ withRaw :: PandocMonad m => LP m a -> LP m (a, [Tok])

11591161withRaw parser = do

11601162 inp <- getInput

11611163 result <- parser

1162− nxt <- option (Tok (0,0) Word "") (lookAhead anyTok)

1164+ nxt <- option (Tok (initialPos "source") Word "") (lookAhead anyTok)

11631165 let raw = takeWhile (/= nxt) inp

11641166 return (result, raw)

11651167

@@ -1739,7 +1741,27 @@ include = do

17391741 then map (maybeAddExtension ".sty") fs

17401742 else map (maybeAddExtension ".tex") fs

17411743 dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

1742− mconcat <$> mapM (insertIncludedFile blocks (tokenize . T.pack) dirs) fs'

1744+ mapM_ (insertIncluded dirs) fs'

1745+ return mempty

1746+

1747+insertIncluded :: PandocMonad m

1748+ => [FilePath]

1749+ -> FilePath

1750+ -> LP m ()

1751+insertIncluded dirs f = do

1752+ pos <- getPosition

1753+ containers <- getIncludeFiles <$> getState

1754+ when (f `elem` containers) $ do

1755+ throwError $ PandocParseError $ "Include file loop at " ++ show pos

1756+ updateState $ addIncludeFile f

1757+ mbcontents <- readFileFromDirs dirs f

1758+ contents <- case mbcontents of

1759+ Just s -> return s

1760+ Nothing -> do

1761+ report $ CouldNotLoadIncludeFile f pos

1762+ return ""

1763+ getInput >>= setInput . (tokenize f (T.pack contents) ++)

1764+ updateState dropLatestIncludeFile

17431765

17441766maybeAddExtension :: String -> FilePath -> FilePath

17451767maybeAddExtension ext fp =

@@ -2394,9 +2416,7 @@ parseTableRow envname prefsufs = do

23942416 >> anyTok)

23952417 suffpos <- getPosition

23962418 option [] (count 1 amp)

2397− return $ map (setpos (sourceLine prefpos, sourceColumn prefpos)) pref

2398− ++ contents ++

2399− map (setpos (sourceLine suffpos, sourceColumn suffpos)) suff

2419+ return $ map (setpos prefpos) pref ++ contents ++ map (setpos suffpos) suff

24002420 rawcells <- sequence (map celltoks prefsufs)

24012421 oldInput <- getInput

24022422 cells <- sequence $ map (\ts -> setInput ts >> parseTableCell) rawcells

src/Text/Pandoc/Readers/LaTeX/Types.hs+4−4

@@ -31,17 +31,17 @@ module Text.Pandoc.Readers.LaTeX.Types ( Tok(..)

3131 , TokType(..)

3232 , Macro(..)

3333 , ExpansionPoint(..)

34− , Line

35− , Column )

34+ , SourcePos

35+ )

3636where

3737import Data.Text (Text)

38−import Text.Parsec.Pos (Line, Column)

38+import Text.Parsec.Pos (SourcePos)

3939

4141 Esc1 | Esc2 | Arg Int

4242 deriving (Eq, Ord, Show)

4343

44−data Tok = Tok (Line, Column) TokType Text

44+data Tok = Tok SourcePos TokType Text

4545 deriving (Eq, Ord, Show)

4646

4747data ExpansionPoint = ExpandWhenDefined | ExpandWhenUsed

test/command/3971.mdnew file+9

@@ -0,0 +1,9 @@

1+```

2+% TEXINPUTS=command pandoc -f latex -t native

3+\documentclass{article}

4+\include{3971b}

5+\code{f}

6+\end{document}

7+^D

8+[Para [Code ("",[],[]) "f"]]

9+```

test/command/3971b.texnew file+2

@@ -0,0 +1,2 @@

1+\newcommand{\code}[1]{\texttt{#1}}

2+\begin{document}