c40857b · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Improved handling of include files in LaTeX reader.

c40857bJohn MacFarlane9 years ago

pandoc.cabal+1

compare with master

unifiedsplit

@@ -148,6 +148,7 @@ extra-source-files:

148 test/command/3510-subdoc.org

149 test/command/3510-export.latex

150 test/command/3510-src.hs

151+ test/command/3971b.tex

151 test/docbook-reader.docbook

152 test/docbook-reader.docbook

152 test/docbook-xref.docbook

153 test/docbook-xref.docbook

153 test/html-reader.html

154 test/html-reader.html

src/Text/Pandoc/Readers/LaTeX.hs+77−57

@@ -71,7 +71,9 @@ import Text.Pandoc.Shared

71import Text.Pandoc.Readers.LaTeX.Types (Macro(..), ExpansionPoint(..), Tok(..),

72 TokType(..))

73import Text.Pandoc.Walk

74+import Text.Pandoc.Error

74−import Text.Pandoc.Error (PandocError(PandocParsecError, PandocMacroLoop))

75+ (PandocError(PandocParsecError, PandocParseError, PandocMacroLoop))

76+import Text.Parsec.Pos

76-- for debugging:

78-- for debugging:

77-- import Text.Pandoc.Extensions (getDefaultExtensions)

79-- import Text.Pandoc.Extensions (getDefaultExtensions)

@@ -85,7 +87,7 @@ readLaTeX :: PandocMonad m

85 -> m Pandoc

87 -> m Pandoc

86readLaTeX opts ltx = do

88readLaTeX opts ltx = do

87 parsed <- runParserT parseLaTeX def{ sOptions = opts } "source"

89 parsed <- runParserT parseLaTeX def{ sOptions = opts } "source"

88− (tokenize (crFilter ltx))

90+ (tokenize "source" (crFilter ltx))

89 case parsed of

91 case parsed of

90 Right result -> return result

92 Right result -> return result

91 Left e -> throwError $ PandocParsecError (T.unpack ltx) e

93 Left e -> throwError $ PandocParsecError (T.unpack ltx) e

@@ -127,7 +129,7 @@ resolveRefs _ x = x

127-- res <- runIOorExplode (runParserT p defaultLaTeXState{

129-- res <- runIOorExplode (runParserT p defaultLaTeXState{

128-- sOptions = def{ readerExtensions =

130-- sOptions = def{ readerExtensions =

129-- enableExtension Ext_raw_tex $

131-- enableExtension Ext_raw_tex $

130−-- getDefaultExtensions "latex" }} "source" (tokenize t))

132+-- getDefaultExtensions "latex" }} "source" (tokenize "source" t))

131-- case res of

133-- case res of

132-- Left e -> error (show e)

134-- Left e -> error (show e)

133-- Right r -> return r

135-- Right r -> return r

@@ -238,7 +240,7 @@ rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s)

238 => LP m a -> ParserT String s m String

240 => LP m a -> ParserT String s m String

239rawLaTeXParser parser = do

241rawLaTeXParser parser = do

240 inp <- getInput

242 inp <- getInput

241− let toks = tokenize $ T.pack inp

243+ let toks = tokenize "source" $ T.pack inp

242 pstate <- getState

244 pstate <- getState

243 let lstate = def{ sOptions = extractReaderOptions pstate }

245 let lstate = def{ sOptions = extractReaderOptions pstate }

244 res <- lift $ runParserT ((,) <$> try (snd <$> withRaw parser) <*> getState)

246 res <- lift $ runParserT ((,) <$> try (snd <$> withRaw parser) <*> getState)

@@ -257,7 +259,7 @@ applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

257 pstate <- getState

259 pstate <- getState

258 let lstate = def{ sOptions = extractReaderOptions pstate

260 let lstate = def{ sOptions = extractReaderOptions pstate

259 , sMacros = extractMacros pstate }

261 , sMacros = extractMacros pstate }

260− res <- runParserT retokenize lstate "math" (tokenize (T.pack s))

262+ res <- runParserT retokenize lstate "math" (tokenize "math" (T.pack s))

261 case res of

263 case res of

262 Left e -> fail (show e)

264 Left e -> fail (show e)

263 Right s' -> return s'

265 Right s' -> return s'

@@ -278,7 +280,7 @@ inlineCommand :: PandocMonad m => ParserT String ParserState m Inlines

278inlineCommand = do

280inlineCommand = do

279 lookAhead (try (char '\\' >> letter) <|> char '$')

281 lookAhead (try (char '\\' >> letter) <|> char '$')

280 inp <- getInput

282 inp <- getInput

281− let toks = tokenize $ T.pack inp

283+ let toks = tokenize "chunk" $ T.pack inp

282 let rawinline = do

284 let rawinline = do

283 (il, raw) <- try $ withRaw (inlineEnvironment <|> inlineCommand')

285 (il, raw) <- try $ withRaw (inlineEnvironment <|> inlineCommand')

284 st <- getState

286 st <- getState

@@ -294,32 +296,33 @@ inlineCommand = do

294 takeP (T.length (untokenize raw))

296 takeP (T.length (untokenize raw))

295 return il

297 return il

296

298

297−tokenize :: Text -> [Tok]

299+tokenize :: SourceName -> Text -> [Tok]

298−tokenize = totoks (1, 1)

300+tokenize sourcename = totoks (initialPos sourcename)

299

301

300−totoks :: (Line, Column) -> Text -> [Tok]

302+totoks :: SourcePos -> Text -> [Tok]

301−totoks (lin,col) t =

303+totoks pos t =

302 case T.uncons t of

304 case T.uncons t of

303 Nothing -> []

305 Nothing -> []

304 Just (c, rest)

306 Just (c, rest)

305 | c == '\n' ->

307 | c == '\n' ->

306− Tok (lin, col) Newline "\n"

308+ Tok pos Newline "\n"

307− : totoks (lin + 1,1) rest

309+ : totoks (setSourceColumn (incSourceLine pos 1) 1) rest

308 | isSpaceOrTab c ->

310 | isSpaceOrTab c ->

309 let (sps, rest') = T.span isSpaceOrTab t

311 let (sps, rest') = T.span isSpaceOrTab t

310− in Tok (lin, col) Spaces sps

312+ in Tok pos Spaces sps

311− : totoks (lin, col + T.length sps) rest'

313+ : totoks (incSourceColumn pos (T.length sps))

314+ rest'

312 | isAlphaNum c ->

315 | isAlphaNum c ->

313 let (ws, rest') = T.span isAlphaNum t

316 let (ws, rest') = T.span isAlphaNum t

314− in Tok (lin, col) Word ws

317+ in Tok pos Word ws

315− : totoks (lin, col + T.length ws) rest'

318+ : totoks (incSourceColumn pos (T.length ws)) rest'

316 | c == '%' ->

319 | c == '%' ->

317 let (cs, rest') = T.break (== '\n') rest

320 let (cs, rest') = T.break (== '\n') rest

318− in Tok (lin, col) Comment ("%" <> cs)

321+ in Tok pos Comment ("%" <> cs)

319− : totoks (lin, col + 1 + T.length cs) rest'

322+ : totoks (incSourceColumn pos (1 + T.length cs)) rest'

320 | c == '\\' ->

323 | c == '\\' ->

321 case T.uncons rest of

324 case T.uncons rest of

322− Nothing -> [Tok (lin, col) Symbol (T.singleton c)]

325+ Nothing -> [Tok pos Symbol (T.singleton c)]

323 Just (d, rest')

326 Just (d, rest')

324 | isLetterOrAt d ->

327 | isLetterOrAt d ->

325 -- \makeatletter is common in macro defs;

328 -- \makeatletter is common in macro defs;

@@ -328,24 +331,24 @@ totoks (lin,col) t =

328 -- probably best for now

331 -- probably best for now

329 let (ws, rest'') = T.span isLetterOrAt rest

332 let (ws, rest'') = T.span isLetterOrAt rest

330 (ss, rest''') = T.span isSpaceOrTab rest''

333 (ss, rest''') = T.span isSpaceOrTab rest''

331− in Tok (lin, col) (CtrlSeq ws) ("\\" <> ws <> ss)

334+ in Tok pos (CtrlSeq ws) ("\\" <> ws <> ss)

332− : totoks (lin,

335+ : totoks (incSourceColumn pos

333− col + 1 + T.length ws + T.length ss) rest'''

336+ (1 + T.length ws + T.length ss)) rest'''

334 | d == '\t' || d == '\n' ->

337 | d == '\t' || d == '\n' ->

335− Tok (lin, col) Symbol ("\\")

338+ Tok pos Symbol ("\\")

336− : totoks (lin, col + 1) rest

339+ : totoks (incSourceColumn pos 1) rest

337 | otherwise ->

340 | otherwise ->

338− Tok (lin, col) (CtrlSeq (T.singleton d)) (T.pack [c,d])

341+ Tok pos (CtrlSeq (T.singleton d)) (T.pack [c,d])

339− : totoks (lin, col + 2) rest'

342+ : totoks (incSourceColumn pos 2) rest'

340 | c == '#' ->

343 | c == '#' ->

341 let (t1, t2) = T.span (\d -> d >= '0' && d <= '9') rest

344 let (t1, t2) = T.span (\d -> d >= '0' && d <= '9') rest

342 in case safeRead (T.unpack t1) of

345 in case safeRead (T.unpack t1) of

343 Just i ->

346 Just i ->

344− Tok (lin, col) (Arg i) ("#" <> t1)

347+ Tok pos (Arg i) ("#" <> t1)

345− : totoks (lin, col + 1 + T.length t1) t2

348+ : totoks (incSourceColumn pos (1 + T.length t1)) t2

346 Nothing ->

349 Nothing ->

347− Tok (lin, col) Symbol ("#")

350+ Tok pos Symbol ("#")

348− : totoks (lin, col + 1) t2

351+ : totoks (incSourceColumn pos 1) t2

349 | c == '^' ->

352 | c == '^' ->

350 case T.uncons rest of

353 case T.uncons rest of

351 Just ('^', rest') ->

354 Just ('^', rest') ->

@@ -354,20 +357,20 @@ totoks (lin,col) t =

354 | isLowerHex d ->

357 | isLowerHex d ->

355 case T.uncons rest'' of

358 case T.uncons rest'' of

356 Just (e, rest''') | isLowerHex e ->

359 Just (e, rest''') | isLowerHex e ->

357− Tok (lin, col) Esc2 (T.pack ['^','^',d,e])

360+ Tok pos Esc2 (T.pack ['^','^',d,e])

358− : totoks (lin, col + 4) rest'''

361+ : totoks (incSourceColumn pos 4) rest'''

359 _ ->

362 _ ->

360− Tok (lin, col) Esc1 (T.pack ['^','^',d])

363+ Tok pos Esc1 (T.pack ['^','^',d])

361− : totoks (lin, col + 3) rest''

364+ : totoks (incSourceColumn pos 3) rest''

362 | d < '\128' ->

365 | d < '\128' ->

363− Tok (lin, col) Esc1 (T.pack ['^','^',d])

366+ Tok pos Esc1 (T.pack ['^','^',d])

364− : totoks (lin, col + 3) rest''

367+ : totoks (incSourceColumn pos 3) rest''

365− _ -> [Tok (lin, col) Symbol ("^"),

368+ _ -> [Tok pos Symbol ("^"),

366− Tok (lin, col + 1) Symbol ("^")]

369+ Tok (incSourceColumn pos 1) Symbol ("^")]

367− _ -> Tok (lin, col) Symbol ("^")

370+ _ -> Tok pos Symbol ("^")

368− : totoks (lin, col + 1) rest

371+ : totoks (incSourceColumn pos 1) rest

369 | otherwise ->

372 | otherwise ->

370− Tok (lin, col) Symbol (T.singleton c) : totoks (lin, col + 1) rest

373+ Tok pos Symbol (T.singleton c) : totoks (incSourceColumn pos 1) rest

371

374

372 where isSpaceOrTab ' ' = True

375 where isSpaceOrTab ' ' = True

373 isSpaceOrTab '\t' = True

376 isSpaceOrTab '\t' = True

@@ -393,8 +396,7 @@ satisfyTok f =

393 where matcher t | f t = Just t

396 where matcher t | f t = Just t

394 | otherwise = Nothing

397 | otherwise = Nothing

395 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

398 updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos

396− updatePos spos _ (Tok (lin,col) _ _ : _) =

399+ updatePos _spos _ (Tok pos _ _ : _) = pos

397− setSourceColumn (setSourceLine spos lin) col

398 updatePos spos _ [] = spos

400 updatePos spos _ [] = spos

399

401

400doMacros :: PandocMonad m => Int -> LP m ()

402doMacros :: PandocMonad m => Int -> LP m ()

@@ -437,7 +439,7 @@ doMacros n = do

437 else doMacros (n + 1)

439 else doMacros (n + 1)

438 ExpandWhenDefined -> return ()

440 ExpandWhenDefined -> return ()

439

441

440−setpos :: (Line, Column) -> Tok -> Tok

442+setpos :: SourcePos -> Tok -> Tok

441setpos spos (Tok _ tt txt) = Tok spos tt txt

443setpos spos (Tok _ tt txt) = Tok spos tt txt

442

444

443anyControlSeq :: PandocMonad m => LP m Tok

445anyControlSeq :: PandocMonad m => LP m Tok

@@ -728,15 +730,15 @@ doverb = do

728

730

729verbTok :: PandocMonad m => Char -> LP m Tok

731verbTok :: PandocMonad m => Char -> LP m Tok

730verbTok stopchar = do

732verbTok stopchar = do

731− t@(Tok (lin, col) toktype txt) <- satisfyTok (not . isNewlineTok)

733+ t@(Tok pos toktype txt) <- satisfyTok (not . isNewlineTok)

732 case T.findIndex (== stopchar) txt of

734 case T.findIndex (== stopchar) txt of

733 Nothing -> return t

735 Nothing -> return t

734 Just i -> do

736 Just i -> do

735 let (t1, t2) = T.splitAt i txt

737 let (t1, t2) = T.splitAt i txt

736 inp <- getInput

738 inp <- getInput

737− setInput $ Tok (lin, col + i) Symbol (T.singleton stopchar)

739+ setInput $ Tok (incSourceColumn pos i) Symbol (T.singleton stopchar)

738− : (totoks (lin, col + i + 1) (T.drop 1 t2)) ++ inp

740+ : (totoks (incSourceColumn pos (i + 1)) (T.drop 1 t2)) ++ inp

739− return $ Tok (lin, col) toktype t1

741+ return $ Tok pos toktype t1

740

742

741dolstinline :: PandocMonad m => LP m Inlines

743dolstinline :: PandocMonad m => LP m Inlines

742dolstinline = do

744dolstinline = do

@@ -1117,16 +1119,16 @@ tok = grouped inline <|> inlineCommand' <|> singleChar'

1117

1119

1118singleChar :: PandocMonad m => LP m Tok

1120singleChar :: PandocMonad m => LP m Tok

1119singleChar = try $ do

1121singleChar = try $ do

1120− Tok (lin,col) toktype t <- satisfyTok (tokTypeIn [Word, Symbol])

1122+ Tok pos toktype t <- satisfyTok (tokTypeIn [Word, Symbol])

1121 guard $ not $ toktype == Symbol &&

1123 guard $ not $ toktype == Symbol &&

1122 T.any (`Set.member` specialChars) t

1124 T.any (`Set.member` specialChars) t

1123 if T.length t > 1

1125 if T.length t > 1

1124 then do

1126 then do

1125 let (t1, t2) = (T.take 1 t, T.drop 1 t)

1127 let (t1, t2) = (T.take 1 t, T.drop 1 t)

1126 inp <- getInput

1128 inp <- getInput

1127− setInput $ (Tok (lin, col + 1) toktype t2) : inp

1129+ setInput $ (Tok (incSourceColumn pos 1) toktype t2) : inp

1128− return $ Tok (lin,col) toktype t1

1130+ return $ Tok pos toktype t1

1129− else return $ Tok (lin,col) toktype t

1131+ else return $ Tok pos toktype t

1130

1132

1131opt :: PandocMonad m => LP m Inlines

1133opt :: PandocMonad m => LP m Inlines

1132opt = bracketed inline

1134opt = bracketed inline

@@ -1159,7 +1161,7 @@ withRaw :: PandocMonad m => LP m a -> LP m (a, [Tok])

1159withRaw parser = do

1161withRaw parser = do

1160 inp <- getInput

1162 inp <- getInput

1161 result <- parser

1163 result <- parser

1162− nxt <- option (Tok (0,0) Word "") (lookAhead anyTok)

1164+ nxt <- option (Tok (initialPos "source") Word "") (lookAhead anyTok)

1163 let raw = takeWhile (/= nxt) inp

1165 let raw = takeWhile (/= nxt) inp

1164 return (result, raw)

1166 return (result, raw)

1165

1167

@@ -1739,7 +1741,27 @@ include = do

1739 then map (maybeAddExtension ".sty") fs

1741 then map (maybeAddExtension ".sty") fs

1740 else map (maybeAddExtension ".tex") fs

1742 else map (maybeAddExtension ".tex") fs

1741 dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

1743 dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

1742− mconcat <$> mapM (insertIncludedFile blocks (tokenize . T.pack) dirs) fs'

1744+ mapM_ (insertIncluded dirs) fs'

1745+ return mempty

1746+

1747+insertIncluded :: PandocMonad m

1748+ => [FilePath]

1749+ -> FilePath

1750+ -> LP m ()

1751+insertIncluded dirs f = do

1752+ pos <- getPosition

1753+ containers <- getIncludeFiles <$> getState

1754+ when (f `elem` containers) $ do

1755+ throwError $ PandocParseError $ "Include file loop at " ++ show pos

1756+ updateState $ addIncludeFile f

1757+ mbcontents <- readFileFromDirs dirs f

1758+ contents <- case mbcontents of

1759+ Just s -> return s

1760+ Nothing -> do

1761+ report $ CouldNotLoadIncludeFile f pos

1762+ return ""

1763+ getInput >>= setInput . (tokenize f (T.pack contents) ++)

1764+ updateState dropLatestIncludeFile

1743

1765

1744maybeAddExtension :: String -> FilePath -> FilePath

1766maybeAddExtension :: String -> FilePath -> FilePath

1745maybeAddExtension ext fp =

1767maybeAddExtension ext fp =

@@ -2394,9 +2416,7 @@ parseTableRow envname prefsufs = do

2394 >> anyTok)

2416 >> anyTok)

2395 suffpos <- getPosition

2417 suffpos <- getPosition

2396 option [] (count 1 amp)

2418 option [] (count 1 amp)

2397− return $ map (setpos (sourceLine prefpos, sourceColumn prefpos)) pref

2419+ return $ map (setpos prefpos) pref ++ contents ++ map (setpos suffpos) suff

2398− ++ contents ++

2399− map (setpos (sourceLine suffpos, sourceColumn suffpos)) suff

2400 rawcells <- sequence (map celltoks prefsufs)

2420 rawcells <- sequence (map celltoks prefsufs)

2401 oldInput <- getInput

2421 oldInput <- getInput

2402 cells <- sequence $ map (\ts -> setInput ts >> parseTableCell) rawcells

2422 cells <- sequence $ map (\ts -> setInput ts >> parseTableCell) rawcells

src/Text/Pandoc/Readers/LaTeX/Types.hs+4−4

@@ -31,17 +31,17 @@ module Text.Pandoc.Readers.LaTeX.Types ( Tok(..)

31 , TokType(..)

32 , Macro(..)

33 , ExpansionPoint(..)

34− , Line

34+ , SourcePos

35− , Column )

35+ )

36where

37import Data.Text (Text)

38−import Text.Parsec.Pos (Line, Column)

38+import Text.Parsec.Pos (SourcePos)

41 Esc1 | Esc2 | Arg Int

42 deriving (Eq, Ord, Show)

44−data Tok = Tok (Line, Column) TokType Text

44+data Tok = Tok SourcePos TokType Text

45 deriving (Eq, Ord, Show)

47data ExpansionPoint = ExpandWhenDefined | ExpandWhenUsed

test/command/3971.mdnew file+9

@@ -0,0 +1,9 @@

1+```

2+% TEXINPUTS=command pandoc -f latex -t native

3+\documentclass{article}

4+\include{3971b}

5+\code{f}

6+\end{document}

7+^D

8+[Para [Code ("",[],[]) "f"]]

9+```

test/command/3971b.texnew file+2

@@ -0,0 +1,2 @@

1+\newcommand{\code}[1]{\texttt{#1}}

2+\begin{document}