5c055b4 · typeable/pandoc

git.iko.soy/typeable/pandoc

Introduce file-scope parsing (parse-before-combine)

Traditionally pandoc operates on multiple files by first concetenating them (around extra line breaks) and then processing the joined file. So it only parses a multi-file document at the document scope. This has the benefit that footnotes and links can be in different files, but it also introduces a couple of difficulties: - it is difficult to join files with footnotes without some sort of preprocessing, which makes it difficult to write academic documents in small pieces. - it makes it impossible to process multiple binary input files, which can't be catted. - it makes it impossible to process files from different input formats. This commit introduces alternative method. Instead of catting the files first, it parses the files first, and then combines the parsed output. This makes it impossible to have links across multiple files, and auto-identified headers won't work correctly if headers in multiple files have the same name. On the other hand, footnotes across multiple files will work correctly and will allow more freedom for input formats. Since ByteStringReaders can currently only read one binary file, and will ignore subsequent files, we also changes the behavior to automatically parse before combining if using the ByteStringReader. If we use one file, it will work as normal. If there is more than one file it will combine them after parsing (assuming that the format is the same). Note that this is intended to be an optional method, defaulting to off. Turn it on with `--file-scope`.

commit5c055b4cf3cdfac534a74c5c5775aa2d58889150 authorJesse Rosenthal <jrosenthal@jhu.edu> dateSat 20 Feb 2016, 21:27 (11 years ago) parent68fd333

2 files changed+25−4

compare with master

unifiedsplit

pandoc.hs+23−4 src/Text/Pandoc/Options.hs+2

pandoc.hs+23−4

@@ -215,6 +215,7 @@ data Opt = Opt

215215 , optExtractMedia :: Maybe FilePath -- ^ Path to extract embedded media

216216 , optTrace :: Bool -- ^ Print debug information

217217 , optTrackChanges :: TrackChanges -- ^ Accept or reject MS Word track-changes.

218+ , optFileScope :: Bool -- ^ Parse input files before combining

218219 , optKaTeXStylesheet :: Maybe String -- ^ Path to stylesheet for KaTeX

219220 , optKaTeXJS :: Maybe String -- ^ Path to js file for KaTeX

220221 }

@@ -278,6 +279,7 @@ defaultOpts = Opt

278279 , optExtractMedia = Nothing

279280 , optTrace = False

280281 , optTrackChanges = AcceptChanges

282+ , optFileScope = False

281283 , optKaTeXStylesheet = Nothing

282284 , optKaTeXJS = Nothing

283285 }

@@ -387,6 +389,11 @@ options =

387389 "accept|reject|all")

388390 "" -- "Accepting or reject MS Word track-changes.""

389391

392+ , Option "" ["file-scope"]

393+ (NoArg

394+ (\opt -> return opt { optFileScope = True }))

395+ "" -- "Parse input files before combining"

396+

390397 , Option "" ["extract-media"]

391398 (ReqArg

392399 (\arg opt ->

@@ -1117,6 +1124,7 @@ convertWithOpts opts args = do

11171124 , optExtractMedia = mbExtractMedia

11181125 , optTrace = trace

11191126 , optTrackChanges = trackChanges

1127+ , optFileScope = fileScope

11201128 , optKaTeXStylesheet = katexStylesheet

11211129 , optKaTeXJS = katexJS

11221130 } = opts

@@ -1269,6 +1277,7 @@ convertWithOpts opts args = do

12691277 , readerDefaultImageExtension = defaultImageExtension

12701278 , readerTrace = trace

12711279 , readerTrackChanges = trackChanges

1280+ , readerFileScope = fileScope

12721281 }

12731282

12741283 when (not (isTextFormat format) && outputFile == "-") $

@@ -1301,13 +1310,23 @@ convertWithOpts opts args = do

13011310 then handleIncludes

13021311 else return . Right

13031312

1304− (doc, media) <- fmap handleError $

1305− case reader of

1313+ let sourceToDoc :: [FilePath] -> IO (Pandoc, MediaBag)

1314+ sourceToDoc sources' = fmap handleError $

1315+ case reader of

13061316 StringReader r-> do

1307− srcs <- convertTabs . intercalate "\n" <$> readSources sources

1317+ srcs <- convertTabs . intercalate "\n" <$> readSources sources'

13081318 doc <- handleIncludes' srcs

13091319 either (return . Left) (\s -> fmap (,mempty) <$> r readerOpts s) doc

1310− ByteStringReader r -> readFiles sources >>= r readerOpts

1320+ ByteStringReader r -> readFiles sources' >>= r readerOpts

1321+

1322+ -- We parse first if fileScope is set OR if the reader is a

1323+ -- BSReader. So, if it's a StringReader AND not fileScope, we

1324+ -- don't.

1325+ (doc, media) <- case reader of

1326+ (StringReader _) | not fileScope -> sourceToDoc sources

1327+ _ -> do

1328+ pairs <- mapM (\s -> sourceToDoc [s]) sources

1329+ return (mconcat $ map fst pairs, mconcat $ map snd pairs)

13111330

13121331 let writerOptions = def { writerStandalone = standalone',

13131332 writerTemplate = templ,

src/Text/Pandoc/Options.hs+2

@@ -264,6 +264,7 @@ data ReaderOptions = ReaderOptions{

264264 , readerDefaultImageExtension :: String -- ^ Default extension for images

265265 , readerTrace :: Bool -- ^ Print debugging info

266266 , readerTrackChanges :: TrackChanges

267+ , readerFileScope :: Bool -- ^ Parse before combining

267268} deriving (Show, Read, Data, Typeable, Generic)

268269

269270instance Default ReaderOptions

@@ -280,6 +281,7 @@ instance Default ReaderOptions

280281 , readerDefaultImageExtension = ""

281282 , readerTrace = False

282283 , readerTrackChanges = AcceptChanges

284+ , readerFileScope = False

283285 }

284286

285287--