400fe31 · typeable/pandoc

git.iko.soy/typeable/pandoc

Allow `--extract-media` to work with non-binary input formats.

If `--extract-media` is supplied with a non-binary input format, pandoc will attempt to extract the contents of all linked images, whether in local files, data: uris, or external uris. They will be named based on the sha1 hash of the contents. Closes #1583, #2289. Notes: - One thing that is slightly subideal with this commit is that identical resources will be downloaded multiple times. To improve this we could have mediabag store an original filename/url + a new name. - We might think about reusing some of this code, since more or less the same thing is done in the Docx, EPUB, PDF writers (with slight variations).

commit400fe3188e3f5a3e48700ae114a0da05ae6e599a authorJohn MacFarlane <jgm@berkeley.edu> dateSun 7 May 2017, 11:45 (9 years ago) parentde0fd90

1 file changed+26−4

compare with master

unifiedsplit

src/Text/Pandoc/App.hs+26−4

src/Text/Pandoc/App.hs+26−4

@@ -45,6 +45,7 @@ import Data.Aeson (eitherDecode', encode)

4545import qualified Data.ByteString as BS

4646import qualified Data.ByteString.Lazy as B

4747import Data.Char (toLower, toUpper)

48+import Data.Digest.Pure.SHA (sha1, showDigest)

4849import qualified Data.Set as Set

4950import Data.Foldable (foldrM)

5051import Data.List (intercalate, isPrefixOf, isSuffixOf, sort)

@@ -68,17 +69,19 @@ import System.IO (stdout)

6869import System.IO.Error (isDoesNotExistError)

6970import Text.Pandoc

7071import Text.Pandoc.Builder (setMeta)

71−import Text.Pandoc.Class (PandocIO, getLog, withMediaBag, getMediaBag)

72+import Text.Pandoc.Class (PandocIO, getLog, withMediaBag, getMediaBag,

73+ fetchItem, insertMedia)

7274import Text.Pandoc.Highlighting (highlightingStyles)

7375import Text.Pandoc.Lua ( runLuaFilter )

7476import Text.Pandoc.MediaBag (extractMediaBag, mediaDirectory)

77+import Text.Pandoc.MIME (extensionFromMimeType)

7578import Text.Pandoc.PDF (makePDF)

7679import Text.Pandoc.Process (pipeProcess)

7780import Text.Pandoc.SelfContained (makeSelfContained, makeDataURI)

7881import Text.Pandoc.Shared (headerShift, openURL, readDataFile,

7982 readDataFileUTF8, safeRead, tabFilter)

8083import qualified Text.Pandoc.UTF8 as UTF8

81−import Text.Pandoc.Walk (walk)

84+import Text.Pandoc.Walk (walkM, walk)

8285import Text.Pandoc.XML (toEntities)

8386import Text.Printf

8487#ifndef _WINDOWS

@@ -413,11 +416,15 @@ convertWithOpts opts = do

413416

414417 runIO' $ do

415418 (doc, media) <- withMediaBag $ sourceToDoc sources >>=

416− (maybe return extractMedia (optExtractMedia opts)

419+ ( (if isJust (optExtractMedia opts)

420+ then fillMedia (writerSourceURL writerOptions)

421+ else return)

422+ >=> maybe return extractMedia (optExtractMedia opts)

417423 >=> return . flip (foldr addMetadata) metadata

418424 >=> applyTransforms transforms

419425 >=> applyLuaFilters datadir (optLuaFilters opts) [format]

420− >=> applyFilters datadir filters' [format])

426+ >=> applyFilters datadir filters' [format]

427+ )

421428

422429 case writer of

423430 ByteStringWriter f -> f writerOptions doc >>= writeFnBinary outputFile

@@ -723,6 +730,21 @@ defaultWriterName x =

723730

724731-- Transformations of a Pandoc document post-parsing:

725732

733+-- | Traverse tree, filling media bag.

734+fillMedia :: Maybe String -> Pandoc -> PandocIO Pandoc

735+fillMedia sourceURL d = walkM handleImage d

736+ where handleImage :: Inline -> PandocIO Inline

737+ handleImage (Image attr lab (src, tit)) = do

738+ (bs, mt) <- fetchItem sourceURL src

739+ let ext = fromMaybe (takeExtension src)

740+ (mt >>= extensionFromMimeType)

741+ let bs' = B.fromChunks [bs]

742+ let basename = showDigest $ sha1 bs'

743+ let fname = basename <.> ext

744+ insertMedia fname mt bs'

745+ return $ Image attr lab (fname, tit)

746+ handleImage x = return x

747+

726748extractMedia :: FilePath -> Pandoc -> PandocIO Pandoc

727749extractMedia dir d = do

728750 media <- getMediaBag