git.iko.soy/typeable/pandoc
on master help sign in
Move filter functions to separate module
commit5d49cbd35e815dd041e54da511bdd0eeafd400c0 authorAlbert Krewinkel <albert@zeitkraut.de> dateWed 10 Jan 2018, 22:26 (9 years ago) parentf130109
6 files changed+273−96
pandoc.cabal+5−1 src/Text/Pandoc/App.hs+5−95 src/Text/Pandoc/Filter.hs+60 src/Text/Pandoc/Filter/Json.hs+97 src/Text/Pandoc/Filter/Lua.hs+53 src/Text/Pandoc/Filter/Path.hs+53
pandoc.cabal+5−1
@@ -501,7 +501,11 @@ library
501501 Text.Pandoc.ImageSize,
502502 Text.Pandoc.BCP47,
503503 Text.Pandoc.Class
504− other-modules: Text.Pandoc.Readers.Docx.Lists,
504+ other-modules: Text.Pandoc.Filter,
505+ Text.Pandoc.Filter.Json,
506+ Text.Pandoc.Filter.Lua,
507+ Text.Pandoc.Filter.Path,
508+ Text.Pandoc.Readers.Docx.Lists,
505509 Text.Pandoc.Readers.Docx.Combine,
506510 Text.Pandoc.Readers.Docx.Parse,
507511 Text.Pandoc.Readers.Docx.Util,
src/Text/Pandoc/App.hs+5−95
@@ -46,12 +46,11 @@ import qualified Control.Exception as E
4646import Control.Monad
4747import Control.Monad.Except (catchError, throwError)
4848import Control.Monad.Trans
49−import Data.Aeson (defaultOptions, eitherDecode', encode)
49+import Data.Aeson (defaultOptions)
5050import Data.Aeson.TH (deriveJSON)
5151import qualified Data.ByteString as BS
5252import qualified Data.ByteString.Lazy as B
5353import Data.Char (toLower, toUpper)
54−import Data.Foldable (foldrM)
5554import Data.List (find, intercalate, isPrefixOf, isSuffixOf, sort)
5655import qualified Data.Map as M
5756import Data.Maybe (fromMaybe, isJust, isNothing)
@@ -73,10 +72,9 @@ import Skylighting (Style, Syntax (..), defaultSyntaxMap, parseTheme,
7372 pygments)
7473import Skylighting.Parser (addSyntaxDefinition, parseSyntaxDefinition)
7574import System.Console.GetOpt
76−import System.Directory (Permissions (..), doesFileExist, findExecutable,
77− getAppUserDataDirectory, getPermissions)
78−import System.Environment (getArgs, getEnvironment, getProgName)
79−import System.Exit (ExitCode (..), exitSuccess)
75+import System.Directory (getAppUserDataDirectory)
76+import System.Environment (getArgs, getProgName)
77+import System.Exit (exitSuccess)
8078import System.FilePath
8179import System.IO (nativeNewline, stdout)
8280import qualified System.IO as IO (Newline (..))
@@ -84,10 +82,9 @@ import System.IO.Error (isDoesNotExistError)
8482import Text.Pandoc
8583import Text.Pandoc.BCP47 (Lang (..), parseBCP47)
8684import Text.Pandoc.Builder (setMeta, deleteMeta)
85+import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), applyFilters)
8786import Text.Pandoc.Highlighting (highlightingStyles)
88−import Text.Pandoc.Lua (LuaException (..), runLuaFilter)
8987import Text.Pandoc.PDF (makePDF)
90−import Text.Pandoc.Process (pipeProcess)
9188import Text.Pandoc.SelfContained (makeDataURI, makeSelfContained)
9289import Text.Pandoc.Shared (eastAsianLineBreakFilter, stripEmptyParagraphs,
9390 headerShift, isURI, ordNub, safeRead, tabFilter)
@@ -538,48 +535,6 @@ type Transform = Pandoc -> Pandoc
538535isTextFormat :: String -> Bool
539536isTextFormat s = s `notElem` ["odt","docx","epub2","epub3","epub","pptx"]
540537
541−externalFilter :: MonadIO m
542− => ReaderOptions -> FilePath -> [String] -> Pandoc -> m Pandoc
543−externalFilter ropts f args' d = liftIO $ do
544− exists <- doesFileExist f
545− isExecutable <- if exists
546− then executable <$> getPermissions f
547− else return True
548− let (f', args'') = if exists
549− then case map toLower (takeExtension f) of
550− _ | isExecutable -> ("." </> f, args')
551− ".py" -> ("python", f:args')
552− ".hs" -> ("runhaskell", f:args')
553− ".pl" -> ("perl", f:args')
554− ".rb" -> ("ruby", f:args')
555− ".php" -> ("php", f:args')
556− ".js" -> ("node", f:args')
557− ".r" -> ("Rscript", f:args')
558− _ -> (f, args')
559− else (f, args')
560− unless (exists && isExecutable) $ do
561− mbExe <- findExecutable f'
562− when (isNothing mbExe) $
563− E.throwIO $ PandocFilterError f ("Could not find executable " ++ f')
564− env <- getEnvironment
565− let env' = Just
566− ( ("PANDOC_VERSION", pandocVersion)
567− : ("PANDOC_READER_OPTIONS", UTF8.toStringLazy (encode ropts))
568− : env )
569− (exitcode, outbs) <- E.handle filterException $
570− pipeProcess env' f' args'' $ encode d
571− case exitcode of
572− ExitSuccess -> either (E.throwIO . PandocFilterError f)
573− return $ eitherDecode' outbs
574− ExitFailure ec -> E.throwIO $ PandocFilterError f
575− ("Filter returned error status " ++ show ec)
576− where filterException :: E.SomeException -> IO a
577− filterException e = E.throwIO $ PandocFilterError f (show e)
578−
579−data Filter = LuaFilter FilePath
580− | JSONFilter FilePath
581− deriving (Show)
582−
583538-- | Data structure for command line options.
584539data Opt = Opt
585540 { optTabStop :: Int -- ^ Number of spaces per tab
@@ -824,50 +779,6 @@ defaultWriterName x =
824779applyTransforms :: Monad m => [Transform] -> Pandoc -> m Pandoc
825780applyTransforms transforms d = return $ foldr ($) d transforms
826781
827− -- First we check to see if a filter is found. If not, and if it's
828− -- not an absolute path, we check to see whether it's in `userdir/filters`.
829− -- If not, we leave it unchanged.
830−expandFilterPath :: PandocMonad m => FilePath -> m FilePath
831−expandFilterPath fp = do
832− mbDatadir <- getUserDataDir
833− fpExists <- fileExists fp
834− if fpExists
835− then return fp
836− else case mbDatadir of
837− Just datadir | isRelative fp -> do
838− let filterPath = datadir </> "filters" </> fp
839− filterPathExists <- fileExists filterPath
840− if filterPathExists
841− then return filterPath
842− else return fp
843− _ -> return fp
844−
845−applyFilters :: ReaderOptions
846− -> [Filter]
847− -> [String]
848− -> Pandoc
849− -> PandocIO Pandoc
850−applyFilters ropts filters args d = do
851− foldrM ($) d $ map (applyFilter ropts args) filters
852−
853−applyFilter :: ReaderOptions
854− -> [String]
855− -> Filter
856− -> Pandoc
857− -> PandocIO Pandoc
858−applyFilter ropts args (LuaFilter f) d = do
859− f' <- expandFilterPath f
860− let format = case args of
861− (x:_) -> x
862− _ -> error "Format not supplied for lua filter"
863− res <- runLuaFilter ropts f' format d
864− case res of
865− Right x -> return x
866− Left (LuaException s) -> E.throw (PandocFilterError f s)
867−applyFilter ropts args (JSONFilter f) d = do
868− f' <- expandFilterPath f
869− liftIO $ externalFilter ropts f' args d
870−
871782readSource :: FilePath -> PandocIO Text
872783readSource "-" = liftIO (UTF8.toText <$> BS.getContents)
873784readSource src = case parseURI src of
@@ -1722,5 +1633,4 @@ deprecatedOption o msg =
17221633-- see https://github.com/jgm/pandoc/pull/4083
17231634-- using generic deriving caused long compilation times
17241635$(deriveJSON defaultOptions ''LineEnding)
1725−$(deriveJSON defaultOptions ''Filter)
17261636$(deriveJSON defaultOptions ''Opt)
src/Text/Pandoc/Filter.hsnew file+60
@@ -0,0 +1,60 @@
1+{-
2+Copyright (C) 2006-2017 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+{-# LANGUAGE TemplateHaskell #-}
19+
20+{- |
21+ Module : Text.Pandoc.Filter
22+ Copyright : Copyright (C) 2006-2017 John MacFarlane
23+ License : GNU GPL, version 2 or above
24+
25+ Maintainer : John MacFarlane <jgm@berkeley@edu>
26+ Stability : alpha
27+ Portability : portable
28+
29+Programmatically modifications of pandoc documents.
30+-}
31+module Text.Pandoc.Filter
32+ ( Filter (..)
33+ , applyFilters
34+ ) where
35+
36+import Data.Aeson (defaultOptions)
37+import Data.Aeson.TH (deriveJSON)
38+import Data.Foldable (foldrM)
39+import Text.Pandoc.Class (PandocIO)
40+import Text.Pandoc.Definition (Pandoc)
41+import Text.Pandoc.Options (ReaderOptions)
42+import qualified Text.Pandoc.Filter.Json as JsonFilter
43+import qualified Text.Pandoc.Filter.Lua as LuaFilter
44+
45+data Filter = LuaFilter FilePath
46+ | JSONFilter FilePath
47+ deriving (Show)
48+
49+applyFilters :: ReaderOptions
50+ -> [Filter]
51+ -> [String]
52+ -> Pandoc
53+ -> PandocIO Pandoc
54+applyFilters ropts filters args d = do
55+ foldrM ($) d $ map applyFilter filters
56+ where
57+ applyFilter (JSONFilter f) = JsonFilter.apply ropts args f
58+ applyFilter (LuaFilter f) = LuaFilter.apply ropts args f
59+
60+$(deriveJSON defaultOptions ''Filter)
src/Text/Pandoc/Filter/Json.hsnew file+97
@@ -0,0 +1,97 @@
1+{-
2+Copyright (C) 2006-2018 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Filter
21+ Copyright : Copyright (C) 2006-2018 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley@edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Programmatically modifications of pandoc documents via JSON filters.
29+-}
30+module Text.Pandoc.Filter.Json (apply) where
31+
32+import Control.Monad (unless, when)
33+import Control.Monad.Trans (MonadIO (liftIO))
34+import Data.Aeson (eitherDecode', encode)
35+import Data.Char (toLower)
36+import Data.Maybe (isNothing)
37+import System.Directory (executable, doesFileExist, findExecutable,
38+ getPermissions)
39+import System.Environment (getEnvironment)
40+import System.Exit (ExitCode (..))
41+import System.FilePath ((</>), takeExtension)
42+import Text.Pandoc.Class (PandocIO)
43+import Text.Pandoc.Error (PandocError (PandocFilterError))
44+import Text.Pandoc.Definition (Pandoc)
45+import Text.Pandoc.Filter.Path (expandFilterPath)
46+import Text.Pandoc.Options (ReaderOptions)
47+import Text.Pandoc.Process (pipeProcess)
48+import Text.Pandoc.Shared (pandocVersion)
49+import qualified Control.Exception as E
50+import qualified Text.Pandoc.UTF8 as UTF8
51+
52+apply :: ReaderOptions
53+ -> [String]
54+ -> FilePath
55+ -> Pandoc
56+ -> PandocIO Pandoc
57+apply ropts args f d = do
58+ f' <- expandFilterPath f
59+ liftIO $ externalFilter ropts f' args d
60+
61+externalFilter :: MonadIO m
62+ => ReaderOptions -> FilePath -> [String] -> Pandoc -> m Pandoc
63+externalFilter ropts f args' d = liftIO $ do
64+ exists <- doesFileExist f
65+ isExecutable <- if exists
66+ then executable <$> getPermissions f
67+ else return True
68+ let (f', args'') = if exists
69+ then case map toLower (takeExtension f) of
70+ _ | isExecutable -> ("." </> f, args')
71+ ".py" -> ("python", f:args')
72+ ".hs" -> ("runhaskell", f:args')
73+ ".pl" -> ("perl", f:args')
74+ ".rb" -> ("ruby", f:args')
75+ ".php" -> ("php", f:args')
76+ ".js" -> ("node", f:args')
77+ ".r" -> ("Rscript", f:args')
78+ _ -> (f, args')
79+ else (f, args')
80+ unless (exists && isExecutable) $ do
81+ mbExe <- findExecutable f'
82+ when (isNothing mbExe) $
83+ E.throwIO $ PandocFilterError f ("Could not find executable " ++ f')
84+ env <- getEnvironment
85+ let env' = Just
86+ ( ("PANDOC_VERSION", pandocVersion)
87+ : ("PANDOC_READER_OPTIONS", UTF8.toStringLazy (encode ropts))
88+ : env )
89+ (exitcode, outbs) <- E.handle filterException $
90+ pipeProcess env' f' args'' $ encode d
91+ case exitcode of
92+ ExitSuccess -> either (E.throwIO . PandocFilterError f)
93+ return $ eitherDecode' outbs
94+ ExitFailure ec -> E.throwIO $ PandocFilterError f
95+ ("Filter returned error status " ++ show ec)
96+ where filterException :: E.SomeException -> IO a
97+ filterException e = E.throwIO $ PandocFilterError f (show e)
src/Text/Pandoc/Filter/Lua.hsnew file+53
@@ -0,0 +1,53 @@
1+{-
2+Copyright (C) 2006-2018 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Filter.Lua
21+ Copyright : Copyright (C) 2006-2018 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley@edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Apply Lua filters to modify a pandoc documents programmatically.
29+-}
30+module Text.Pandoc.Filter.Lua (apply) where
31+
32+import Control.Exception (throw)
33+import Text.Pandoc.Class (PandocIO)
34+import Text.Pandoc.Definition (Pandoc)
35+import Text.Pandoc.Error (PandocError (PandocFilterError))
36+import Text.Pandoc.Filter.Path (expandFilterPath)
37+import Text.Pandoc.Lua (LuaException (..), runLuaFilter)
38+import Text.Pandoc.Options (ReaderOptions)
39+
40+apply :: ReaderOptions
41+ -> [String]
42+ -> FilePath
43+ -> Pandoc
44+ -> PandocIO Pandoc
45+apply ropts args f d = do
46+ f' <- expandFilterPath f
47+ let format = case args of
48+ (x:_) -> x
49+ _ -> error "Format not supplied for lua filter"
50+ res <- runLuaFilter ropts f' format d
51+ case res of
52+ Right x -> return x
53+ Left (LuaException s) -> throw (PandocFilterError f s)
src/Text/Pandoc/Filter/Path.hsnew file+53
@@ -0,0 +1,53 @@
1+{-
2+Copyright (C) 2006-2018 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Filter.Path
21+ Copyright : Copyright (C) 2006-2018 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley@edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Expand paths of filters, searching the data directory.
29+-}
30+module Text.Pandoc.Filter.Path
31+ ( expandFilterPath
32+ ) where
33+
34+import Text.Pandoc.Class (PandocMonad, fileExists, getUserDataDir)
35+import System.FilePath ((</>), isRelative)
36+
37+ -- First we check to see if a filter is found. If not, and if it's
38+ -- not an absolute path, we check to see whether it's in `userdir/filters`.
39+ -- If not, we leave it unchanged.
40+expandFilterPath :: PandocMonad m => FilePath -> m FilePath
41+expandFilterPath fp = do
42+ mbDatadir <- getUserDataDir
43+ fpExists <- fileExists fp
44+ if fpExists
45+ then return fp
46+ else case mbDatadir of
47+ Just datadir | isRelative fp -> do
48+ let filterPath = datadir </> "filters" </> fp
49+ filterPathExists <- fileExists filterPath
50+ if filterPathExists
51+ then return filterPath
52+ else return fp
53+ _ -> return fp