git.iko.soy/typeable/pandoc
on master help sign in
Implement defaults file inheritance (#6924)

Allow defaults files to inherit options from other defaults files by specifying them with the following syntax: `defaults: [list of defaults files or single defaults file]`.

commit385b6a3b215124fd2dfa044b8847d69a6cf14a73 authorDavid Martschenko <62178322+davidmrt98@users.noreply.github.com> dateTue 5 Jan 2021, 19:15 (6 years ago) parentea479bf
14 files changed+189−33
MANUAL.txt+10 pandoc.cabal+7 src/Text/Pandoc/App/CommandLineOptions.hs+10−26 src/Text/Pandoc/App/Opt.hs+129−7 test/command/defaults-inheritance-1.md+6 test/command/defaults-inheritance-2.md+5 test/command/defaults-inheritance-3.md+6 test/command/defaults3.yaml+4 test/command/defaults4.yaml+3 test/command/defaults5.yaml+2 test/command/defaults6.yaml+2 test/command/defaults7.yaml+2 test/command/defaults8.yaml+2 test/command/defaults9.yaml+1
MANUAL.txt+10
@@ -1507,6 +1507,16 @@ input-files:
15071507- content.md
15081508# or you may use input-file: with a single value
15091509
1510+# Include options from the specified defaults files.
1511+# The files will be searched for first in the working directory
1512+# and then in the defaults subdirectory of the user data directory.
1513+# The files are included in the same order in which they appear in
1514+# the list. Options specified in this defaults file always have
1515+# priority over the included ones.
1516+defaults:
1517+- defsA
1518+- defsB
1519+
15101520template: letter
15111521standalone: true
15121522self-contained: false
pandoc.cabal+7
@@ -216,6 +216,13 @@ extra-source-files:
216216 test/command/01.csv
217217 test/command/defaults1.yaml
218218 test/command/defaults2.yaml
219+ test/command/defaults3.yaml
220+ test/command/defaults4.yaml
221+ test/command/defaults5.yaml
222+ test/command/defaults6.yaml
223+ test/command/defaults7.yaml
224+ test/command/defaults8.yaml
225+ test/command/defaults9.yaml
219226 test/command/3533-rst-csv-tables.csv
220227 test/command/3880.txt
221228 test/command/5182.txt
src/Text/Pandoc/App/CommandLineOptions.hs+10−26
@@ -25,6 +25,7 @@ module Text.Pandoc.App.CommandLineOptions (
2525import Control.Monad
2626import Control.Monad.Trans
2727import Control.Monad.Except (throwError)
28+import Control.Monad.State.Strict
2829import Data.Aeson.Encode.Pretty (encodePretty', Config(..), keyOrder,
2930 defConfig, Indent(..), NumberFormat(..))
3031import Data.Bifunctor (second)
@@ -46,10 +47,12 @@ import System.FilePath
4647import System.IO (stdout)
4748import Text.DocTemplates (Context (..), ToContext (toVal), Val (..))
4849import Text.Pandoc
49−import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..), addMeta)
50+import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..),
51+ DefaultsState (..), addMeta, applyDefaults,
52+ fullDefaultsPath)
5053import Text.Pandoc.Filter (Filter (..))
5154import Text.Pandoc.Highlighting (highlightingStyles)
52−import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs, findM)
55+import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs)
5356import Text.Printf
5457
5558#ifdef EMBED_DATA_FILES
@@ -64,7 +67,6 @@ import qualified Data.ByteString as BS
6467import qualified Data.ByteString.Lazy as B
6568import qualified Data.Map as M
6669import qualified Data.Text as T
67−import qualified Data.YAML as Y
6870import qualified Text.Pandoc.UTF8 as UTF8
6971
7072parseOptions :: [OptDescr (Opt -> IO Opt)] -> Opt -> IO Opt
@@ -166,7 +168,11 @@ options =
166168
167169 , Option "d" ["defaults"]
168170 (ReqArg
169− (\arg opt -> applyDefaults opt arg
171+ (\arg opt -> runIOorExplode $ do
172+ let defsState = DefaultsState { curDefaults = Nothing,
173+ inheritanceGraph = [] }
174+ fp <- fullDefaultsPath (optDataDir opt) arg
175+ evalStateT (applyDefaults opt fp) defsState
170176 )
171177 "FILE")
172178 ""
@@ -1012,28 +1018,6 @@ writersNames = sort
10121018splitField :: String -> (String, String)
10131019splitField = second (tailDef "true") . break (`elemText` ":=")
10141020
1015−-- | Apply defaults from --defaults file.
1016−applyDefaults :: Opt -> FilePath -> IO Opt
1017−applyDefaults opt file = runIOorExplode $ do
1018− let fp = if null (takeExtension file)
1019− then addExtension file "yaml"
1020− else file
1021− setVerbosity $ optVerbosity opt
1022− dataDirs <- liftIO defaultUserDataDirs
1023− let fps = fp : case optDataDir opt of
1024− Nothing -> map (</> ("defaults" </> fp))
1025− dataDirs
1026− Just dd -> [dd </> "defaults" </> fp]
1027− fp' <- fromMaybe fp <$> findM fileExists fps
1028− inp <- readFileLazy fp'
1029− case Y.decode1 inp of
1030− Right (f :: Opt -> Opt) -> return $ f opt
1031− Left (errpos, errmsg) -> throwError $
1032− PandocParseError $ T.pack $
1033− "Error parsing " ++ fp' ++ " line " ++
1034− show (Y.posLine errpos) ++ " column " ++
1035− show (Y.posColumn errpos) ++ ":\n" ++ errmsg
1036−
10371021lookupHighlightStyle :: PandocMonad m => String -> m Style
10381022lookupHighlightStyle s
10391023 | takeExtension s == ".theme" = -- attempt to load KDE theme
src/Text/Pandoc/App/Opt.hs+129−7
@@ -20,10 +20,17 @@ module Text.Pandoc.App.Opt (
2020 Opt(..)
2121 , LineEnding (..)
2222 , IpynbOutput (..)
23+ , DefaultsState (..)
2324 , defaultOpts
2425 , addMeta
26+ , applyDefaults
27+ , fullDefaultsPath
2528 ) where
29+import Control.Monad.Except (MonadIO, liftIO, throwError, (>=>), foldM)
30+import Control.Monad.State.Strict (StateT, modify, gets)
31+import System.FilePath ( addExtension, (</>), takeExtension )
2632import Data.Char (isLower, toLower)
33+import Data.Maybe (fromMaybe)
2734import GHC.Generics hiding (Meta)
2835import Text.Pandoc.Builder (setMeta)
2936import Text.Pandoc.Filter (Filter (..))
@@ -34,7 +41,9 @@ import Text.Pandoc.Options (TopLevelDivision (TopLevelDefault),
3441 ReferenceLocation (EndOfDocument),
3542 ObfuscationMethod (NoObfuscation),
3643 CiteMethod (Citeproc))
37−import Text.Pandoc.Shared (camelCaseStrToHyphenated)
44+import Text.Pandoc.Class (readFileLazy, fileExists, setVerbosity, PandocMonad)
45+import Text.Pandoc.Error (PandocError (PandocParseError, PandocSomeError))
46+import Text.Pandoc.Shared (camelCaseStrToHyphenated, defaultUserDataDirs, findM, ordNub)
3847import qualified Text.Pandoc.Parsing as P
3948import Text.Pandoc.Readers.Metadata (yamlMap)
4049import Text.Pandoc.Class.PandocPure
@@ -150,16 +159,77 @@ data Opt = Opt
150159 } deriving (Generic, Show)
151160
152161instance FromYAML (Opt -> Opt) where
153− parseYAML (Mapping _ _ m) =
154− foldr (.) id <$> mapM doOpt (M.toList m)
162+ parseYAML (Mapping _ _ m) = chain doOpt (M.toList m)
155163 parseYAML n = failAtNode n "Expected a mapping"
156164
165+data DefaultsState = DefaultsState
166+ {
167+ curDefaults :: Maybe FilePath -- currently parsed file
168+ , inheritanceGraph :: [[FilePath]] -- defaults file inheritance graph
169+ } deriving (Show)
170+
171+instance (PandocMonad m, MonadIO m)
172+ => FromYAML (Opt -> StateT DefaultsState m Opt) where
173+ parseYAML (Mapping _ _ m) = do
174+ let opts = M.mapKeys toText m
175+ dataDir <- case M.lookup "data-dir" opts of
176+ Nothing -> return Nothing
177+ Just v -> Just . unpack <$> parseYAML v
178+ f <- parseOptions $ M.toList m
179+ case M.lookup "defaults" opts of
180+ Just v -> do
181+ g <- parseDefaults v dataDir
182+ return $ g >=> f
183+ Nothing -> return f
184+ where
185+ toText (Scalar _ (SStr s)) = s
186+ toText _ = ""
187+ parseYAML n = failAtNode n "Expected a mapping"
188+
189+parseDefaults :: (PandocMonad m, MonadIO m)
190+ => Node Pos
191+ -> Maybe FilePath
192+ -> Parser (Opt -> StateT DefaultsState m Opt)
193+parseDefaults n dataDir = parseDefsNames n >>= \ds -> return $ \o -> do
194+ -- get parent defaults:
195+ defsParent <- gets $ fromMaybe "" . curDefaults
196+ -- get child defaults:
197+ defsChildren <- mapM (fullDefaultsPath dataDir) ds
198+ -- expand parent in defaults inheritance graph by children:
199+ defsGraph <- gets inheritanceGraph
200+ let defsGraphExp = expand defsGraph defsChildren defsParent
201+ modify $ \defsState -> defsState{ inheritanceGraph = defsGraphExp }
202+ -- check for cyclic inheritance:
203+ if cyclic defsGraphExp
204+ then throwError $
205+ PandocSomeError $ T.pack $
206+ "Error: Circular defaults file reference in " ++
207+ "'" ++ defsParent ++ "'"
208+ else foldM applyDefaults o defsChildren
209+ where parseDefsNames x = (parseYAML x >>= \xs -> return $ map unpack xs)
210+ <|> (parseYAML x >>= \x' -> return [unpack x'])
211+
212+parseOptions :: Monad m
213+ => [(Node Pos, Node Pos)]
214+ -> Parser (Opt -> StateT DefaultsState m Opt)
215+parseOptions ns = do
216+ f <- chain doOpt' ns
217+ return $ return . f
218+
219+chain :: Monad m => (a -> m (b -> b)) -> [a] -> m (b -> b)
220+chain f = foldM g id
221+ where g o n = f n >>= \o' -> return $ o' . o
222+
223+doOpt' :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
224+doOpt' (k',v) = do
225+ k <- parseStringKey k'
226+ case k of
227+ "defaults" -> return id
228+ _ -> doOpt (k',v)
229+
157230doOpt :: (Node Pos, Node Pos) -> Parser (Opt -> Opt)
158231doOpt (k',v) = do
159− k <- case k' of
160− Scalar _ (SStr t) -> return t
161− Scalar _ _ -> failAtNode k' "Non-string key"
162− _ -> failAtNode k' "Non-scalar key"
232+ k <- parseStringKey k'
163233 case k of
164234 "tab-stop" ->
165235 parseYAML v >>= \x -> return (\o -> o{ optTabStop = x })
@@ -494,6 +564,12 @@ defaultOpts = Opt
494564 , optStripComments = False
495565 }
496566
567+parseStringKey :: Node Pos -> Parser Text
568+parseStringKey k = case k of
569+ Scalar _ (SStr t) -> return t
570+ Scalar _ _ -> failAtNode k "Non-string key"
571+ _ -> failAtNode k "Non-scalar key"
572+
497573yamlToMeta :: Node Pos -> Parser Meta
498574yamlToMeta (Mapping _ _ m) =
499575 either (fail . show) return $ runEverything (yamlMap pMetaString m)
@@ -524,6 +600,52 @@ readMetaValue s
524600 | s == "FALSE" = MetaBool False
525601 | otherwise = MetaString $ T.pack s
526602
603+-- | Apply defaults from --defaults file.
604+applyDefaults :: (PandocMonad m, MonadIO m)
605+ => Opt
606+ -> FilePath
607+ -> StateT DefaultsState m Opt
608+applyDefaults opt file = do
609+ setVerbosity $ optVerbosity opt
610+ modify $ \defsState -> defsState{ curDefaults = Just file }
611+ inp <- readFileLazy file
612+ case decode1 inp of
613+ Right f -> f opt
614+ Left (errpos, errmsg) -> throwError $
615+ PandocParseError $ T.pack $
616+ "Error parsing " ++ file ++ " line " ++
617+ show (posLine errpos) ++ " column " ++
618+ show (posColumn errpos) ++ ":\n" ++ errmsg
619+
620+fullDefaultsPath :: (PandocMonad m, MonadIO m)
621+ => Maybe FilePath
622+ -> FilePath
623+ -> m FilePath
624+fullDefaultsPath dataDir file = do
625+ let fp = if null (takeExtension file)
626+ then addExtension file "yaml"
627+ else file
628+ dataDirs <- liftIO defaultUserDataDirs
629+ let fps = fp : case dataDir of
630+ Nothing -> map (</> ("defaults" </> fp))
631+ dataDirs
632+ Just dd -> [dd </> "defaults" </> fp]
633+ fromMaybe fp <$> findM fileExists fps
634+
635+-- | In a list of lists, append another list in front of every list which
636+-- starts with specific element.
637+expand :: Ord a => [[a]] -> [a] -> a -> [[a]]
638+expand [] ns n = fmap (\x -> x : [n]) ns
639+expand ps ns n = concatMap (ext n ns) ps
640+ where
641+ ext x xs p = case p of
642+ (l : _) | x == l -> fmap (: p) xs
643+ _ -> [p]
644+
645+cyclic :: Ord a => [[a]] -> Bool
646+cyclic = any hasDuplicate
647+ where
648+ hasDuplicate xs = length (ordNub xs) /= length xs
527649
528650-- see https://github.com/jgm/pandoc/pull/4083
529651-- using generic deriving caused long compilation times
test/command/defaults-inheritance-1.mdnew file+6
@@ -0,0 +1,6 @@
1+```
2+% pandoc -d command/defaults3
3+# Header
4+^D
5+# Header
6+```
test/command/defaults-inheritance-2.mdnew file+5
@@ -0,0 +1,5 @@
1+```
2+% pandoc -d command/defaults6
3+^D
4+Error: Circular defaults file reference in 'command/defaults7.yaml'
5+```
test/command/defaults-inheritance-3.mdnew file+6
@@ -0,0 +1,6 @@
1+```
2+% pandoc -d command/defaults8
3+<h1>Header</h1>
4+^D
5+# Header
6+```
test/command/defaults3.yamlnew file+4
@@ -0,0 +1,4 @@
1+defaults:
2+ - command/defaults4
3+ - command/defaults5
4+to: markdown
test/command/defaults4.yamlnew file+3
@@ -0,0 +1,3 @@
1+from: html
2+defaults:
3+ - command/defaults5
test/command/defaults5.yamlnew file+2
@@ -0,0 +1,2 @@
1+from: markdown
2+to: html
test/command/defaults6.yamlnew file+2
@@ -0,0 +1,2 @@
1+defaults:
2+ - command/defaults7
test/command/defaults7.yamlnew file+2
@@ -0,0 +1,2 @@
1+defaults:
2+ - command/defaults6
test/command/defaults8.yamlnew file+2
@@ -0,0 +1,2 @@
1+from: html
2+defaults: command/defaults9
test/command/defaults9.yamlnew file+1
@@ -0,0 +1 @@
1+to: markdown