git.iko.soy/typeable/pandoc
on master help sign in
Add lua filter functions to walk inline and block elements.

Refactored some code from Text.Pandoc.Lua.PandocModule into new internal module Text.Pandoc.Lua.Filter. Add `walk_inline` and `walk_block` in pandoc lua module.

commit6174b5bea5e8c4c35c191bd62f1f42e4d7fce69e authorJohn MacFarlane <jgm@berkeley.edu> dateSat 11 Nov 2017, 11:01 (9 years ago) parent5bedd62
5 files changed+241−149
doc/lua-filters.md+48−1 pandoc.cabal+1 src/Text/Pandoc/Lua.hs+3−147 src/Text/Pandoc/Lua/Filter.hs+168 src/Text/Pandoc/Lua/PandocModule.hs+21−1
doc/lua-filters.md+48−1
@@ -165,6 +165,8 @@ those elements accessible through the filter function parameter.
165165
166166Some pandoc functions have been made available in lua:
167167
168+- `walk_block` and `walk_inline` allow filters to be applied
169+ inside specific block or inline elements.
168170- `read` allows filters to parse strings into pandoc documents
169171- `pipe` runs an external command with input from and output to
170172 strings
@@ -333,6 +335,20 @@ will output:
333335</dl>
334336```
335337
338+## Uppercasing text inside all headers
339+
340+This filter uses `walk_block` to transform inline elements
341+inside headers, converting all their text into uppercase.
342+
343+``` lua
344+function Header(el)
345+ return pandoc.walk_block(el, {
346+ Str = function(el)
347+ return pandoc.Str(el.text:upper())
348+ end })
349+end
350+```
351+
336352## Converting ABC code to music notation
337353
338354This filter replaces code blocks with class `abc` with
@@ -1070,6 +1086,38 @@ Lua functions for pandoc scripts.
10701086
10711087## Helper Functions
10721088
1089+[`walk_block (element, filter)`]{#walk_block}
1090+
1091+: Apply a filter inside a block element, walking its
1092+ contents.
1093+
1094+ Parameters:
1095+
1096+ `element`:
1097+ : the block element
1098+
1099+ `filter`:
1100+ : a lua filter (table of functions) to be applied
1101+ within the block element
1102+
1103+ Returns: the transformed block element
1104+
1105+[`walk_inline (element, filter)`]{#walk_inline}
1106+
1107+: Apply a filter inside an inline element, walking its
1108+ contents.
1109+
1110+ Parameters:
1111+
1112+ `element`:
1113+ : the inline element
1114+
1115+ `filter`:
1116+ : a lua filter (table of functions) to be applied
1117+ within the inline element
1118+
1119+ Returns: the transformed inline element
1120+
10731121[`read (markup[, format])`]{#read}
10741122
10751123: Parse the given string into a Pandoc document.
@@ -1142,7 +1190,6 @@ Lua functions for pandoc scripts.
11421190
11431191 local output = pandoc.pipe("sed", {"-e","s/a/b/"}, "abc")
11441192
1145−
11461193# Submodule mediabag
11471194
11481195The submodule `mediabag` allows accessing pandoc's media
pandoc.cabal+1
@@ -478,6 +478,7 @@ library
478478 Text.Pandoc.Readers.Org.Parsing,
479479 Text.Pandoc.Readers.Org.Shared,
480480 Text.Pandoc.Lua.PandocModule,
481+ Text.Pandoc.Lua.Filter,
481482 Text.Pandoc.Lua.StackInstances,
482483 Text.Pandoc.Lua.Util,
483484 Text.Pandoc.CSS,
src/Text/Pandoc/Lua.hs+3−147
@@ -33,25 +33,18 @@ Pandoc lua utils.
3333-}
3434module Text.Pandoc.Lua (LuaException (..), pushPandocModule, runLuaFilter) where
3535
36−import Control.Monad (mplus, unless, when, (>=>))
36+import Control.Monad (when, (>=>))
3737import Control.Monad.Identity (Identity)
3838import Control.Monad.Trans (MonadIO (..))
39−import Data.Data (Data, DataType, dataTypeConstrs, dataTypeName, dataTypeOf,
40− showConstr, toConstr, tyconUQname)
41−import Data.Foldable (foldrM)
4239import Data.IORef (IORef, newIORef, readIORef)
43−import Data.Map (Map)
44−import Data.Maybe (isJust)
45−import Foreign.Lua (FromLuaStack (peek), Lua, LuaException (..), StackIndex,
40+import Foreign.Lua (FromLuaStack (peek), Lua, LuaException (..),
4641 Status (OK), ToLuaStack (push))
4742import Text.Pandoc.Class (CommonState, PandocIO, getCommonState, getMediaBag,
4843 setMediaBag)
4944import Text.Pandoc.Definition
5045import Text.Pandoc.Lua.PandocModule (pushMediaBagModule, pushPandocModule)
46+import Text.Pandoc.Lua.Filter (LuaFilter, walkMWithLuaFilter)
5147import Text.Pandoc.MediaBag (MediaBag)
52−import Text.Pandoc.Walk (walkM)
53−
54−import qualified Data.Map as Map
5548import qualified Foreign.Lua as Lua
5649
5750runLuaFilter :: Maybe FilePath -> FilePath -> String
@@ -109,142 +102,5 @@ pushGlobalFilter = do
109102runAll :: [LuaFilter] -> Pandoc -> Lua Pandoc
110103runAll = foldr ((>=>) . walkMWithLuaFilter) return
111104
112−walkMWithLuaFilter :: LuaFilter -> Pandoc -> Lua Pandoc
113−walkMWithLuaFilter (LuaFilter fnMap) =
114− walkInlines >=> walkBlocks >=> walkMeta >=> walkPandoc
115− where
116− walkInlines :: Pandoc -> Lua Pandoc
117− walkInlines =
118− if hasOneOf inlineFilterNames
119− then walkM (mconcatMapM (tryFilter fnMap :: Inline -> Lua [Inline]))
120− else return
121−
122− walkBlocks :: Pandoc -> Lua Pandoc
123− walkBlocks =
124− if hasOneOf blockFilterNames
125− then walkM (mconcatMapM (tryFilter fnMap :: Block -> Lua [Block]))
126− else return
127−
128− walkMeta :: Pandoc -> Lua Pandoc
129− walkMeta =
130− case Map.lookup "Meta" fnMap of
131− Just fn -> walkM (\(Pandoc meta blocks) -> do
132− meta' <- runFilterFunction fn meta *> singleElement meta
133− return $ Pandoc meta' blocks)
134− Nothing -> return
135−
136− walkPandoc :: Pandoc -> Lua Pandoc
137− walkPandoc =
138− case foldl mplus Nothing (map (`Map.lookup` fnMap) pandocFilterNames) of
139− Just fn -> \x -> runFilterFunction fn x *> singleElement x
140− Nothing -> return
141−
142− mconcatMapM f = fmap mconcat . mapM f
143− hasOneOf = any (\k -> isJust (Map.lookup k fnMap))
144−
145−constructorsFor :: DataType -> [String]
146−constructorsFor x = map show (dataTypeConstrs x)
147−
148−inlineFilterNames :: [String]
149−inlineFilterNames = "Inline" : constructorsFor (dataTypeOf (Str []))
150−
151−blockFilterNames :: [String]
152−blockFilterNames = "Block" : constructorsFor (dataTypeOf (Para []))
153−
154−metaFilterName :: String
155−metaFilterName = "Meta"
156−
157−pandocFilterNames :: [String]
158−pandocFilterNames = ["Pandoc", "Doc"]
159−
160−type FunctionMap = Map String LuaFilterFunction
161−newtype LuaFilter = LuaFilter FunctionMap
162−newtype LuaFilterFunction = LuaFilterFunction { functionIndex :: Int }
163−
164−-- | Try running a filter for the given element
165−tryFilter :: (Data a, FromLuaStack a, ToLuaStack a)
166− => FunctionMap -> a -> Lua [a]
167−tryFilter fnMap x =
168− let filterFnName = showConstr (toConstr x)
169− catchAllName = tyconUQname $ dataTypeName (dataTypeOf x)
170− in
171− case Map.lookup filterFnName fnMap `mplus` Map.lookup catchAllName fnMap of
172− Just fn -> runFilterFunction fn x *> elementOrList x
173− Nothing -> return [x]
174−
175−instance FromLuaStack LuaFilter where
176− peek idx =
177− let constrs = metaFilterName : pandocFilterNames
178− ++ blockFilterNames
179− ++ inlineFilterNames
180− fn c acc = do
181− Lua.getfield idx c
182− filterFn <- Lua.tryLua (peek (-1))
183− Lua.pop 1
184− return $ case filterFn of
185− Left _ -> acc
186− Right f -> (c, f) : acc
187− in LuaFilter . Map.fromList <$> foldrM fn [] constrs
188−
189−-- | Push a value to the stack via a lua filter function. The filter function is
190−-- called with given element as argument and is expected to return an element.
191−-- Alternatively, the function can return nothing or nil, in which case the
192−-- element is left unchanged.
193−runFilterFunction :: ToLuaStack a => LuaFilterFunction -> a -> Lua ()
194−runFilterFunction lf x = do
195− pushFilterFunction lf
196− push x
197− z <- Lua.pcall 1 1 Nothing
198− when (z /= OK) $ do
199− let addPrefix = ("Error while running filter function: " ++)
200− Lua.throwTopMessageAsError' addPrefix
201−
202−elementOrList :: FromLuaStack a => a -> Lua [a]
203−elementOrList x = do
204− let topOfStack = Lua.StackIndex (-1)
205− elementUnchanged <- Lua.isnil topOfStack
206− if elementUnchanged
207− then [x] <$ Lua.pop 1
208− else do
209− mbres <- Lua.peekEither topOfStack
210− case mbres of
211− Right res -> [res] <$ Lua.pop 1
212− Left _ -> Lua.toList topOfStack <* Lua.pop 1
213−
214−singleElement :: FromLuaStack a => a -> Lua a
215−singleElement x = do
216− elementUnchanged <- Lua.isnil (-1)
217− if elementUnchanged
218− then x <$ Lua.pop 1
219− else do
220− mbres <- Lua.peekEither (-1)
221− case mbres of
222− Right res -> res <$ Lua.pop 1
223− Left err -> do
224− Lua.pop 1
225− Lua.throwLuaError $
226− "Error while trying to get a filter's return " ++
227− "value from lua stack.\n" ++ err
228−
229−-- | Push the filter function to the top of the stack.
230−pushFilterFunction :: LuaFilterFunction -> Lua ()
231−pushFilterFunction lf =
232− -- The function is stored in a lua registry table, retrieve it from there.
233− Lua.rawgeti Lua.registryindex (functionIndex lf)
234−
235−registerFilterFunction :: StackIndex -> Lua LuaFilterFunction
236−registerFilterFunction idx = do
237− isFn <- Lua.isfunction idx
238− unless isFn . Lua.throwLuaError $ "Not a function at index " ++ show idx
239− Lua.pushvalue idx
240− refIdx <- Lua.ref Lua.registryindex
241− return $ LuaFilterFunction refIdx
242−
243105instance (FromLuaStack a) => FromLuaStack (Identity a) where
244106 peek = fmap return . peek
245−
246−instance ToLuaStack LuaFilterFunction where
247− push = pushFilterFunction
248−
249−instance FromLuaStack LuaFilterFunction where
250− peek = registerFilterFunction
src/Text/Pandoc/Lua/Filter.hsnew file+168
@@ -0,0 +1,168 @@
1+{-# LANGUAGE FlexibleContexts #-}
2+
3+module Text.Pandoc.Lua.Filter ( LuaFilterFunction
4+ , LuaFilter
5+ , tryFilter
6+ , runFilterFunction
7+ , walkMWithLuaFilter
8+ , walkInlines
9+ , walkBlocks
10+ , blockElementNames
11+ , inlineElementNames
12+ ) where
13+import Control.Monad (mplus, unless, when, (>=>))
14+import Text.Pandoc.Definition
15+import Data.Foldable (foldrM)
16+import Data.Map (Map)
17+import qualified Data.Map as Map
18+import qualified Foreign.Lua as Lua
19+import Foreign.Lua (FromLuaStack (peek), Lua, StackIndex,
20+ Status (OK), ToLuaStack (push))
21+import Text.Pandoc.Walk (walkM, Walkable)
22+import Data.Data (Data, DataType, dataTypeConstrs, dataTypeName, dataTypeOf,
23+ showConstr, toConstr, tyconUQname)
24+import Text.Pandoc.Lua.StackInstances()
25+
26+type FunctionMap = Map String LuaFilterFunction
27+
28+newtype LuaFilterFunction = LuaFilterFunction { functionIndex :: Int }
29+
30+instance ToLuaStack LuaFilterFunction where
31+ push = pushFilterFunction
32+
33+instance FromLuaStack LuaFilterFunction where
34+ peek = registerFilterFunction
35+
36+newtype LuaFilter = LuaFilter FunctionMap
37+
38+instance FromLuaStack LuaFilter where
39+ peek idx =
40+ let constrs = metaFilterName : pandocFilterNames
41+ ++ blockElementNames
42+ ++ inlineElementNames
43+ fn c acc = do
44+ Lua.getfield idx c
45+ filterFn <- Lua.tryLua (peek (-1))
46+ Lua.pop 1
47+ return $ case filterFn of
48+ Left _ -> acc
49+ Right f -> (c, f) : acc
50+ in LuaFilter . Map.fromList <$> foldrM fn [] constrs
51+
52+-- | Push the filter function to the top of the stack.
53+pushFilterFunction :: LuaFilterFunction -> Lua ()
54+pushFilterFunction lf =
55+ -- The function is stored in a lua registry table, retrieve it from there.
56+ Lua.rawgeti Lua.registryindex (functionIndex lf)
57+
58+registerFilterFunction :: StackIndex -> Lua LuaFilterFunction
59+registerFilterFunction idx = do
60+ isFn <- Lua.isfunction idx
61+ unless isFn . Lua.throwLuaError $ "Not a function at index " ++ show idx
62+ Lua.pushvalue idx
63+ refIdx <- Lua.ref Lua.registryindex
64+ return $ LuaFilterFunction refIdx
65+
66+elementOrList :: FromLuaStack a => a -> Lua [a]
67+elementOrList x = do
68+ let topOfStack = Lua.StackIndex (-1)
69+ elementUnchanged <- Lua.isnil topOfStack
70+ if elementUnchanged
71+ then [x] <$ Lua.pop 1
72+ else do
73+ mbres <- Lua.peekEither topOfStack
74+ case mbres of
75+ Right res -> [res] <$ Lua.pop 1
76+ Left _ -> Lua.toList topOfStack <* Lua.pop 1
77+
78+-- | Try running a filter for the given element
79+tryFilter :: (Data a, FromLuaStack a, ToLuaStack a)
80+ => LuaFilter -> a -> Lua [a]
81+tryFilter (LuaFilter fnMap) x =
82+ let filterFnName = showConstr (toConstr x)
83+ catchAllName = tyconUQname $ dataTypeName (dataTypeOf x)
84+ in
85+ case Map.lookup filterFnName fnMap `mplus` Map.lookup catchAllName fnMap of
86+ Just fn -> runFilterFunction fn x *> elementOrList x
87+ Nothing -> return [x]
88+
89+-- | Push a value to the stack via a lua filter function. The filter function is
90+-- called with given element as argument and is expected to return an element.
91+-- Alternatively, the function can return nothing or nil, in which case the
92+-- element is left unchanged.
93+runFilterFunction :: ToLuaStack a => LuaFilterFunction -> a -> Lua ()
94+runFilterFunction lf x = do
95+ pushFilterFunction lf
96+ push x
97+ z <- Lua.pcall 1 1 Nothing
98+ when (z /= OK) $ do
99+ let addPrefix = ("Error while running filter function: " ++)
100+ Lua.throwTopMessageAsError' addPrefix
101+
102+walkMWithLuaFilter :: LuaFilter -> Pandoc -> Lua Pandoc
103+walkMWithLuaFilter f =
104+ walkInlines f >=> walkBlocks f >=> walkMeta f >=> walkPandoc f
105+
106+mconcatMapM :: Monad m => (a -> m [a]) -> [a] -> m [a]
107+mconcatMapM f = fmap mconcat . mapM f
108+
109+hasOneOf :: LuaFilter -> [String] -> Bool
110+hasOneOf (LuaFilter fnMap) = any (\k -> Map.member k fnMap)
111+
112+walkInlines :: Walkable [Inline] a => LuaFilter -> a -> Lua a
113+walkInlines f =
114+ if f `hasOneOf` inlineElementNames
115+ then walkM (mconcatMapM (tryFilter f :: Inline -> Lua [Inline]))
116+ else return
117+
118+walkBlocks :: Walkable [Block] a => LuaFilter -> a -> Lua a
119+walkBlocks f =
120+ if f `hasOneOf` blockElementNames
121+ then walkM (mconcatMapM (tryFilter f :: Block -> Lua [Block]))
122+ else return
123+
124+walkMeta :: LuaFilter -> Pandoc -> Lua Pandoc
125+walkMeta (LuaFilter fnMap) =
126+ case Map.lookup "Meta" fnMap of
127+ Just fn -> walkM (\(Pandoc meta blocks) -> do
128+ meta' <- runFilterFunction fn meta *> singleElement meta
129+ return $ Pandoc meta' blocks)
130+ Nothing -> return
131+
132+walkPandoc :: LuaFilter -> Pandoc -> Lua Pandoc
133+walkPandoc (LuaFilter fnMap) =
134+ case foldl mplus Nothing (map (`Map.lookup` fnMap) pandocFilterNames) of
135+ Just fn -> \x -> runFilterFunction fn x *> singleElement x
136+ Nothing -> return
137+
138+constructorsFor :: DataType -> [String]
139+constructorsFor x = map show (dataTypeConstrs x)
140+
141+inlineElementNames :: [String]
142+inlineElementNames = "Inline" : constructorsFor (dataTypeOf (Str []))
143+
144+blockElementNames :: [String]
145+blockElementNames = "Block" : constructorsFor (dataTypeOf (Para []))
146+
147+metaFilterName :: String
148+metaFilterName = "Meta"
149+
150+pandocFilterNames :: [String]
151+pandocFilterNames = ["Pandoc", "Doc"]
152+
153+singleElement :: FromLuaStack a => a -> Lua a
154+singleElement x = do
155+ elementUnchanged <- Lua.isnil (-1)
156+ if elementUnchanged
157+ then x <$ Lua.pop 1
158+ else do
159+ mbres <- Lua.peekEither (-1)
160+ case mbres of
161+ Right res -> res <$ Lua.pop 1
162+ Left err -> do
163+ Lua.pop 1
164+ Lua.throwLuaError $
165+ "Error while trying to get a filter's return " ++
166+ "value from lua stack.\n" ++ err
167+
168+
src/Text/Pandoc/Lua/PandocModule.hs+21−1
@@ -1,3 +1,5 @@
1+{-# LANGUAGE ScopedTypeVariables #-}
2+{-# LANGUAGE FlexibleContexts #-}
13{-
24Copyright © 2017 Albert Krewinkel <tarleb+pandoc@moltkeplatz.de>
35
@@ -38,13 +40,15 @@ import Data.Digest.Pure.SHA (sha1, showDigest)
3840import Data.IORef
3941import Data.Maybe (fromMaybe)
4042import Data.Text (pack)
41−import Foreign.Lua (FromLuaStack, Lua, NumResults, liftIO)
43+import Foreign.Lua (ToLuaStack, FromLuaStack, Lua, NumResults, liftIO)
4244import Foreign.Lua.FunctionCalling (ToHaskellFunction)
4345import System.Exit (ExitCode (..))
4446import Text.Pandoc.Class (CommonState (..), fetchItem, putCommonState,
4547 readDataFile, runIO, runIOorExplode, setMediaBag,
4648 setUserDataDir)
4749import Text.Pandoc.Lua.StackInstances ()
50+import Text.Pandoc.Definition (Block, Inline)
51+import Text.Pandoc.Walk (Walkable)
4852import Text.Pandoc.MIME (MimeType)
4953import Text.Pandoc.Options (ReaderOptions (readerExtensions))
5054import Text.Pandoc.Process (pipeProcess)
@@ -53,6 +57,7 @@ import Text.Pandoc.Readers (Reader (..), getReader)
5357import qualified Data.ByteString.Lazy as BL
5458import qualified Foreign.Lua as Lua
5559import qualified Text.Pandoc.MediaBag as MB
60+import Text.Pandoc.Lua.Filter (walkInlines, walkBlocks, LuaFilter)
5661
5762-- | Push the "pandoc" on the lua stack.
5863pushPandocModule :: Maybe FilePath -> Lua ()
@@ -63,12 +68,27 @@ pushPandocModule datadir = do
6368 addFunction "_pipe" pipeFn
6469 addFunction "_read" readDoc
6570 addFunction "sha1" sha1HashFn
71+ addFunction "walk_block" walkBlock
72+ addFunction "walk_inline" walkInline
6673
6774-- | Get the string representation of the pandoc module
6875pandocModuleScript :: Maybe FilePath -> IO String
6976pandocModuleScript datadir = unpack <$>
7077 runIOorExplode (setUserDataDir datadir >> readDataFile "pandoc.lua")
7178
79+walkElement :: (ToLuaStack a, Walkable [Inline] a, Walkable [Block] a)
80+ => a -> LuaFilter -> Lua NumResults
81+walkElement x f = do
82+ x' <- walkInlines f x >>= walkBlocks f
83+ Lua.push x'
84+ return 1
85+
86+walkInline :: Inline -> LuaFilter -> Lua NumResults
87+walkInline = walkElement
88+
89+walkBlock :: Block -> LuaFilter -> Lua NumResults
90+walkBlock = walkElement
91+
7292readDoc :: String -> String -> Lua NumResults
7393readDoc formatSpec content = do
7494 case getReader formatSpec of