git.iko.soy/typeable/pandoc
on master help sign in
Merge pull request #5312 from tarleb/pandoc-mediabag-extension

Pandoc mediabag extension

commitd68799570859e64d2fdefb6dde15abfb6553b1a5 authorJohn MacFarlane <jgm@berkeley.edu> dateThu 30 May 2019, 10:45 (7 years ago) parent8507d98f722237
7 files changed+236−6
compare with master
unifiedsplit
doc/lua-filters.md+45−1 pandoc.cabal+1 src/Text/Pandoc/Lua/Marshaling/MediaBag.hs+75 src/Text/Pandoc/Lua/Module/MediaBag.hs+20−2 src/Text/Pandoc/MediaBag.hs+15 test/Tests/Lua/Module.hs+8−3 test/lua/module/pandoc-mediabag.lua+72
doc/lua-filters.md+45−1
@@ -2448,9 +2448,26 @@ The module is loaded as part of module `pandoc` and can either be
24482448accessed via the `pandoc.mediabag` field, or explicitly required,
24492449e.g.:
24502450
2451−
24522451 local mb = require 'pandoc.mediabag'
24532452
2453+### delete {#mediabag-delete}
2454+
2455+`delete (filepath)`
2456+
2457+Removes a single entry from the media bag.
2458+
2459+Parameters:
2460+
2461+`filepath`:
2462+: filename of the item to be deleted. The media bag will be
2463+ left unchanged if no entry with the given filename exists.
2464+
2465+### empty {#mediabag-empty}
2466+
2467+`empty ()`
2468+
2469+Clear-out the media bag, deleting all items.
2470+
24542471### insert {#mediabag-insert}
24552472
24562473`insert (filepath, mime_type, contents)`
@@ -2475,6 +2492,33 @@ Usage:
24752492 local contents = "Hello, World!"
24762493 pandoc.mediabag(fp, mt, contents)
24772494
2495+### iter {#mediabag-iter}
2496+
2497+`items ()`
2498+
2499+Returns an iterator triple to be used with Lua's generic `for`
2500+statement. The iterator returns the filepath, MIME type, and
2501+content of a media bag item on each invocation. Items are
2502+processed one-by-one to avoid excessive memory use.
2503+
2504+This function should be used only when full access to all items,
2505+including their contents, is required. For all other cases,
2506+[`list`](#mediabag-list) should be preferred.
2507+
2508+Returns:
2509+
2510+ - The iterator function; must be called with the iterator state
2511+ and the current iterator value.
2512+ - Iterator state – an opaque value to be passed to the iterator
2513+ function.
2514+ - Initial iterator value.
2515+
2516+Usage:
2517+
2518+ for fp, mt, contents in pandoc.mediabag.items() do
2519+ -- print(fp, mt, contents)
2520+ end
2521+
24782522### list {#mediabag-list}
24792523
24802524`list ()`
pandoc.cabal+1
@@ -591,6 +591,7 @@ library
591591 Text.Pandoc.Lua.Marshaling.AST,
592592 Text.Pandoc.Lua.Marshaling.AnyValue,
593593 Text.Pandoc.Lua.Marshaling.CommonState,
594+ Text.Pandoc.Lua.Marshaling.MediaBag,
594595 Text.Pandoc.Lua.Marshaling.ReaderOptions,
595596 Text.Pandoc.Lua.Marshaling.Version,
596597 Text.Pandoc.Lua.Module.MediaBag,
src/Text/Pandoc/Lua/Marshaling/MediaBag.hsnew file+75
@@ -0,0 +1,75 @@
1+{-# LANGUAGE NoImplicitPrelude #-}
2+{- |
3+ Module : Text.Pandoc.Lua.Marshaling.MediaBag
4+ Copyright : © 2012-2019 John MacFarlane
5+ © 2017-2019 Albert Krewinkel
6+ License : GNU GPL, version 2 or above
7+ Maintainer : Albert Krewinkel <tarleb+pandoc@moltkeplatz.de>
8+ Stability : alpha
9+
10+Instances to marshal (push) and unmarshal (peek) media data.
11+-}
12+module Text.Pandoc.Lua.Marshaling.MediaBag (pushIterator) where
13+
14+import Prelude
15+import Foreign.Ptr (Ptr)
16+import Foreign.StablePtr (StablePtr, deRefStablePtr, newStablePtr)
17+import Foreign.Lua (Lua, NumResults, Peekable, Pushable, StackIndex)
18+import Foreign.Lua.Types.Peekable (reportValueOnFailure)
19+import Foreign.Lua.Userdata (ensureUserdataMetatable, pushAnyWithMetatable,
20+ toAnyWithName)
21+import Text.Pandoc.MediaBag (MediaBag, mediaItems)
22+import Text.Pandoc.MIME (MimeType)
23+import Text.Pandoc.Lua.Marshaling.AnyValue (AnyValue (..))
24+
25+import qualified Data.ByteString.Lazy as BL
26+import qualified Foreign.Lua as Lua
27+import qualified Foreign.Storable as Storable
28+
29+-- | A list of 'MediaBag' items.
30+newtype MediaItems = MediaItems [(String, MimeType, BL.ByteString)]
31+
32+instance Pushable MediaItems where
33+ push = pushMediaItems
34+
35+instance Peekable MediaItems where
36+ peek = peekMediaItems
37+
38+-- | Push an iterator triple to be used with Lua's @for@ loop construct.
39+-- Each iterator invokation returns a tripple consisting of an item's
40+-- filename, MIME type, and content.
41+pushIterator :: MediaBag -> Lua NumResults
42+pushIterator mb = do
43+ Lua.pushHaskellFunction nextItem
44+ Lua.push (MediaItems $ mediaItems mb)
45+ Lua.pushnil
46+ return 3
47+
48+-- | Lua type name for @'MediaItems'@.
49+mediaItemsTypeName :: String
50+mediaItemsTypeName = "pandoc MediaItems"
51+
52+-- | Push a @MediaItems@ element to the stack.
53+pushMediaItems :: MediaItems -> Lua ()
54+pushMediaItems xs = pushAnyWithMetatable pushMT xs
55+ where
56+ pushMT = ensureUserdataMetatable mediaItemsTypeName (return ())
57+
58+-- | Retrieve a @MediaItems@ element from the stack.
59+peekMediaItems :: StackIndex -> Lua MediaItems
60+peekMediaItems = reportValueOnFailure mediaItemsTypeName
61+ (`toAnyWithName` mediaItemsTypeName)
62+
63+-- | Retrieve a list of items from an iterator state, return the first
64+-- item (if present), and advance the state.
65+nextItem :: Ptr (StablePtr MediaItems) -> AnyValue -> Lua NumResults
66+nextItem ptr _ = do
67+ (MediaItems items) <- Lua.liftIO $ deRefStablePtr =<< Storable.peek ptr
68+ case items of
69+ [] -> 2 <$ (Lua.pushnil *> Lua.pushnil)
70+ (key, mt, content):xs -> do
71+ Lua.liftIO $ Storable.poke ptr =<< newStablePtr (MediaItems xs)
72+ Lua.push key
73+ Lua.push mt
74+ Lua.push content
75+ return 3
src/Text/Pandoc/Lua/Module/MediaBag.hs+20−2
@@ -20,6 +20,7 @@ import Foreign.Lua (Lua, NumResults, Optional, liftIO)
2020import Text.Pandoc.Class (CommonState (..), fetchItem, putCommonState,
2121 runIOorExplode, setMediaBag)
2222import Text.Pandoc.Lua.Marshaling ()
23+import Text.Pandoc.Lua.Marshaling.MediaBag (pushIterator)
2324import Text.Pandoc.Lua.Util (addFunction)
2425import Text.Pandoc.MIME (MimeType)
2526
@@ -33,7 +34,10 @@ import qualified Text.Pandoc.MediaBag as MB
3334pushModule :: Lua NumResults
3435pushModule = do
3536 Lua.newtable
37+ addFunction "delete" delete
38+ addFunction "empty" empty
3639 addFunction "insert" insertMediaFn
40+ addFunction "items" items
3741 addFunction "lookup" lookupMediaFn
3842 addFunction "list" mediaDirectoryFn
3943 addFunction "fetch" fetch
@@ -58,6 +62,16 @@ setCommonState st = do
5862modifyCommonState :: (CommonState -> CommonState) -> Lua ()
5963modifyCommonState f = getCommonState >>= setCommonState . f
6064
65+-- | Delete a single item from the media bag.
66+delete :: FilePath -> Lua NumResults
67+delete fp = 0 <$ modifyCommonState
68+ (\st -> st { stMediaBag = MB.deleteMedia fp (stMediaBag st) })
69+
70+-- | Delete all items from the media bag.
71+empty :: Lua NumResults
72+empty = 0 <$ modifyCommonState (\st -> st { stMediaBag = mempty })
73+
74+-- | Insert a new item into the media bag.
6175insertMediaFn :: FilePath
6276 -> Optional MimeType
6377 -> BL.ByteString
@@ -66,15 +80,19 @@ insertMediaFn fp optionalMime contents = do
6680 modifyCommonState $ \st ->
6781 let mb = MB.insertMedia fp (Lua.fromOptional optionalMime) contents
6882 (stMediaBag st)
69− in st { stMediaBag = mb}
83+ in st { stMediaBag = mb }
7084 return 0
7185
86+-- | Returns iterator values to be used with a Lua @for@ loop.
87+items :: Lua NumResults
88+items = stMediaBag <$> getCommonState >>= pushIterator
89+
7290lookupMediaFn :: FilePath
7391 -> Lua NumResults
7492lookupMediaFn fp = do
7593 res <- MB.lookupMedia fp . stMediaBag <$> getCommonState
7694 case res of
77− Nothing -> Lua.pushnil *> return 1
95+ Nothing -> 1 <$ Lua.pushnil
7896 Just (mimeType, contents) -> do
7997 Lua.push mimeType
8098 Lua.push contents
src/Text/Pandoc/MediaBag.hs+15
@@ -16,9 +16,11 @@ interface for interacting with it.
1616-}
1717module Text.Pandoc.MediaBag (
1818 MediaBag,
19+ deleteMedia,
1920 lookupMedia,
2021 insertMedia,
2122 mediaDirectory,
23+ mediaItems
2224 ) where
2325import Prelude
2426import qualified Data.ByteString.Lazy as BL
@@ -40,6 +42,14 @@ newtype MediaBag = MediaBag (M.Map [String] (MimeType, BL.ByteString))
4042instance Show MediaBag where
4143 show bag = "MediaBag " ++ show (mediaDirectory bag)
4244
45+-- | Delete a media item from a 'MediaBag', or do nothing if no item corresponds
46+-- to the given path.
47+deleteMedia :: FilePath -- ^ relative path and canonical name of resource
48+ -> MediaBag
49+ -> MediaBag
50+deleteMedia fp (MediaBag mediamap) =
51+ MediaBag $ M.delete (splitDirectories fp) mediamap
52+
4353-- | Insert a media item into a 'MediaBag', replacing any existing
4454-- value with the same name.
4555insertMedia :: FilePath -- ^ relative path and canonical name of resource
@@ -66,3 +76,8 @@ mediaDirectory :: MediaBag -> [(String, MimeType, Int)]
6676mediaDirectory (MediaBag mediamap) =
6777 M.foldrWithKey (\fp (mime,contents) ->
6878 ((Posix.joinPath fp, mime, fromIntegral $ BL.length contents):)) [] mediamap
79+
80+mediaItems :: MediaBag -> [(String, MimeType, BL.ByteString)]
81+mediaItems (MediaBag mediamap) =
82+ M.foldrWithKey (\fp (mime,contents) ->
83+ ((Posix.joinPath fp, mime, contents):)) [] mediamap
test/Tests/Lua/Module.hs+8−3
@@ -19,9 +19,14 @@ import Tests.Lua (runLuaTest)
1919
2020tests :: [TestTree]
2121tests =
22− [ testPandocLua "pandoc" ("lua" </> "module" </> "pandoc.lua")
23− , testPandocLua "pandoc.types" ("lua" </> "module" </> "pandoc-types.lua")
24− , testPandocLua "pandoc.util" ("lua" </> "module" </> "pandoc-utils.lua")
22+ [ testPandocLua "pandoc"
23+ ("lua" </> "module" </> "pandoc.lua")
24+ , testPandocLua "pandoc.mediabag"
25+ ("lua" </> "module" </> "pandoc-mediabag.lua")
26+ , testPandocLua "pandoc.types"
27+ ("lua" </> "module" </> "pandoc-types.lua")
28+ , testPandocLua "pandoc.util"
29+ ("lua" </> "module" </> "pandoc-utils.lua")
2530 ]
2631
2732testPandocLua :: TestName -> FilePath -> TestTree
test/lua/module/pandoc-mediabag.luanew file+72
@@ -0,0 +1,72 @@
1+local tasty = require 'tasty'
2+
3+local test = tasty.test_case
4+local group = tasty.test_group
5+local assert = tasty.assert
6+
7+local mediabag = require 'pandoc.mediabag'
8+
9+return {
10+ group 'insert' {
11+ test('insert adds an item to the mediabag', function ()
12+ local fp = "media/hello.txt"
13+ local mt = "text/plain"
14+ local contents = "Hello, World!"
15+ assert.are_same(mediabag.list(), {})
16+ mediabag.insert(fp, mt, contents)
17+ assert.are_same(
18+ mediabag.list(),
19+ {{['path'] = fp, ['type'] = mt, ['length'] = 13}}
20+ )
21+ mediabag.empty() -- clean up
22+ end),
23+ test('is idempotent', function ()
24+ local fp = "media/hello.txt"
25+ local mt = "text/plain"
26+ local contents = "Hello, World!"
27+ mediabag.insert(fp, mt, contents)
28+ mediabag.insert(fp, mt, contents)
29+ assert.are_same(
30+ mediabag.list(),
31+ {{['path'] = fp, ['type'] = mt, ['length'] = 13}}
32+ )
33+ mediabag.empty() -- clean up
34+ end),
35+ },
36+
37+ group 'delete' {
38+ test('removes an item', function ()
39+ assert.are_same(mediabag.list(), {})
40+ mediabag.insert('test.html', 'text/html', '<aside>Who cares?</aside>')
41+ mediabag.insert('test.css', 'text/plain', 'aside { color: red; }')
42+ assert.are_equal(#mediabag.list(), 2)
43+ mediabag.delete('test.html')
44+ assert.are_same(
45+ mediabag.list(),
46+ {{['path'] = 'test.css', ['type'] = 'text/plain', ['length'] = 21}}
47+ )
48+ mediabag.empty() -- clean up
49+ end),
50+ },
51+
52+ group 'items' {
53+ test('iterates over all items', function ()
54+ local input_items = {
55+ ['test.html'] = {'text/html', '<aside>Really?</aside>'},
56+ ['test.css'] = {'text/plain', 'aside { color: red; }'},
57+ ['test.js'] = {'application/javascript', 'alert("HI MOM!")'}
58+ }
59+ -- fill mediabag
60+ for name, v in pairs(input_items) do
61+ mediabag.insert(name, v[1], v[2])
62+ end
63+
64+ local seen_items = {}
65+ for fp, mt, c in mediabag.items() do
66+ seen_items[fp] = {mt, c}
67+ end
68+ assert.are_same(seen_items, input_items)
69+ mediabag.empty() -- clean up
70+ end)
71+ }
72+}