git.iko.soy/typeable/pandoc
on master help sign in
Add yamlToRefs, yamlBsToRefs.

T.P.Readers.Markdown now exports yamlToRefs. [API change] T.P.Readers.Metadata exports yamlBsToRefs. [API change] These allow specifying an id filter so we parse only references that are used in the document. Improves timing with a 3M yaml references file from 36s to 17s.

commit6a32ea71ea9af8613b8585e874fef8d5f29665fa authorJohn MacFarlane <jgm@berkeley.edu> dateMon 5 Oct 2020, 21:07 (6 years ago) parent89e4f1b
3 files changed+74−10
compare with master
unifiedsplit
src/Text/Pandoc/Citeproc.hs+5−7 src/Text/Pandoc/Readers/Markdown.hs+25−2 src/Text/Pandoc/Readers/Metadata.hs+44−1
src/Text/Pandoc/Citeproc.hs+5−7
@@ -18,7 +18,7 @@ import Text.Pandoc.Citeproc.Locator (parseLocator)
1818import Text.Pandoc.Citeproc.CslJson (cslJsonToReferences)
1919import Text.Pandoc.Citeproc.BibTeX (readBibtexString, Variant(..))
2020import Text.Pandoc.Citeproc.MetaValue (metaValueToReference, metaValueToText)
21−import Text.Pandoc.Readers.Markdown (yamlToMeta)
21+import Text.Pandoc.Readers.Markdown (yamlToRefs)
2222import Text.Pandoc.Class (setResourcePath, getResourcePath, getUserDataDir)
2323import Data.ByteString (ByteString)
2424import qualified Data.ByteString.Lazy as L
@@ -213,12 +213,10 @@ getRefs locale format idpred raw =
213213 (return . filter (idpred . unItemId . referenceId)) .
214214 cslJsonToReferences $ raw
215215 Format_yaml -> do
216− meta <- yamlToMeta def{ readerExtensions = pandocExtensions }
217− (L.fromStrict raw)
218− case lookupMeta "references" meta of
219− Just (MetaList rs) ->
220− return $ mapMaybe (metaValueToReference idpred) rs
221− _ -> throwError $ PandocAppError "No references field"
216+ rs <- yamlToRefs idpred
217+ def{ readerExtensions = pandocExtensions }
218+ (L.fromStrict raw)
219+ return $ mapMaybe (metaValueToReference idpred . MetaMap) rs
222220
223221-- localized quotes
224222convertQuotes :: Locale -> Inline -> Inline
src/Text/Pandoc/Readers/Markdown.hs+25−2
@@ -13,7 +13,10 @@
1313
1414Conversion of markdown-formatted plain text to 'Pandoc' document.
1515-}
16−module Text.Pandoc.Readers.Markdown ( readMarkdown, yamlToMeta ) where
16+module Text.Pandoc.Readers.Markdown (
17+ readMarkdown,
18+ yamlToMeta,
19+ yamlToRefs ) where
1720
1821import Control.Monad
1922import Control.Monad.Except (throwError)
@@ -44,7 +47,7 @@ import Text.Pandoc.Readers.LaTeX (applyMacros, rawLaTeXBlock, rawLaTeXInline)
4447import Text.Pandoc.Shared
4548import qualified Text.Pandoc.UTF8 as UTF8
4649import Text.Pandoc.XML (fromEntities)
47−import Text.Pandoc.Readers.Metadata (yamlBsToMeta)
50+import Text.Pandoc.Readers.Metadata (yamlBsToMeta, yamlBsToRefs)
4851
4952type MarkdownParser m = ParserT Text ParserState m
5053
@@ -75,6 +78,26 @@ yamlToMeta opts bstr = do
7578 Right result -> return result
7679 Left e -> throwError e
7780
81+-- | Read a YAML string and extract references from the
82+-- 'references' field, filter using an id predicate and
83+-- parsing fields as Markdown.
84+yamlToRefs :: PandocMonad m
85+ => (Text -> Bool)
86+ -> ReaderOptions
87+ -> BL.ByteString
88+ -> m [M.Map Text MetaValue]
89+yamlToRefs idpred opts bstr = do
90+ let parser = do
91+ refs <- yamlBsToRefs (fmap B.toMetaValue <$> parseBlocks) idpred bstr
92+ return $ runF refs defaultParserState
93+ parsed <- readWithM parser def{ stateOptions = opts } ""
94+ case parsed of
95+ Right result -> return result
96+ Left e -> throwError e
97+
98+
99+
100+
78101--
79102-- Constants and data structure definitions
80103--
src/Text/Pandoc/Readers/Metadata.hs+44−1
@@ -1,4 +1,5 @@
11{-# LANGUAGE OverloadedStrings #-}
2+{-# LANGUAGE ScopedTypeVariables #-}
23{- |
34 Module : Text.Pandoc.Readers.Metadata
45 Copyright : Copyright (C) 2006-2020 John MacFarlane
@@ -10,7 +11,10 @@
1011
1112Parse YAML/JSON metadata to 'Pandoc' 'Meta'.
1213-}
13−module Text.Pandoc.Readers.Metadata ( yamlBsToMeta, yamlMap ) where
14+module Text.Pandoc.Readers.Metadata (
15+ yamlBsToMeta,
16+ yamlBsToRefs,
17+ yamlMap ) where
1418
1519import Control.Monad
1620import Control.Monad.Except (throwError)
@@ -47,6 +51,45 @@ yamlBsToMeta pMetaValue bstr = do
4751 (T.pack err') pos
4852 return . return $ mempty
4953
54+-- Returns filtered list of references.
55+yamlBsToRefs :: PandocMonad m
56+ => ParserT Text ParserState m (F MetaValue)
57+ -> (Text -> Bool) -- ^ Filter for id
58+ -> BL.ByteString
59+ -> ParserT Text ParserState m (F [M.Map Text MetaValue])
60+yamlBsToRefs pMetaValue idpred bstr = do
61+ pos <- getPosition
62+ case YAML.decodeNode' YAML.failsafeSchemaResolver False False bstr of
63+ Right (YAML.Doc (YAML.Mapping _ _ o):_)
64+ -> case YAML.parseEither (o YAML..: "references") of
65+ Right ns -> do
66+ let g n = case YAML.parseEither (n YAML..: "id") of
67+ Right t -> idpred t ||
68+ case YAML.parseEither (n YAML..:
69+ "other-ids") of
70+ Right (oids :: [Text]) ->
71+ any idpred oids
72+ _ -> False
73+ _ -> False
74+ sequence <$> mapM (yamlMap pMetaValue) (filter g ns)
75+ Left _ -> do
76+ logMessage $ CouldNotParseYamlMetadata
77+ ("expecting 'references' field") pos
78+ return . return $ mempty
79+
80+ Right [] -> return . return $ mempty
81+ Right [YAML.Doc (YAML.Scalar _ YAML.SNull)]
82+ -> return . return $ mempty
83+ Right _ -> do logMessage $ CouldNotParseYamlMetadata "not an object"
84+ pos
85+ return . return $ mempty
86+ Left (_pos, err')
87+ -> do logMessage $ CouldNotParseYamlMetadata
88+ (T.pack err') pos
89+ return . return $ mempty
90+
91+
92+
5093nodeToKey :: PandocMonad m
5194 => YAML.Node YE.Pos
5295 -> m Text