git.iko.soy/typeable/pandoc
on master help sign in
Use HsYAML instead of yaml for translations, YAML metadata.

yaml wraps a C library; HsYAML is pure Haskell. Closes #4747. Advances #4535.

commite49b8304e43d8381a2c7693643ab648f32482359 authorJohn MacFarlane <jgm@berkeley.edu> dateFri 29 Jun 2018, 22:32 (8 years ago) parent39dc3b9
8 files changed+94−77
MANUAL.txt+3−1 pandoc.cabal+3−6 src/Text/Pandoc/App.hs+6−5 src/Text/Pandoc/Readers/Markdown.hs+46−56 src/Text/Pandoc/Translations.hs+31−8 src/Text/Pandoc/Writers/Markdown.hs+1−1 stack.lts9.yaml+2 stack.yaml+2
MANUAL.txt+3−1
@@ -3059,7 +3059,9 @@ Metadata will be taken from the fields of the YAML object and added to any
30593059existing document metadata. Metadata can contain lists and objects (nested
30603060arbitrarily), but all string scalars will be interpreted as Markdown. Fields
30613061with names ending in an underscore will be ignored by pandoc. (They may be
3062−given a role by external processors.)
3062+given a role by external processors.) Field names must not be
3063+interpretable as YAML numbers or boolean values (so, for
3064+example, `yes`, `True`, and `15` cannot be used as field names).
30633065
30643066A document may contain multiple metadata blocks. The metadata fields will
30653067be combined through a *left-biased union*: if two metadata blocks attempt
pandoc.cabal+3−6
@@ -370,7 +370,6 @@ library
370370 temporary >= 1.1 && < 1.4,
371371 blaze-html >= 0.9 && < 0.10,
372372 blaze-markup >= 0.8 && < 0.9,
373− scientific >= 0.2 && < 0.4,
374373 vector >= 0.10 && < 0.13,
375374 hslua >= 0.9.5 && < 0.9.6,
376375 hslua-module-text >= 0.1.2 && < 0.2,
@@ -387,12 +386,10 @@ library
387386 http-client >= 0.4.30 && < 0.6,
388387 http-client-tls >= 0.2.4 && < 0.4,
389388 http-types >= 0.8 && < 0.13,
390− case-insensitive >= 1.2 && < 1.3
389+ case-insensitive >= 1.2 && < 1.3,
390+ HsYAML >= 0.1.1.1 && < 0.2
391391 if impl(ghc < 8.0)
392− build-depends: semigroups == 0.18.*,
393− yaml >= 0.8.11 && < 0.8.31
394− else
395− build-depends: yaml >= 0.8.11 && < 0.9
392+ build-depends: semigroups == 0.18.*
396393 if impl(ghc < 8.4)
397394 hs-source-dirs: prelude
398395 other-modules: Prelude
src/Text/Pandoc/App.hs+6−5
@@ -62,8 +62,7 @@ import qualified Data.Text as T
6262import qualified Data.Text.Lazy as TL
6363import qualified Data.Text.Lazy.Encoding as TE
6464import qualified Data.Text.Encoding.Error as TE
65−import Data.Yaml (decodeEither')
66−import qualified Data.Yaml as Yaml
65+import qualified Data.YAML as YAML
6766import GHC.Generics
6867import Network.URI (URI (..), parseURI)
6968#ifdef EMBED_DATA_FILES
@@ -702,9 +701,11 @@ removeMetaKeys :: [(String,String)] -> Pandoc -> Pandoc
702701removeMetaKeys kvs pdc = foldr (deleteMeta . fst) pdc kvs
703702
704703readMetaValue :: String -> MetaValue
705−readMetaValue s = case decodeEither' (UTF8.fromString s) of
706− Right (Yaml.String t) -> MetaString $ T.unpack t
707− Right (Yaml.Bool b) -> MetaBool b
704+readMetaValue s = case YAML.decodeStrict (UTF8.fromString s) of
705+ Right [YAML.Scalar (YAML.SStr t)]
706+ -> MetaString $ T.unpack t
707+ Right [YAML.Scalar (YAML.SBool b)]
708+ -> MetaBool b
708709 _ -> MetaString s
709710
710711-- Determine default reader based on source file extensions
src/Text/Pandoc/Readers/Markdown.hs+46−56
@@ -37,18 +37,14 @@ import Prelude
3737import Control.Monad
3838import Control.Monad.Except (throwError)
3939import Data.Char (isAlphaNum, isPunctuation, isSpace, toLower)
40−import qualified Data.HashMap.Strict as H
4140import Data.List (intercalate, sortBy, transpose, elemIndex)
4241import qualified Data.Map as M
4342import Data.Maybe
4443import Data.Ord (comparing)
45−import Data.Scientific (base10Exponent, coefficient)
4644import qualified Data.Set as Set
4745import Data.Text (Text)
4846import qualified Data.Text as T
49−import qualified Data.Vector as V
50−import Data.Yaml (ParseException (..), YamlException (..), YamlMark (..))
51−import qualified Data.Yaml as Yaml
47+import qualified Data.YAML as YAML
5248import System.FilePath (addExtension, takeExtension)
5349import Text.HTML.TagSoup
5450import Text.Pandoc.Builder (Blocks, Inlines)
@@ -246,47 +242,38 @@ yamlMetaBlock = try $ do
246242 -- by including --- and ..., we allow yaml blocks with just comments:
247243 let rawYaml = unlines ("---" : (rawYamlLines ++ ["..."]))
248244 optional blanklines
249− case Yaml.decodeEither' $ UTF8.fromString rawYaml of
250− Right (Yaml.Object hashmap) -> do
251− let alist = H.toList hashmap
252− mapM_ (\(k, v) ->
253− if ignorable k
254− then return ()
255− else do
256− v' <- yamlToMeta v
257− let k' = T.unpack k
258− updateState $ \st -> st{ stateMeta' =
259− do m <- stateMeta' st
260− -- if there's already a value, leave it unchanged
261− case lookupMeta k' m of
262− Just _ -> return m
263− Nothing -> do
264− v'' <- v'
265− return $ B.setMeta (T.unpack k) v'' m}
245+ case YAML.decodeStrict (UTF8.fromString rawYaml) of
246+ Right (YAML.Mapping _ hashmap : _) -> do
247+ let alist = M.toList hashmap
248+ mapM_ (\(k', v) ->
249+ case YAML.parseEither (YAML.parseYAML k') of
250+ Left e -> fail e
251+ Right k -> do
252+ if ignorable k
253+ then return ()
254+ else do
255+ v' <- yamlToMeta v
256+ let k' = T.unpack k
257+ updateState $ \st -> st{ stateMeta' =
258+ do m <- stateMeta' st
259+ -- if there's already a value, leave it unchanged
260+ case lookupMeta k' m of
261+ Just _ -> return m
262+ Nothing -> do
263+ v'' <- v'
264+ return $ B.setMeta (T.unpack k) v'' m}
266265 ) alist
267− Right Yaml.Null -> return ()
266+ Right [] -> return ()
267+ Right (YAML.Scalar YAML.SNull:_) -> return ()
268268 Right _ -> do
269− logMessage $
270− CouldNotParseYamlMetadata "not an object"
271− pos
272− return ()
269+ logMessage $
270+ CouldNotParseYamlMetadata "not an object"
271+ pos
272+ return ()
273273 Left err' -> do
274− case err' of
275− InvalidYaml (Just YamlParseException{
276− yamlProblem = problem
277− , yamlContext = _ctxt
278− , yamlProblemMark = Yaml.YamlMark {
279− yamlLine = yline
280− , yamlColumn = ycol
281− }}) ->
282− logMessage $ CouldNotParseYamlMetadata
283− problem (setSourceLine
284− (setSourceColumn pos
285− (sourceColumn pos + ycol))
286− (sourceLine pos + 1 + yline))
287− _ -> logMessage $ CouldNotParseYamlMetadata
288− (show err') pos
289− return ()
274+ logMessage $ CouldNotParseYamlMetadata
275+ err' pos
276+ return ()
290277 return mempty
291278
292279-- ignore fields ending with _
@@ -313,22 +300,25 @@ toMetaValue x =
313300 -- `|` or `>` will.
314301
315302yamlToMeta :: PandocMonad m
316− => Yaml.Value -> MarkdownParser m (F MetaValue)
317−yamlToMeta (Yaml.String t) = toMetaValue t
318−yamlToMeta (Yaml.Number n)
319− -- avoid decimal points for numbers that don't need them:
320− | base10Exponent n >= 0 = return $ return $ MetaString $ show
321− $ coefficient n * (10 ^ base10Exponent n)
322− | otherwise = return $ return $ MetaString $ show n
323−yamlToMeta (Yaml.Bool b) = return $ return $ MetaBool b
324−yamlToMeta (Yaml.Array xs) = do
325− xs' <- mapM yamlToMeta (V.toList xs)
303+ => YAML.Node -> MarkdownParser m (F MetaValue)
304+yamlToMeta (YAML.Scalar x) =
305+ case x of
306+ YAML.SStr t -> toMetaValue t
307+ YAML.SBool b -> return $ return $ MetaBool b
308+ YAML.SFloat d -> return $ return $ MetaString (show d)
309+ YAML.SInt i -> return $ return $ MetaString (show i)
310+ _ -> return $ return $ MetaString ""
311+yamlToMeta (YAML.Sequence _ xs) = do
312+ xs' <- mapM yamlToMeta xs
326313 return $ do
327314 xs'' <- sequence xs'
328315 return $ B.toMetaValue xs''
329−yamlToMeta (Yaml.Object o) = do
330− let alist = H.toList o
331− foldM (\m (k,v) ->
316+yamlToMeta (YAML.Mapping _ o) = do
317+ let alist = M.toList o
318+ foldM (\m (k',v) ->
319+ case YAML.parseEither (YAML.parseYAML k') of
320+ Left e -> fail e
321+ Right k -> do
332322 if ignorable k
333323 then return m
334324 else do
src/Text/Pandoc/Translations.hs+31−8
@@ -48,11 +48,12 @@ module Text.Pandoc.Translations (
4848 )
4949where
5050import Prelude
51−import Data.Aeson.Types (typeMismatch)
51+import Data.Aeson.Types (Value(..), FromJSON(..))
52+import qualified Data.Aeson.Types as Aeson
5253import qualified Data.HashMap.Strict as HM
5354import qualified Data.Map as M
5455import Data.Text as T
55−import Data.Yaml as Yaml
56+import qualified Data.YAML as YAML
5657import GHC.Generics (Generic)
5758import Text.Pandoc.Shared (safeRead)
5859import qualified Text.Pandoc.UTF8 as UTF8
@@ -90,7 +91,15 @@ instance FromJSON Term where
9091 Just t' -> pure t'
9192 Nothing -> fail $ "Invalid Term name " ++
9293 show t
93− parseJSON invalid = typeMismatch "Term" invalid
94+ parseJSON invalid = Aeson.typeMismatch "Term" invalid
95+
96+instance YAML.FromYAML Term where
97+ parseYAML (YAML.Scalar (YAML.SStr t)) =
98+ case safeRead (T.unpack t) of
99+ Just t' -> pure t'
100+ Nothing -> fail $ "Invalid Term name " ++
101+ show t
102+ parseYAML invalid = YAML.typeMismatch "Term" invalid
94103
95104instance FromJSON Translations where
96105 parseJSON (Object hm) = do
@@ -102,14 +111,28 @@ instance FromJSON Translations where
102111 Just t ->
103112 case v of
104113 (String s) -> return (t, T.unpack $ T.strip s)
105− inv -> typeMismatch "String" inv
106− parseJSON invalid = typeMismatch "Translations" invalid
114+ inv -> Aeson.typeMismatch "String" inv
115+ parseJSON invalid = Aeson.typeMismatch "Translations" invalid
116+
117+instance YAML.FromYAML Translations where
118+ parseYAML = YAML.withMap "Translations" $
119+ \tr -> Translations .M.fromList <$> mapM addItem (M.toList tr)
120+ where addItem (n@(YAML.Scalar (YAML.SStr k)), v) =
121+ case safeRead (T.unpack k) of
122+ Nothing -> YAML.typeMismatch "Term" n
123+ Just t ->
124+ case v of
125+ (YAML.Scalar (YAML.SStr s)) ->
126+ return (t, T.unpack (T.strip s))
127+ n' -> YAML.typeMismatch "String" n'
128+ addItem (n, _) = YAML.typeMismatch "String" n
107129
108130lookupTerm :: Term -> Translations -> Maybe String
109131lookupTerm t (Translations tm) = M.lookup t tm
110132
111133readTranslations :: String -> Either String Translations
112134readTranslations s =
113− case Yaml.decodeEither' $ UTF8.fromString s of
114− Left err' -> Left $ prettyPrintParseException err'
115− Right t -> Right t
135+ case YAML.decodeStrict $ UTF8.fromString s of
136+ Left err' -> Left err'
137+ Right (t:_) -> Right t
138+ Right [] -> Left "empty YAML document"
src/Text/Pandoc/Writers/Markdown.hs+1−1
@@ -50,7 +50,7 @@ import qualified Data.Set as Set
5050import Data.Text (Text)
5151import qualified Data.Text as T
5252import qualified Data.Vector as V
53−import Data.Yaml (Value (Array, Bool, Number, Object, String))
53+import Data.Aeson (Value (Array, Bool, Number, Object, String))
5454import Network.HTTP (urlEncode)
5555import Text.HTML.TagSoup (Tag (..), isTagText, parseTags)
5656import Text.Pandoc.Class (PandocMonad, report)
stack.lts9.yaml+2
@@ -27,4 +27,6 @@ extra-deps:
2727- pandoc-types-1.17.5
2828- haddock-library-1.6.0
2929- texmath-0.11
30+- HsYAML-0.1.1.1
31+- text-1.2.3.0
3032resolver: lts-9.14
stack.yaml+2
@@ -22,6 +22,8 @@ extra-deps:
2222- hslua-module-text-0.1.2.1
2323- texmath-0.11
2424- haddock-library-1.6.0
25+- HsYAML-0.1.1.1
26+- text-1.2.3.0
2527ghc-options:
2628 "$locals": -fhide-source-paths -XNoImplicitPrelude
2729resolver: lts-10.10