git.iko.soy/typeable/pandoc
on master help sign in
Added gfm (GitHub-flavored CommonMark) as an input and output format.

This uses bindings to GitHub's fork of cmark, so it should parse gfm exactly as GitHub does (excepting certain postprocessing steps, involving notifications, emojis, etc.). * Added Text.Pandoc.Readers.GFM (exporting readGFM) * Added Text.Pandoc.Writers.GFM (exporting writeGFM) * Added `gfm` as input and output forma

commitb22bc02c14e46cd745fdb104b369bb1d04a9f3c7 authorJohn MacFarlane <jgm@berkeley.edu> dateThu 3 Aug 2017, 10:50 (9 years ago) parent38b6ada
7 files changed+332−2
pandoc.cabal+3 src/Text/Pandoc/Readers.hs+3 src/Text/Pandoc/Readers/CommonMark.hs+2−2 src/Text/Pandoc/Readers/GFM.hs+131 src/Text/Pandoc/Writers.hs+3 src/Text/Pandoc/Writers/GFM.hs+189 stack.yaml+1
pandoc.cabal+3
@@ -321,6 +321,7 @@ Library
321321 JuicyPixels >= 3.1.6.1 && < 3.3,
322322 Glob >= 0.7 && < 0.8,
323323 cmark >= 0.5 && < 0.6,
324+ cmark-gfm >= 0.1 && < 0.2,
324325 doctemplates >= 0.1 && < 0.2,
325326 http-client >= 0.4.30 && < 0.6,
326327 http-client-tls >= 0.2.4 && < 0.4,
@@ -370,6 +371,7 @@ Library
370371 Text.Pandoc.Readers.LaTeX.Types,
371372 Text.Pandoc.Readers.Markdown,
372373 Text.Pandoc.Readers.CommonMark,
374+ Text.Pandoc.Readers.GFM,
373375 Text.Pandoc.Readers.MediaWiki,
374376 Text.Pandoc.Readers.Vimwiki,
375377 Text.Pandoc.Readers.RST,
@@ -401,6 +403,7 @@ Library
401403 Text.Pandoc.Writers.Ms,
402404 Text.Pandoc.Writers.Markdown,
403405 Text.Pandoc.Writers.CommonMark,
406+ Text.Pandoc.Writers.GFM,
404407 Text.Pandoc.Writers.Haddock,
405408 Text.Pandoc.Writers.RST,
406409 Text.Pandoc.Writers.Org,
src/Text/Pandoc/Readers.hs+3
@@ -45,6 +45,7 @@ module Text.Pandoc.Readers
4545 , readOdt
4646 , readMarkdown
4747 , readCommonMark
48+ , readGFM
4849 , readMediaWiki
4950 , readVimwiki
5051 , readRST
@@ -76,6 +77,7 @@ import Text.Pandoc.Error
7677import Text.Pandoc.Extensions
7778import Text.Pandoc.Options
7879import Text.Pandoc.Readers.CommonMark
80+import Text.Pandoc.Readers.GFM
7981import Text.Pandoc.Readers.DocBook
8082import Text.Pandoc.Readers.Docx
8183import Text.Pandoc.Readers.EPUB
@@ -117,6 +119,7 @@ readers = [ ("native" , TextReader readNative)
117119 ,("markdown_github" , TextReader readMarkdown)
118120 ,("markdown_mmd", TextReader readMarkdown)
119121 ,("commonmark" , TextReader readCommonMark)
122+ ,("gfm" , TextReader readGFM)
120123 ,("rst" , TextReader readRST)
121124 ,("mediawiki" , TextReader readMediaWiki)
122125 ,("vimwiki" , TextReader readVimwiki)
src/Text/Pandoc/Readers/CommonMark.hs+2−2
@@ -44,8 +44,8 @@ readCommonMark :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
4444readCommonMark opts s = return $
4545 nodeToPandoc $ commonmarkToNode opts' s
4646 where opts' = if extensionEnabled Ext_smart (readerExtensions opts)
47− then [optNormalize, optSmart]
48− else [optNormalize]
47+ then [optSmart]
48+ else []
4949
5050nodeToPandoc :: Node -> Pandoc
5151nodeToPandoc (Node _ DOCUMENT nodes) =
src/Text/Pandoc/Readers/GFM.hsnew file+131
@@ -0,0 +1,131 @@
1+{-
2+Copyright (C) 2017 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Readers.GFM
21+ Copyright : Copyright (C) 2017 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley.edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Conversion of GitHub flavored CommonMark text to 'Pandoc' document.
29+
30+CommonMark is a strongly specified variant of Markdown: http://commonmark.org.
31+-}
32+module Text.Pandoc.Readers.GFM (readGFM)
33+where
34+
35+import CMarkGFM
36+import Data.List (groupBy)
37+import Data.Text (Text, unpack)
38+import Text.Pandoc.Class (PandocMonad)
39+import Text.Pandoc.Definition
40+import Text.Pandoc.Options
41+
42+-- | Parse a CommonMark formatted string into a 'Pandoc' structure.
43+readGFM :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
44+readGFM opts s = return $
45+ nodeToPandoc $ commonmarkToNode opts' exts s
46+ where opts' = [optSmart | enabled Ext_smart]
47+ exts = [extStrikethrough | enabled Ext_strikeout] ++
48+ [extTable | enabled Ext_pipe_tables] ++
49+ [extAutolink | enabled Ext_autolink_bare_uris]
50+ -- extTagfilter: no pandoc extension for this
51+ enabled x = extensionEnabled x (readerExtensions opts)
52+
53+nodeToPandoc :: Node -> Pandoc
54+nodeToPandoc (Node _ DOCUMENT nodes) =
55+ Pandoc nullMeta $ foldr addBlock [] nodes
56+nodeToPandoc n = -- shouldn't happen
57+ Pandoc nullMeta $ foldr addBlock [] [n]
58+
59+addBlocks :: [Node] -> [Block]
60+addBlocks = foldr addBlock []
61+
62+addBlock :: Node -> [Block] -> [Block]
63+addBlock (Node _ PARAGRAPH nodes) =
64+ (Para (addInlines nodes) :)
65+addBlock (Node _ THEMATIC_BREAK _) =
66+ (HorizontalRule :)
67+addBlock (Node _ BLOCK_QUOTE nodes) =
68+ (BlockQuote (addBlocks nodes) :)
69+addBlock (Node _ (HTML_BLOCK t) _) =
70+ (RawBlock (Format "html") (unpack t) :)
71+-- Note: the cmark parser will never generate CUSTOM_BLOCK,
72+-- so we don't need to handle it:
73+addBlock (Node _ (CUSTOM_BLOCK _onEnter _onExit) _nodes) =
74+ id
75+addBlock (Node _ (CODE_BLOCK info t) _) =
76+ (CodeBlock ("", take 1 (words (unpack info)), []) (unpack t) :)
77+addBlock (Node _ (HEADING lev) nodes) =
78+ (Header lev ("",[],[]) (addInlines nodes) :)
79+addBlock (Node _ (LIST listAttrs) nodes) =
80+ (constructor (map (setTightness . addBlocks . children) nodes) :)
81+ where constructor = case listType listAttrs of
82+ BULLET_LIST -> BulletList
83+ ORDERED_LIST -> OrderedList
84+ (start, DefaultStyle, delim)
85+ start = listStart listAttrs
86+ setTightness = if listTight listAttrs
87+ then map paraToPlain
88+ else id
89+ paraToPlain (Para xs) = Plain (xs)
90+ paraToPlain x = x
91+ delim = case listDelim listAttrs of
92+ PERIOD_DELIM -> Period
93+ PAREN_DELIM -> OneParen
94+addBlock (Node _ ITEM _) = id -- handled in LIST
95+addBlock _ = id
96+
97+children :: Node -> [Node]
98+children (Node _ _ ns) = ns
99+
100+addInlines :: [Node] -> [Inline]
101+addInlines = foldr addInline []
102+
103+addInline :: Node -> [Inline] -> [Inline]
104+addInline (Node _ (TEXT t) _) = (map toinl clumps ++)
105+ where raw = unpack t
106+ clumps = groupBy samekind raw
107+ samekind ' ' ' ' = True
108+ samekind ' ' _ = False
109+ samekind _ ' ' = False
110+ samekind _ _ = True
111+ toinl (' ':_) = Space
112+ toinl xs = Str xs
113+addInline (Node _ LINEBREAK _) = (LineBreak :)
114+addInline (Node _ SOFTBREAK _) = (SoftBreak :)
115+addInline (Node _ (HTML_INLINE t) _) =
116+ (RawInline (Format "html") (unpack t) :)
117+-- Note: the cmark parser will never generate CUSTOM_BLOCK,
118+-- so we don't need to handle it:
119+addInline (Node _ (CUSTOM_INLINE _onEnter _onExit) _nodes) =
120+ id
121+addInline (Node _ (CODE t) _) =
122+ (Code ("",[],[]) (unpack t) :)
123+addInline (Node _ EMPH nodes) =
124+ (Emph (addInlines nodes) :)
125+addInline (Node _ STRONG nodes) =
126+ (Strong (addInlines nodes) :)
127+addInline (Node _ (LINK url title) nodes) =
128+ (Link nullAttr (addInlines nodes) (unpack url, unpack title) :)
129+addInline (Node _ (IMAGE url title) nodes) =
130+ (Image nullAttr (addInlines nodes) (unpack url, unpack title) :)
131+addInline _ = id
src/Text/Pandoc/Writers.hs+3
@@ -38,6 +38,7 @@ module Text.Pandoc.Writers
3838 , writeAsciiDoc
3939 , writeBeamer
4040 , writeCommonMark
41+ , writeGFM
4142 , writeConTeXt
4243 , writeCustom
4344 , writeDZSlides
@@ -89,6 +90,7 @@ import Text.Pandoc.Definition
8990import Text.Pandoc.Options
9091import Text.Pandoc.Writers.AsciiDoc
9192import Text.Pandoc.Writers.CommonMark
93+import Text.Pandoc.Writers.GFM
9294import Text.Pandoc.Writers.ConTeXt
9395import Text.Pandoc.Writers.Custom
9496import Text.Pandoc.Writers.Docbook
@@ -172,6 +174,7 @@ writers = [
172174 ,("asciidoc" , TextWriter writeAsciiDoc)
173175 ,("haddock" , TextWriter writeHaddock)
174176 ,("commonmark" , TextWriter writeCommonMark)
177+ ,("gfm" , TextWriter writeGFM)
175178 ,("tei" , TextWriter writeTEI)
176179 ,("muse" , TextWriter writeMuse)
177180 ]
src/Text/Pandoc/Writers/GFM.hsnew file+189
@@ -0,0 +1,189 @@
1+{-
2+Copyright (C) 2015 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Writers.GFM
21+ Copyright : Copyright (C) 2017 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley.edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Conversion of 'Pandoc' documents to CommonMark.
29+
30+CommonMark: <http://commonmark.org>
31+-}
32+module Text.Pandoc.Writers.GFM (writeGFM) where
33+
34+import CMarkGFM
35+import Control.Monad.State.Strict (State, get, modify, runState)
36+import Data.Foldable (foldrM)
37+import Data.Text (Text)
38+import qualified Data.Text as T
39+import Text.Pandoc.Class (PandocMonad)
40+import Text.Pandoc.Definition
41+import Text.Pandoc.Options
42+import Text.Pandoc.Shared (isTightList, linesToPara)
43+import Text.Pandoc.Templates (renderTemplate')
44+import Text.Pandoc.Walk (walkM)
45+import Text.Pandoc.Writers.HTML (writeHtml5String)
46+import Text.Pandoc.Writers.Shared
47+
48+-- | Convert Pandoc to GitHub flavored CommonMark.
49+writeGFM :: PandocMonad m => WriterOptions -> Pandoc -> m Text
50+writeGFM opts (Pandoc meta blocks) = do
51+ let (blocks', notes) = runState (walkM processNotes blocks) []
52+ notes' = if null notes
53+ then []
54+ else [OrderedList (1, Decimal, Period) $ reverse notes]
55+ main <- blocksToCommonMark opts (blocks' ++ notes')
56+ metadata <- metaToJSON opts
57+ (blocksToCommonMark opts)
58+ (inlinesToCommonMark opts)
59+ meta
60+ let context = defField "body" main $ metadata
61+ case writerTemplate opts of
62+ Nothing -> return main
63+ Just tpl -> renderTemplate' tpl context
64+
65+processNotes :: Inline -> State [[Block]] Inline
66+processNotes (Note bs) = do
67+ modify (bs :)
68+ notes <- get
69+ return $ Str $ "[" ++ show (length notes) ++ "]"
70+processNotes x = return x
71+
72+node :: NodeType -> [Node] -> Node
73+node = Node Nothing
74+
75+blocksToCommonMark :: PandocMonad m => WriterOptions -> [Block] -> m Text
76+blocksToCommonMark opts bs = do
77+ let cmarkOpts = [optHardBreaks | isEnabled Ext_hard_line_breaks opts]
78+ colwidth = if writerWrapText opts == WrapAuto
79+ then Just $ writerColumns opts
80+ else Nothing
81+ nodes <- blocksToNodes bs
82+ return $
83+ nodeToCommonmark cmarkOpts colwidth $
84+ node DOCUMENT nodes
85+
86+inlinesToCommonMark :: PandocMonad m => WriterOptions -> [Inline] -> m Text
87+inlinesToCommonMark opts ils = return $
88+ nodeToCommonmark cmarkOpts colwidth $ node PARAGRAPH (inlinesToNodes ils)
89+ where cmarkOpts = [optHardBreaks | isEnabled Ext_hard_line_breaks opts]
90+ colwidth = if writerWrapText opts == WrapAuto
91+ then Just $ writerColumns opts
92+ else Nothing
93+
94+blocksToNodes :: PandocMonad m => [Block] -> m [Node]
95+blocksToNodes = foldrM blockToNodes []
96+
97+blockToNodes :: PandocMonad m => Block -> [Node] -> m [Node]
98+blockToNodes (Plain xs) ns = return (node PARAGRAPH (inlinesToNodes xs) : ns)
99+blockToNodes (Para xs) ns = return (node PARAGRAPH (inlinesToNodes xs) : ns)
100+blockToNodes (LineBlock lns) ns = blockToNodes (linesToPara lns) ns
101+blockToNodes (CodeBlock (_,classes,_) xs) ns = return $
102+ (node (CODE_BLOCK (T.pack (unwords classes)) (T.pack xs)) [] : ns)
103+blockToNodes (RawBlock fmt xs) ns
104+ | fmt == Format "html" = return (node (HTML_BLOCK (T.pack xs)) [] : ns)
105+ | otherwise = return (node (CUSTOM_BLOCK (T.pack xs) (T.empty)) [] : ns)
106+blockToNodes (BlockQuote bs) ns = do
107+ nodes <- blocksToNodes bs
108+ return (node BLOCK_QUOTE nodes : ns)
109+blockToNodes (BulletList items) ns = do
110+ nodes <- mapM blocksToNodes items
111+ return (node (LIST ListAttributes{
112+ listType = BULLET_LIST,
113+ listDelim = PERIOD_DELIM,
114+ listTight = isTightList items,
115+ listStart = 1 }) (map (node ITEM) nodes) : ns)
116+blockToNodes (OrderedList (start, _sty, delim) items) ns = do
117+ nodes <- mapM blocksToNodes items
118+ return (node (LIST ListAttributes{
119+ listType = ORDERED_LIST,
120+ listDelim = case delim of
121+ OneParen -> PAREN_DELIM
122+ TwoParens -> PAREN_DELIM
123+ _ -> PERIOD_DELIM,
124+ listTight = isTightList items,
125+ listStart = start }) (map (node ITEM) nodes) : ns)
126+blockToNodes HorizontalRule ns = return (node THEMATIC_BREAK [] : ns)
127+blockToNodes (Header lev _ ils) ns = return (node (HEADING lev) (inlinesToNodes ils) : ns)
128+blockToNodes (Div _ bs) ns = do
129+ nodes <- blocksToNodes bs
130+ return (nodes ++ ns)
131+blockToNodes (DefinitionList items) ns = blockToNodes (BulletList items') ns
132+ where items' = map dlToBullet items
133+ dlToBullet (term, ((Para xs : ys) : zs)) =
134+ Para (term ++ [LineBreak] ++ xs) : ys ++ concat zs
135+ dlToBullet (term, ((Plain xs : ys) : zs)) =
136+ Plain (term ++ [LineBreak] ++ xs) : ys ++ concat zs
137+ dlToBullet (term, xs) =
138+ Para term : concat xs
139+blockToNodes t@(Table _ _ _ _ _) ns = do
140+ s <- writeHtml5String def $! Pandoc nullMeta [t]
141+ return (node (HTML_BLOCK s) [] : ns)
142+blockToNodes Null ns = return ns
143+
144+inlinesToNodes :: [Inline] -> [Node]
145+inlinesToNodes = foldr inlineToNodes []
146+
147+inlineToNodes :: Inline -> [Node] -> [Node]
148+inlineToNodes (Str s) = (node (TEXT (T.pack s)) [] :)
149+inlineToNodes Space = (node (TEXT (T.pack " ")) [] :)
150+inlineToNodes LineBreak = (node LINEBREAK [] :)
151+inlineToNodes SoftBreak = (node SOFTBREAK [] :)
152+inlineToNodes (Emph xs) = (node EMPH (inlinesToNodes xs) :)
153+inlineToNodes (Strong xs) = (node STRONG (inlinesToNodes xs) :)
154+inlineToNodes (Strikeout xs) =
155+ ((node (HTML_INLINE (T.pack "<s>")) [] : inlinesToNodes xs ++
156+ [node (HTML_INLINE (T.pack "</s>")) []]) ++ )
157+inlineToNodes (Superscript xs) =
158+ ((node (HTML_INLINE (T.pack "<sup>")) [] : inlinesToNodes xs ++
159+ [node (HTML_INLINE (T.pack "</sup>")) []]) ++ )
160+inlineToNodes (Subscript xs) =
161+ ((node (HTML_INLINE (T.pack "<sub>")) [] : inlinesToNodes xs ++
162+ [node (HTML_INLINE (T.pack "</sub>")) []]) ++ )
163+inlineToNodes (SmallCaps xs) =
164+ ((node (HTML_INLINE (T.pack "<span class=\"smallcaps\">")) []
165+ : inlinesToNodes xs ++
166+ [node (HTML_INLINE (T.pack "</span>")) []]) ++ )
167+inlineToNodes (Link _ ils (url,tit)) =
168+ (node (LINK (T.pack url) (T.pack tit)) (inlinesToNodes ils) :)
169+inlineToNodes (Image _ ils (url,tit)) =
170+ (node (IMAGE (T.pack url) (T.pack tit)) (inlinesToNodes ils) :)
171+inlineToNodes (RawInline fmt xs)
172+ | fmt == Format "html" = (node (HTML_INLINE (T.pack xs)) [] :)
173+ | otherwise = (node (CUSTOM_INLINE (T.pack xs) (T.empty)) [] :)
174+inlineToNodes (Quoted qt ils) =
175+ ((node (TEXT start) [] : inlinesToNodes ils ++ [node (TEXT end) []]) ++)
176+ where (start, end) = case qt of
177+ SingleQuote -> (T.pack "‘", T.pack "’")
178+ DoubleQuote -> (T.pack "“", T.pack "”")
179+inlineToNodes (Code _ str) = (node (CODE (T.pack str)) [] :)
180+inlineToNodes (Math mt str) =
181+ case mt of
182+ InlineMath ->
183+ (node (HTML_INLINE (T.pack ("\\(" ++ str ++ "\\)"))) [] :)
184+ DisplayMath ->
185+ (node (HTML_INLINE (T.pack ("\\[" ++ str ++ "\\]"))) [] :)
186+inlineToNodes (Span _ ils) = (inlinesToNodes ils ++)
187+inlineToNodes (Cite _ ils) = (inlinesToNodes ils ++)
188+inlineToNodes (Note _) = id -- should not occur
189+-- we remove Note elements in preprocessing
stack.yaml+1
@@ -11,4 +11,5 @@ extra-deps:
1111- skylighting-0.3.3
1212- texmath-0.9.4.1
1313- cmark-0.5.6
14+- cmark-gfm-0.1.0
1415resolver: lts-8.16