git.iko.soy/typeable/pandoc
on master help sign in
Added gfm (GitHub-flavored CommonMark) as an input and output format.

This uses bindings to GitHub's fork of cmark, so it should parse gfm exactly as GitHub does (excepting certain postprocessing steps, involving notifications, emojis, etc.). * Added Text.Pandoc.Readers.GFM (exporting readGFM) * Added Text.Pandoc.Writers.GFM (exporting writeGFM) * Added `gfm` as input and output forma Note that tables are currently always rendered as HTML in the writer; this can be improved when CMarkGFM supports tables in output.

commit2c81c4c218be757ff2610e3b47548f0642ce5c10 authorJohn MacFarlane <jgm@berkeley.edu> dateThu 3 Aug 2017, 10:50 (9 years ago) parent8995281
7 files changed+386−2
pandoc.cabal+3 src/Text/Pandoc/Readers.hs+3 src/Text/Pandoc/Readers/CommonMark.hs+2−2 src/Text/Pandoc/Readers/GFM.hs+185 src/Text/Pandoc/Writers.hs+3 src/Text/Pandoc/Writers/GFM.hs+189 stack.yaml+1
pandoc.cabal+3
@@ -321,6 +321,7 @@ Library
321321 JuicyPixels >= 3.1.6.1 && < 3.3,
322322 Glob >= 0.7 && < 0.8,
323323 cmark >= 0.5 && < 0.6,
324+ cmark-gfm >= 0.1.1 && < 0.2,
324325 doctemplates >= 0.1 && < 0.2,
325326 http-client >= 0.4.30 && < 0.6,
326327 http-client-tls >= 0.2.4 && < 0.4,
@@ -370,6 +371,7 @@ Library
370371 Text.Pandoc.Readers.LaTeX.Types,
371372 Text.Pandoc.Readers.Markdown,
372373 Text.Pandoc.Readers.CommonMark,
374+ Text.Pandoc.Readers.GFM,
373375 Text.Pandoc.Readers.MediaWiki,
374376 Text.Pandoc.Readers.Vimwiki,
375377 Text.Pandoc.Readers.RST,
@@ -401,6 +403,7 @@ Library
401403 Text.Pandoc.Writers.Ms,
402404 Text.Pandoc.Writers.Markdown,
403405 Text.Pandoc.Writers.CommonMark,
406+ Text.Pandoc.Writers.GFM,
404407 Text.Pandoc.Writers.Haddock,
405408 Text.Pandoc.Writers.RST,
406409 Text.Pandoc.Writers.Org,
src/Text/Pandoc/Readers.hs+3
@@ -45,6 +45,7 @@ module Text.Pandoc.Readers
4545 , readOdt
4646 , readMarkdown
4747 , readCommonMark
48+ , readGFM
4849 , readMediaWiki
4950 , readVimwiki
5051 , readRST
@@ -76,6 +77,7 @@ import Text.Pandoc.Error
7677import Text.Pandoc.Extensions
7778import Text.Pandoc.Options
7879import Text.Pandoc.Readers.CommonMark
80+import Text.Pandoc.Readers.GFM
7981import Text.Pandoc.Readers.DocBook
8082import Text.Pandoc.Readers.Docx
8183import Text.Pandoc.Readers.EPUB
@@ -117,6 +119,7 @@ readers = [ ("native" , TextReader readNative)
117119 ,("markdown_github" , TextReader readMarkdown)
118120 ,("markdown_mmd", TextReader readMarkdown)
119121 ,("commonmark" , TextReader readCommonMark)
122+ ,("gfm" , TextReader readGFM)
120123 ,("rst" , TextReader readRST)
121124 ,("mediawiki" , TextReader readMediaWiki)
122125 ,("vimwiki" , TextReader readVimwiki)
src/Text/Pandoc/Readers/CommonMark.hs+2−2
@@ -44,8 +44,8 @@ readCommonMark :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
4444readCommonMark opts s = return $
4545 nodeToPandoc $ commonmarkToNode opts' s
4646 where opts' = if extensionEnabled Ext_smart (readerExtensions opts)
47− then [optNormalize, optSmart]
48− else [optNormalize]
47+ then [optSmart]
48+ else []
4949
5050nodeToPandoc :: Node -> Pandoc
5151nodeToPandoc (Node _ DOCUMENT nodes) =
src/Text/Pandoc/Readers/GFM.hsnew file+185
@@ -0,0 +1,185 @@
1+{-
2+Copyright (C) 2017 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Readers.GFM
21+ Copyright : Copyright (C) 2017 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley.edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Conversion of GitHub flavored CommonMark text to 'Pandoc' document.
29+
30+CommonMark is a strongly specified variant of Markdown: http://commonmark.org.
31+-}
32+module Text.Pandoc.Readers.GFM (readGFM)
33+where
34+
35+import CMarkGFM
36+import Data.List (groupBy)
37+import Data.Text (Text, unpack)
38+import Text.Pandoc.Class (PandocMonad)
39+import Text.Pandoc.Definition
40+import Text.Pandoc.Options
41+
42+-- | Parse a CommonMark formatted string into a 'Pandoc' structure.
43+readGFM :: PandocMonad m => ReaderOptions -> Text -> m Pandoc
44+readGFM opts s = return $
45+ nodeToPandoc $ commonmarkToNode opts' exts s
46+ where opts' = [optSmart | enabled Ext_smart]
47+ exts = [extStrikethrough, extTable, extAutolink]
48+ enabled x = extensionEnabled x (readerExtensions opts)
49+
50+nodeToPandoc :: Node -> Pandoc
51+nodeToPandoc (Node _ DOCUMENT nodes) =
52+ Pandoc nullMeta $ foldr addBlock [] nodes
53+nodeToPandoc n = -- shouldn't happen
54+ Pandoc nullMeta $ foldr addBlock [] [n]
55+
56+addBlocks :: [Node] -> [Block]
57+addBlocks = foldr addBlock []
58+
59+addBlock :: Node -> [Block] -> [Block]
60+addBlock (Node _ PARAGRAPH nodes) =
61+ (Para (addInlines nodes) :)
62+addBlock (Node _ THEMATIC_BREAK _) =
63+ (HorizontalRule :)
64+addBlock (Node _ BLOCK_QUOTE nodes) =
65+ (BlockQuote (addBlocks nodes) :)
66+addBlock (Node _ (HTML_BLOCK t) _) =
67+ (RawBlock (Format "html") (unpack t) :)
68+-- Note: the cmark parser will never generate CUSTOM_BLOCK,
69+-- so we don't need to handle it:
70+addBlock (Node _ (CUSTOM_BLOCK _onEnter _onExit) _nodes) =
71+ id
72+addBlock (Node _ (CODE_BLOCK info t) _) =
73+ (CodeBlock ("", take 1 (words (unpack info)), []) (unpack t) :)
74+addBlock (Node _ (HEADING lev) nodes) =
75+ (Header lev ("",[],[]) (addInlines nodes) :)
76+addBlock (Node _ (LIST listAttrs) nodes) =
77+ (constructor (map (setTightness . addBlocks . children) nodes) :)
78+ where constructor = case listType listAttrs of
79+ BULLET_LIST -> BulletList
80+ ORDERED_LIST -> OrderedList
81+ (start, DefaultStyle, delim)
82+ start = listStart listAttrs
83+ setTightness = if listTight listAttrs
84+ then map paraToPlain
85+ else id
86+ paraToPlain (Para xs) = Plain (xs)
87+ paraToPlain x = x
88+ delim = case listDelim listAttrs of
89+ PERIOD_DELIM -> Period
90+ PAREN_DELIM -> OneParen
91+addBlock (Node _ (TABLE alignments) nodes) = do
92+ (Table [] aligns widths headers rows :)
93+ where aligns = map fromTableCellAlignment alignments
94+ fromTableCellAlignment NoAlignment = AlignDefault
95+ fromTableCellAlignment LeftAligned = AlignLeft
96+ fromTableCellAlignment RightAligned = AlignRight
97+ fromTableCellAlignment CenterAligned = AlignCenter
98+ widths = replicate numcols 0.0
99+ numcols = if null rows'
100+ then 0
101+ else maximum $ map length rows'
102+ rows' = map toRow $ filter isRow nodes
103+ (headers, rows) = case rows' of
104+ (h:rs) -> (h, rs)
105+ [] -> ([], [])
106+ isRow (Node _ TABLE_ROW _) = True
107+ isRow _ = False
108+ isCell (Node _ TABLE_CELL _) = True
109+ isCell _ = False
110+ toRow (Node _ TABLE_ROW ns) = map toCell $ filter isCell ns
111+ toRow (Node _ t _) = error $ "toRow encountered non-row " ++ show t
112+ toCell (Node _ TABLE_CELL []) = []
113+ toCell (Node _ TABLE_CELL (n:ns))
114+ | isBlockNode n = addBlocks (n:ns)
115+ | otherwise = [Plain (addInlines (n:ns))]
116+ toCell (Node _ t _) = error $ "toCell encountered non-cell " ++ show t
117+addBlock (Node _ TABLE_ROW _) = id -- handled in TABLE
118+addBlock (Node _ TABLE_CELL _) = id -- handled in TABLE
119+addBlock _ = id
120+
121+isBlockNode :: Node -> Bool
122+isBlockNode (Node _ nodetype _) =
123+ case nodetype of
124+ DOCUMENT -> True
125+ THEMATIC_BREAK -> True
126+ PARAGRAPH -> True
127+ BLOCK_QUOTE -> True
128+ HTML_BLOCK _ -> True
129+ CUSTOM_BLOCK _ _ -> True
130+ CODE_BLOCK _ _ -> True
131+ HEADING _ -> True
132+ LIST _ -> True
133+ ITEM -> True
134+ TEXT _ -> False
135+ SOFTBREAK -> False
136+ LINEBREAK -> False
137+ HTML_INLINE _ -> False
138+ CUSTOM_INLINE _ _ -> False
139+ CODE _ -> False
140+ EMPH -> False
141+ STRONG -> False
142+ LINK _ _ -> False
143+ IMAGE _ _ -> False
144+ STRIKETHROUGH -> False
145+ TABLE _ -> False
146+ TABLE_ROW -> False
147+ TABLE_CELL -> False
148+
149+children :: Node -> [Node]
150+children (Node _ _ ns) = ns
151+
152+addInlines :: [Node] -> [Inline]
153+addInlines = foldr addInline []
154+
155+addInline :: Node -> [Inline] -> [Inline]
156+addInline (Node _ (TEXT t) _) = (map toinl clumps ++)
157+ where raw = unpack t
158+ clumps = groupBy samekind raw
159+ samekind ' ' ' ' = True
160+ samekind ' ' _ = False
161+ samekind _ ' ' = False
162+ samekind _ _ = True
163+ toinl (' ':_) = Space
164+ toinl xs = Str xs
165+addInline (Node _ LINEBREAK _) = (LineBreak :)
166+addInline (Node _ SOFTBREAK _) = (SoftBreak :)
167+addInline (Node _ (HTML_INLINE t) _) =
168+ (RawInline (Format "html") (unpack t) :)
169+-- Note: the cmark parser will never generate CUSTOM_BLOCK,
170+-- so we don't need to handle it:
171+addInline (Node _ (CUSTOM_INLINE _onEnter _onExit) _nodes) =
172+ id
173+addInline (Node _ (CODE t) _) =
174+ (Code ("",[],[]) (unpack t) :)
175+addInline (Node _ EMPH nodes) =
176+ (Emph (addInlines nodes) :)
177+addInline (Node _ STRONG nodes) =
178+ (Strong (addInlines nodes) :)
179+addInline (Node _ STRIKETHROUGH nodes) =
180+ (Strikeout (addInlines nodes) :)
181+addInline (Node _ (LINK url title) nodes) =
182+ (Link nullAttr (addInlines nodes) (unpack url, unpack title) :)
183+addInline (Node _ (IMAGE url title) nodes) =
184+ (Image nullAttr (addInlines nodes) (unpack url, unpack title) :)
185+addInline _ = id
src/Text/Pandoc/Writers.hs+3
@@ -38,6 +38,7 @@ module Text.Pandoc.Writers
3838 , writeAsciiDoc
3939 , writeBeamer
4040 , writeCommonMark
41+ , writeGFM
4142 , writeConTeXt
4243 , writeCustom
4344 , writeDZSlides
@@ -89,6 +90,7 @@ import Text.Pandoc.Definition
8990import Text.Pandoc.Options
9091import Text.Pandoc.Writers.AsciiDoc
9192import Text.Pandoc.Writers.CommonMark
93+import Text.Pandoc.Writers.GFM
9294import Text.Pandoc.Writers.ConTeXt
9395import Text.Pandoc.Writers.Custom
9496import Text.Pandoc.Writers.Docbook
@@ -172,6 +174,7 @@ writers = [
172174 ,("asciidoc" , TextWriter writeAsciiDoc)
173175 ,("haddock" , TextWriter writeHaddock)
174176 ,("commonmark" , TextWriter writeCommonMark)
177+ ,("gfm" , TextWriter writeGFM)
175178 ,("tei" , TextWriter writeTEI)
176179 ,("muse" , TextWriter writeMuse)
177180 ]
src/Text/Pandoc/Writers/GFM.hsnew file+189
@@ -0,0 +1,189 @@
1+{-
2+Copyright (C) 2015 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Writers.GFM
21+ Copyright : Copyright (C) 2017 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley.edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Conversion of 'Pandoc' documents to CommonMark.
29+
30+CommonMark: <http://commonmark.org>
31+-}
32+module Text.Pandoc.Writers.GFM (writeGFM) where
33+
34+import CMarkGFM
35+import Control.Monad.State.Strict (State, get, modify, runState)
36+import Data.Foldable (foldrM)
37+import Data.Text (Text)
38+import qualified Data.Text as T
39+import Text.Pandoc.Class (PandocMonad)
40+import Text.Pandoc.Definition
41+import Text.Pandoc.Options
42+import Text.Pandoc.Shared (isTightList, linesToPara)
43+import Text.Pandoc.Templates (renderTemplate')
44+import Text.Pandoc.Walk (walkM)
45+import Text.Pandoc.Writers.HTML (writeHtml5String)
46+import Text.Pandoc.Writers.Shared
47+
48+-- | Convert Pandoc to GitHub flavored CommonMark.
49+writeGFM :: PandocMonad m => WriterOptions -> Pandoc -> m Text
50+writeGFM opts (Pandoc meta blocks) = do
51+ let (blocks', notes) = runState (walkM processNotes blocks) []
52+ notes' = if null notes
53+ then []
54+ else [OrderedList (1, Decimal, Period) $ reverse notes]
55+ main <- blocksToCommonMark opts (blocks' ++ notes')
56+ metadata <- metaToJSON opts
57+ (blocksToCommonMark opts)
58+ (inlinesToCommonMark opts)
59+ meta
60+ let context = defField "body" main $ metadata
61+ case writerTemplate opts of
62+ Nothing -> return main
63+ Just tpl -> renderTemplate' tpl context
64+
65+processNotes :: Inline -> State [[Block]] Inline
66+processNotes (Note bs) = do
67+ modify (bs :)
68+ notes <- get
69+ return $ Str $ "[" ++ show (length notes) ++ "]"
70+processNotes x = return x
71+
72+node :: NodeType -> [Node] -> Node
73+node = Node Nothing
74+
75+blocksToCommonMark :: PandocMonad m => WriterOptions -> [Block] -> m Text
76+blocksToCommonMark opts bs = do
77+ let cmarkOpts = [optHardBreaks | isEnabled Ext_hard_line_breaks opts]
78+ colwidth = if writerWrapText opts == WrapAuto
79+ then Just $ writerColumns opts
80+ else Nothing
81+ nodes <- blocksToNodes bs
82+ return $
83+ nodeToCommonmark cmarkOpts colwidth $
84+ node DOCUMENT nodes
85+
86+inlinesToCommonMark :: PandocMonad m => WriterOptions -> [Inline] -> m Text
87+inlinesToCommonMark opts ils = return $
88+ nodeToCommonmark cmarkOpts colwidth $ node PARAGRAPH (inlinesToNodes ils)
89+ where cmarkOpts = [optHardBreaks | isEnabled Ext_hard_line_breaks opts]
90+ colwidth = if writerWrapText opts == WrapAuto
91+ then Just $ writerColumns opts
92+ else Nothing
93+
94+blocksToNodes :: PandocMonad m => [Block] -> m [Node]
95+blocksToNodes = foldrM blockToNodes []
96+
97+blockToNodes :: PandocMonad m => Block -> [Node] -> m [Node]
98+blockToNodes (Plain xs) ns = return (node PARAGRAPH (inlinesToNodes xs) : ns)
99+blockToNodes (Para xs) ns = return (node PARAGRAPH (inlinesToNodes xs) : ns)
100+blockToNodes (LineBlock lns) ns = blockToNodes (linesToPara lns) ns
101+blockToNodes (CodeBlock (_,classes,_) xs) ns = return $
102+ (node (CODE_BLOCK (T.pack (unwords classes)) (T.pack xs)) [] : ns)
103+blockToNodes (RawBlock fmt xs) ns
104+ | fmt == Format "html" = return (node (HTML_BLOCK (T.pack xs)) [] : ns)
105+ | otherwise = return (node (CUSTOM_BLOCK (T.pack xs) (T.empty)) [] : ns)
106+blockToNodes (BlockQuote bs) ns = do
107+ nodes <- blocksToNodes bs
108+ return (node BLOCK_QUOTE nodes : ns)
109+blockToNodes (BulletList items) ns = do
110+ nodes <- mapM blocksToNodes items
111+ return (node (LIST ListAttributes{
112+ listType = BULLET_LIST,
113+ listDelim = PERIOD_DELIM,
114+ listTight = isTightList items,
115+ listStart = 1 }) (map (node ITEM) nodes) : ns)
116+blockToNodes (OrderedList (start, _sty, delim) items) ns = do
117+ nodes <- mapM blocksToNodes items
118+ return (node (LIST ListAttributes{
119+ listType = ORDERED_LIST,
120+ listDelim = case delim of
121+ OneParen -> PAREN_DELIM
122+ TwoParens -> PAREN_DELIM
123+ _ -> PERIOD_DELIM,
124+ listTight = isTightList items,
125+ listStart = start }) (map (node ITEM) nodes) : ns)
126+blockToNodes HorizontalRule ns = return (node THEMATIC_BREAK [] : ns)
127+blockToNodes (Header lev _ ils) ns = return (node (HEADING lev) (inlinesToNodes ils) : ns)
128+blockToNodes (Div _ bs) ns = do
129+ nodes <- blocksToNodes bs
130+ return (nodes ++ ns)
131+blockToNodes (DefinitionList items) ns = blockToNodes (BulletList items') ns
132+ where items' = map dlToBullet items
133+ dlToBullet (term, ((Para xs : ys) : zs)) =
134+ Para (term ++ [LineBreak] ++ xs) : ys ++ concat zs
135+ dlToBullet (term, ((Plain xs : ys) : zs)) =
136+ Plain (term ++ [LineBreak] ++ xs) : ys ++ concat zs
137+ dlToBullet (term, xs) =
138+ Para term : concat xs
139+blockToNodes t@(Table _ _ _ _ _) ns = do
140+ s <- writeHtml5String def $! Pandoc nullMeta [t]
141+ return (node (HTML_BLOCK s) [] : ns)
142+blockToNodes Null ns = return ns
143+
144+inlinesToNodes :: [Inline] -> [Node]
145+inlinesToNodes = foldr inlineToNodes []
146+
147+inlineToNodes :: Inline -> [Node] -> [Node]
148+inlineToNodes (Str s) = (node (TEXT (T.pack s)) [] :)
149+inlineToNodes Space = (node (TEXT (T.pack " ")) [] :)
150+inlineToNodes LineBreak = (node LINEBREAK [] :)
151+inlineToNodes SoftBreak = (node SOFTBREAK [] :)
152+inlineToNodes (Emph xs) = (node EMPH (inlinesToNodes xs) :)
153+inlineToNodes (Strong xs) = (node STRONG (inlinesToNodes xs) :)
154+inlineToNodes (Strikeout xs) =
155+ ((node (HTML_INLINE (T.pack "<s>")) [] : inlinesToNodes xs ++
156+ [node (HTML_INLINE (T.pack "</s>")) []]) ++ )
157+inlineToNodes (Superscript xs) =
158+ ((node (HTML_INLINE (T.pack "<sup>")) [] : inlinesToNodes xs ++
159+ [node (HTML_INLINE (T.pack "</sup>")) []]) ++ )
160+inlineToNodes (Subscript xs) =
161+ ((node (HTML_INLINE (T.pack "<sub>")) [] : inlinesToNodes xs ++
162+ [node (HTML_INLINE (T.pack "</sub>")) []]) ++ )
163+inlineToNodes (SmallCaps xs) =
164+ ((node (HTML_INLINE (T.pack "<span class=\"smallcaps\">")) []
165+ : inlinesToNodes xs ++
166+ [node (HTML_INLINE (T.pack "</span>")) []]) ++ )
167+inlineToNodes (Link _ ils (url,tit)) =
168+ (node (LINK (T.pack url) (T.pack tit)) (inlinesToNodes ils) :)
169+inlineToNodes (Image _ ils (url,tit)) =
170+ (node (IMAGE (T.pack url) (T.pack tit)) (inlinesToNodes ils) :)
171+inlineToNodes (RawInline fmt xs)
172+ | fmt == Format "html" = (node (HTML_INLINE (T.pack xs)) [] :)
173+ | otherwise = (node (CUSTOM_INLINE (T.pack xs) (T.empty)) [] :)
174+inlineToNodes (Quoted qt ils) =
175+ ((node (TEXT start) [] : inlinesToNodes ils ++ [node (TEXT end) []]) ++)
176+ where (start, end) = case qt of
177+ SingleQuote -> (T.pack "‘", T.pack "’")
178+ DoubleQuote -> (T.pack "“", T.pack "”")
179+inlineToNodes (Code _ str) = (node (CODE (T.pack str)) [] :)
180+inlineToNodes (Math mt str) =
181+ case mt of
182+ InlineMath ->
183+ (node (HTML_INLINE (T.pack ("\\(" ++ str ++ "\\)"))) [] :)
184+ DisplayMath ->
185+ (node (HTML_INLINE (T.pack ("\\[" ++ str ++ "\\]"))) [] :)
186+inlineToNodes (Span _ ils) = (inlinesToNodes ils ++)
187+inlineToNodes (Cite _ ils) = (inlinesToNodes ils ++)
188+inlineToNodes (Note _) = id -- should not occur
189+-- we remove Note elements in preprocessing
stack.yaml+1
@@ -11,4 +11,5 @@ extra-deps:
1111- skylighting-0.3.3
1212- texmath-0.9.4.1
1313- cmark-0.5.6
14+- cmark-gfm-0.1.1
1415resolver: lts-8.16