git.iko.soy/typeable/pandoc
on master help sign in
Added CommonMark reader using cmark (libcmark bindings).

- Added commonmark as an input format. - Added `Text.Pandoc.Readers.CommonMark.readCommonMark`. - For now, we use the markdown writer to generate benchmark text for the CommonMark reader. We can change this when we get a writer.

commite0d234e54d18a82a7c90aa3946f890140e200051 authorJohn MacFarlane <jgm@berkeley.edu> dateMon 16 Mar 2015, 22:20 (12 years ago) parent5721a5d
5 files changed+150−18
README+14−11 benchmark/benchmark-pandoc.hs+12−6 pandoc.cabal+3−1 src/Text/Pandoc.hs+3 src/Text/Pandoc/Readers/CommonMark.hs+118
README+14−11
@@ -12,17 +12,18 @@ Description
1212
1313Pandoc is a [Haskell] library for converting from one markup format to
1414another, and a command-line tool that uses this library. It can read
15−[markdown] and (subsets of) [Textile], [reStructuredText], [HTML],
16−[LaTeX], [MediaWiki markup], [TWiki markup], [Haddock markup], [OPML],
17−[Emacs Org-mode], [DocBook], [txt2tags], [EPUB] and [Word docx]; and
18−it can write plain text, [markdown], [reStructuredText], [XHTML],
19−[HTML 5], [LaTeX] (including [beamer] slide shows), [ConTeXt], [RTF],
20−[OPML], [DocBook], [OpenDocument], [ODT], [Word docx], [GNU Texinfo],
21−[MediaWiki markup], [DokuWiki markup], [Haddock markup], [EPUB] (v2 or v3),
22−[FictionBook2], [Textile], [groff man] pages, [Emacs Org-Mode], [AsciiDoc],
23−[InDesign ICML], and [Slidy], [Slideous], [DZSlides], [reveal.js] or
24−[S5] HTML slide shows. It can also produce [PDF] output on systems where
25−LaTeX is installed.
15+[Markdown], [CommonMark], and (subsets of) [Textile],
16+[reStructuredText], [HTML], [LaTeX], [MediaWiki markup], [TWiki
17+markup], [Haddock markup], [OPML], [Emacs Org-mode], [DocBook],
18+[txt2tags], [EPUB] and [Word docx]; and it can write plain text,
19+[Markdown], [reStructuredText], [XHTML], [HTML 5], [LaTeX] (including
20+[beamer] slide shows), [ConTeXt], [RTF], [OPML], [DocBook],
21+[OpenDocument], [ODT], [Word docx], [GNU Texinfo], [MediaWiki markup],
22+[DokuWiki markup], [Haddock markup], [EPUB] (v2 or v3),
23+[FictionBook2], [Textile], [groff man] pages, [Emacs Org-Mode],
24+[AsciiDoc], [InDesign ICML], and [Slidy], [Slideous], [DZSlides],
25+[reveal.js] or [S5] HTML slide shows. It can also produce [PDF] output
26+on systems where LaTeX is installed.
2627
2728Pandoc's enhanced version of markdown includes syntax for footnotes,
2829tables, flexible ordered lists, definition lists, fenced code blocks,
@@ -159,6 +160,7 @@ General options
159160 extended markdown), `markdown_strict` (original unextended markdown),
160161 `markdown_phpextra` (PHP Markdown Extra extended markdown),
161162 `markdown_github` (github extended markdown),
163+ `commonmark` (CommonMark markdown),
162164 `textile` (Textile), `rst` (reStructuredText), `html` (HTML),
163165 `docbook` (DocBook), `t2t` (txt2tags), `docx` (docx), `epub` (EPUB),
164166 `opml` (OPML), `org` (Emacs Org-mode), `mediawiki` (MediaWiki markup),
@@ -3238,3 +3240,4 @@ Rosenthal.
32383240[EPUB]: http://idpf.org/epub
32393241[EPUBspine]: http://www.idpf.org/epub/301/spec/epub-publications.html#sec-spine-elem
32403242[KaTeX]: https://github.com/Khan/KaTeX
3243+[CommonMark]: http://commonmark.org
benchmark/benchmark-pandoc.hs+12−6
@@ -26,12 +26,18 @@ import Debug.Trace (trace)
2626readerBench :: Pandoc
2727 -> (String, ReaderOptions -> String -> IO Pandoc)
2828 -> Maybe Benchmark
29−readerBench doc (name, reader) = case lookup name writers of
30− Just (PureStringWriter writer) ->
31− let inp = writer def{ writerWrapText = True} doc
32− in return $ bench (name ++ " reader") $ nfIO $
33− (reader def{ readerSmart = True }) inp
34− _ -> trace ("\nCould not find writer for " ++ name ++ "\n") Nothing
29+readerBench doc (name, reader) =
30+ case lookup name writers of
31+ Just (PureStringWriter writer) ->
32+ let inp = writer def{ writerWrapText = True} doc
33+ in return $ bench (name ++ " reader") $ nfIO $
34+ (reader def{ readerSmart = True }) inp
35+ _ | name == "commonmark" ->
36+ let inp = writeMarkdown def{ writerWrapText = True} doc
37+ in return $ bench (name ++ " reader") $ nfIO $
38+ (reader def{ readerSmart = True }) inp
39+ | otherwise -> trace ("\nCould not find writer for " ++ name ++
40+ "\n") Nothing
3541
3642writerBench :: Pandoc
3743 -> (String, WriterOptions -> Pandoc -> String)
pandoc.cabal+3−1
@@ -254,7 +254,8 @@ Library
254254 old-time,
255255 deepseq-generics >= 0.1 && < 0.2,
256256 JuicyPixels >= 3.1.6.1 && < 3.3,
257− filemanip >= 0.3 && < 0.4
257+ filemanip >= 0.3 && < 0.4,
258+ cmark >= 0.3 && < 0.4
258259 if flag(old-locale)
259260 Build-Depends: old-locale >= 1 && < 1.1,
260261 time >= 1.2 && < 1.5
@@ -292,6 +293,7 @@ Library
292293 Text.Pandoc.Readers.HTML,
293294 Text.Pandoc.Readers.LaTeX,
294295 Text.Pandoc.Readers.Markdown,
296+ Text.Pandoc.Readers.CommonMark,
295297 Text.Pandoc.Readers.MediaWiki,
296298 Text.Pandoc.Readers.RST,
297299 Text.Pandoc.Readers.Org,
src/Text/Pandoc.hs+3
@@ -66,6 +66,7 @@ module Text.Pandoc
6666 , mkStringReader
6767 , readDocx
6868 , readMarkdown
69+ , readCommonMark
6970 , readMediaWiki
7071 , readRST
7172 , readOrg
@@ -124,6 +125,7 @@ import Text.Pandoc.Definition
124125import Text.Pandoc.Generic
125126import Text.Pandoc.JSON
126127import Text.Pandoc.Readers.Markdown
128+import Text.Pandoc.Readers.CommonMark
127129import Text.Pandoc.Readers.MediaWiki
128130import Text.Pandoc.Readers.RST
129131import Text.Pandoc.Readers.Org
@@ -225,6 +227,7 @@ readers = [ ("native" , StringReader $ \_ s -> return $ readNative s)
225227 ,("markdown_phpextra" , mkStringReaderWithWarnings readMarkdownWithWarnings)
226228 ,("markdown_github" , mkStringReaderWithWarnings readMarkdownWithWarnings)
227229 ,("markdown_mmd", mkStringReaderWithWarnings readMarkdownWithWarnings)
230+ ,("commonmark" , mkStringReader readCommonMark)
228231 ,("rst" , mkStringReaderWithWarnings readRSTWithWarnings )
229232 ,("mediawiki" , mkStringReader readMediaWiki)
230233 ,("docbook" , mkStringReader readDocBook)
src/Text/Pandoc/Readers/CommonMark.hsnew file+118
@@ -0,0 +1,118 @@
1+{-
2+Copyright (C) 2015 John MacFarlane <jgm@berkeley.edu>
3+
4+This program is free software; you can redistribute it and/or modify
5+it under the terms of the GNU General Public License as published by
6+the Free Software Foundation; either version 2 of the License, or
7+(at your option) any later version.
8+
9+This program is distributed in the hope that it will be useful,
10+but WITHOUT ANY WARRANTY; without even the implied warranty of
11+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+GNU General Public License for more details.
13+
14+You should have received a copy of the GNU General Public License
15+along with this program; if not, write to the Free Software
16+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17+-}
18+
19+{- |
20+ Module : Text.Pandoc.Readers.CommonMark
21+ Copyright : Copyright (C) 2015 John MacFarlane
22+ License : GNU GPL, version 2 or above
23+
24+ Maintainer : John MacFarlane <jgm@berkeley.edu>
25+ Stability : alpha
26+ Portability : portable
27+
28+Conversion of CommonMark-formatted plain text to 'Pandoc' document.
29+
30+CommonMark is a strongly specified variant of Markdown: http://commonmark.org.
31+-}
32+module Text.Pandoc.Readers.CommonMark (readCommonMark)
33+where
34+
35+import CMark
36+import Data.Text (unpack, pack)
37+import Data.List (groupBy)
38+import Text.Pandoc.Definition
39+import Text.Pandoc.Options
40+
41+-- | Parse a CommonMark formatted string into a 'Pandoc' structure.
42+readCommonMark :: ReaderOptions -> String -> Pandoc
43+readCommonMark opts = nodeToPandoc . commonmarkToNode opts' . pack
44+ where opts' = if readerSmart opts
45+ then [optNormalize, optSmart]
46+ else [optNormalize]
47+
48+nodeToPandoc :: Node -> Pandoc
49+nodeToPandoc (Node _ DOCUMENT nodes) =
50+ Pandoc nullMeta $ foldr addBlock [] nodes
51+nodeToPandoc n = -- shouldn't happen
52+ Pandoc nullMeta $ foldr addBlock [] [n]
53+
54+addBlocks :: [Node] -> [Block]
55+addBlocks = foldr addBlock []
56+
57+addBlock :: Node -> [Block] -> [Block]
58+addBlock (Node _ PARAGRAPH nodes) =
59+ (Para (addInlines nodes) :)
60+addBlock (Node _ HRULE _) =
61+ (HorizontalRule :)
62+addBlock (Node _ BLOCK_QUOTE nodes) =
63+ (BlockQuote (addBlocks nodes) :)
64+addBlock (Node _ (HTML t) _) =
65+ (RawBlock (Format "html") (unpack t) :)
66+addBlock (Node _ (CODE_BLOCK info t) _) =
67+ (CodeBlock ("", take 1 (words (unpack info)), []) (unpack t) :)
68+addBlock (Node _ (HEADER lev) nodes) =
69+ (Header lev ("",[],[]) (addInlines nodes) :)
70+addBlock (Node _ (LIST listAttrs) nodes) =
71+ (constructor (map (setTightness . addBlocks . children) nodes) :)
72+ where constructor = case listType listAttrs of
73+ BULLET_LIST -> BulletList
74+ ORDERED_LIST -> OrderedList
75+ (start, DefaultStyle, delim)
76+ start = listStart listAttrs
77+ setTightness = if listTight listAttrs
78+ then map paraToPlain
79+ else id
80+ paraToPlain (Para xs) = Plain (xs)
81+ paraToPlain x = x
82+ delim = case listDelim listAttrs of
83+ PERIOD_DELIM -> Period
84+ PAREN_DELIM -> OneParen
85+addBlock (Node _ ITEM nodes) = id -- handled in LIST
86+addBlock _ = id
87+
88+children :: Node -> [Node]
89+children (Node _ _ ns) = ns
90+
91+addInlines :: [Node] -> [Inline]
92+addInlines = foldr addInline []
93+
94+addInline :: Node -> [Inline] -> [Inline]
95+addInline (Node _ (TEXT t) _) = (map toinl clumps ++)
96+ where raw = unpack t
97+ clumps = groupBy samekind raw
98+ samekind ' ' ' ' = True
99+ samekind ' ' _ = False
100+ samekind _ ' ' = False
101+ samekind _ _ = True
102+ toinl (' ':_) = Space
103+ toinl xs = Str xs
104+addInline (Node _ LINEBREAK _) = (LineBreak :)
105+addInline (Node _ SOFTBREAK _) = (Space :)
106+addInline (Node _ (INLINE_HTML t) _) =
107+ (RawInline (Format "html") (unpack t) :)
108+addInline (Node _ (CODE t) _) =
109+ (Code ("",[],[]) (unpack t) :)
110+addInline (Node _ EMPH nodes) =
111+ (Emph (addInlines nodes) :)
112+addInline (Node _ STRONG nodes) =
113+ (Strong (addInlines nodes) :)
114+addInline (Node _ (LINK url title) nodes) =
115+ (Link (addInlines nodes) (unpack url, unpack title) :)
116+addInline (Node _ (IMAGE url title) nodes) =
117+ (Image (addInlines nodes) (unpack url, unpack title) :)
118+addInline _ = id