git.iko.soy/typeable/pandoc
on master help sign in
punctuation handling, and more html-specific handling
commitc3866f3c6671300f27ae40927fe6f8a2a85cb28b authorpaul.rivier <paul.r.ml@gmail.com> dateFri 3 Dec 2010, 17:26 (16 years ago) parentd724c6b
4 files changed+81−56
compare with master
unifiedsplit
src/Text/Pandoc/Readers/Markdown.hs+2−1 src/Text/Pandoc/Readers/Textile.hs+31−7 tests/textile-reader.native+33−35 tests/textile-reader.textile+15−13
src/Text/Pandoc/Readers/Markdown.hs+2−1
@@ -28,7 +28,8 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2828Conversion of markdown-formatted plain text to 'Pandoc' document.
2929-}
3030module Text.Pandoc.Readers.Markdown (
31− readMarkdown
31+ readMarkdown,
32+ smartPunctuation
3233 ) where
3334
3435import Data.List ( transpose, isSuffixOf, sortBy, findIndex, intercalate )
src/Text/Pandoc/Readers/Textile.hs+31−7
@@ -26,7 +26,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2626 Portability : portable
2727
2828Conversion from Textile to 'Pandoc' document, based on the spec
29−available at ahttp://redcloth.org/hobix.com/textile/
29+available at http://redcloth.org/textile.
3030
3131Implemented and parsed:
3232 - Paragraphs
@@ -34,7 +34,7 @@ Implemented and parsed:
3434 - Lists
3535 - blockquote
3636 - Inlines : strong, emph, cite, code, deleted, superscript,
37− subscript, links
37+ subscript, links, smart punctuation
3838
3939Implemented but discarded:
4040 - HTML-specific and CSS-specific attributes
@@ -42,7 +42,20 @@ Implemented but discarded:
4242Left to be implemented:
4343 - Pandoc Meta Information (title, author, date)
4444 - footnotes
45− - should autolink be shared through Parsing.hs ?
45+ - dimension sign
46+ - registered, trademark, and copyright symbols
47+ - acronyms
48+ - uppercase
49+ - definition lists
50+ - continued blocks (ex bq..)
51+ -
52+
53+
54+
55+TODO : refactor common patterns across readers :
56+ - autolink
57+ - smartPunctuation
58+ - more ...
4659
4760-}
4861
@@ -55,6 +68,7 @@ import Text.Pandoc.Shared
5568import Text.Pandoc.Parsing
5669import Text.Pandoc.Readers.HTML ( htmlTag, htmlEndTag, -- find code blocks
5770 rawHtmlBlock, rawHtmlInline )
71+import Text.Pandoc.Readers.Markdown (smartPunctuation)
5872import Text.ParserCombinators.Parsec
5973import Data.Char ( digitToInt )
6074
@@ -71,7 +85,7 @@ readTextile state s = (readWith parseTextile) state (s ++ "\n\n")
7185
7286-- | Special chars border strings parsing
7387specialChars :: [Char]
74−specialChars = "\\[]*#_@~<>!?-+^&'\";:|"
88+specialChars = "\\[]<>*#_@~-+^&,.;:!?|\"'%"
7589
7690-- | Generate a Pandoc ADT from a textile document
7791parseTextile :: GenParser Char ParserState Pandoc
@@ -275,6 +289,8 @@ inlines = manyTill inline newline
275289inlineParsers :: [GenParser Char ParserState Inline]
276290inlineParsers = [ autoLink
277291 , str
292+ , htmlSpan
293+ , smartPunctuation -- from markdown reader
278294 , whitespace
279295 , endline
280296 , rawHtmlInline
@@ -296,17 +312,25 @@ inlineParsers = [ autoLink
296312str :: GenParser Char ParserState Inline
297313str = many1 (noneOf (specialChars ++ "\t\n ")) >>= return . Str
298314
315+-- | Textile allows HTML span infos, we discard them
316+htmlSpan :: GenParser Char ParserState Inline
317+htmlSpan = try $ do
318+ char '%'
319+ _ <- attributes
320+ content <- manyTill anyChar (char '%')
321+ return $ Str content
322+
299323-- | Some number of space chars
300324whitespace :: GenParser Char ParserState Inline
301325whitespace = many1 spaceChar >> return Space <?> "whitespace"
302326
303−-- | In Textile, an endline character that can be treated as a space,
304−-- not a structural break
327+-- | In Textile, an isolated endline character is a line break
305328endline :: GenParser Char ParserState Inline
306329endline = try $ do
307330 newline >> notFollowedBy blankline
308− return Space
331+ return LineBreak
309332
333+-- | Textile standard link syntax is label:"target"
310334link :: GenParser Char ParserState Inline
311335link = try $ do
312336 name <- surrounded (char '"') inline
tests/textile-reader.native+33−35
@@ -1,5 +1,5 @@
11Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
2−[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc.",Space,Str "Most",Space,Str "of",Space,Str "them",Space,Str "are",Space,Str "adapted",Space,Str "from",Space,Str "John",Space,Str "Gruber",Str "'",Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite."]
2+[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc",Space,Str "Textile",Space,Str "Reader",Str ".",Space,Str "Part",Space,Str "of",Space,Str "it",Space,Str "comes",LineBreak,Str "from",Space,Str "John",Space,Str "Gruber",Str "'",Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite",Str "."]
33, Para [Strikeout [Str "-"],Str "-",Str "-"]
44, Header 1 [Str "Headers"]
55, Header 2 [Str "Level",Space,Str "2",Space,Str "with",Space,Str "an",Space,Link [Str "embeded",Space,Str "link"] ("http://www.example.com","")]
@@ -8,19 +8,19 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
88, Header 5 [Str "Level",Space,Str "5"]
99, Header 6 [Str "Level",Space,Str "6"]
1010, Header 1 [Str "Paragraphs"]
11−, Para [Str "Here",Str "'",Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph."]
12−, Para [Str "Line",Space,Str "break",Space,Str "are",Space,Str "not",Space,Str "paragraph",Space,Str "break",Space,Str "in",Space,Str "textile,",Space,Str "so",Space,Str "you",Space,Str "can",Space,Str "wrap",Space,Str "your",Space,Str "very",Space,Str "long",Space,Str "paragraph",Space,Str "with",Space,Str "your",Space,Str "favourite",Space,Str "text",Space,Str "editor,",Space,Str "it",Space,Str "will",Space,Str "be",Space,Str "rendered",Space,Str "as",Space,Str "a",Space,Str "single",Space,Str "one."]
13−, Para [Str "Here",Str "'",Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet."]
11+, Para [Str "Here",Str "'",Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph",Str "."]
12+, Para [Str "Line",Space,Str "breaks",Space,Str "are",Space,Str "preserved",Space,Str "in",Space,Str "textile",Str ",",Space,Str "so",Space,Str "you",Space,Str "can",Space,Str "not",Space,Str "wrap",Space,Str "your",Space,Str "very",LineBreak,Str "long",Space,Str "paragraph",Space,Str "with",Space,Str "your",Space,Str "favourite",Space,Str "text",Space,Str "editor",Space,Str "and",Space,Str "have",Space,Str "it",Space,Str "rendered",LineBreak,Str "with",Space,Str "no",Space,Str "break",Str "."]
13+, Para [Str "Here",Str "'",Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet",Str "."]
1414, BulletList
15− [ [ Plain [Str "criminey."] ]
15+ [ [ Plain [Str "criminey",Str "."] ]
1616 ]
17−, Para [Str "There",Space,Str "should",Space,Str "be",Space,Str "a",Space,Str "hard",Space,Str "line",Space,Str "break"]
18−, Para [Str "here."]
17+, Para [Str "There",Space,Str "should",Space,Str "be",Space,Str "a",Space,Str "paragraph",Space,Str "break",Space,Str "between",Space,Str "here"]
18+, Para [Str "and",Space,Str "here",Str "."]
1919, Header 1 [Str "Block",Space,Str "Quotes"]
2020, BlockQuote
21− [ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "famous",Space,Str "quote",Space,Str "from",Space,Str "somebody.",Space,Str "He",Space,Str "had",Space,Str "a",Space,Str "lot",Space,Str "of",Space,Str "things",Space,Str "to",Space,Str "say,",Space,Str "so",Space,Str "the",Space,Str "text",Space,Str "is",Space,Str "really",Space,Str "really",Space,Str "long",Space,Str "and",Space,Str "spans",Space,Str "on",Space,Str "multiple",Space,Str "lines."] ]
21+ [ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "famous",Space,Str "quote",Space,Str "from",Space,Str "somebody",Str ".",Space,Str "He",Space,Str "had",Space,Str "a",Space,Str "lot",Space,Str "of",Space,Str "things",Space,Str "to",LineBreak,Str "say",Str ",",Space,Str "so",Space,Str "the",Space,Str "text",Space,Str "is",Space,Str "really",Space,Str "really",Space,Str "long",Space,Str "and",Space,Str "spans",Space,Str "on",Space,Str "multiple",Space,Str "lines",Str "."] ]
2222
23−, Para [Str "And",Space,Str "a",Space,Str "following",Space,Str "paragraph."]
23+, Para [Str "And",Space,Str "a",Space,Str "following",Space,Str "paragraph",Str "."]
2424, Header 1 [Str "Code",Space,Str "Blocks"]
2525, Para [Str "Code",Str ":"]
2626, CodeBlock ("",[],[]) "\n ---- (should be four hyphens)\n\n sub status {\n print \"working\";\n }\n\n this code block is indented by one tab\n"
@@ -43,24 +43,22 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
4343, BulletList
4444 [ [ Plain [Str "ui",Space,Str "1"]
4545 , BulletList
46− [ [ Plain [Str "ui",Space,Str "1.1"]
46+ [ [ Plain [Str "ui",Space,Str "1",Str ".",Str "1"]
4747 , OrderedList (1,DefaultStyle,DefaultDelim)
48− [ [ Plain [Str "oi",Space,Str "1.1.1"] ]
49− , [ Plain [Str "oi",Space,Str "1.1.2"] ] ] ], [ Plain [Str "ui",Space,Str "1.2"] ] ] ], [ Plain [Str "ui",Space,Str "2"]
48+ [ [ Plain [Str "oi",Space,Str "1",Str ".",Str "1",Str ".",Str "1"] ]
49+ , [ Plain [Str "oi",Space,Str "1",Str ".",Str "1",Str ".",Str "2"] ] ] ], [ Plain [Str "ui",Space,Str "1",Str ".",Str "2"] ] ] ], [ Plain [Str "ui",Space,Str "2"]
5050 , OrderedList (1,DefaultStyle,DefaultDelim)
51− [ [ Plain [Str "oi",Space,Str "2.1"]
51+ [ [ Plain [Str "oi",Space,Str "2",Str ".",Str "1"]
5252 , BulletList
53− [ [ Plain [Str "ui",Space,Str "2.1.1"] ]
54− , [ Plain [Str "ui",Space,Str "2.1.2"] ] ] ] ] ] ]
53+ [ [ Plain [Str "ui",Space,Str "2",Str ".",Str "1",Str ".",Str "1"] ]
54+ , [ Plain [Str "ui",Space,Str "2",Str ".",Str "1",Str ".",Str "2"] ] ] ] ] ] ]
5555, Header 1 [Str "Inline",Space,Str "Markup"]
56−, Para [Str "This",Space,Str "is",Space,Emph [Str "emphasized"],Str ",",Space,Str "and",Space,Str "so",Space,Emph [Str "is",Space,Str "this"],Str "."]
57−, Para [Str "This",Space,Str "is",Space,Strong [Str "strong"],Str ",",Space,Str "and",Space,Str "so",Space,Strong [Str "is",Space,Str "this"],Str "."]
58−, Para [Str "A",Space,Link [Strong [Str "strong",Space,Str "link"]] ("http://www.foobar.com",""),Str "."]
59−, Para [Emph [Strong [Str "This",Space,Str "is",Space,Str "strong",Space,Str "and",Space,Str "em."]]]
60−, Para [Str "So",Space,Str "is",Space,Strong [Emph [Str "this"]],Space,Str "word",Space,Str "and",Space,Emph [Strong [Str "that",Space,Str "one"]],Str "."]
61−, Para [Strikeout [Str "This",Space,Str "is",Space,Str "strikeout",Space,Str "and",Space,Strong [Str "strong"]]]
62−, Para [Str "Superscripts",Str ":",Space,Str "a",Superscript [Str "bc"],Str "d",Space,Str "a",Superscript [Strong [Str "hello"]],Space,Str "a",Superscript [Str "hello",Space,Str "there"],Str "."]
63−, Para [Str "Subscripts",Str ":",Space,Str "H",Subscript [Str "2"],Str "O,",Space,Str "H",Subscript [Str "23"],Str "O,",Space,Str "H",Subscript [Str "many",Space,Str "of",Space,Str "them"],Str "O."]
56+, Para [Str "This",Space,Str "is",Space,Emph [Str "emphasized"],Str ",",Space,Str "and",Space,Str "so",Space,Emph [Str "is",Space,Str "this"],Str ".",LineBreak,Str "This",Space,Str "is",Space,Strong [Str "strong"],Str ",",Space,Str "and",Space,Str "so",Space,Strong [Str "is",Space,Str "this"],Str ".",LineBreak,Str "A",Space,Link [Strong [Str "strong",Space,Str "link"]] ("http://www.foobar.com",""),Str "."]
57+, Para [Emph [Strong [Str "This",Space,Str "is",Space,Str "strong",Space,Str "and",Space,Str "em",Str "."]],LineBreak,Str "So",Space,Str "is",Space,Strong [Emph [Str "this"]],Space,Str "word",Space,Str "and",Space,Emph [Strong [Str "that",Space,Str "one"]],Str ".",LineBreak,Strikeout [Str "This",Space,Str "is",Space,Str "strikeout",Space,Str "and",Space,Strong [Str "strong"]]]
58+, Para [Str "Superscripts",Str ":",Space,Str "a",Superscript [Str "bc"],Str "d",Space,Str "a",Superscript [Strong [Str "hello"]],Space,Str "a",Superscript [Str "hello",Space,Str "there"],Str ".",LineBreak,Str "Subscripts",Str ":",Space,Str "H",Subscript [Str "2"],Str "O",Str ",",Space,Str "H",Subscript [Str "23"],Str "O",Str ",",Space,Str "H",Subscript [Str "many",Space,Str "of",Space,Str "them"],Str "O",Str "."]
59+, Para [Str "Dashes",Space,Str ":",Space,Str "How",Space,Str "cool",Space,Str "-",Str "-",Space,Str "automatic",Space,Str "dashes",Str "."]
60+, Para [Str "Elipses",Space,Str ":",Space,Str "He",Space,Str "thought",Space,Str "and",Space,Str "thought",Space,Str ".",Str ".",Str ".",Space,Str "and",Space,Str "then",Space,Str "thought",Space,Str "some",Space,Str "more",Str "."]
61+, Para [Str "Quotes",Space,Str "and",Space,Str "apostrophes",Space,Str ":",Space,Str "\"",Str "I",Str "'",Str "d",Space,Str "like",Space,Str "to",Space,Str "thank",Space,Str "you",Str "\"",Space,Str "for",Space,Str "example",Str "."]
6462, Header 1 [Str "Links"]
6563, Header 2 [Str "Explicit"]
6664, Para [Str "Just",Space,Str "a",Space,Link [Str "url"] ("http://www.url.com","")]
@@ -84,7 +82,7 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
8482 [ [ Plain [Str "bella"] ]
8583 , [ Plain [Str "45"] ]
8684 , [ Plain [Str "f"] ] ] ]
87−, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Str "..."]
85+, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Str ".",Str ".",Str "."]
8886, Header 2 [Str "With",Space,Str "headers"]
8987, Table [] [AlignDefault,AlignDefault,AlignDefault] [0.0,0.0,0.0]
9088 [ [ Plain [Str "name"] ]
@@ -100,11 +98,11 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
10098 , [ Plain [Str "45"] ]
10199 , [ Plain [Str "f"] ] ] ]
102100, Header 1 [Str "Images"]
103−, Para [Str "Textile",Space,Str "inline",Space,Str "image",Space,Str "syntax,",Space,Str "like",Space,Str "here",Space,Image [Str "this is the alt text"] ("this_is_an_image.png","this is the alt text"),Space,Str "and",Space,Str "here",Space,Image [Str ""] ("this_is_an_image.png",""),Str "."]
101+, Para [Str "Textile",Space,Str "inline",Space,Str "image",Space,Str "syntax",Str ",",Space,Str "like",Space,LineBreak,Str "here",Space,Image [Str "this is the alt text"] ("this_is_an_image.png","this is the alt text"),LineBreak,Str "and",Space,Str "here",Space,Image [Str ""] ("this_is_an_image.png",""),Str "."]
104102, Header 1 [Str "Attributes"]
105103, Header 2 [Str "HTML",Space,Str "and",Space,Str "CSS",Space,Str "attributes",Space,Str "are",Space,Str "ignored"]
106−, Para [Str "as",Space,Str "well",Space,Str "as",Space,Strong [Str "inline",Space,Str "attributes"]]
107−, Para [Str "and",Space,Str "paragraph",Space,Str "attributes,",Space,Str "and",Space,Str "table",Space,Str "attributes."]
104+, Para [Str "as",Space,Str "well",Space,Str "as",Space,Strong [Str "inline",Space,Str "attributes"],Space,Str "of",Space,Str " all kind"]
105+, Para [Str "and",Space,Str "paragraph",Space,Str "attributes",Str ",",Space,Str "and",Space,Str "table",Space,Str "attributes",Str "."]
108106, Table [] [AlignDefault,AlignDefault,AlignDefault] [0.0,0.0,0.0]
109107 [
110108 ] [
@@ -115,16 +113,16 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
115113 , [ Plain [Str "24"] ]
116114 , [ Plain [Str "f"] ] ] ]
117115, Header 1 [Str "Raw",Space,Str "HTML"]
118−, Para [Str "However,",Space,HtmlInline "<strong>",Space,Str "raw",Space,Str "HTML",Space,Str "inlines",Space,HtmlInline "</strong>",Space,Str "are",Space,Str "accepted,",Space,Str "as",Space,Str "well",Space,Str "as",Space,Str ":"]
119−, RawHtml "<div class=\"foobar\">"
120−, Para [Str "any",Space,Strong [Str "Raw",Space,Str "HTML",Space,Str "Block"],Space,Str "with",Space,Str "bold"]
121−, RawHtml "</div>"
116+, Para [Str "However",Str ",",Space,Str "raw",Space,Str "HTML",Space,Str "inlines",Space,Str "are",Space,Str "accepted",Str ",",Space,Str "as",Space,Str "well",Space,Str "as",Space,Str ":"]
117+, Null
118+, Para [Str "any",Space,Strong [Str "Raw",Space,Str "HTML",Space,Str "Block"],Space,Str "with",Space,Str "bold",LineBreak]
119+, Null
122120, Para [Str "Html",Space,Str "blocks",Space,Str "can",Space,Str "be"]
123−, RawHtml "<div>"
121+, Null
124122, Para [Str "inlined"]
125−, RawHtml "</div>"
126−, Para [Str "as",Space,Str "well."]
123+, Null
124+, Para [Str "as",Space,Str "well",Str "."]
127125, BulletList
128126 [ [ Plain [Str "this",Space,Str "<",Str "div",Str ">",Space,Str "won",Str "'",Str "t",Space,Str "produce",Space,Str "raw",Space,Str "html",Space,Str "blocks",Space,Str "<",Str "/div",Str ">"] ]
129− , [ Plain [Str "but",Space,Str "this",Space,HtmlInline "<strong>",Space,Str "will",Space,Str "produce",Space,Str "inline",Space,Str "html",Space,HtmlInline "</strong>"] ] ]
127+ , [ Plain [Str "but",Space,Str "this",Space,Str "",Space,Str "will",Space,Str "produce",Space,Str "inline",Space,Str "html",Space,Str ""] ] ]
130128, Para [Str "Can",Space,Str "you",Space,Str "prove",Space,Str "that",Space,Str "2",Space,Str "<",Space,Str "3",Space,Str "?"] ]
tests/textile-reader.textile+15−13
@@ -1,5 +1,5 @@
1−This is a set of tests for pandoc. Most of them are adapted from John
2−Gruber's markdown test suite.
1+This is a set of tests for pandoc Textile Reader. Part of it comes
2+from John Gruber's markdown test suite.
33
44-----
55
@@ -20,17 +20,18 @@ h1. Paragraphs
2020
2121Here's a regular paragraph.
2222
23−Line break are not paragraph break in textile, so you can wrap your
24−very long paragraph with your favourite text editor, it will be
25−rendered as a single one.
23+Line breaks are preserved in textile, so you can not wrap your very
24+long paragraph with your favourite text editor and have it rendered
25+with no break.
26+
2627
2728Here's one with a bullet.
2829
2930* criminey.
3031
31−There should be a hard line break
32+There should be a paragraph break between here
3233
33−here.
34+and here.
3435
3536h1. Block Quotes
3637
@@ -96,21 +97,22 @@ h2. Nested
9697h1. Inline Markup
9798
9899This is _emphasized_, and so __is this__.
99−
100100This is *strong*, and so **is this**.
101−
102101A "*strong link*":http://www.foobar.com.
103102
104103_*This is strong and em.*_
105−
106104So is *_this_* word and __**that one**__.
107−
108105-This is strikeout and *strong*-
109106
110107Superscripts: a^bc^d a^*hello*^ a^hello there^.
111−
112108Subscripts: H~2~O, H~23~O, H~many of them~O.
113109
110+Dashes : How cool -- automatic dashes.
111+
112+Elipses : He thought and thought ... and then thought some more.
113+
114+Quotes and apostrophes : "I'd like to thank you" for example.
115+
114116
115117h1. Links
116118
@@ -154,7 +156,7 @@ h1. Attributes
154156
155157h2{color:red}. HTML and CSS attributes are ignored
156158
157−as well as *(foo)inline attributes*
159+as well as *(foo)inline attributes* of %{color:red} all kind%
158160
159161p{color:green}. and paragraph attributes, and table attributes.
160162