git.iko.soy/typeable/pandoc
on master help sign in
Moved smartPunctuation from Markdown to Parsing.

+ Parameterized smartPunctuation on an inline parser. + Handle smartPunctuation in Textile reader.

commit50ca61ef4977217889b386f84abd0ba190fac3f2 authorJohn MacFarlane <jgm@berkeley.edu> dateTue 7 Dec 2010, 19:03 (16 years ago) parentf917b46
4 files changed+107−110
compare with master
unifiedsplit
src/Text/Pandoc/Parsing.hs+92−3 src/Text/Pandoc/Readers/Markdown.hs+2−89 src/Text/Pandoc/Readers/Textile.hs+5−10 tests/textile-reader.native+8−8
src/Text/Pandoc/Parsing.hs+92−3
@@ -67,17 +67,18 @@ module Text.Pandoc.Parsing ( (>>~),
6767 Key,
6868 toKey,
6969 fromKey,
70− lookupKeySrc )
70+ lookupKeySrc,
71+ smartPunctuation )
7172where
7273
7374import Text.Pandoc.Definition
7475import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)
7576import Text.ParserCombinators.Parsec
7677import Text.Pandoc.CharacterReferences ( characterReference )
77−import Data.Char ( toLower, toUpper, ord, isAscii )
78+import Data.Char ( toLower, toUpper, ord, isAscii, isAlphaNum )
7879import Data.List ( intercalate, transpose )
7980import Network.URI ( parseURI, URI (..), isAllowedInURI )
80−import Control.Monad ( join, liftM )
81+import Control.Monad ( join, liftM, guard )
8182import Text.Pandoc.Shared
8283import qualified Data.Map as M
8384import Text.TeXMath.Macros (Macro)
@@ -678,3 +679,91 @@ lookupKeySrc table key = case M.lookup key table of
678679 Nothing -> Nothing
679680 Just src -> Just src
680681
682+-- | Fail unless we're in "smart typography" mode.
683+failUnlessSmart :: GenParser tok ParserState ()
684+failUnlessSmart = getState >>= guard . stateSmart
685+
686+smartPunctuation :: GenParser Char ParserState Inline
687+ -> GenParser Char ParserState Inline
688+smartPunctuation inlineParser = do
689+ failUnlessSmart
690+ choice [ quoted inlineParser, apostrophe, dash, ellipses ]
691+
692+apostrophe :: GenParser Char ParserState Inline
693+apostrophe = (char '\'' <|> char '\8217') >> return Apostrophe
694+
695+quoted :: GenParser Char ParserState Inline
696+ -> GenParser Char ParserState Inline
697+quoted inlineParser = doubleQuoted inlineParser <|> singleQuoted inlineParser
698+
699+withQuoteContext :: QuoteContext
700+ -> (GenParser Char ParserState Inline)
701+ -> GenParser Char ParserState Inline
702+withQuoteContext context parser = do
703+ oldState <- getState
704+ let oldQuoteContext = stateQuoteContext oldState
705+ setState oldState { stateQuoteContext = context }
706+ result <- parser
707+ newState <- getState
708+ setState newState { stateQuoteContext = oldQuoteContext }
709+ return result
710+
711+singleQuoted :: GenParser Char ParserState Inline
712+ -> GenParser Char ParserState Inline
713+singleQuoted inlineParser = try $ do
714+ singleQuoteStart
715+ withQuoteContext InSingleQuote $ many1Till inlineParser singleQuoteEnd >>=
716+ return . Quoted SingleQuote . normalizeSpaces
717+
718+doubleQuoted :: GenParser Char ParserState Inline
719+ -> GenParser Char ParserState Inline
720+doubleQuoted inlineParser = try $ do
721+ doubleQuoteStart
722+ withQuoteContext InDoubleQuote $ do
723+ contents <- manyTill inlineParser doubleQuoteEnd
724+ return . Quoted DoubleQuote . normalizeSpaces $ contents
725+
726+failIfInQuoteContext :: QuoteContext -> GenParser tok ParserState ()
727+failIfInQuoteContext context = do
728+ st <- getState
729+ if stateQuoteContext st == context
730+ then fail "already inside quotes"
731+ else return ()
732+
733+singleQuoteStart :: GenParser Char ParserState ()
734+singleQuoteStart = do
735+ failIfInQuoteContext InSingleQuote
736+ try $ do oneOf "'\8216"
737+ notFollowedBy (oneOf ")!],.;:-? \t\n")
738+ notFollowedBy (try (oneOfStrings ["s","t","m","ve","ll","re"] >>
739+ satisfy (not . isAlphaNum)))
740+ -- possess/contraction
741+ return ()
742+
743+singleQuoteEnd :: GenParser Char st ()
744+singleQuoteEnd = try $ do
745+ oneOf "'\8217"
746+ notFollowedBy alphaNum
747+
748+doubleQuoteStart :: GenParser Char ParserState ()
749+doubleQuoteStart = do
750+ failIfInQuoteContext InDoubleQuote
751+ try $ do oneOf "\"\8220"
752+ notFollowedBy (oneOf " \t\n")
753+
754+doubleQuoteEnd :: GenParser Char st ()
755+doubleQuoteEnd = oneOf "\"\8221" >> return ()
756+
757+ellipses :: GenParser Char st Inline
758+ellipses = oneOfStrings ["...", " . . . ", ". . .", " . . ."] >> return Ellipses
759+
760+dash :: GenParser Char st Inline
761+dash = enDash <|> emDash
762+
763+enDash :: GenParser Char st Inline
764+enDash = try $ char '-' >> notFollowedBy (noneOf "0123456789") >> return EnDash
765+
766+emDash :: GenParser Char st Inline
767+emDash = oneOfStrings ["---", "--"] >> return EmDash
768+
769+
src/Text/Pandoc/Readers/Markdown.hs+2−89
@@ -27,10 +27,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
2727
2828Conversion of markdown-formatted plain text to 'Pandoc' document.
2929-}
30−module Text.Pandoc.Readers.Markdown (
31− readMarkdown,
32− smartPunctuation
33− ) where
30+module Text.Pandoc.Readers.Markdown ( readMarkdown ) where
3431
3532import Data.List ( transpose, isSuffixOf, sortBy, findIndex, intercalate )
3633import qualified Data.Map as M
@@ -108,12 +105,6 @@ failUnlessBeginningOfLine = do
108105 pos <- getPosition
109106 if sourceColumn pos == 1 then return () else fail "not beginning of line"
110107
111−-- | Fail unless we're in "smart typography" mode.
112−failUnlessSmart :: GenParser tok ParserState ()
113−failUnlessSmart = do
114− state <- getState
115− if stateSmart state then return () else pzero
116−
117108-- | Parse a sequence of inline elements between square brackets,
118109-- including inlines between balanced pairs of square brackets.
119110inlinesInBalancedBrackets :: GenParser Char ParserState Inline
@@ -906,7 +897,7 @@ inline = choice inlineParsers <?> "inline"
906897
907898inlineParsers :: [GenParser Char ParserState Inline]
908899inlineParsers = [ str
909− , smartPunctuation
900+ , smartPunctuation inline
910901 , whitespace
911902 , endline
912903 , code
@@ -1047,84 +1038,6 @@ subscript = failIfStrict >> enclosed (char '~') (char '~')
10471038 (notFollowedBy spaceChar >> inline) >>= -- may not contain Space
10481039 return . Subscript
10491040
1050−smartPunctuation :: GenParser Char ParserState Inline
1051−smartPunctuation = failUnlessSmart >>
1052− choice [ quoted, apostrophe, dash, ellipses ]
1053−
1054−apostrophe :: GenParser Char ParserState Inline
1055−apostrophe = (char '\'' <|> char '\8217') >> return Apostrophe
1056−
1057−quoted :: GenParser Char ParserState Inline
1058−quoted = doubleQuoted <|> singleQuoted
1059−
1060−withQuoteContext :: QuoteContext
1061− -> (GenParser Char ParserState Inline)
1062− -> GenParser Char ParserState Inline
1063−withQuoteContext context parser = do
1064− oldState <- getState
1065− let oldQuoteContext = stateQuoteContext oldState
1066− setState oldState { stateQuoteContext = context }
1067− result <- parser
1068− newState <- getState
1069− setState newState { stateQuoteContext = oldQuoteContext }
1070− return result
1071−
1072−singleQuoted :: GenParser Char ParserState Inline
1073−singleQuoted = try $ do
1074− singleQuoteStart
1075− withQuoteContext InSingleQuote $ many1Till inline singleQuoteEnd >>=
1076− return . Quoted SingleQuote . normalizeSpaces
1077−
1078−doubleQuoted :: GenParser Char ParserState Inline
1079−doubleQuoted = try $ do
1080− doubleQuoteStart
1081− withQuoteContext InDoubleQuote $ do
1082− contents <- manyTill inline doubleQuoteEnd
1083− return . Quoted DoubleQuote . normalizeSpaces $ contents
1084−
1085−failIfInQuoteContext :: QuoteContext -> GenParser tok ParserState ()
1086−failIfInQuoteContext context = do
1087− st <- getState
1088− if stateQuoteContext st == context
1089− then fail "already inside quotes"
1090− else return ()
1091−
1092−singleQuoteStart :: GenParser Char ParserState ()
1093−singleQuoteStart = do
1094− failIfInQuoteContext InSingleQuote
1095− try $ do oneOf "'\8216"
1096− notFollowedBy (oneOf ")!],.;:-? \t\n")
1097− notFollowedBy (try (oneOfStrings ["s","t","m","ve","ll","re"] >>
1098− satisfy (not . isAlphaNum)))
1099− -- possess/contraction
1100− return ()
1101−
1102−singleQuoteEnd :: GenParser Char st ()
1103−singleQuoteEnd = try $ do
1104− oneOf "'\8217"
1105− notFollowedBy alphaNum
1106−
1107−doubleQuoteStart :: GenParser Char ParserState ()
1108−doubleQuoteStart = do
1109− failIfInQuoteContext InDoubleQuote
1110− try $ do oneOf "\"\8220"
1111− notFollowedBy (oneOf " \t\n")
1112−
1113−doubleQuoteEnd :: GenParser Char st ()
1114−doubleQuoteEnd = oneOf "\"\8221" >> return ()
1115−
1116−ellipses :: GenParser Char st Inline
1117−ellipses = oneOfStrings ["...", " . . . ", ". . .", " . . ."] >> return Ellipses
1118−
1119−dash :: GenParser Char st Inline
1120−dash = enDash <|> emDash
1121−
1122−enDash :: GenParser Char st Inline
1123−enDash = try $ char '-' >> notFollowedBy (noneOf "0123456789") >> return EnDash
1124−
1125−emDash :: GenParser Char st Inline
1126−emDash = oneOfStrings ["---", "--"] >> return EmDash
1127−
11281041whitespace :: GenParser Char ParserState Inline
11291042whitespace = spaceChar >>
11301043 ( (spaceChar >> skipMany spaceChar >> option Space (endline >> return LineBreak))
src/Text/Pandoc/Readers/Textile.hs+5−10
@@ -34,21 +34,17 @@ Implemented and parsed:
3434 - Lists
3535 - blockquote
3636 - Inlines : strong, emph, cite, code, deleted, superscript,
37− subscript, links, smart punctuation
37+ subscript, links
3838
3939Implemented but discarded:
4040 - HTML-specific and CSS-specific attributes
4141
4242Left to be implemented:
43− - Pandoc Meta Information (title, author, date)
4443 - footnotes
4544 - dimension sign
46− - uppercase
45+ - all caps
4746 - definition lists
4847 - continued blocks (ex bq..)
49− -
50−
51−
5248
5349TODO : refactor common patterns across readers :
5450 - autolink
@@ -58,9 +54,8 @@ TODO : refactor common patterns across readers :
5854-}
5955
6056
61−module Text.Pandoc.Readers.Textile (
62− readTextile
63− ) where
57+module Text.Pandoc.Readers.Textile ( readTextile) where
58+
6459import Text.Pandoc.Definition
6560import Text.Pandoc.Shared
6661import Text.Pandoc.Parsing
@@ -313,7 +308,6 @@ inlineParsers = [ autoLink
313308 , mark
314309 , str
315310 , htmlSpan
316−-- , smartPunctuation -- from markdown reader
317311 , whitespace
318312 , endline
319313 , rawHtmlInline
@@ -328,6 +322,7 @@ inlineParsers = [ autoLink
328322 , simpleInline (char '~') Subscript
329323 , link
330324 , image
325+ , smartPunctuation inline
331326 , symbol
332327 ]
333328
tests/textile-reader.native+8−8
@@ -1,5 +1,5 @@
11Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
2−[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc",Space,Str "Textile",Space,Str "Reader",Str ".",Space,Str "Part",Space,Str "of",Space,Str "it",Space,Str "comes",LineBreak,Str "from",Space,Str "John",Space,Str "Gruber",Str "'",Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite",Str "."]
2+[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc",Space,Str "Textile",Space,Str "Reader",Str ".",Space,Str "Part",Space,Str "of",Space,Str "it",Space,Str "comes",LineBreak,Str "from",Space,Str "John",Space,Str "Gruber",Apostrophe,Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite",Str "."]
33, HorizontalRule
44, Header 1 [Str "Headers"]
55, Header 2 [Str "Level",Space,Str "2",Space,Str "with",Space,Str "an",Space,Link [Str "embeded",Space,Str "link"] ("http://www.example.com","")]
@@ -8,9 +8,9 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
88, Header 5 [Str "Level",Space,Str "5"]
99, Header 6 [Str "Level",Space,Str "6"]
1010, Header 1 [Str "Paragraphs"]
11−, Para [Str "Here",Str "'",Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph",Str "."]
11+, Para [Str "Here",Apostrophe,Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph",Str "."]
1212, Para [Str "Line",Space,Str "breaks",Space,Str "are",Space,Str "preserved",Space,Str "in",Space,Str "textile",Str ",",Space,Str "so",Space,Str "you",Space,Str "can",Space,Str "not",Space,Str "wrap",Space,Str "your",Space,Str "very",LineBreak,Str "long",Space,Str "paragraph",Space,Str "with",Space,Str "your",Space,Str "favourite",Space,Str "text",Space,Str "editor",Space,Str "and",Space,Str "have",Space,Str "it",Space,Str "rendered",LineBreak,Str "with",Space,Str "no",Space,Str "break",Str "."]
13−, Para [Str "Here",Str "'",Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet",Str "."]
13+, Para [Str "Here",Apostrophe,Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet",Str "."]
1414, BulletList
1515 [ [ Plain [Str "criminey",Str "."] ]
1616 ]
@@ -56,9 +56,9 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
5656, Para [Str "This",Space,Str "is",Space,Emph [Str "emphasized"],Str ",",Space,Str "and",Space,Str "so",Space,Emph [Str "is",Space,Str "this"],Str ".",LineBreak,Str "This",Space,Str "is",Space,Strong [Str "strong"],Str ",",Space,Str "and",Space,Str "so",Space,Strong [Str "is",Space,Str "this"],Str ".",LineBreak,Str "A",Space,Link [Strong [Str "strong",Space,Str "link"]] ("http://www.foobar.com",""),Str "."]
5757, Para [Emph [Strong [Str "This",Space,Str "is",Space,Str "strong",Space,Str "and",Space,Str "em",Str "."]],LineBreak,Str "So",Space,Str "is",Space,Strong [Emph [Str "this"]],Space,Str "word",Space,Str "and",Space,Emph [Strong [Str "that",Space,Str "one"]],Str ".",LineBreak,Strikeout [Str "This",Space,Str "is",Space,Str "strikeout",Space,Str "and",Space,Strong [Str "strong"]]]
5858, Para [Str "Superscripts",Str ":",Space,Str "a",Superscript [Str "bc"],Str "d",Space,Str "a",Superscript [Strong [Str "hello"]],Space,Str "a",Superscript [Str "hello",Space,Str "there"],Str ".",LineBreak,Str "Subscripts",Str ":",Space,Str "H",Subscript [Str "2"],Str "O",Str ",",Space,Str "H",Subscript [Str "23"],Str "O",Str ",",Space,Str "H",Subscript [Str "many",Space,Str "of",Space,Str "them"],Str "O",Str "."]
59−, Para [Str "Dashes",Space,Str ":",Space,Str "How",Space,Str "cool",Space,Str "-",Str "-",Space,Str "automatic",Space,Str "dashes",Str "."]
60−, Para [Str "Elipses",Space,Str ":",Space,Str "He",Space,Str "thought",Space,Str "and",Space,Str "thought",Space,Str ".",Str ".",Str ".",Space,Str "and",Space,Str "then",Space,Str "thought",Space,Str "some",Space,Str "more",Str "."]
61−, Para [Str "Quotes",Space,Str "and",Space,Str "apostrophes",Space,Str ":",Space,Str "\"",Str "I",Str "'",Str "d",Space,Str "like",Space,Str "to",Space,Str "thank",Space,Str "you",Str "\"",Space,Str "for",Space,Str "example",Str "."]
59+, Para [Str "Dashes",Space,Str ":",Space,Str "How",Space,Str "cool",Space,EmDash,Space,Str "automatic",Space,Str "dashes",Str "."]
60+, Para [Str "Elipses",Space,Str ":",Space,Str "He",Space,Str "thought",Space,Str "and",Space,Str "thought",Space,Ellipses,Space,Str "and",Space,Str "then",Space,Str "thought",Space,Str "some",Space,Str "more",Str "."]
61+, Para [Str "Quotes",Space,Str "and",Space,Str "apostrophes",Space,Str ":",Space,Quoted DoubleQuote [Str "I",Apostrophe,Str "d",Space,Str "like",Space,Str "to",Space,Str "thank",Space,Str "you"],Space,Str "for",Space,Str "example",Str "."]
6262, Header 1 [Str "Links"]
6363, Header 2 [Str "Explicit"]
6464, Para [Str "Just",Space,Str "a",Space,Link [Str "url"] ("http://www.url.com","")]
@@ -82,7 +82,7 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
8282 [ [ Plain [Str "bella"] ]
8383 , [ Plain [Str "45"] ]
8484 , [ Plain [Str "f"] ] ] ]
85−, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Str ".",Str ".",Str "."]
85+, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Ellipses]
8686, Header 2 [Str "With",Space,Str "headers"]
8787, Table [] [AlignDefault,AlignDefault,AlignDefault] [0.0,0.0,0.0]
8888 [ [ Plain [Str "name"] ]
@@ -123,7 +123,7 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})
123123, RawHtml "</div>"
124124, Para [Str "as",Space,Str "well",Str "."]
125125, BulletList
126− [ [ Plain [Str "this",Space,Str "<",Str "div",Str ">",Space,Str "won",Str "'",Str "t",Space,Str "produce",Space,Str "raw",Space,Str "html",Space,Str "blocks",Space,Str "<",Str "/div",Str ">"] ]
126+ [ [ Plain [Str "this",Space,Str "<",Str "div",Str ">",Space,Str "won",Apostrophe,Str "t",Space,Str "produce",Space,Str "raw",Space,Str "html",Space,Str "blocks",Space,Str "<",Str "/div",Str ">"] ]
127127 , [ Plain [Str "but",Space,Str "this",Space,HtmlInline "<strong>",Space,Str "will",Space,Str "produce",Space,Str "inline",Space,Str "html",Space,HtmlInline "</strong>"] ] ]
128128, Para [Str "Can",Space,Str "you",Space,Str "prove",Space,Str "that",Space,Str "2",Space,Str "<",Space,Str "3",Space,Str "?"]
129129, Header 1 [Str "Acronyms",Space,Str "and",Space,Str "marks"]