50ca61e · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Moved smartPunctuation from Markdown to Parsing.

50ca61eJohn MacFarlane16 years ago

src/Text/Pandoc/Parsing.hs+92−3

compare with master

unifiedsplit

@@ -67,17 +67,18 @@ module Text.Pandoc.Parsing ( (>>~),

67 Key,

68 toKey,

69 fromKey,

70− lookupKeySrc )

70+ lookupKeySrc,

71+ smartPunctuation )

71where

72where

73import Text.Pandoc.Definition

74import Text.Pandoc.Definition

74import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)

75import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)

75import Text.ParserCombinators.Parsec

76import Text.ParserCombinators.Parsec

76import Text.Pandoc.CharacterReferences ( characterReference )

77import Text.Pandoc.CharacterReferences ( characterReference )

77−import Data.Char ( toLower, toUpper, ord, isAscii )

78+import Data.Char ( toLower, toUpper, ord, isAscii, isAlphaNum )

78import Data.List ( intercalate, transpose )

79import Data.List ( intercalate, transpose )

79import Network.URI ( parseURI, URI (..), isAllowedInURI )

80import Network.URI ( parseURI, URI (..), isAllowedInURI )

80−import Control.Monad ( join, liftM )

81+import Control.Monad ( join, liftM, guard )

81import Text.Pandoc.Shared

82import Text.Pandoc.Shared

82import qualified Data.Map as M

83import qualified Data.Map as M

83import Text.TeXMath.Macros (Macro)

84import Text.TeXMath.Macros (Macro)

@@ -678,3 +679,91 @@ lookupKeySrc table key = case M.lookup key table of

678 Nothing -> Nothing

679 Nothing -> Nothing

679 Just src -> Just src

680 Just src -> Just src

680

681

682+-- | Fail unless we're in "smart typography" mode.

683+failUnlessSmart :: GenParser tok ParserState ()

684+failUnlessSmart = getState >>= guard . stateSmart

685+

686+smartPunctuation :: GenParser Char ParserState Inline

687+ -> GenParser Char ParserState Inline

688+smartPunctuation inlineParser = do

689+ failUnlessSmart

690+ choice [ quoted inlineParser, apostrophe, dash, ellipses ]

691+

692+apostrophe :: GenParser Char ParserState Inline

693+apostrophe = (char '\'' <|> char '\8217') >> return Apostrophe

694+

695+quoted :: GenParser Char ParserState Inline

696+ -> GenParser Char ParserState Inline

697+quoted inlineParser = doubleQuoted inlineParser <|> singleQuoted inlineParser

698+

699+withQuoteContext :: QuoteContext

700+ -> (GenParser Char ParserState Inline)

701+ -> GenParser Char ParserState Inline

702+withQuoteContext context parser = do

703+ oldState <- getState

704+ let oldQuoteContext = stateQuoteContext oldState

705+ setState oldState { stateQuoteContext = context }

706+ result <- parser

707+ newState <- getState

708+ setState newState { stateQuoteContext = oldQuoteContext }

709+ return result

710+

711+singleQuoted :: GenParser Char ParserState Inline

712+ -> GenParser Char ParserState Inline

713+singleQuoted inlineParser = try $ do

714+ singleQuoteStart

715+ withQuoteContext InSingleQuote $ many1Till inlineParser singleQuoteEnd >>=

716+ return . Quoted SingleQuote . normalizeSpaces

717+

718+doubleQuoted :: GenParser Char ParserState Inline

719+ -> GenParser Char ParserState Inline

720+doubleQuoted inlineParser = try $ do

721+ doubleQuoteStart

722+ withQuoteContext InDoubleQuote $ do

723+ contents <- manyTill inlineParser doubleQuoteEnd

724+ return . Quoted DoubleQuote . normalizeSpaces $ contents

725+

726+failIfInQuoteContext :: QuoteContext -> GenParser tok ParserState ()

727+failIfInQuoteContext context = do

728+ st <- getState

729+ if stateQuoteContext st == context

730+ then fail "already inside quotes"

731+ else return ()

732+

733+singleQuoteStart :: GenParser Char ParserState ()

734+singleQuoteStart = do

735+ failIfInQuoteContext InSingleQuote

736+ try $ do oneOf "'\8216"

737+ notFollowedBy (oneOf ")!],.;:-? \t\n")

738+ notFollowedBy (try (oneOfStrings ["s","t","m","ve","ll","re"] >>

739+ satisfy (not . isAlphaNum)))

740+ -- possess/contraction

741+ return ()

742+

743+singleQuoteEnd :: GenParser Char st ()

744+singleQuoteEnd = try $ do

745+ oneOf "'\8217"

746+ notFollowedBy alphaNum

747+

748+doubleQuoteStart :: GenParser Char ParserState ()

749+doubleQuoteStart = do

750+ failIfInQuoteContext InDoubleQuote

751+ try $ do oneOf "\"\8220"

752+ notFollowedBy (oneOf " \t\n")

753+

754+doubleQuoteEnd :: GenParser Char st ()

755+doubleQuoteEnd = oneOf "\"\8221" >> return ()

756+

757+ellipses :: GenParser Char st Inline

758+ellipses = oneOfStrings ["...", " . . . ", ". . .", " . . ."] >> return Ellipses

759+

760+dash :: GenParser Char st Inline

761+dash = enDash <|> emDash

762+

763+enDash :: GenParser Char st Inline

764+enDash = try $ char '-' >> notFollowedBy (noneOf "0123456789") >> return EnDash

765+

766+emDash :: GenParser Char st Inline

767+emDash = oneOfStrings ["---", "--"] >> return EmDash

768+

769+

src/Text/Pandoc/Readers/Markdown.hs+2−89

@@ -27,10 +27,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

28Conversion of markdown-formatted plain text to 'Pandoc' document.

29-}

30−module Text.Pandoc.Readers.Markdown (

30+module Text.Pandoc.Readers.Markdown ( readMarkdown ) where

31− readMarkdown,

32− smartPunctuation

33− ) where

35import Data.List ( transpose, isSuffixOf, sortBy, findIndex, intercalate )

32import Data.List ( transpose, isSuffixOf, sortBy, findIndex, intercalate )

36import qualified Data.Map as M

33import qualified Data.Map as M

@@ -108,12 +105,6 @@ failUnlessBeginningOfLine = do

108 pos <- getPosition

105 pos <- getPosition

109 if sourceColumn pos == 1 then return () else fail "not beginning of line"

106 if sourceColumn pos == 1 then return () else fail "not beginning of line"

110

107

111−-- | Fail unless we're in "smart typography" mode.

112−failUnlessSmart :: GenParser tok ParserState ()

113−failUnlessSmart = do

114− state <- getState

115− if stateSmart state then return () else pzero

116−

117-- | Parse a sequence of inline elements between square brackets,

108-- | Parse a sequence of inline elements between square brackets,

118-- including inlines between balanced pairs of square brackets.

109-- including inlines between balanced pairs of square brackets.

119inlinesInBalancedBrackets :: GenParser Char ParserState Inline

110inlinesInBalancedBrackets :: GenParser Char ParserState Inline

@@ -906,7 +897,7 @@ inline = choice inlineParsers <?> "inline"

906

897

907inlineParsers :: [GenParser Char ParserState Inline]

898inlineParsers :: [GenParser Char ParserState Inline]

908inlineParsers = [ str

899inlineParsers = [ str

909− , smartPunctuation

900+ , smartPunctuation inline

910 , whitespace

901 , whitespace

911 , endline

902 , endline

912 , code

903 , code

@@ -1047,84 +1038,6 @@ subscript = failIfStrict >> enclosed (char '~') (char '~')

1047 (notFollowedBy spaceChar >> inline) >>= -- may not contain Space

1038 (notFollowedBy spaceChar >> inline) >>= -- may not contain Space

1048 return . Subscript

1039 return . Subscript

1049

1040

1050−smartPunctuation :: GenParser Char ParserState Inline

1051−smartPunctuation = failUnlessSmart >>

1052− choice [ quoted, apostrophe, dash, ellipses ]

1053−

1054−apostrophe :: GenParser Char ParserState Inline

1055−apostrophe = (char '\'' <|> char '\8217') >> return Apostrophe

1056−

1057−quoted :: GenParser Char ParserState Inline

1058−quoted = doubleQuoted <|> singleQuoted

1059−

1060−withQuoteContext :: QuoteContext

1061− -> (GenParser Char ParserState Inline)

1062− -> GenParser Char ParserState Inline

1063−withQuoteContext context parser = do

1064− oldState <- getState

1065− let oldQuoteContext = stateQuoteContext oldState

1066− setState oldState { stateQuoteContext = context }

1067− result <- parser

1068− newState <- getState

1069− setState newState { stateQuoteContext = oldQuoteContext }

1070− return result

1071−

1072−singleQuoted :: GenParser Char ParserState Inline

1073−singleQuoted = try $ do

1074− singleQuoteStart

1075− withQuoteContext InSingleQuote $ many1Till inline singleQuoteEnd >>=

1076− return . Quoted SingleQuote . normalizeSpaces

1077−

1078−doubleQuoted :: GenParser Char ParserState Inline

1079−doubleQuoted = try $ do

1080− doubleQuoteStart

1081− withQuoteContext InDoubleQuote $ do

1082− contents <- manyTill inline doubleQuoteEnd

1083− return . Quoted DoubleQuote . normalizeSpaces $ contents

1084−

1085−failIfInQuoteContext :: QuoteContext -> GenParser tok ParserState ()

1086−failIfInQuoteContext context = do

1087− st <- getState

1088− if stateQuoteContext st == context

1089− then fail "already inside quotes"

1090− else return ()

1091−

1092−singleQuoteStart :: GenParser Char ParserState ()

1093−singleQuoteStart = do

1094− failIfInQuoteContext InSingleQuote

1095− try $ do oneOf "'\8216"

1096− notFollowedBy (oneOf ")!],.;:-? \t\n")

1097− notFollowedBy (try (oneOfStrings ["s","t","m","ve","ll","re"] >>

1098− satisfy (not . isAlphaNum)))

1099− -- possess/contraction

1100− return ()

1101−

1102−singleQuoteEnd :: GenParser Char st ()

1103−singleQuoteEnd = try $ do

1104− oneOf "'\8217"

1105− notFollowedBy alphaNum

1106−

1107−doubleQuoteStart :: GenParser Char ParserState ()

1108−doubleQuoteStart = do

1109− failIfInQuoteContext InDoubleQuote

1110− try $ do oneOf "\"\8220"

1111− notFollowedBy (oneOf " \t\n")

1112−

1113−doubleQuoteEnd :: GenParser Char st ()

1114−doubleQuoteEnd = oneOf "\"\8221" >> return ()

1115−

1116−ellipses :: GenParser Char st Inline

1117−ellipses = oneOfStrings ["...", " . . . ", ". . .", " . . ."] >> return Ellipses

1118−

1119−dash :: GenParser Char st Inline

1120−dash = enDash <|> emDash

1121−

1122−enDash :: GenParser Char st Inline

1123−enDash = try $ char '-' >> notFollowedBy (noneOf "0123456789") >> return EnDash

1124−

1125−emDash :: GenParser Char st Inline

1126−emDash = oneOfStrings ["---", "--"] >> return EmDash

1127−

1128whitespace :: GenParser Char ParserState Inline

1041whitespace :: GenParser Char ParserState Inline

1129whitespace = spaceChar >>

1042whitespace = spaceChar >>

1130 ( (spaceChar >> skipMany spaceChar >> option Space (endline >> return LineBreak))

1043 ( (spaceChar >> skipMany spaceChar >> option Space (endline >> return LineBreak))

src/Text/Pandoc/Readers/Textile.hs+5−10

@@ -34,21 +34,17 @@ Implemented and parsed:

34 - Lists

35 - blockquote

36 - Inlines : strong, emph, cite, code, deleted, superscript,

37− subscript, links, smart punctuation

37+ subscript, links

39Implemented but discarded:

40 - HTML-specific and CSS-specific attributes

42Left to be implemented:

43− - Pandoc Meta Information (title, author, date)

44 - footnotes

43 - footnotes

45 - dimension sign

44 - dimension sign

46− - uppercase

45+ - all caps

47 - definition lists

46 - definition lists

48 - continued blocks (ex bq..)

47 - continued blocks (ex bq..)

49− -

50−

51−

53TODO : refactor common patterns across readers :

49TODO : refactor common patterns across readers :

54 - autolink

50 - autolink

@@ -58,9 +54,8 @@ TODO : refactor common patterns across readers :

58-}

54-}

61−module Text.Pandoc.Readers.Textile (

57+module Text.Pandoc.Readers.Textile ( readTextile) where

62− readTextile

58+

63− ) where

64import Text.Pandoc.Definition

59import Text.Pandoc.Definition

65import Text.Pandoc.Shared

60import Text.Pandoc.Shared

66import Text.Pandoc.Parsing

61import Text.Pandoc.Parsing

@@ -313,7 +308,6 @@ inlineParsers = [ autoLink

313 , mark

308 , mark

314 , str

309 , str

315 , htmlSpan

310 , htmlSpan

316−-- , smartPunctuation -- from markdown reader

317 , whitespace

311 , whitespace

318 , endline

312 , endline

319 , rawHtmlInline

313 , rawHtmlInline

@@ -328,6 +322,7 @@ inlineParsers = [ autoLink

328 , simpleInline (char '~') Subscript

322 , simpleInline (char '~') Subscript

329 , link

323 , link

330 , image

324 , image

325+ , smartPunctuation inline

331 , symbol

326 , symbol

332 ]

327 ]

333

328

tests/textile-reader.native+8−8

@@ -1,5 +1,5 @@

1Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})

2−[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc",Space,Str "Textile",Space,Str "Reader",Str ".",Space,Str "Part",Space,Str "of",Space,Str "it",Space,Str "comes",LineBreak,Str "from",Space,Str "John",Space,Str "Gruber",Str "'",Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite",Str "."]

2+[ Para [Str "This",Space,Str "is",Space,Str "a",Space,Str "set",Space,Str "of",Space,Str "tests",Space,Str "for",Space,Str "pandoc",Space,Str "Textile",Space,Str "Reader",Str ".",Space,Str "Part",Space,Str "of",Space,Str "it",Space,Str "comes",LineBreak,Str "from",Space,Str "John",Space,Str "Gruber",Apostrophe,Str "s",Space,Str "markdown",Space,Str "test",Space,Str "suite",Str "."]

3, HorizontalRule

4, Header 1 [Str "Headers"]

5, Header 2 [Str "Level",Space,Str "2",Space,Str "with",Space,Str "an",Space,Link [Str "embeded",Space,Str "link"] ("http://www.example.com","")]

@@ -8,9 +8,9 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})

8, Header 5 [Str "Level",Space,Str "5"]

9, Header 6 [Str "Level",Space,Str "6"]

10, Header 1 [Str "Paragraphs"]

11−, Para [Str "Here",Str "'",Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph",Str "."]

11+, Para [Str "Here",Apostrophe,Str "s",Space,Str "a",Space,Str "regular",Space,Str "paragraph",Str "."]

12, Para [Str "Line",Space,Str "breaks",Space,Str "are",Space,Str "preserved",Space,Str "in",Space,Str "textile",Str ",",Space,Str "so",Space,Str "you",Space,Str "can",Space,Str "not",Space,Str "wrap",Space,Str "your",Space,Str "very",LineBreak,Str "long",Space,Str "paragraph",Space,Str "with",Space,Str "your",Space,Str "favourite",Space,Str "text",Space,Str "editor",Space,Str "and",Space,Str "have",Space,Str "it",Space,Str "rendered",LineBreak,Str "with",Space,Str "no",Space,Str "break",Str "."]

13−, Para [Str "Here",Str "'",Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet",Str "."]

13+, Para [Str "Here",Apostrophe,Str "s",Space,Str "one",Space,Str "with",Space,Str "a",Space,Str "bullet",Str "."]

14, BulletList

15 [ [ Plain [Str "criminey",Str "."] ]

16 ]

@@ -56,9 +56,9 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})

56, Para [Str "This",Space,Str "is",Space,Emph [Str "emphasized"],Str ",",Space,Str "and",Space,Str "so",Space,Emph [Str "is",Space,Str "this"],Str ".",LineBreak,Str "This",Space,Str "is",Space,Strong [Str "strong"],Str ",",Space,Str "and",Space,Str "so",Space,Strong [Str "is",Space,Str "this"],Str ".",LineBreak,Str "A",Space,Link [Strong [Str "strong",Space,Str "link"]] ("http://www.foobar.com",""),Str "."]

57, Para [Emph [Strong [Str "This",Space,Str "is",Space,Str "strong",Space,Str "and",Space,Str "em",Str "."]],LineBreak,Str "So",Space,Str "is",Space,Strong [Emph [Str "this"]],Space,Str "word",Space,Str "and",Space,Emph [Strong [Str "that",Space,Str "one"]],Str ".",LineBreak,Strikeout [Str "This",Space,Str "is",Space,Str "strikeout",Space,Str "and",Space,Strong [Str "strong"]]]

58, Para [Str "Superscripts",Str ":",Space,Str "a",Superscript [Str "bc"],Str "d",Space,Str "a",Superscript [Strong [Str "hello"]],Space,Str "a",Superscript [Str "hello",Space,Str "there"],Str ".",LineBreak,Str "Subscripts",Str ":",Space,Str "H",Subscript [Str "2"],Str "O",Str ",",Space,Str "H",Subscript [Str "23"],Str "O",Str ",",Space,Str "H",Subscript [Str "many",Space,Str "of",Space,Str "them"],Str "O",Str "."]

59−, Para [Str "Dashes",Space,Str ":",Space,Str "How",Space,Str "cool",Space,Str "-",Str "-",Space,Str "automatic",Space,Str "dashes",Str "."]

59+, Para [Str "Dashes",Space,Str ":",Space,Str "How",Space,Str "cool",Space,EmDash,Space,Str "automatic",Space,Str "dashes",Str "."]

60−, Para [Str "Elipses",Space,Str ":",Space,Str "He",Space,Str "thought",Space,Str "and",Space,Str "thought",Space,Str ".",Str ".",Str ".",Space,Str "and",Space,Str "then",Space,Str "thought",Space,Str "some",Space,Str "more",Str "."]

60+, Para [Str "Elipses",Space,Str ":",Space,Str "He",Space,Str "thought",Space,Str "and",Space,Str "thought",Space,Ellipses,Space,Str "and",Space,Str "then",Space,Str "thought",Space,Str "some",Space,Str "more",Str "."]

61−, Para [Str "Quotes",Space,Str "and",Space,Str "apostrophes",Space,Str ":",Space,Str "\"",Str "I",Str "'",Str "d",Space,Str "like",Space,Str "to",Space,Str "thank",Space,Str "you",Str "\"",Space,Str "for",Space,Str "example",Str "."]

61+, Para [Str "Quotes",Space,Str "and",Space,Str "apostrophes",Space,Str ":",Space,Quoted DoubleQuote [Str "I",Apostrophe,Str "d",Space,Str "like",Space,Str "to",Space,Str "thank",Space,Str "you"],Space,Str "for",Space,Str "example",Str "."]

62, Header 1 [Str "Links"]

63, Header 2 [Str "Explicit"]

64, Para [Str "Just",Space,Str "a",Space,Link [Str "url"] ("http://www.url.com","")]

@@ -82,7 +82,7 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})

82 [ [ Plain [Str "bella"] ]

83 , [ Plain [Str "45"] ]

84 , [ Plain [Str "f"] ] ] ]

85−, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Str ".",Str ".",Str "."]

85+, Para [Str "and",Space,Str "some",Space,Str "text",Space,Str "following",Space,Ellipses]

86, Header 2 [Str "With",Space,Str "headers"]

87, Table [] [AlignDefault,AlignDefault,AlignDefault] [0.0,0.0,0.0]

88 [ [ Plain [Str "name"] ]

@@ -123,7 +123,7 @@ Pandoc (Meta {docTitle = [Str ""], docAuthors = [[Str ""]], docDate = [Str ""]})

123, RawHtml "</div>"

124, Para [Str "as",Space,Str "well",Str "."]

125, BulletList

126− [ [ Plain [Str "this",Space,Str "<",Str "div",Str ">",Space,Str "won",Str "'",Str "t",Space,Str "produce",Space,Str "raw",Space,Str "html",Space,Str "blocks",Space,Str "<",Str "/div",Str ">"] ]

126+ [ [ Plain [Str "this",Space,Str "<",Str "div",Str ">",Space,Str "won",Apostrophe,Str "t",Space,Str "produce",Space,Str "raw",Space,Str "html",Space,Str "blocks",Space,Str "<",Str "/div",Str ">"] ]

127 , [ Plain [Str "but",Space,Str "this",Space,HtmlInline "<strong>",Space,Str "will",Space,Str "produce",Space,Str "inline",Space,Str "html",Space,HtmlInline "</strong>"] ] ]

128, Para [Str "Can",Space,Str "you",Space,Str "prove",Space,Str "that",Space,Str "2",Space,Str "<",Space,Str "3",Space,Str "?"]

129, Header 1 [Str "Acronyms",Space,Str "and",Space,Str "marks"]