141affd · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

More changes in entity handling: Instead of using entities for characters

141affdfiddlosopher20 years ago

src/Text/Pandoc/Entities.hs+30−31

compare with master

unifiedsplit

@@ -31,9 +31,9 @@ and vice versa.

31module Text.Pandoc.Entities (

32 charToEntity,

33 charToNumericalEntity,

34− specialCharToEntity,

35 encodeEntities,

34 encodeEntities,

36 decodeEntities,

35 decodeEntities,

36+ escapeSGMLChar,

37 stringToSGML,

38 characterEntity

39 ) where

@@ -54,17 +54,6 @@ charToEntity char =

54charToNumericalEntity :: Char -> String

55charToNumericalEntity ch = "&#" ++ show (ord ch) ++ ";"

57−-- | Escape special character to SGML entity.

58−specialCharToEntity :: Bool -- ^ Use numerical entities only.

59− -> Char -- ^ Character to convert.

60− -> [Char]

61−specialCharToEntity numericalEntities c =

62− if (c `elem` "&<>\"") || (ord c > 127)

63− then if numericalEntities

64− then charToNumericalEntity c

65− else charToEntity c

66− else [c]

67−

68-- | Parse SGML character entity.

57-- | Parse SGML character entity.

69characterEntity :: GenParser Char st Char

58characterEntity :: GenParser Char st Char

70characterEntity = namedEntity <|> hexEntity <|> decimalEntity <?> "SGML entity"

59characterEntity = namedEntity <|> hexEntity <|> decimalEntity <?> "SGML entity"

@@ -97,18 +86,27 @@ decimalEntity = try $ do

97 end <- char ';'

86 end <- char ';'

98 return $ chr $ read body

87 return $ chr $ read body

100−-- | Escape string as needed for SGML. Entity references are not preserved.

89+-- | Escape one character as needed for SGML.

101−encodeEntities :: Bool -- ^ Use only numerical entities.

102− -> String -- ^ String to convert.

103− -> String

90+escapeSGMLChar :: Char -> String

104−encodeEntities numericalEntities =

91+escapeSGMLChar x =

105− concatMap (specialCharToEntity numericalEntities)

92+ case x of

93+ '&' -> "&"

94+ '<' -> "<"

95+ '>' -> ">"

96+ '"' -> """

97+ c -> [c]

106

107−-- | Escape string as needed for SGML, using only numerical entities.

99+-- | True if the character needs to be escaped.

100+needsEscaping :: Char -> Bool

101+needsEscaping c = c `elem` "&<>\""

102+

108−-- Entity references are not preserved.

103+-- | Escape string as needed for SGML. Entity references are not preserved.

109−encodeEntitiesNumerical :: String -> String

104+encodeEntities :: String -> String

110−encodeEntitiesNumerical =

105+encodeEntities "" = ""

111− concatMap (\c -> "&#" ++ show (ord c) ++ ";")

106+encodeEntities str =

107+ case break needsEscaping str of

108+ (okay, "") -> okay

109+ (okay, (c:cs)) -> okay ++ escapeSGMLChar c ++ encodeEntities cs

112

110

113-- | Convert entities in a string to characters.

111-- | Convert entities in a string to characters.

114decodeEntities :: String -> String

112decodeEntities :: String -> String

@@ -118,18 +116,19 @@ decodeEntities str =

118 Right result -> result

116 Right result -> result

119

117

120-- | Escape string for SGML, preserving entity references.

118-- | Escape string for SGML, preserving entity references.

121−stringToSGML :: Bool -- ^ Use only numerical entities.

122− -> String -- ^ String to convert.

123− -> String

119+stringToSGML :: String -> String

124−stringToSGML numericalEntities str =

120+stringToSGML str =

125− let nonentity = do

121+ let regular = do

122+ str <- many1 (satisfy (not . needsEscaping))

123+ return str

124+ special = do

126 notFollowedBy characterEntity

125 notFollowedBy characterEntity

127 c <- anyChar

126 c <- anyChar

128− return $ specialCharToEntity numericalEntities c

127+ return $ escapeSGMLChar c

129 entity = do

128 entity = do

130 ent <- manyTill anyChar (char ';')

129 ent <- manyTill anyChar (char ';')

131− return (ent ++ ";") in

130+ return (ent ++ ";") in

132− case parse (many (nonentity <|> entity)) str str of

131+ case parse (many (regular <|> special <|> entity)) str str of

133 Left err -> error $ "\nError: " ++ show err

132 Left err -> error $ "\nError: " ++ show err

134 Right result -> concat result

133 Right result -> concat result

135

134

src/Text/Pandoc/Shared.hs+2−2

@@ -524,8 +524,8 @@ replaceRefLinksInline keytable other = other

524-- | Return a text object with a string of formatted SGML attributes.

525attributeList :: [(String, String)] -> Doc

526attributeList = text . concatMap

527− (\(a, b) -> " " ++ stringToSGML True a ++ "=\"" ++

527+ (\(a, b) -> " " ++ stringToSGML a ++ "=\"" ++

528− stringToSGML True b ++ "\"")

528+ stringToSGML b ++ "\"")

529

530-- | Put the supplied contents between start and end tags of tagType,

531-- with specified attributes and (if specified) indentation.

src/Text/Pandoc/Writers/Docbook.hs+10−10

@@ -64,8 +64,8 @@ authorToDocbook name = inTagsIndented "author" $

64 then -- last name first

65 let (lastname, rest) = break (==',') name

66 firstname = removeLeadingSpace rest in

67− inTagsSimple "firstname" (text $ stringToSGML True firstname) <>

67+ inTagsSimple "firstname" (text $ stringToSGML firstname) <>

68− inTagsSimple "surname" (text $ stringToSGML True lastname)

68+ inTagsSimple "surname" (text $ stringToSGML lastname)

69 else -- last name last

70 let namewords = words name

71 lengthname = length namewords

@@ -73,8 +73,8 @@ authorToDocbook name = inTagsIndented "author" $

73 0 -> ("","")

74 1 -> ("", name)

75 n -> (joinWithSep " " (take (n-1) namewords), last namewords) in

76− inTagsSimple "firstname" (text $ stringToSGML True firstname) $$

76+ inTagsSimple "firstname" (text $ stringToSGML firstname) $$

77− inTagsSimple "surname" (text $ stringToSGML True lastname)

77+ inTagsSimple "surname" (text $ stringToSGML lastname)

79-- | Convert Pandoc document to string in Docbook format.

80writeDocbook :: WriterOptions -> Pandoc -> String

@@ -86,7 +86,7 @@ writeDocbook opts (Pandoc (Meta title authors date) blocks) =

86 then inTagsIndented "articleinfo" $

87 (inTagsSimple "title" (wrap opts title)) $$

88 (vcat (map authorToDocbook authors)) $$

89− (inTagsSimple "date" (text $ stringToSGML True date))

89+ (inTagsSimple "date" (text $ stringToSGML date))

90 else empty

91 blocks' = replaceReferenceLinks blocks

92 (noteBlocks, blocks'') = partition isNoteBlock blocks'

@@ -141,7 +141,7 @@ blockToDocbook opts (Para lst) =

141blockToDocbook opts (BlockQuote blocks) =

142 inTagsIndented "blockquote" (blocksToDocbook opts blocks)

143blockToDocbook opts (CodeBlock str) =

144− text "<screen>\n" <> text (encodeEntities True str) <> text "\n</screen>"

144+ text "<screen>\n" <> text (encodeEntities str) <> text "\n</screen>"

145blockToDocbook opts (BulletList lst) =

146 inTagsIndented "itemizedlist" $ listItemsToDocbook opts lst

147blockToDocbook opts (OrderedList lst) =

@@ -198,7 +198,7 @@ inlinesToDocbook opts lst = hcat (map (inlineToDocbook opts) lst)

198

199-- | Convert an inline element to Docbook.

200inlineToDocbook :: WriterOptions -> Inline -> Doc

201−inlineToDocbook opts (Str str) = text $ stringToSGML True str

201+inlineToDocbook opts (Str str) = text $ stringToSGML str

202inlineToDocbook opts (Emph lst) =

203 inTagsSimple "emphasis" (inlinesToDocbook opts lst)

204inlineToDocbook opts (Strong lst) =

@@ -211,7 +211,7 @@ inlineToDocbook opts Ellipses = text "…"

211inlineToDocbook opts EmDash = text "—"

212inlineToDocbook opts EnDash = text "–"

213inlineToDocbook opts (Code str) =

214− inTagsSimple "literal" $ text (encodeEntities True str)

214+ inTagsSimple "literal" $ text (encodeEntities str)

215inlineToDocbook opts (TeX str) = inlineToDocbook opts (Code str)

216inlineToDocbook opts (HtmlInline str) = empty

217inlineToDocbook opts LineBreak =

@@ -219,7 +219,7 @@ inlineToDocbook opts LineBreak =

219inlineToDocbook opts Space = char ' '

220inlineToDocbook opts (Link txt (Src src tit)) =

221 if isPrefixOf "mailto:" src

222− then inTagsSimple "email" $ text (encodeEntities True $ drop 7 src)

222+ then inTagsSimple "email" $ text (encodeEntities $ drop 7 src)

223 else inTags False "ulink" [("url", src)] $ inlinesToDocbook opts txt

224inlineToDocbook opts (Link text (Ref ref)) = empty -- shouldn't occur

225inlineToDocbook opts (Image alt (Src src tit)) =

@@ -227,7 +227,7 @@ inlineToDocbook opts (Image alt (Src src tit)) =

227 then empty

228 else inTagsIndented "objectinfo" $

229 inTagsIndented "title"

230− (text $ stringToSGML True tit) in

230+ (text $ stringToSGML tit) in

231 inTagsIndented "inlinemediaobject" $

232 inTagsIndented "imageobject" $

233 titleDoc $$ selfClosingTag "imagedata" [("fileref", src)]

src/Text/Pandoc/Writers/HTML.hs+9−9

@@ -127,11 +127,11 @@ htmlHeader opts (Meta title authors date) =

127 then empty

128 else selfClosingTag "meta" [("name", "author"),

129 ("content",

130− joinWithSep ", " (map (stringToSGML False) authors))]

130+ joinWithSep ", " (map stringToSGML authors))]

131 datetext = if (date == "")

132 then empty

133 else selfClosingTag "meta" [("name", "date"),

134− ("content", stringToSGML False date)] in

134+ ("content", stringToSGML date)] in

135 text (writerHeader opts) $$ authortext $$ datetext $$ titletext $$

136 text "</head>\n<body>"

137

@@ -168,7 +168,7 @@ blockToHtml opts (Note ref lst) =

168 (text "↩")

169blockToHtml opts (Key _ _) = empty

170blockToHtml opts (CodeBlock str) =

171− text "<pre><code>" <> text (encodeEntities False str) <> text "\n</code></pre>"

171+ text "<pre><code>" <> text (encodeEntities str) <> text "\n</code></pre>"

172blockToHtml opts (RawHtml str) = text str

173blockToHtml opts (BulletList lst) =

174 let attribs = if (writerIncremental opts)

@@ -234,7 +234,7 @@ inlineToHtml opts (Emph lst) =

234inlineToHtml opts (Strong lst) =

235 inTagsSimple "strong" (inlineListToHtml opts lst)

236inlineToHtml opts (Code str) =

237− inTagsSimple "code" $ text (encodeEntities False str)

237+ inTagsSimple "code" $ text (encodeEntities str)

238inlineToHtml opts (Quoted SingleQuote lst) =

239 text "‘" <> (inlineListToHtml opts lst) <> text "’"

240inlineToHtml opts (Quoted DoubleQuote lst) =

@@ -243,16 +243,16 @@ inlineToHtml opts EmDash = text "—"

243inlineToHtml opts EnDash = text "–"

244inlineToHtml opts Ellipses = text "…"

245inlineToHtml opts Apostrophe = text "’"

246−inlineToHtml opts (Str str) = text $ stringToSGML False str

246+inlineToHtml opts (Str str) = text $ stringToSGML str

247−inlineToHtml opts (TeX str) = text $ encodeEntities False str

247+inlineToHtml opts (TeX str) = text $ encodeEntities str

248inlineToHtml opts (HtmlInline str) = text str

249inlineToHtml opts (LineBreak) = selfClosingTag "br" []

250inlineToHtml opts Space = space

251inlineToHtml opts (Link txt (Src src tit)) =

252− let title = stringToSGML False tit in

252+ let title = stringToSGML tit in

253 if (isPrefixOf "mailto:" src)

254 then obfuscateLink opts txt src

255− else inTags False "a" ([("href", encodeEntities False src)] ++

255+ else inTags False "a" ([("href", encodeEntities src)] ++

256 if null tit then [] else [("title", title)])

257 (inlineListToHtml opts txt)

258inlineToHtml opts (Link txt (Ref ref)) =

@@ -260,7 +260,7 @@ inlineToHtml opts (Link txt (Ref ref)) =

260 (inlineListToHtml opts ref) <> char ']'

261 -- this is what markdown does, for better or worse

262inlineToHtml opts (Image alt (Src source tit)) =

263− let title = stringToSGML False tit

263+ let title = stringToSGML tit

264 alternate = render $ inlineListToHtml opts alt in

265 selfClosingTag "img" $ [("src", source)] ++

266 (if null alternate then [] else [("alt", alternate)]) ++

tests/writer.docbook+30−30

@@ -89,7 +89,7 @@

89 </para>

90 <screen>

91sub status {

92− print "working";

92+ print "working";

93}

94</screen>

95 <para>

@@ -122,7 +122,7 @@ sub status {

122 </blockquote>

123 </blockquote>

124 <para>

125− This should not be a block quote: 2 > 1.

125+ This should not be a block quote: 2 > 1.

126 </para>

127 <para>

128 Box-style:

@@ -133,7 +133,7 @@ sub status {

133 </para>

134 <screen>

135sub status {

136− print "working";

136+ print "working";

137}

138</screen>

139 </blockquote>

@@ -177,7 +177,7 @@ sub status {

177---- (should be four hyphens)

178

179sub status {

180− print "working";

180+ print "working";

181}

182

183this code block is indented by one tab

@@ -188,7 +188,7 @@ this code block is indented by one tab

188 <screen>

189 this code block is indented by two tabs

190

191−These should not be escaped: \$ \\ \> \[ \{

191+These should not be escaped: \$ \\ \> \[ \{

192</screen>

193 </section>

194 <section>

@@ -577,9 +577,9 @@ These should not be escaped: \$ \\ \> \[ \{

577 word.

578 </para>

579 <para>

580− This is code: <literal>></literal>, <literal>$</literal>,

580+ This is code: <literal>></literal>, <literal>$</literal>,

581 <literal>\</literal>, <literal>\$</literal>,

582− <literal><html></literal>.

582+ <literal><html></literal>.

583 </para>

584 </section>

585 <section>

@@ -602,7 +602,7 @@ These should not be escaped: \$ \\ \> \[ \{

602 </para>

603 <para>

604 Here is some quoted <quote><literal>code</literal></quote> and a

605− <quote><ulink url="http://example.com/?foo=1&bar=2">quoted link</ulink></quote>.

605+ <quote><ulink url="http://example.com/?foo=1&bar=2">quoted link</ulink></quote>.

606 </para>

607 <para>

608 Some dashes: one—two—three—four—five.

@@ -691,9 +691,9 @@ These should not be escaped: \$ \\ \> \[ \{

691 </para>

692 <para>

693 <literal>\begin{tabular}{|l|l|}\hline

694−Animal & Number \\ \hline

694+Animal & Number \\ \hline

695−Dog & 2 \\

695+Dog & 2 \\

696−Cat & 1 \\ \hline

696+Cat & 1 \\ \hline

697\end{tabular}</literal>

698 </para>

699 </section>

@@ -705,44 +705,44 @@ Cat & 1 \\ \hline

705 <itemizedlist>

706 <listitem>

707 <para>

708− I hat: Î

708+ I hat: Î

709 </para>

710 </listitem>

711 <listitem>

712 <para>

713− o umlaut: ö

713+ o umlaut: ö

714 </para>

715 </listitem>

716 <listitem>

717 <para>

718− section: §

718+ section: §

719 </para>

720 </listitem>

721 <listitem>

722 <para>

723− set membership: ∈

723+ set membership: ∈

724 </para>

725 </listitem>

726 <listitem>

727 <para>

729 </para>

730 </listitem>

731 </itemizedlist>

732 <para>

733− AT&T has an ampersand in their name.

733+ AT&T has an ampersand in their name.

734 </para>

735 <para>

736− AT&T is another way to write it.

736+ AT&T is another way to write it.

737 </para>

738 <para>

739− This & that.

739+ This & that.

740 </para>

741 <para>

742− 4 < 5.

742+ 4 < 5.

743 </para>

744 <para>

745− 6 > 5.

745+ 6 > 5.

746 </para>

747 <para>

748 Backslash: \

@@ -775,7 +775,7 @@ Cat & 1 \\ \hline

775 Right paren: )

776 </para>

777 <para>

778− Greater-than: >

778+ Greater-than: >

779 </para>

780 <para>

781 Hash: #

@@ -868,25 +868,25 @@ Cat & 1 \\ \hline

868 <title>With ampersands</title>

869 <para>

870 Here's a

871− <ulink url="http://example.com/?foo=1&bar=2">link with an ampersand in the URL</ulink>.

871+ <ulink url="http://example.com/?foo=1&bar=2">link with an ampersand in the URL</ulink>.

872 </para>

873 <para>

874 Here's a link with an amersand in the link text:

875− <ulink url="http://att.com/">AT&T</ulink>.

875+ <ulink url="http://att.com/">AT&T</ulink>.

876 </para>

877 <para>

878− Here's an <ulink url="/script?foo=1&bar=2">inline link</ulink>.

878+ Here's an <ulink url="/script?foo=1&bar=2">inline link</ulink>.

879 </para>

880 <para>

881 Here's an

882− <ulink url="/script?foo=1&bar=2">inline link in pointy braces</ulink>.

882+ <ulink url="/script?foo=1&bar=2">inline link in pointy braces</ulink>.

883 </para>

884 </section>

885 <section>

886 <title>Autolinks</title>

887 <para>

888 With an ampersand:

889− <ulink url="http://example.com/?foo=1&bar=2">http://example.com/?foo=1&bar=2</ulink>

889+ <ulink url="http://example.com/?foo=1&bar=2">http://example.com/?foo=1&bar=2</ulink>

890 </para>

891 <itemizedlist>

892 <listitem>

@@ -916,10 +916,10 @@ Cat & 1 \\ \hline

916 </blockquote>

917 <para>

918 Auto-links should not occur here:

919− <literal><http://example.com/></literal>

919+ <literal><http://example.com/></literal>

920 </para>

921 <screen>

922−or here: <http://example.com/>

922+or here: <http://example.com/>

923</screen>

924 </section>

925 </section>

@@ -970,7 +970,7 @@ or here: <http://example.com/>

970 footnote (as with list items).

971 </para>

972 <screen>

973− { <code> }

973+ { <code> }

974</screen>

975 <para>

976 If you want, you can indent every line, but you can also be lazy

tests/writer.html+5−5

@@ -557,11 +557,11 @@ Cat & 1 \\ \hline

557 Here is some unicode:

558</p>

559<ul>

560− <li>I hat: Î</li>

560+ <li>I hat: Î</li>

561− <li>o umlaut: ö</li>

561+ <li>o umlaut: ö</li>

562− <li>section: §</li>

562+ <li>section: §</li>

563− <li>set membership: ∈</li>

563+ <li>set membership: ∈</li>

565</ul>

566<p>

567 AT&T has an ampersand in their name.