141affd · typeable/pandoc

git.iko.soy/typeable/pandoc

More changes in entity handling: Instead of using entities for characters

above 128 in HTML and Docbook output, we now just use unicode. After all, we're declaring UTF-8 content in the header. This makes the HTML and docbook files produced by pandoc much more readable and editable. Changes to Entities.hs: + Removed specialCharToEntity + Added escapeSGMLChar (which just escapes the basic four, <>&") + Modified encodeEntities and stringToSGML to use escapeSGMLChar + Removed encodeEntitiesNumerical + Rewrote encodeEntities for better performance + Rewrote stringToSGML for better performance git-svn-id: https://pandoc.googlecode.com/svn/trunk@516 788f1e2b-df1e-0410-8736-df70ead52e1b

commit141affdb5140478464bf3c7331f6be4cf9454dd6 authorfiddlosopher <fiddlosopher@788f1e2b-df1e-0410-8736-df70ead52e1b> dateSat 27 Jan 2007, 22:13 (20 years ago) parentd064171

6 files changed+86−87

compare with master

unifiedsplit

src/Text/Pandoc/Entities.hs+30−31 src/Text/Pandoc/Shared.hs+2−2 src/Text/Pandoc/Writers/Docbook.hs+10−10 src/Text/Pandoc/Writers/HTML.hs+9−9 tests/writer.docbook+30−30 tests/writer.html+5−5

src/Text/Pandoc/Entities.hs+30−31

@@ -31,9 +31,9 @@ and vice versa.

3131module Text.Pandoc.Entities (

3232 charToEntity,

3333 charToNumericalEntity,

34− specialCharToEntity,

3534 encodeEntities,

3635 decodeEntities,

36+ escapeSGMLChar,

3737 stringToSGML,

3838 characterEntity

3939 ) where

@@ -54,17 +54,6 @@ charToEntity char =

5454charToNumericalEntity :: Char -> String

5555charToNumericalEntity ch = "&#" ++ show (ord ch) ++ ";"

5656

57−-- | Escape special character to SGML entity.

58−specialCharToEntity :: Bool -- ^ Use numerical entities only.

59− -> Char -- ^ Character to convert.

60− -> [Char]

61−specialCharToEntity numericalEntities c =

62− if (c `elem` "&<>\"") || (ord c > 127)

63− then if numericalEntities

64− then charToNumericalEntity c

65− else charToEntity c

66− else [c]

67−

6857-- | Parse SGML character entity.

6958characterEntity :: GenParser Char st Char

7059characterEntity = namedEntity <|> hexEntity <|> decimalEntity <?> "SGML entity"

@@ -97,18 +86,27 @@ decimalEntity = try $ do

9786 end <- char ';'

9887 return $ chr $ read body

9988

100−-- | Escape string as needed for SGML. Entity references are not preserved.

101−encodeEntities :: Bool -- ^ Use only numerical entities.

102− -> String -- ^ String to convert.

103− -> String

104−encodeEntities numericalEntities =

105− concatMap (specialCharToEntity numericalEntities)

89+-- | Escape one character as needed for SGML.

90+escapeSGMLChar :: Char -> String

91+escapeSGMLChar x =

92+ case x of

93+ '&' -> "&"

94+ '<' -> "<"

95+ '>' -> ">"

96+ '"' -> """

97+ c -> [c]

10698

107−-- | Escape string as needed for SGML, using only numerical entities.

108−-- Entity references are not preserved.

109−encodeEntitiesNumerical :: String -> String

110−encodeEntitiesNumerical =

111− concatMap (\c -> "&#" ++ show (ord c) ++ ";")

99+-- | True if the character needs to be escaped.

100+needsEscaping :: Char -> Bool

101+needsEscaping c = c `elem` "&<>\""

102+

103+-- | Escape string as needed for SGML. Entity references are not preserved.

104+encodeEntities :: String -> String

105+encodeEntities "" = ""

106+encodeEntities str =

107+ case break needsEscaping str of

108+ (okay, "") -> okay

109+ (okay, (c:cs)) -> okay ++ escapeSGMLChar c ++ encodeEntities cs

112110

113111-- | Convert entities in a string to characters.

114112decodeEntities :: String -> String

@@ -118,18 +116,19 @@ decodeEntities str =

118116 Right result -> result

119117

120118-- | Escape string for SGML, preserving entity references.

121−stringToSGML :: Bool -- ^ Use only numerical entities.

122− -> String -- ^ String to convert.

123− -> String

124−stringToSGML numericalEntities str =

125− let nonentity = do

119+stringToSGML :: String -> String

120+stringToSGML str =

121+ let regular = do

122+ str <- many1 (satisfy (not . needsEscaping))

123+ return str

124+ special = do

126125 notFollowedBy characterEntity

127126 c <- anyChar

128− return $ specialCharToEntity numericalEntities c

127+ return $ escapeSGMLChar c

129128 entity = do

130129 ent <- manyTill anyChar (char ';')

131− return (ent ++ ";") in

132− case parse (many (nonentity <|> entity)) str str of

130+ return (ent ++ ";") in

131+ case parse (many (regular <|> special <|> entity)) str str of

133132 Left err -> error $ "\nError: " ++ show err

134133 Right result -> concat result

135134

src/Text/Pandoc/Shared.hs+2−2

@@ -524,8 +524,8 @@ replaceRefLinksInline keytable other = other

524524-- | Return a text object with a string of formatted SGML attributes.

525525attributeList :: [(String, String)] -> Doc

526526attributeList = text . concatMap

527− (\(a, b) -> " " ++ stringToSGML True a ++ "=\"" ++

528− stringToSGML True b ++ "\"")

527+ (\(a, b) -> " " ++ stringToSGML a ++ "=\"" ++

528+ stringToSGML b ++ "\"")

529529

530530-- | Put the supplied contents between start and end tags of tagType,

531531-- with specified attributes and (if specified) indentation.

src/Text/Pandoc/Writers/Docbook.hs+10−10

@@ -64,8 +64,8 @@ authorToDocbook name = inTagsIndented "author" $

6464 then -- last name first

6565 let (lastname, rest) = break (==',') name

6666 firstname = removeLeadingSpace rest in

67− inTagsSimple "firstname" (text $ stringToSGML True firstname) <>

68− inTagsSimple "surname" (text $ stringToSGML True lastname)

67+ inTagsSimple "firstname" (text $ stringToSGML firstname) <>

68+ inTagsSimple "surname" (text $ stringToSGML lastname)

6969 else -- last name last

7070 let namewords = words name

7171 lengthname = length namewords

@@ -73,8 +73,8 @@ authorToDocbook name = inTagsIndented "author" $

7373 0 -> ("","")

7474 1 -> ("", name)

7575 n -> (joinWithSep " " (take (n-1) namewords), last namewords) in

76− inTagsSimple "firstname" (text $ stringToSGML True firstname) $$

77− inTagsSimple "surname" (text $ stringToSGML True lastname)

76+ inTagsSimple "firstname" (text $ stringToSGML firstname) $$

77+ inTagsSimple "surname" (text $ stringToSGML lastname)

7878

7979-- | Convert Pandoc document to string in Docbook format.

8080writeDocbook :: WriterOptions -> Pandoc -> String

@@ -86,7 +86,7 @@ writeDocbook opts (Pandoc (Meta title authors date) blocks) =

8686 then inTagsIndented "articleinfo" $

8787 (inTagsSimple "title" (wrap opts title)) $$

8888 (vcat (map authorToDocbook authors)) $$

89− (inTagsSimple "date" (text $ stringToSGML True date))

89+ (inTagsSimple "date" (text $ stringToSGML date))

9090 else empty

9191 blocks' = replaceReferenceLinks blocks

9292 (noteBlocks, blocks'') = partition isNoteBlock blocks'

@@ -141,7 +141,7 @@ blockToDocbook opts (Para lst) =

141141blockToDocbook opts (BlockQuote blocks) =

142142 inTagsIndented "blockquote" (blocksToDocbook opts blocks)

143143blockToDocbook opts (CodeBlock str) =

144− text "<screen>\n" <> text (encodeEntities True str) <> text "\n</screen>"

144+ text "<screen>\n" <> text (encodeEntities str) <> text "\n</screen>"

145145blockToDocbook opts (BulletList lst) =

146146 inTagsIndented "itemizedlist" $ listItemsToDocbook opts lst

147147blockToDocbook opts (OrderedList lst) =

@@ -198,7 +198,7 @@ inlinesToDocbook opts lst = hcat (map (inlineToDocbook opts) lst)

198198

199199-- | Convert an inline element to Docbook.

200200inlineToDocbook :: WriterOptions -> Inline -> Doc

201−inlineToDocbook opts (Str str) = text $ stringToSGML True str

201+inlineToDocbook opts (Str str) = text $ stringToSGML str

202202inlineToDocbook opts (Emph lst) =

203203 inTagsSimple "emphasis" (inlinesToDocbook opts lst)

204204inlineToDocbook opts (Strong lst) =

@@ -211,7 +211,7 @@ inlineToDocbook opts Ellipses = text "…"

211211inlineToDocbook opts EmDash = text "—"

212212inlineToDocbook opts EnDash = text "–"

213213inlineToDocbook opts (Code str) =

214− inTagsSimple "literal" $ text (encodeEntities True str)

214+ inTagsSimple "literal" $ text (encodeEntities str)

215215inlineToDocbook opts (TeX str) = inlineToDocbook opts (Code str)

216216inlineToDocbook opts (HtmlInline str) = empty

217217inlineToDocbook opts LineBreak =

@@ -219,7 +219,7 @@ inlineToDocbook opts LineBreak =

219219inlineToDocbook opts Space = char ' '

220220inlineToDocbook opts (Link txt (Src src tit)) =

221221 if isPrefixOf "mailto:" src

222− then inTagsSimple "email" $ text (encodeEntities True $ drop 7 src)

222+ then inTagsSimple "email" $ text (encodeEntities $ drop 7 src)

223223 else inTags False "ulink" [("url", src)] $ inlinesToDocbook opts txt

224224inlineToDocbook opts (Link text (Ref ref)) = empty -- shouldn't occur

225225inlineToDocbook opts (Image alt (Src src tit)) =

@@ -227,7 +227,7 @@ inlineToDocbook opts (Image alt (Src src tit)) =

227227 then empty

228228 else inTagsIndented "objectinfo" $

229229 inTagsIndented "title"

230− (text $ stringToSGML True tit) in

230+ (text $ stringToSGML tit) in

231231 inTagsIndented "inlinemediaobject" $

232232 inTagsIndented "imageobject" $

233233 titleDoc $$ selfClosingTag "imagedata" [("fileref", src)]

src/Text/Pandoc/Writers/HTML.hs+9−9

@@ -127,11 +127,11 @@ htmlHeader opts (Meta title authors date) =

127127 then empty

128128 else selfClosingTag "meta" [("name", "author"),

129129 ("content",

130− joinWithSep ", " (map (stringToSGML False) authors))]

130+ joinWithSep ", " (map stringToSGML authors))]

131131 datetext = if (date == "")

132132 then empty

133133 else selfClosingTag "meta" [("name", "date"),

134− ("content", stringToSGML False date)] in

134+ ("content", stringToSGML date)] in

135135 text (writerHeader opts) $$ authortext $$ datetext $$ titletext $$

136136 text "</head>\n<body>"

137137

@@ -168,7 +168,7 @@ blockToHtml opts (Note ref lst) =

168168 (text "↩")

169169blockToHtml opts (Key _ _) = empty

170170blockToHtml opts (CodeBlock str) =

171− text "<pre><code>" <> text (encodeEntities False str) <> text "\n</code></pre>"

171+ text "<pre><code>" <> text (encodeEntities str) <> text "\n</code></pre>"

172172blockToHtml opts (RawHtml str) = text str

173173blockToHtml opts (BulletList lst) =

174174 let attribs = if (writerIncremental opts)

@@ -234,7 +234,7 @@ inlineToHtml opts (Emph lst) =

234234inlineToHtml opts (Strong lst) =

235235 inTagsSimple "strong" (inlineListToHtml opts lst)

236236inlineToHtml opts (Code str) =

237− inTagsSimple "code" $ text (encodeEntities False str)

237+ inTagsSimple "code" $ text (encodeEntities str)

238238inlineToHtml opts (Quoted SingleQuote lst) =

239239 text "‘" <> (inlineListToHtml opts lst) <> text "’"

240240inlineToHtml opts (Quoted DoubleQuote lst) =

@@ -243,16 +243,16 @@ inlineToHtml opts EmDash = text "—"

243243inlineToHtml opts EnDash = text "–"

244244inlineToHtml opts Ellipses = text "…"

245245inlineToHtml opts Apostrophe = text "’"

246−inlineToHtml opts (Str str) = text $ stringToSGML False str

247−inlineToHtml opts (TeX str) = text $ encodeEntities False str

246+inlineToHtml opts (Str str) = text $ stringToSGML str

247+inlineToHtml opts (TeX str) = text $ encodeEntities str

248248inlineToHtml opts (HtmlInline str) = text str

249249inlineToHtml opts (LineBreak) = selfClosingTag "br" []

250250inlineToHtml opts Space = space

251251inlineToHtml opts (Link txt (Src src tit)) =

252− let title = stringToSGML False tit in

252+ let title = stringToSGML tit in

253253 if (isPrefixOf "mailto:" src)

254254 then obfuscateLink opts txt src

255− else inTags False "a" ([("href", encodeEntities False src)] ++

255+ else inTags False "a" ([("href", encodeEntities src)] ++

256256 if null tit then [] else [("title", title)])

257257 (inlineListToHtml opts txt)

258258inlineToHtml opts (Link txt (Ref ref)) =

@@ -260,7 +260,7 @@ inlineToHtml opts (Link txt (Ref ref)) =

260260 (inlineListToHtml opts ref) <> char ']'

261261 -- this is what markdown does, for better or worse

262262inlineToHtml opts (Image alt (Src source tit)) =

263− let title = stringToSGML False tit

263+ let title = stringToSGML tit

264264 alternate = render $ inlineListToHtml opts alt in

265265 selfClosingTag "img" $ [("src", source)] ++

266266 (if null alternate then [] else [("alt", alternate)]) ++

tests/writer.docbook+30−30

@@ -89,7 +89,7 @@

8989 </para>

9090 <screen>

9191sub status {

92− print "working";

92+ print "working";

9393}

9494</screen>

9595 <para>

@@ -122,7 +122,7 @@ sub status {

122122 </blockquote>

123123 </blockquote>

124124 <para>

125− This should not be a block quote: 2 > 1.

125+ This should not be a block quote: 2 > 1.

126126 </para>

127127 <para>

128128 Box-style:

@@ -133,7 +133,7 @@ sub status {

133133 </para>

134134 <screen>

135135sub status {

136− print "working";

136+ print "working";

137137}

138138</screen>

139139 </blockquote>

@@ -177,7 +177,7 @@ sub status {

177177---- (should be four hyphens)

178178

179179sub status {

180− print "working";

180+ print "working";

181181}

182182

183183this code block is indented by one tab

@@ -188,7 +188,7 @@ this code block is indented by one tab

188188 <screen>

189189 this code block is indented by two tabs

190190

191−These should not be escaped: \$ \\ \> \[ \{

191+These should not be escaped: \$ \\ \> \[ \{

192192</screen>

193193 </section>

194194 <section>

@@ -577,9 +577,9 @@ These should not be escaped: \$ \\ \> \[ \{

577577 word.

578578 </para>

579579 <para>

580− This is code: <literal>></literal>, <literal>$</literal>,

580+ This is code: <literal>></literal>, <literal>$</literal>,

581581 <literal>\</literal>, <literal>\$</literal>,

582− <literal><html></literal>.

582+ <literal><html></literal>.

583583 </para>

584584 </section>

585585 <section>

@@ -602,7 +602,7 @@ These should not be escaped: \$ \\ \> \[ \{

602602 </para>

603603 <para>

604604 Here is some quoted <quote><literal>code</literal></quote> and a

605− <quote><ulink url="http://example.com/?foo=1&bar=2">quoted link</ulink></quote>.

605+ <quote><ulink url="http://example.com/?foo=1&bar=2">quoted link</ulink></quote>.

606606 </para>

607607 <para>

608608 Some dashes: one—two—three—four—five.

@@ -691,9 +691,9 @@ These should not be escaped: \$ \\ \> \[ \{

691691 </para>

692692 <para>

693693 <literal>\begin{tabular}{|l|l|}\hline

694−Animal & Number \\ \hline

695−Dog & 2 \\

696−Cat & 1 \\ \hline

694+Animal & Number \\ \hline

695+Dog & 2 \\

696+Cat & 1 \\ \hline

697697\end{tabular}</literal>

698698 </para>

699699 </section>

@@ -705,44 +705,44 @@ Cat & 1 \\ \hline

705705 <itemizedlist>

706706 <listitem>

707707 <para>

708− I hat: Î

708+ I hat: Î

709709 </para>

710710 </listitem>

711711 <listitem>

712712 <para>

713− o umlaut: ö

713+ o umlaut: ö

714714 </para>

715715 </listitem>

716716 <listitem>

717717 <para>

718− section: §

718+ section: §

719719 </para>

720720 </listitem>

721721 <listitem>

722722 <para>

723− set membership: ∈

723+ set membership: ∈

724724 </para>

725725 </listitem>

726726 <listitem>

727727 <para>

729729 </para>

730730 </listitem>

731731 </itemizedlist>

732732 <para>

733− AT&T has an ampersand in their name.

733+ AT&T has an ampersand in their name.

734734 </para>

735735 <para>

736− AT&T is another way to write it.

736+ AT&T is another way to write it.

737737 </para>

738738 <para>

739− This & that.

739+ This & that.

740740 </para>

741741 <para>

742− 4 < 5.

742+ 4 < 5.

743743 </para>

744744 <para>

745− 6 > 5.

745+ 6 > 5.

746746 </para>

747747 <para>

748748 Backslash: \

@@ -775,7 +775,7 @@ Cat & 1 \\ \hline

775775 Right paren: )

776776 </para>

777777 <para>

778− Greater-than: >

778+ Greater-than: >

779779 </para>

780780 <para>

781781 Hash: #

@@ -868,25 +868,25 @@ Cat & 1 \\ \hline

868868 <title>With ampersands</title>

869869 <para>

870870 Here's a

871− <ulink url="http://example.com/?foo=1&bar=2">link with an ampersand in the URL</ulink>.

871+ <ulink url="http://example.com/?foo=1&bar=2">link with an ampersand in the URL</ulink>.

872872 </para>

873873 <para>

874874 Here's a link with an amersand in the link text:

875− <ulink url="http://att.com/">AT&T</ulink>.

875+ <ulink url="http://att.com/">AT&T</ulink>.

876876 </para>

877877 <para>

878− Here's an <ulink url="/script?foo=1&bar=2">inline link</ulink>.

878+ Here's an <ulink url="/script?foo=1&bar=2">inline link</ulink>.

879879 </para>

880880 <para>

881881 Here's an

882− <ulink url="/script?foo=1&bar=2">inline link in pointy braces</ulink>.

882+ <ulink url="/script?foo=1&bar=2">inline link in pointy braces</ulink>.

883883 </para>

884884 </section>

885885 <section>

886886 <title>Autolinks</title>

887887 <para>

888888 With an ampersand:

889− <ulink url="http://example.com/?foo=1&bar=2">http://example.com/?foo=1&bar=2</ulink>

889+ <ulink url="http://example.com/?foo=1&bar=2">http://example.com/?foo=1&bar=2</ulink>

890890 </para>

891891 <itemizedlist>

892892 <listitem>

@@ -916,10 +916,10 @@ Cat & 1 \\ \hline

916916 </blockquote>

917917 <para>

918918 Auto-links should not occur here:

919− <literal><http://example.com/></literal>

919+ <literal><http://example.com/></literal>

920920 </para>

921921 <screen>

922−or here: <http://example.com/>

922+or here: <http://example.com/>

923923</screen>

924924 </section>

925925 </section>

@@ -970,7 +970,7 @@ or here: <http://example.com/>

970970 footnote (as with list items).

971971 </para>

972972 <screen>

973− { <code> }

973+ { <code> }

974974</screen>

975975 <para>

976976 If you want, you can indent every line, but you can also be lazy

tests/writer.html+5−5

@@ -557,11 +557,11 @@ Cat & 1 \\ \hline

557557 Here is some unicode:

558558</p>

559559<ul>

560− <li>I hat: Î</li>

561− <li>o umlaut: ö</li>

562− <li>section: §</li>

563− <li>set membership: ∈</li>

560+ <li>I hat: Î</li>

561+ <li>o umlaut: ö</li>

562+ <li>section: §</li>

563+ <li>set membership: ∈</li>

565565</ul>

566566<p>

567567 AT&T has an ampersand in their name.