8efb897 · typeable/pandoc

git.iko.soy/typeable/pandoc

Groff writer character escaping changes.

T.P.GroffChar: replaced `essentialEscapes` with `manEscapes`, which includes all the escapes mentioned in the groff_man manual. T.P.Writers.Groff: removed escapeCode; changed parameter on escapeString from Bool to new type `EscapeMode`. Rewrote `escapeString`.

commit8efb8975ed641ddd075954e1ccc7f71eca1d3c16 authorJohn MacFarlane <jgm@berkeley.edu> dateTue 23 Oct 2018, 21:38 (8 years ago) parent556e3ee

8 files changed+101−86

compare with master

unifiedsplit

src/Text/Pandoc/GroffChar.hs+14−9 src/Text/Pandoc/Writers/Groff.hs+28−24 src/Text/Pandoc/Writers/Man.hs+13−7 src/Text/Pandoc/Writers/Ms.hs+15−15 test/command/4550.md+1−1 test/tables.ms+4−4 test/writer.man+3−3 test/writer.ms+23−23

src/Text/Pandoc/GroffChar.hs+14−9

@@ -31,24 +31,29 @@ Groff character escaping/unescaping.

3131-}

3232

3333module Text.Pandoc.GroffChar (

34− essentialEscapes

34+ manEscapes

3535 , characterCodes

3636 , combiningAccents

3737 ) where

3838import Prelude

39−import qualified Data.Map as Map

4039

41−essentialEscapes :: Map.Map Char String

42−essentialEscapes = Map.fromList

43− [ ('\160', "\\~")

40+-- | These are the escapes specifically mentioned in groff_man(7).

41+manEscapes :: [(Char, String)]

42+manEscapes =

43+ [ ('\160', "\\ ")

4444 , ('\'', "\\[aq]")

45− , ('`', "\\[ga]")

45+ , ('‘', "\\[oq]")

46+ , ('’', "\\[cq]")

4647 , ('"', "\\[dq]")

47− , ('~', "\\[ti]")

48+ , ('“', "\\[lq]")

49+ , ('”', "\\[rq]")

50+ , ('—', "\\[em]")

51+ , ('–', "\\[en]")

52+ , ('`', "\\[ga]")

4853 , ('^', "\\[ha]")

49− , ('@', "\\[at]")

50− , ('\\', "\\[rs]")

54+ , ('~', "\\[ti]")

5155 , ('-', "\\-") -- minus; - will be interpreted as hyphen U+2010

56+ , ('\\', "\\[rs]")

5257 , ('\x2026', "\\&...") -- because u2026 doesn't render on tty

5358 ]

5459

src/Text/Pandoc/Writers/Groff.hs+28−24

@@ -34,22 +34,21 @@ module Text.Pandoc.Writers.Groff (

3434 , defaultWriterState

3535 , MS

3636 , Note

37+ , EscapeMode(..)

3738 , escapeString

38− , escapeCode

3939 , withFontFeature

4040 ) where

4141import Prelude

4242import Data.Char (ord, isAscii)

4343import Control.Monad.State.Strict

44−import Data.List (intercalate)

4544import qualified Data.Map as Map

4645import Data.Maybe (fromMaybe, isJust, catMaybes)

4746import Text.Pandoc.Class (PandocMonad)

4847import Text.Pandoc.Definition

4948import Text.Pandoc.Pretty

5049import Text.Printf (printf)

51−import Text.Pandoc.GroffChar (essentialEscapes, characterCodes,

52− combiningAccents)

50+import Text.Pandoc.GroffChar (manEscapes,

51+ characterCodes, combiningAccents)

5352

5453data WriterState = WriterState { stHasInlineMath :: Bool

5554 , stFirstPara :: Bool

@@ -80,33 +79,38 @@ type Note = [Block]

8079

8180type MS = StateT WriterState

8281

82+data EscapeMode = AllowUTF8 -- ^ use preferred man escapes

83+ | AsciiOnly -- ^ escape everything

84+ deriving Show

85+

8386combiningAccentsMap :: Map.Map Char String

8487combiningAccentsMap = Map.fromList combiningAccents

8588

89+essentialEscapes :: Map.Map Char String

90+essentialEscapes = Map.fromList manEscapes

91+

8692-- | Escape special characters for groff.

87−escapeString :: Bool -> String -> String

93+escapeString :: EscapeMode -> String -> String

8894escapeString _ [] = []

89−escapeString useAscii (x:xs) =

95+escapeString escapeMode ('\n':'.':xs) =

96+ '\n':'\\':'&':'.':escapeString escapeMode xs

97+escapeString escapeMode (x:xs) =

9098 case Map.lookup x essentialEscapes of

91− Just s -> s ++ escapeString useAscii xs

99+ Just s -> s ++ escapeString escapeMode xs

92100 Nothing

93− | isAscii x || not useAscii -> x : escapeString useAscii xs

94− | otherwise ->

95− let accents = catMaybes $ takeWhile isJust

96− (map (\c -> Map.lookup c combiningAccentsMap) xs)

97− rest = drop (length accents) xs

98− s = case Map.lookup x characterCodeMap of

99− Just t -> "\\[" <> unwords (t:accents) <> "]"

100− Nothing -> "\\[" <> unwords

101− (printf "u%04X" (ord x) : accents) <> "]"

102− in s ++ escapeString useAscii rest

103−

104−-- | Escape a literal (code) section for groff.

105−escapeCode :: Bool -> String -> String

106−escapeCode useAscii = intercalate "\n" . map escapeLine . lines

107− where escapeLine xs = case xs of

108− ('.':_) -> "\\%" ++ escapeString useAscii xs

109− _ -> escapeString useAscii xs

101+ | isAscii x -> x : escapeString escapeMode xs

102+ | otherwise ->

103+ case escapeMode of

104+ AllowUTF8 -> x : escapeString escapeMode xs

105+ AsciiOnly ->

106+ let accents = catMaybes $ takeWhile isJust

107+ (map (\c -> Map.lookup c combiningAccentsMap) xs)

108+ rest = drop (length accents) xs

109+ s = case Map.lookup x characterCodeMap of

110+ Just t -> "\\[" <> unwords (t:accents) <> "]"

111+ Nothing -> "\\[" <> unwords

112+ (printf "u%04X" (ord x) : accents) <> "]"

113+ in s ++ escapeString escapeMode rest

110114

111115characterCodeMap :: Map.Map Char String

112116characterCodeMap = Map.fromList characterCodes

src/Text/Pandoc/Writers/Man.hs+13−7

@@ -97,6 +97,9 @@ pandocToMan opts (Pandoc meta blocks) = do

9797 Nothing -> return main

9898 Just tpl -> renderTemplate' tpl context

9999

100+escString :: WriterOptions -> String -> String

101+escString _ = escapeString AsciiOnly -- for better portability

102+

100103-- | Return man representation of notes.

101104notesToMan :: PandocMonad m => WriterOptions -> [[Block]] -> StateT WriterState m Doc

102105notesToMan opts notes =

@@ -143,11 +146,14 @@ blockToMan opts (Header level _ inlines) = do

143146 1 -> ".SH "

144147 _ -> ".SS "

145148 return $ text heading <> contents

146−blockToMan _ (CodeBlock _ str) = return $

149+blockToMan opts (CodeBlock _ str) = return $

147150 text ".IP" $$

148151 text ".nf" $$

149152 text "\\f[C]" $$

150− text (escapeCode True str) $$

153+ ((case str of

154+ '.':_ -> text "\\&"

155+ _ -> mempty) <>

156+ text (escString opts str)) $$

151157 text "\\f[R]" $$

152158 text ".fi"

153159blockToMan opts (BlockQuote blocks) = do

@@ -296,11 +302,11 @@ inlineToMan opts (Quoted DoubleQuote lst) = do

296302 return $ text "\\[lq]" <> contents <> text "\\[rq]"

297303inlineToMan opts (Cite _ lst) =

298304 inlineListToMan opts lst

299−inlineToMan _ (Code _ str) =

300− withFontFeature 'C' (return (text $ escapeCode True str))

301−inlineToMan _ (Str str@('.':_)) =

302− return $ afterBreak "\\&" <> text (escapeString True str)

303−inlineToMan _ (Str str) = return $ text $ escapeString True str

305+inlineToMan opts (Code _ str) =

306+ withFontFeature 'C' (return (text $ escString opts str))

307+inlineToMan opts (Str str@('.':_)) =

308+ return $ afterBreak "\\&" <> text (escString opts str)

309+inlineToMan opts (Str str) = return $ text $ escString opts str

304310inlineToMan opts (Math InlineMath str) =

305311 lift (texMathToInlines InlineMath str) >>= inlineListToMan opts

306312inlineToMan opts (Math DisplayMath str) = do

src/Text/Pandoc/Writers/Ms.hs+15−15

@@ -69,9 +69,6 @@ writeMs :: PandocMonad m => WriterOptions -> Pandoc -> m Text

6969writeMs opts document =

7070 evalStateT (pandocToMs opts document) defaultWriterState

7171

72−escString :: WriterOptions -> String -> String

73−escString opts = escapeString (writerPreferAscii opts)

74−

7572-- | Return groff ms representation of document.

7673pandocToMs :: PandocMonad m => WriterOptions -> Pandoc -> MS m Text

7774pandocToMs opts (Pandoc meta blocks) = do

@@ -87,8 +84,8 @@ pandocToMs opts (Pandoc meta blocks) = do

8784 body <- blockListToMs opts blocks

8885 let main = render' body

8986 hasInlineMath <- gets stHasInlineMath

90− let titleMeta = (escString opts . stringify) $ docTitle meta

91− let authorsMeta = map (escString opts . stringify) $ docAuthors meta

87+ let titleMeta = (escapeStr opts . stringify) $ docTitle meta

88+ let authorsMeta = map (escapeStr opts . stringify) $ docAuthors meta

9289 hasHighlighting <- gets stHighlighting

9390 let highlightingMacros = if hasHighlighting

9491 then case writerHighlightStyle opts of

@@ -108,6 +105,10 @@ pandocToMs opts (Pandoc meta blocks) = do

108105 Nothing -> return main

109106 Just tpl -> renderTemplate' tpl context

110107

108+escapeStr :: WriterOptions -> String -> String

109+escapeStr opts =

110+ escapeString (if writerPreferAscii opts then AsciiOnly else AllowUTF8)

111+

111112escapeUri :: String -> String

112113escapeUri = escapeURIString (\c -> c /= '@' && isAllowedInURI c)

113114

@@ -121,11 +122,11 @@ toSmallCaps :: WriterOptions -> String -> String

121122toSmallCaps _ [] = []

122123toSmallCaps opts (c:cs)

123124 | isLower c = let (lowers,rest) = span isLower (c:cs)

124− in "\\s-2" ++ escString opts (map toUpper lowers) ++

125+ in "\\s-2" ++ escapeStr opts (map toUpper lowers) ++

125126 "\\s0" ++ toSmallCaps opts rest

126127 | isUpper c = let (uppers,rest) = span isUpper (c:cs)

127− in escString opts uppers ++ toSmallCaps opts rest

128− | otherwise = escapeString (writerPreferAscii opts) [c] ++ toSmallCaps opts cs

128+ in escapeStr opts uppers ++ toSmallCaps opts rest

129+ | otherwise = escapeStr opts [c] ++ toSmallCaps opts cs

129130

130131-- We split inline lists into sentences, and print one sentence per

131132-- line. groff/troff treats the line-ending period differently.

@@ -162,7 +163,7 @@ blockToMs opts (Para [Image attr alt (src,_tit)])

162163 _ -> empty

163164 capt <- inlineListToMs' opts alt

164165 return $ nowrap (text ".PSPIC -C " <>

165− doubleQuotes (text (escString opts src)) <>

166+ doubleQuotes (text (escapeStr opts src)) <>

166167 sizeAttrs) $$

167168 text ".ce 1000" $$

168169 capt $$

@@ -200,7 +201,7 @@ blockToMs opts (Header level (ident,classes,_) inlines) = do

200201 (if null secnum

201202 then ""

202203 else " ") ++

203− escString opts (stringify inlines))

204+ escapeStr opts (stringify inlines))

204205 let backlink = nowrap (text ".pdfhref L -D " <>

205206 doubleQuotes (text (toAscii ident)) <> space <> text "\\") <> cr <>

206207 text " -- "

@@ -409,7 +410,7 @@ inlineToMs opts (Str str) = do

409410 smallcaps <- gets stSmallCaps

410411 if smallcaps

411412 then return $ shim <> text (toSmallCaps opts str)

412− else return $ shim <> text (escString opts str)

413+ else return $ shim <> text (escapeStr opts str)

413414inlineToMs opts (Math InlineMath str) = do

414415 modify $ \st -> st{ stHasInlineMath = True }

415416 res <- convertMath writeEqn InlineMath str

@@ -453,7 +454,7 @@ inlineToMs opts (Link _ txt (src, _)) = do

453454 text " -- " <> doubleQuotes (nowrap contents) <> cr <> text "\\&"

454455inlineToMs opts (Image _ alternate (_, _)) =

455456 return $ char '[' <> text "IMAGE: " <>

456− text (escString opts (stringify alternate))

457+ text (escapeStr opts (stringify alternate))

457458 <> char ']'

458459inlineToMs _ (Note contents) = do

459460 modify $ \st -> st{ stNotes = contents : stNotes st }

@@ -540,15 +541,14 @@ msFormatter opts _fmtopts =

540541 where fmtLine = hcat . map fmtToken

541542 fmtToken (toktype, tok) = text "\\*" <>

542543 brackets (text (show toktype) <> text " \""

543− <> text (escapeCode (writerPreferAscii opts)

544− (T.unpack tok)) <> text "\"")

544+ <> text (escapeStr opts (T.unpack tok)) <> text "\"")

545545

546546highlightCode :: PandocMonad m => WriterOptions -> Attr -> String -> MS m Doc

547547highlightCode opts attr str =

548548 case highlight (writerSyntaxMap opts) (msFormatter opts) attr str of

549549 Left msg -> do

550550 unless (null msg) $ report $ CouldNotHighlight msg

551− return $ text (escapeCode (writerPreferAscii opts) str)

551+ return $ text (escapeStr opts str)

552552 Right h -> do

553553 modify (\st -> st{ stHighlighting = True })

554554 return h

test/command/4550.md+1−1

@@ -3,5 +3,5 @@

33A ‘simple’ example

44^D

55.LP

6−A ‘simple’ example

6+A \[oq]simple\[cq] example

77```

test/tables.ms+4−4

@@ -135,7 +135,7 @@ T}

135135.LP

136136Multiline table with caption:

137137.PP

138−Here’s the caption. It may span multiple lines.

138+Here\[cq]s the caption. It may span multiple lines.

139139.TS

140140delim(@@) tab( );

141141cw(10.5n) lw(9.6n) rw(11.4n) lw(24.5n).

@@ -165,7 +165,7 @@ row

165165T} T{

1661665.0

167167T} T{

168−Here’s another one.

168+Here\[cq]s another one.

169169Note the blank line between rows.

170170T}

171171.TE

@@ -201,7 +201,7 @@ row

201201T} T{

2022025.0

203203T} T{

204−Here’s another one.

204+Here\[cq]s another one.

205205Note the blank line between rows.

206206T}

207207.TE

@@ -261,7 +261,7 @@ row

261261T} T{

2622625.0

263263T} T{

264−Here’s another one.

264+Here\[cq]s another one.

265265Note the blank line between rows.

266266T}

267267.TE

test/writer.man+3−3

@@ -308,7 +308,7 @@ Nested.

308308.PP

309309Should not be a list item:

310310.PP

311−M.A.\~2007

311+M.A.\ 2007

312312.PP

313313B.

314314Williams

@@ -492,9 +492,9 @@ This is code: \f[C]>\f[R], \f[C]$\f[R], \f[C]\[rs]\f[R], \f[C]\[rs]$\f[R],

492492.PP

493493[STRIKEOUT:This is \f[I]strikeout\f[R].]

494494.PP

495−Superscripts: a^bc^d a^\f[I]hello\f[R]^ a^hello\~there^.

495+Superscripts: a^bc^d a^\f[I]hello\f[R]^ a^hello\ there^.

496496.PP

497−Subscripts: H~2~O, H~23~O, H~many\~of\~them~O.

497+Subscripts: H~2~O, H~23~O, H~many\ of\ them~O.

498498.PP

499499These should not be superscripts or subscripts, because of the unescaped

500500spaces: a\[ha]b c\[ha]d, a\[ti]b c\[ti]d.

test/writer.ms+23−23

@@ -75,7 +75,7 @@ July 17, 2006

7575.1C

7676.LP

7777This is a set of tests for pandoc.

78−Most of them are adapted from John Gruber’s markdown test suite.

78+Most of them are adapted from John Gruber\[cq]s markdown test suite.

7979.HLINE

8080.SH 1

8181Headers

@@ -126,7 +126,7 @@ Paragraphs

126126.pdfhref O 1 "Paragraphs"

127127.pdfhref M "paragraphs"

128128.LP

129−Here’s a regular paragraph.

129+Here\[cq]s a regular paragraph.

130130.PP

131131In Markdown 1.0.0 and earlier.

132132Version 8.

@@ -134,7 +134,7 @@ This line turns into a list item.

134134Because a hard\-wrapped line in the middle of a paragraph looked like a list

135135item.

136136.PP

137−Here’s one with a bullet.

137+Here\[cq]s one with a bullet.

138138* criminey.

139139.PP

140140There should be a hard line break

@@ -314,7 +314,7 @@ Item 1, graf one.

314314.PP

315315Item 1.

316316graf two.

317−The quick brown fox jumped over the lazy dog’s back.

317+The quick brown fox jumped over the lazy dog\[cq]s back.

318318.RE

319319.IP " 2." 4

320320Item 2.

@@ -335,7 +335,7 @@ Tab

335335.RE

336336.RE

337337.LP

338−Here’s another:

338+Here\[cq]s another:

339339.IP " 1." 4

340340First

341341.IP " 2." 4

@@ -431,7 +431,7 @@ Nested.

431431.LP

432432Should not be a list item:

433433.PP

434−M.A.\~2007

434+M.A.\ 2007

435435.PP

436436B.

437437Williams

@@ -570,7 +570,7 @@ Interpreted markdown in a table:

570570This is \f[I]emphasized\f[R]

571571And this is \f[B]strong\f[R]

572572.PP

573−Here’s a simple block:

573+Here\[cq]s a simple block:

574574.LP

575575foo

576576.LP

@@ -617,7 +617,7 @@ Code:

617617\f[]

618618.fi

619619.LP

620−Hr’s:

620+Hr\[cq]s:

621621.HLINE

622622.SH 1

623623Inline Markup

@@ -646,9 +646,9 @@ This is code: \f[C]>\f[R], \f[C]$\f[R], \f[C]\[rs]\f[R], \f[C]\[rs]$\f[R],

646646.PP

647647\m[strikecolor]This is \f[I]strikeout\f[R].\m[]

648648.PP

649−Superscripts: a\*{bc\*}d a\*{\f[I]hello\f[R]\*} a\*{hello\~there\*}.

649+Superscripts: a\*{bc\*}d a\*{\f[I]hello\f[R]\*} a\*{hello\ there\*}.

650650.PP

651−Subscripts: H\*<2\*>O, H\*<23\*>O, H\*<many\~of\~them\*>O.

651+Subscripts: H\*<2\*>O, H\*<23\*>O, H\*<many\ of\ them\*>O.

652652.PP

653653These should not be superscripts or subscripts, because of the unescaped

654654spaces: a\[ha]b c\[ha]d, a\[ti]b c\[ti]d.

@@ -666,16 +666,16 @@ Smart quotes, ellipses, dashes

666666`Oak,' `elm,' and `beech' are names of trees.

667667So is `pine.'

668668.PP

669−`He said, \[lq]I want to go.\[rq]' Were you alive in the 70’s?

669+`He said, \[lq]I want to go.\[rq]' Were you alive in the 70\[cq]s?

670670.PP

671671Here is some quoted `\f[C]code\f[R]' and a \[lq]\c

672672.pdfhref W -D "http://example.com/?foo=1&bar=2" -A "\c" \

673673 -- "quoted link"

674674\&\[rq].

675675.PP

676−Some dashes: one—two — three—four — five.

676+Some dashes: one\[em]two \[em] three\[em]four \[em] five.

677677.PP

678−Dashes between numbers: 5–7, 255–66, 1987–1999.

678+Dashes between numbers: 5\[en]7, 255\[en]66, 1987\[en]1999.

679679.PP

680680Ellipses\&...and\&...and\&....

681681.HLINE

@@ -695,14 +695,14 @@ LaTeX

695695.IP \[bu] 3

696696@p@\-Tree

697697.IP \[bu] 3

698−Here’s some display math:

698+Here\[cq]s some display math:

699699.EQ

700700d over {d x} f ( x ) = lim sub {h -> 0} {f ( x + h ) \[u2212] f ( x )} over h

701701.EN

702702.IP \[bu] 3

703−Here’s one that has a line break in it: @alpha + omega times x sup 2@.

703+Here\[cq]s one that has a line break in it: @alpha + omega times x sup 2@.

704704.LP

705−These shouldn’t be math:

705+These shouldn\[cq]t be math:

706706.IP \[bu] 3

707707To get the famous equation, write \f[C]$e = mc\[ha]2$\f[R].

708708.IP \[bu] 3

@@ -714,7 +714,7 @@ Shoes ($20) and socks ($5).

714714.IP \[bu] 3

715715Escaped \f[C]$\f[R]: $73 \f[I]this should be emphasized\f[R] 23$.

716716.LP

717−Here’s a LaTeX table:

717+Here\[cq]s a LaTeX table:

718718.HLINE

719719.SH 1

720720Special Characters

@@ -885,22 +885,22 @@ With ampersands

885885.pdfhref O 2 "With ampersands"

886886.pdfhref M "with-ampersands"

887887.LP

888−Here’s a \c

888+Here\[cq]s a \c

889889.pdfhref W -D "http://example.com/?foo=1&bar=2" -A "\c" \

890890 -- "link with an ampersand in the URL"

891891\&.

892892.PP

893−Here’s a link with an amersand in the link text: \c

893+Here\[cq]s a link with an amersand in the link text: \c

894894.pdfhref W -D "http://att.com/" -A "\c" \

895895 -- "AT&T"

896896\&.

897897.PP

898−Here’s an \c

898+Here\[cq]s an \c

899899.pdfhref W -D "/script?foo=1&bar=2" -A "\c" \

900900 -- "inline link"

901901\&.

902902.PP

903−Here’s an \c

903+Here\[cq]s an \c

904904.pdfhref W -D "/script?foo=1&bar=2" -A "\c" \

905905 -- "inline link in pointy braces"

906906\&.

@@ -925,7 +925,7 @@ It should.

925925.LP

926926An e\-mail address: \c

927927.pdfhref W -D "mailto:nobody%40nowhere.net" -A "\c" \

928− -- "nobody\[at]nowhere.net"

928+ -- "nobody@nowhere.net"

929929\&

930930.RS

931931.LP

@@ -967,7 +967,7 @@ It need not be placed at the end of the document.

967967.FE

968968and another.\**

969969.FS

970−Here’s the long note.

970+Here\[cq]s the long note.

971971This one contains multiple blocks.

972972.PP

973973Subsequent blocks are indented to show that they belong to the footnote (as