6207bde · typeable/pandoc

git.iko.soy/typeable/pandoc

CommonMark writer: add plain text fallbacks. (#4531)

Previously, the writer would unconditionally emit HTMLish output for subscripts, superscripts, strikeouts (if the strikeout extension is disabled) and small caps, even with raw_html disabled. Now there are plain-text (and, where possible, fancy Unicode) fallbacks for all of these corresponding (mostly) to the Markdown fallbacks, and the HTMLish output is only used when raw_html is enabled. This commit adds exported functions `toSuperscript` and `toSubscript` to `Text.Pandoc.Writers.Shared`. [API change] Closes #4528.

commit6207bdeb681142e9fa3731e6e0ee7fa8e6c120f5 authorquasicomputational <quasicomputational@gmail.com> dateSat 6 Oct 2018, 05:33 (8 years ago) parenta26b3a2

5 files changed+237−41

compare with master

unifiedsplit

MANUAL.txt+8−2 src/Text/Pandoc/Writers/CommonMark.hs+42−10 src/Text/Pandoc/Writers/Markdown.hs+1−28 src/Text/Pandoc/Writers/Shared.hs+30−1 test/command/4528.md+156

MANUAL.txt+8−2

@@ -3381,8 +3381,14 @@ Markdown allows it, but it has been made an extension so that it can

33813381be disabled if desired.)

33823382

33833383The raw HTML is passed through unchanged in HTML, S5, Slidy, Slideous,

3384−DZSlides, EPUB, Markdown, Emacs Org mode, and Textile output, and suppressed

3385−in other formats.

3384+DZSlides, EPUB, Markdown, CommonMark, Emacs Org mode, and Textile

3385+output, and suppressed in other formats.

3386+

3387+In the CommonMark format, if `raw_html` is enabled, superscripts,

3388+subscripts, strikeouts and small capitals will be represented as HTML.

3389+Otherwise, plain-text fallbacks will be used. Note that even if

3390+`raw_html` is disabled, tables will be rendered with HTML syntax if

3391+they cannot use pipe syntax.

33863392

33873393#### Extension: `markdown_in_html_blocks` ####

33883394

src/Text/Pandoc/Writers/CommonMark.hs+42−10

@@ -45,7 +45,7 @@ import Network.HTTP (urlEncode)

4545import Text.Pandoc.Class (PandocMonad)

4646import Text.Pandoc.Definition

4747import Text.Pandoc.Options

48−import Text.Pandoc.Shared (isTightList, linesToPara, substitute)

48+import Text.Pandoc.Shared (isTightList, linesToPara, substitute, capitalize)

4949import Text.Pandoc.Templates (renderTemplate')

5050import Text.Pandoc.Walk (query, walk, walkM)

5151import Text.Pandoc.Writers.HTML (writeHtml5String, tagWithAttributes)

@@ -253,18 +253,34 @@ inlineToNodes opts (Strong xs) = (node STRONG (inlinesToNodes opts xs) :)

253253inlineToNodes opts (Strikeout xs) =

254254 if isEnabled Ext_strikeout opts

255255 then (node (CUSTOM_INLINE "~~" "~~") (inlinesToNodes opts xs) :)

256− else ((node (HTML_INLINE (T.pack "<s>")) [] : inlinesToNodes opts xs ++

257− [node (HTML_INLINE (T.pack "</s>")) []]) ++ )

256+ else if isEnabled Ext_raw_html opts

257+ then ((node (HTML_INLINE (T.pack "<s>")) [] : inlinesToNodes opts xs ++

258+ [node (HTML_INLINE (T.pack "</s>")) []]) ++ )

259+ else (inlinesToNodes opts xs ++)

258260inlineToNodes opts (Superscript xs) =

259− ((node (HTML_INLINE (T.pack "")) [] : inlinesToNodes opts xs ++

260− [node (HTML_INLINE (T.pack "")) []]) ++ )

261+ if isEnabled Ext_raw_html opts

262+ then ((node (HTML_INLINE (T.pack "")) [] : inlinesToNodes opts xs ++

263+ [node (HTML_INLINE (T.pack "")) []]) ++ )

264+ else case traverse toSuperscriptInline xs of

265+ Nothing ->

266+ ((node (TEXT (T.pack "^(")) [] : inlinesToNodes opts xs ++

267+ [node (TEXT (T.pack ")")) []]) ++ )

268+ Just xs' -> (inlinesToNodes opts xs' ++)

261269inlineToNodes opts (Subscript xs) =

262− ((node (HTML_INLINE (T.pack "")) [] : inlinesToNodes opts xs ++

263− [node (HTML_INLINE (T.pack "")) []]) ++ )

270+ if isEnabled Ext_raw_html opts

271+ then ((node (HTML_INLINE (T.pack "")) [] : inlinesToNodes opts xs ++

272+ [node (HTML_INLINE (T.pack "")) []]) ++ )

273+ else case traverse toSubscriptInline xs of

274+ Nothing ->

275+ ((node (TEXT (T.pack "_(")) [] : inlinesToNodes opts xs ++

276+ [node (TEXT (T.pack ")")) []]) ++ )

277+ Just xs' -> (inlinesToNodes opts xs' ++)

264278inlineToNodes opts (SmallCaps xs) =

265− ((node (HTML_INLINE (T.pack "")) []

266− : inlinesToNodes opts xs ++

267− [node (HTML_INLINE (T.pack "")) []]) ++ )

279+ if isEnabled Ext_raw_html opts

280+ then ((node (HTML_INLINE (T.pack "")) []

281+ : inlinesToNodes opts xs ++

282+ [node (HTML_INLINE (T.pack "")) []]) ++ )

283+ else (inlinesToNodes opts (capitalize xs) ++)

268284inlineToNodes opts (Link _ ils (url,tit)) =

269285 (node (LINK (T.pack url) (T.pack tit)) (inlinesToNodes opts ils) :)

270286-- title beginning with fig: indicates implicit figure

@@ -319,3 +335,19 @@ inlineToNodes opts (Span attr ils) =

319335inlineToNodes opts (Cite _ ils) = (inlinesToNodes opts ils ++)

320336inlineToNodes _ (Note _) = id -- should not occur

321337-- we remove Note elements in preprocessing

338+

339+toSubscriptInline :: Inline -> Maybe Inline

340+toSubscriptInline Space = Just Space

341+toSubscriptInline (Span attr ils) = Span attr <$> traverse toSubscriptInline ils

342+toSubscriptInline (Str s) = Str <$> traverse toSubscript s

343+toSubscriptInline LineBreak = Just LineBreak

344+toSubscriptInline SoftBreak = Just SoftBreak

345+toSubscriptInline _ = Nothing

346+

347+toSuperscriptInline :: Inline -> Maybe Inline

348+toSuperscriptInline Space = Just Space

349+toSuperscriptInline (Span attr ils) = Span attr <$> traverse toSuperscriptInline ils

350+toSuperscriptInline (Str s) = Str <$> traverse toSuperscript s

351+toSuperscriptInline LineBreak = Just LineBreak

352+toSuperscriptInline SoftBreak = Just SoftBreak

353+toSuperscriptInline _ = Nothing

src/Text/Pandoc/Writers/Markdown.hs+1−28

@@ -38,7 +38,7 @@ module Text.Pandoc.Writers.Markdown (writeMarkdown, writePlain) where

3838import Prelude

3939import Control.Monad.Reader

4040import Control.Monad.State.Strict

41−import Data.Char (chr, isPunctuation, isSpace, ord, isAlphaNum)

41+import Data.Char (isPunctuation, isSpace, isAlphaNum)

4242import Data.Default

4343import qualified Data.HashMap.Strict as H

4444import Data.List (find, group, intersperse, sortBy, stripPrefix, transpose)

@@ -1249,33 +1249,6 @@ makeMathPlainer = walk go

12491249 go (Emph xs) = Span nullAttr xs

12501250 go x = x

12511251

1252−toSuperscript :: Char -> Maybe Char

1253−toSuperscript '1' = Just '\x00B9'

1254−toSuperscript '2' = Just '\x00B2'

1255−toSuperscript '3' = Just '\x00B3'

1256−toSuperscript '+' = Just '\x207A'

1257−toSuperscript '-' = Just '\x207B'

1258−toSuperscript '=' = Just '\x207C'

1259−toSuperscript '(' = Just '\x207D'

1260−toSuperscript ')' = Just '\x207E'

1261−toSuperscript c

1262− | c >= '0' && c <= '9' =

1263− Just $ chr (0x2070 + (ord c - 48))

1264− | isSpace c = Just c

1265− | otherwise = Nothing

1266−

1267−toSubscript :: Char -> Maybe Char

1268−toSubscript '+' = Just '\x208A'

1269−toSubscript '-' = Just '\x208B'

1270−toSubscript '=' = Just '\x208C'

1271−toSubscript '(' = Just '\x208D'

1272−toSubscript ')' = Just '\x208E'

1273−toSubscript c

1274− | c >= '0' && c <= '9' =

1275− Just $ chr (0x2080 + (ord c - 48))

1276− | isSpace c = Just c

1277− | otherwise = Nothing

1278−

12791252lineBreakToSpace :: Inline -> Inline

12801253lineBreakToSpace LineBreak = Space

12811254lineBreakToSpace SoftBreak = Space

src/Text/Pandoc/Writers/Shared.hs+30−1

@@ -48,12 +48,15 @@ module Text.Pandoc.Writers.Shared (

4848 , lookupMetaString

4949 , stripLeadingTrailingSpace

5050 , groffEscape

51+ , toSubscript

52+ , toSuperscript

5153 )

5254where

5355import Prelude

5456import Control.Monad (zipWithM)

5557import Data.Aeson (FromJSON (..), Result (..), ToJSON (..), Value (Object),

5658 encode, fromJSON)

59+import Data.Char (chr, ord, isAscii, isSpace)

5760import qualified Data.HashMap.Strict as H

5861import Data.List (groupBy, intersperse, transpose)

5962import qualified Data.Map as M

@@ -68,7 +71,6 @@ import Text.Pandoc.Shared (stringify)

6871import Text.Pandoc.UTF8 (toStringLazy)

6972import Text.Pandoc.XML (escapeStringForXML)

7073import Text.Printf (printf)

71−import Data.Char (isAscii, ord)

7274

7375-- | Create JSON value for template from a 'Meta' and an association list

7476-- of variables, specified at the command line or in the writer.

@@ -392,3 +394,30 @@ groffEscape = T.concatMap toUchar

392394 | isAscii c = T.singleton c

393395 | otherwise = T.pack $ printf "\\[u%04X]" (ord c)

394396

397+

398+toSuperscript :: Char -> Maybe Char

399+toSuperscript '1' = Just '\x00B9'

400+toSuperscript '2' = Just '\x00B2'

401+toSuperscript '3' = Just '\x00B3'

402+toSuperscript '+' = Just '\x207A'

403+toSuperscript '-' = Just '\x207B'

404+toSuperscript '=' = Just '\x207C'

405+toSuperscript '(' = Just '\x207D'

406+toSuperscript ')' = Just '\x207E'

407+toSuperscript c

408+ | c >= '0' && c <= '9' =

409+ Just $ chr (0x2070 + (ord c - 48))

410+ | isSpace c = Just c

411+ | otherwise = Nothing

412+

413+toSubscript :: Char -> Maybe Char

414+toSubscript '+' = Just '\x208A'

415+toSubscript '-' = Just '\x208B'

416+toSubscript '=' = Just '\x208C'

417+toSubscript '(' = Just '\x208D'

418+toSubscript ')' = Just '\x208E'

419+toSubscript c

420+ | c >= '0' && c <= '9' =

421+ Just $ chr (0x2080 + (ord c - 48))

422+ | isSpace c = Just c

423+ | otherwise = Nothing

test/command/4528.mdnew file+156

@@ -0,0 +1,156 @@

1+# Rendering small caps, superscripts and subscripts with and without `raw_html`

3+## Small caps

5+```

6+% pandoc --wrap=none -f latex -t commonmark-raw_html

7+This has \textsc{small caps} in it.

8+^D

9+This has SMALL CAPS in it.

10+```

11+

12+```

13+% pandoc --wrap=none -f latex -t commonmark+raw_html

14+This has \textsc{small caps} in it.

15+^D

16+This has small caps in it.

17+```

18+```

19+

20+```

21+% pandoc --wrap=none -f latex -t markdown_strict+raw_html

22+This has \textsc{small caps} in it.

23+^D

24+This has small caps in it.

25+```

26+

27+## Strikeout

28+

29+```

30+% pandoc --wrap=none -f html -t commonmark-raw_html-strikeout

31+This has <s>strikeout</s> in it.

32+^D

33+This has strikeout in it.

34+

35+```

36+% pandoc --wrap=none -f html -t commonmark+raw_html-strikeout

37+This has <s>strikeout</s> in it.

38+^D

39+This has <s>strikeout</s> in it.

40+```

41+

42+```

43+% pandoc --wrap=none -f html -t commonmark-raw_html+strikeout

44+This has <s>strikeout</s> in it.

45+^D

46+This has ~~strikeout~~ in it.

47+```

48+

49+```

50+% pandoc --wrap=none -f html -t commonmark+raw_html+strikeout

51+This has <s>strikeout</s> in it.

52+^D

53+This has ~~strikeout~~ in it.

54+```

55+

56+```

57+% pandoc --wrap=none -f html -t markdown_strict-raw_html-strikeout

58+This has <s>strikeout</s> in it.

59+^D

60+This has strikeout in it.

61+```

62+

63+```

64+% pandoc --wrap=none -f html -t markdown_strict+raw_html-strikeout

65+This has <s>strikeout</s> in it.

66+^D

67+This has <s>strikeout</s> in it.

68+```

69+

70+```

71+% pandoc --wrap=none -f html -t markdown_strict-raw_html+strikeout

72+This has <s>strikeout</s> in it.

73+^D

74+This has ~~strikeout~~ in it.

75+```

76+

77+```

78+% pandoc --wrap=none -f html -t markdown_strict+raw_html+strikeout

79+This has <s>strikeout</s> in it.

80+^D

81+This has ~~strikeout~~ in it.

82+```

83+

84+## Superscript

85+

86+```

87+% pandoc --wrap=none -f html -t commonmark-raw_html

88+This has superscript in it and 2 3 again. With emphasis: 2 3. With letters: foo. With a span: 2.

89+^D

90+This has ^(superscript) in it and ² ³ again. With emphasis: ^(*2* 3). With letters: ^(foo). With a span: ².

91+```

92+

93+```

94+% pandoc --wrap=none -f html -t commonmark+raw_html

95+This has superscript in it and 2 again.

96+^D

97+This has superscript in it and 2 again.

98+```

99+

100+```

101+% pandoc --wrap=none -f html -t markdown_strict-raw_html-superscript

102+This has superscript in it and 2 again.

103+^D

104+This has ^(superscript) in it and ² again.

105+```

106+

107+```

108+% pandoc --wrap=none -f html -t markdown_strict+raw_html-superscript

109+This has superscript in it and 2 again.

110+^D

111+This has superscript in it and 2 again.

112+```

113+

114+```

115+% pandoc --wrap=none -f html -t markdown_strict+raw_html+superscript

116+This has superscript in it and 2 again.

117+^D

118+This has ^superscript^ in it and ^2^ again.

119+```

120+

121+## Subscript

122+

123+```

124+% pandoc --wrap=none -f html -t commonmark-raw_html

125+This has subscript in it and 2 3 again. With emphasis: 2 3. With letters: foo. With a span: 2.

126+^D

127+This has \_(subscript) in it and ₂ ₃ again. With emphasis: \_(*2* 3). With letters: \_(foo). With a span: ₂.

128+```

129+

130+```

131+% pandoc --wrap=none -f html -t commonmark+raw_html

132+This has subscript in it and 2 again.

133+^D

134+This has subscript in it and 2 again.

135+```

136+

137+```

138+% pandoc --wrap=none -f html -t markdown_strict-raw_html-subscript

139+This has subscript in it and 2 again.

140+^D

141+This has _(subscript) in it and ₂ again.

142+```

143+

144+```

145+% pandoc --wrap=none -f html -t markdown_strict+raw_html-subscript

146+This has subscript in it and 2 again.

147+^D

148+This has subscript in it and 2 again.

149+```

150+

151+```

152+% pandoc --wrap=none -f html -t markdown_strict+raw_html+subscript

153+This has subscript in it and 2 again.

154+^D

155+This has ~subscript~ in it and ~2~ again.

156+```