a747268 · typeable/pandoc

git.iko.soy/typeable/pandoc

CommonMark writer: respect --ascii (#5043)

commita747268823890c1a82593c88e813fa649f6bc247 authorquasicomputational <quasicomputational@gmail.com> dateMon 5 Nov 2018, 17:33 (8 years ago) parent249fd40

3 files changed+60−8

compare with master

unifiedsplit

MANUAL.txt+6−7 src/Text/Pandoc/Writers/CommonMark.hs+18−1 test/command/ascii.md+36

MANUAL.txt+6−7

@@ -859,13 +859,12 @@ Options affecting specific writers {.options}

859859

860860`--ascii`

861861

862−: Use only ASCII characters in output. Currently supported for

863− XML and HTML formats (which use entities instead of

864− UTF-8 when this option is selected), Markdown (which uses

865− entities), roff ms (which use hexadecimal escapes), and to

866− a limited degree LaTeX (which uses standard commands for

867− accented characters when possible). roff man output uses

868− ASCII by default.

862+: Use only ASCII characters in output. Currently supported for XML

863+ and HTML formats (which use entities instead of UTF-8 when this

864+ option is selected), CommonMark and Markdown (which uses

865+ entities), roff ms (which use hexadecimal escapes), and to a

866+ limited degree LaTeX (which uses standard commands for accented

867+ characters when possible). roff man output uses ASCII by default.

869868

870869`--reference-links`

871870

src/Text/Pandoc/Writers/CommonMark.hs+18−1

@@ -36,6 +36,7 @@ module Text.Pandoc.Writers.CommonMark (writeCommonMark) where

3636import Prelude

3737import CMarkGFM

3838import Control.Monad.State.Strict (State, get, modify, runState)

39+import Data.Char (isAscii)

3940import Data.Foldable (foldrM)

4041import Data.List (transpose)

4142import Data.Monoid (Any (..))

@@ -50,6 +51,7 @@ import Text.Pandoc.Templates (renderTemplate')

5051import Text.Pandoc.Walk (query, walk, walkM)

5152import Text.Pandoc.Writers.HTML (writeHtml5String, tagWithAttributes)

5253import Text.Pandoc.Writers.Shared

54+import Text.Pandoc.XML (toHtml5Entities)

5355

5456-- | Convert Pandoc to CommonMark.

5557writeCommonMark :: PandocMonad m => WriterOptions -> Pandoc -> m Text

@@ -238,7 +240,7 @@ inlinesToNodes :: WriterOptions -> [Inline] -> [Node]

238240inlinesToNodes opts = foldr (inlineToNodes opts) []

239241

240242inlineToNodes :: WriterOptions -> Inline -> [Node] -> [Node]

241−inlineToNodes opts (Str s) = (node (TEXT (T.pack s')) [] :)

243+inlineToNodes opts (Str s) = stringToNodes opts s'

242244 where s' = if isEnabled Ext_smart opts

243245 then unsmartify opts s

244246 else s

@@ -336,6 +338,21 @@ inlineToNodes opts (Cite _ ils) = (inlinesToNodes opts ils ++)

336338inlineToNodes _ (Note _) = id -- should not occur

337339-- we remove Note elements in preprocessing

338340

341+stringToNodes :: WriterOptions -> String -> [Node] -> [Node]

342+stringToNodes opts s

343+ | not (writerPreferAscii opts) = (node (TEXT (T.pack s)) [] :)

344+ | otherwise = step s

345+ where

346+ step input =

347+ let (ascii, rest) = span isAscii input

348+ this = node (TEXT (T.pack ascii)) []

349+ nodes = case rest of

350+ [] -> id

351+ (nonAscii : rest') ->

352+ let escaped = toHtml5Entities (T.singleton nonAscii)

353+ in (node (HTML_INLINE escaped) [] :) . step rest'

354+ in (this :) . nodes

355+

339356toSubscriptInline :: Inline -> Maybe Inline

340357toSubscriptInline Space = Just Space

341358toSubscriptInline (Span attr ils) = Span attr <$> traverse toSubscriptInline ils

test/command/ascii.md+36

@@ -51,3 +51,39 @@ pandoc -t markdown-smart --ascii

5151“äé&imath;å”

5252```

5353

54+# CommonMark tests

55+

56+```

57+% pandoc -f commonmark -t commonmark --ascii

58+hello … ok? … bye

59+^D

60+hello &mldr; ok? &mldr; bye

61+```

62+

63+```

64+% pandoc -f commonmark+smart -t commonmark-smart --ascii --wrap=none

65+"hi"...dog's breath---cat 5--6

66+^D

67+“hi”&mldr;dog’s breath—cat 5–6

68+```

69+

70+```

71+% pandoc -f commonmark+smart -t commonmark+smart --ascii

72+"hi"...dog's breath---cat 5--6

73+^D

74+"hi"...dog's breath---cat 5--6

75+```

76+

77+```

78+% pandoc -f commonmark -t commonmark --ascii

79+foo Ӓ bar

80+^D

81+foo Ӓ bar

82+```

83+

84+```

85+% pandoc -f commonmark -t commonmark --ascii

86+\[foo\](bar)

87+^D

88+\[foo\](bar)

89+```