a747268 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

CommonMark writer: respect --ascii (#5043)

a747268quasicomputational8 years ago

MANUAL.txt+6−7

compare with master

unifiedsplit

@@ -859,13 +859,12 @@ Options affecting specific writers {.options}

859

860`--ascii`

861

862−: Use only ASCII characters in output. Currently supported for

862+: Use only ASCII characters in output. Currently supported for XML

863− XML and HTML formats (which use entities instead of

863+ and HTML formats (which use entities instead of UTF-8 when this

864− UTF-8 when this option is selected), Markdown (which uses

864+ option is selected), CommonMark and Markdown (which uses

865− entities), roff ms (which use hexadecimal escapes), and to

865+ entities), roff ms (which use hexadecimal escapes), and to a

866− a limited degree LaTeX (which uses standard commands for

866+ limited degree LaTeX (which uses standard commands for accented

867− accented characters when possible). roff man output uses

867+ characters when possible). roff man output uses ASCII by default.

868− ASCII by default.

869

868

870`--reference-links`

869`--reference-links`

871

870

src/Text/Pandoc/Writers/CommonMark.hs+18−1

@@ -36,6 +36,7 @@ module Text.Pandoc.Writers.CommonMark (writeCommonMark) where

36import Prelude

37import CMarkGFM

38import Control.Monad.State.Strict (State, get, modify, runState)

39+import Data.Char (isAscii)

39import Data.Foldable (foldrM)

40import Data.Foldable (foldrM)

40import Data.List (transpose)

41import Data.List (transpose)

41import Data.Monoid (Any (..))

42import Data.Monoid (Any (..))

@@ -50,6 +51,7 @@ import Text.Pandoc.Templates (renderTemplate')

50import Text.Pandoc.Walk (query, walk, walkM)

51import Text.Pandoc.Walk (query, walk, walkM)

51import Text.Pandoc.Writers.HTML (writeHtml5String, tagWithAttributes)

52import Text.Pandoc.Writers.HTML (writeHtml5String, tagWithAttributes)

52import Text.Pandoc.Writers.Shared

53import Text.Pandoc.Writers.Shared

54+import Text.Pandoc.XML (toHtml5Entities)

54-- | Convert Pandoc to CommonMark.

56-- | Convert Pandoc to CommonMark.

55writeCommonMark :: PandocMonad m => WriterOptions -> Pandoc -> m Text

57writeCommonMark :: PandocMonad m => WriterOptions -> Pandoc -> m Text

@@ -238,7 +240,7 @@ inlinesToNodes :: WriterOptions -> [Inline] -> [Node]

238inlinesToNodes opts = foldr (inlineToNodes opts) []

240inlinesToNodes opts = foldr (inlineToNodes opts) []

239

241

240inlineToNodes :: WriterOptions -> Inline -> [Node] -> [Node]

242inlineToNodes :: WriterOptions -> Inline -> [Node] -> [Node]

241−inlineToNodes opts (Str s) = (node (TEXT (T.pack s')) [] :)

243+inlineToNodes opts (Str s) = stringToNodes opts s'

242 where s' = if isEnabled Ext_smart opts

244 where s' = if isEnabled Ext_smart opts

243 then unsmartify opts s

245 then unsmartify opts s

244 else s

246 else s

@@ -336,6 +338,21 @@ inlineToNodes opts (Cite _ ils) = (inlinesToNodes opts ils ++)

336inlineToNodes _ (Note _) = id -- should not occur

338inlineToNodes _ (Note _) = id -- should not occur

337-- we remove Note elements in preprocessing

339-- we remove Note elements in preprocessing

338

340

341+stringToNodes :: WriterOptions -> String -> [Node] -> [Node]

342+stringToNodes opts s

343+ | not (writerPreferAscii opts) = (node (TEXT (T.pack s)) [] :)

344+ | otherwise = step s

345+ where

346+ step input =

347+ let (ascii, rest) = span isAscii input

348+ this = node (TEXT (T.pack ascii)) []

349+ nodes = case rest of

350+ [] -> id

351+ (nonAscii : rest') ->

352+ let escaped = toHtml5Entities (T.singleton nonAscii)

353+ in (node (HTML_INLINE escaped) [] :) . step rest'

354+ in (this :) . nodes

355+

339toSubscriptInline :: Inline -> Maybe Inline

356toSubscriptInline :: Inline -> Maybe Inline

340toSubscriptInline Space = Just Space

357toSubscriptInline Space = Just Space

341toSubscriptInline (Span attr ils) = Span attr <$> traverse toSubscriptInline ils

358toSubscriptInline (Span attr ils) = Span attr <$> traverse toSubscriptInline ils

test/command/ascii.md+36

@@ -51,3 +51,39 @@ pandoc -t markdown-smart --ascii

51“äé&imath;å”

52```

54+# CommonMark tests

55+

56+```

57+% pandoc -f commonmark -t commonmark --ascii

58+hello … ok? … bye

59+^D

60+hello &mldr; ok? &mldr; bye

61+```

62+

63+```

64+% pandoc -f commonmark+smart -t commonmark-smart --ascii --wrap=none

65+"hi"...dog's breath---cat 5--6

66+^D

67+“hi”&mldr;dog’s breath—cat 5–6

68+```

69+

70+```

71+% pandoc -f commonmark+smart -t commonmark+smart --ascii

72+"hi"...dog's breath---cat 5--6

73+^D

74+"hi"...dog's breath---cat 5--6

75+```

76+

77+```

78+% pandoc -f commonmark -t commonmark --ascii

79+foo Ӓ bar

80+^D

81+foo Ӓ bar

82+```

83+

84+```

85+% pandoc -f commonmark -t commonmark --ascii

86+\[foo\](bar)

87+^D

88+\[foo\](bar)

89+```