ec30fb3 · typeable/pandoc

git.iko.soy/typeable/pandoc

Wrap emojis in span nodes (#4759)

Text.Pandoc.Emoji now exports `emojiToInline`, which returns a Span inline containing the emoji character and some attributes with metadata (class `emoji`, attribute `data-emoji` with emoji name). Previously, emojis (as supported in Markdown and CommonMark readers, e.g ":smile:") were simply translated into the corresponding unicode code point. By wrapping them in Span nodes, we make it possible to do special handling such as giving them a special font in HTML output. We also open up the possibility of treating them differently when the `--ascii` option is selected (though that is not part of this commit). Closes #4743.

commitec30fb37c12fc5d1a248971831414891cf6dcbe7 authorAnders Waldenborg <anders@0x63.nu> dateMon 16 Jul 2018, 00:14 (8 years ago) parentef07db6

6 files changed+56−21

compare with master

unifiedsplit

src/Text/Pandoc/Emoji.hs+6−1 src/Text/Pandoc/Readers/CommonMark.hs+17−14 src/Text/Pandoc/Readers/Markdown.hs+4−4 test/Tests/Readers/Markdown.hs+3−1 test/command/4743.md+25 test/command/gfm.md+1−1

src/Text/Pandoc/Emoji.hs+6−1

@@ -28,9 +28,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

2828

2929Emoji symbol lookup from canonical string identifier.

3030-}

31−module Text.Pandoc.Emoji ( emojis ) where

31+module Text.Pandoc.Emoji ( emojis, emojiToInline ) where

3232import Prelude

3333import qualified Data.Map as M

34+import Text.Pandoc.Definition (Inline (Span, Str))

3435

3536emojis :: M.Map String String

3637emojis = M.fromList

@@ -905,3 +906,7 @@ emojis = M.fromList

905906 ,("zero","0\65039\8419")

906907 ,("zzz","\128164")

907908 ]

909+

910+emojiToInline :: String -> Maybe Inline

911+emojiToInline emojikey = makeSpan <$> M.lookup emojikey emojis

912+ where makeSpan = Span ("", ["emoji"], [("data-emoji", emojikey)]) . (:[]) . Str

src/Text/Pandoc/Readers/CommonMark.hs+17−14

@@ -44,7 +44,7 @@ import Data.Text (Text, unpack)

4444import Text.Pandoc.Asciify (toAsciiChar)

4545import Text.Pandoc.Class (PandocMonad)

4646import Text.Pandoc.Definition

47−import Text.Pandoc.Emoji (emojis)

47+import Text.Pandoc.Emoji (emojiToInline)

4848import Text.Pandoc.Options

4949import Text.Pandoc.Shared (stringify)

5050import Text.Pandoc.Walk (walkM)

@@ -61,16 +61,19 @@ readCommonMark opts s = return $

6161 [ extTable | isEnabled Ext_pipe_tables opts ] ++

6262 [ extAutolink | isEnabled Ext_autolink_bare_uris opts ]

6363

64−convertEmojis :: String -> String

65−convertEmojis (':':xs) =

64+convertEmojis :: String -> [Inline]

65+convertEmojis s@(':':xs) =

6666 case break (==':') xs of

6767 (ys,':':zs) ->

68− case Map.lookup ys emojis of

69− Just s -> s ++ convertEmojis zs

70− Nothing -> ':' : ys ++ convertEmojis (':':zs)

71− _ -> ':':xs

72−convertEmojis (x:xs) = x : convertEmojis xs

73−convertEmojis [] = []

68+ case emojiToInline ys of

69+ Just em -> em : convertEmojis zs

70+ Nothing -> Str (':' : ys) : convertEmojis (':':zs)

71+ _ -> [Str s]

72+convertEmojis s =

73+ case break (==':') s of

74+ ("","") -> []

75+ (_,"") -> [Str s]

76+ (xs,ys) -> Str xs:convertEmojis ys

7477

7578addHeaderIdentifiers :: ReaderOptions -> Pandoc -> Pandoc

7679addHeaderIdentifiers opts doc = evalState (walkM (addHeaderId opts) doc) mempty

@@ -205,17 +208,17 @@ addInlines :: ReaderOptions -> [Node] -> [Inline]

205208addInlines opts = foldr (addInline opts) []

206209

207210addInline :: ReaderOptions -> Node -> [Inline] -> [Inline]

208−addInline opts (Node _ (TEXT t) _) = (map toinl clumps ++)

211+addInline opts (Node _ (TEXT t) _) = (foldr ((++) . toinl) [] clumps ++)

209212 where raw = unpack t

210213 clumps = groupBy samekind raw

211214 samekind ' ' ' ' = True

212215 samekind ' ' _ = False

213216 samekind _ ' ' = False

214217 samekind _ _ = True

215− toinl (' ':_) = Space

216− toinl xs = Str $ if isEnabled Ext_emoji opts

217− then convertEmojis xs

218− else xs

218+ toinl (' ':_) = [Space]

219+ toinl xs = if isEnabled Ext_emoji opts

220+ then convertEmojis xs

221+ else [Str xs]

219222addInline _ (Node _ LINEBREAK _) = (LineBreak :)

220223addInline opts (Node _ SOFTBREAK _)

221224 | isEnabled Ext_hard_line_breaks opts = (LineBreak :)

src/Text/Pandoc/Readers/Markdown.hs+4−4

@@ -51,7 +51,7 @@ import Text.Pandoc.Builder (Blocks, Inlines)

5151import qualified Text.Pandoc.Builder as B

5252import Text.Pandoc.Class (PandocMonad (..), report)

5353import Text.Pandoc.Definition

54−import Text.Pandoc.Emoji (emojis)

54+import Text.Pandoc.Emoji (emojiToInline)

5555import Text.Pandoc.Error

5656import Text.Pandoc.Logging

5757import Text.Pandoc.Options

@@ -2027,9 +2027,9 @@ emoji = try $ do

20272027 char ':'

20282028 emojikey <- many1 (oneOf emojiChars)

20292029 char ':'

2030− case M.lookup emojikey emojis of

2031− Just s -> return (return (B.str s))

2032− Nothing -> mzero

2030+ case emojiToInline emojikey of

2031+ Just i -> return (return $ B.singleton i)

2032+ Nothing -> mzero

20332033

20342034-- Citations

20352035

test/Tests/Readers/Markdown.hs+3−1

@@ -199,7 +199,9 @@ tests = [ testGroup "inline code"

199199 ]

200200 , testGroup "emoji"

201201 [ test markdownGH "emoji symbols" $

202− ":smile: and :+1:" =?> para (text "😄 and 👍")

202+ ":smile: and :+1:" =?> para (spanWith ("", ["emoji"], [("data-emoji", "smile")]) "😄" <>

203+ space <> str "and" <> space <>

204+ spanWith ("", ["emoji"], [("data-emoji", "+1")]) "👍")

203205 ]

204206 , "unbalanced brackets" =:

205207 "[[[[[[[[[[[[hi" =?> para (text "[[[[[[[[[[[[hi")

test/command/4743.mdnew file+25

@@ -0,0 +1,25 @@

1+Test that emojis are wrapped in Span

3+```

4+% pandoc -f commonmark+emoji -t native

5+My:thumbsup:emoji:heart:

6+^D

7+[Para [Str "My",Span ("",["emoji"],[("data-emoji","thumbsup")]) [Str "\128077"],Str "emoji",Span ("",["emoji"],[("data-emoji","heart")]) [Str "\10084\65039"]]]

8+```

10+```

11+% pandoc -f markdown+emoji -t native

12+My:thumbsup:emoji:heart:

13+^D

14+[Para [Str "My",Span ("",["emoji"],[("data-emoji","thumbsup")]) [Str "\128077"],Str "emoji",Span ("",["emoji"],[("data-emoji","heart")]) [Str "\10084\65039"]]]

15+```

16+

17+```

18+% pandoc -f commonmark+emoji -t html

19+:zero: header

20+=============

21+My:thumbsup:emoji:heart:x :hearts: xyz

22+^D

23+<h1>0️⃣ header</h1>

24+My👍emoji❤️x ♥️ xyz

25+```

test/command/gfm.md+1−1

@@ -38,7 +38,7 @@ gfm tests:

3838% pandoc -f gfm -t native

3939My:thumbsup:emoji:heart:

4040^D

41−[Para [Str "My\128077emoji\10084\65039"]]

41+[Para [Str "My",Span ("",["emoji"],[("data-emoji","thumbsup")]) [Str "\128077"],Str "emoji",Span ("",["emoji"],[("data-emoji","heart")]) [Str "\10084\65039"]]]

4242```

4343

4444```