44120ea · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Implemented `east_asian_line_breaks` extension.

44120eaJohn MacFarlane11 years ago

README+9−1

compare with master

unifiedsplit

@@ -3224,7 +3224,15 @@ treated as spaces or as hard line breaks. This option is intended for

3224use with East Asian languages where spaces are not used between words,

3225but text is divided into lines for readability.

3226

3227−#### Extension: `emoji` ####

3227+#### Extension: `east_asian_line_breaks` ####

3228+

3229+Causes newlines within a paragraph to be ignored, rather than

3230+being treated as spaces or as hard line breaks, when they occur

3231+between two East Asian wide characters. This is a better choice

3232+than `ignore_line_breaks` for texts that include a mix of East

3233+Asian wide characters and other characters.

3234+

3235+##### Extension: `emoji` ####

3228

3236

3229Parses textual emojis like `:smile:` as Unicode emoticons.

3237Parses textual emojis like `:smile:` as Unicode emoticons.

3230

3238

src/Text/Pandoc/Options.hs+2

@@ -106,6 +106,8 @@ data Extension =

106 | Ext_subscript -- ^ Subscript using ~this~ syntax

107 | Ext_hard_line_breaks -- ^ All newlines become hard line breaks

108 | Ext_ignore_line_breaks -- ^ Newlines in paragraphs are ignored

109+ | Ext_east_asian_line_breaks -- ^ Newlines in paragraphs are ignored between

110+ -- East Asian wide characters

109 | Ext_literate_haskell -- ^ Enable literate Haskell conventions

111 | Ext_literate_haskell -- ^ Enable literate Haskell conventions

110 | Ext_abbreviations -- ^ PHP markdown extra abbreviation definitions

112 | Ext_abbreviations -- ^ PHP markdown extra abbreviation definitions

111 | Ext_emoji -- ^ Support emoji like :smile:

113 | Ext_emoji -- ^ Support emoji like :smile:

src/Text/Pandoc/Readers/Markdown.hs+15−1

@@ -40,6 +40,7 @@ import Data.Char ( isSpace, isAlphaNum, toLower )

40import Data.Maybe

41import Text.Pandoc.Definition

42import Text.Pandoc.Emoji (emojis)

43+import Text.Pandoc.Generic (bottomUp)

43import qualified Data.Text as T

44import qualified Data.Text as T

44import Data.Text (Text)

45import Data.Text (Text)

45import qualified Data.Yaml as Yaml

46import qualified Data.Yaml as Yaml

@@ -51,6 +52,7 @@ import qualified Data.Vector as V

51import Text.Pandoc.Builder (Inlines, Blocks, trimInlines)

52import Text.Pandoc.Builder (Inlines, Blocks, trimInlines)

52import Text.Pandoc.Options

53import Text.Pandoc.Options

53import Text.Pandoc.Shared

54import Text.Pandoc.Shared

55+import Text.Pandoc.Pretty (charWidth)

54import Text.Pandoc.XML (fromEntities)

56import Text.Pandoc.XML (fromEntities)

55import Text.Pandoc.Parsing hiding (tableWith)

57import Text.Pandoc.Parsing hiding (tableWith)

56import Text.Pandoc.Readers.LaTeX ( rawLaTeXInline, rawLaTeXBlock )

58import Text.Pandoc.Readers.LaTeX ( rawLaTeXInline, rawLaTeXBlock )

@@ -356,7 +358,19 @@ parseMarkdown = do

356 st <- getState

358 st <- getState

357 let meta = runF (stateMeta' st) st

359 let meta = runF (stateMeta' st) st

358 let Pandoc _ bs = B.doc $ runF blocks st

360 let Pandoc _ bs = B.doc $ runF blocks st

361+ eastAsianLineBreaks <- option False $

362+ True <$ guardEnabled Ext_east_asian_line_breaks

363+ return $ (if eastAsianLineBreaks

364+ then bottomUp softBreakFilter

359− return $ Pandoc meta bs

365+ else id) $ Pandoc meta bs

366+

367+softBreakFilter :: [Inline] -> [Inline]

368+softBreakFilter (x:SoftBreak:y:zs) =

369+ case (stringify x, stringify y) of

370+ (xs@(_:_), (c:_))

371+ | charWidth (last xs) == 2 && charWidth c == 2 -> x:y:zs

372+ _ -> x:SoftBreak:y:zs

373+softBreakFilter xs = xs

360

374

361referenceKey :: MarkdownParser (F Blocks)

375referenceKey :: MarkdownParser (F Blocks)

362referenceKey = try $ do

376referenceKey = try $ do