44120ea · typeable/pandoc

git.iko.soy/typeable/pandoc

Implemented `east_asian_line_breaks` extension.

Text.Pandoc.Options: Added `Ext_east_asian_line_breaks` constructor to `Extension` (API change). This extension is like `ignore_line_breaks`, but smarter -- it only ignores line breaks between two East Asian wide characters. This makes it better suited for writing with a mix of East Asian and non-East Asian scripts. Closes #2586.

commit44120ea7165546152af88fd442c52ab0f201052e authorJohn MacFarlane <jgm@berkeley.edu> dateSat 12 Dec 2015, 17:28 (11 years ago) parent60d383e

3 files changed+26−2

compare with master

unifiedsplit

README+9−1 src/Text/Pandoc/Options.hs+2 src/Text/Pandoc/Readers/Markdown.hs+15−1

README+9−1

@@ -3224,7 +3224,15 @@ treated as spaces or as hard line breaks. This option is intended for

32243224use with East Asian languages where spaces are not used between words,

32253225but text is divided into lines for readability.

32263226

3227−#### Extension: `emoji` ####

3227+#### Extension: `east_asian_line_breaks` ####

3228+

3229+Causes newlines within a paragraph to be ignored, rather than

3230+being treated as spaces or as hard line breaks, when they occur

3231+between two East Asian wide characters. This is a better choice

3232+than `ignore_line_breaks` for texts that include a mix of East

3233+Asian wide characters and other characters.

3234+

3235+##### Extension: `emoji` ####

32283236

32293237Parses textual emojis like `:smile:` as Unicode emoticons.

32303238

src/Text/Pandoc/Options.hs+2

@@ -106,6 +106,8 @@ data Extension =

106106 | Ext_subscript -- ^ Subscript using ~this~ syntax

107107 | Ext_hard_line_breaks -- ^ All newlines become hard line breaks

108108 | Ext_ignore_line_breaks -- ^ Newlines in paragraphs are ignored

109+ | Ext_east_asian_line_breaks -- ^ Newlines in paragraphs are ignored between

110+ -- East Asian wide characters

109111 | Ext_literate_haskell -- ^ Enable literate Haskell conventions

110112 | Ext_abbreviations -- ^ PHP markdown extra abbreviation definitions

111113 | Ext_emoji -- ^ Support emoji like :smile:

src/Text/Pandoc/Readers/Markdown.hs+15−1

@@ -40,6 +40,7 @@ import Data.Char ( isSpace, isAlphaNum, toLower )

4040import Data.Maybe

4141import Text.Pandoc.Definition

4242import Text.Pandoc.Emoji (emojis)

43+import Text.Pandoc.Generic (bottomUp)

4344import qualified Data.Text as T

4445import Data.Text (Text)

4546import qualified Data.Yaml as Yaml

@@ -51,6 +52,7 @@ import qualified Data.Vector as V

5152import Text.Pandoc.Builder (Inlines, Blocks, trimInlines)

5253import Text.Pandoc.Options

5354import Text.Pandoc.Shared

55+import Text.Pandoc.Pretty (charWidth)

5456import Text.Pandoc.XML (fromEntities)

5557import Text.Pandoc.Parsing hiding (tableWith)

5658import Text.Pandoc.Readers.LaTeX ( rawLaTeXInline, rawLaTeXBlock )

@@ -356,7 +358,19 @@ parseMarkdown = do

356358 st <- getState

357359 let meta = runF (stateMeta' st) st

358360 let Pandoc _ bs = B.doc $ runF blocks st

359− return $ Pandoc meta bs

361+ eastAsianLineBreaks <- option False $

362+ True <$ guardEnabled Ext_east_asian_line_breaks

363+ return $ (if eastAsianLineBreaks

364+ then bottomUp softBreakFilter

365+ else id) $ Pandoc meta bs

366+

367+softBreakFilter :: [Inline] -> [Inline]

368+softBreakFilter (x:SoftBreak:y:zs) =

369+ case (stringify x, stringify y) of

370+ (xs@(_:_), (c:_))

371+ | charWidth (last xs) == 2 && charWidth c == 2 -> x:y:zs

372+ _ -> x:SoftBreak:y:zs

373+softBreakFilter xs = xs

360374

361375referenceKey :: MarkdownParser (F Blocks)

362376referenceKey = try $ do