git.iko.soy/typeable/pandoc
on master help sign in
New treatment of dashes in --smart mode.

* `---` is always em-dash, `--` is always en-dash. * pandoc no longer tries to guess when `-` should be en-dash. * A new option, `--old-dashes`, is provided for legacy documents. Rationale: The rules for en-dash are too complex and language-dependent for a guesser to work reliably. This change gives users greater control. The alternative of using unicode isn't very good, since unicode em- and en- dashes are barely distinguishable in a monospace font.

commitda8425598a8ab4a98388e8ee346a2ae7ec540aa0 authorJohn MacFarlane <fiddlosopher@gmail.com> dateSun 1 Jan 2012, 13:48 (15 years ago) parent3cf60c7
6 files changed+56−15
compare with master
unifiedsplit
README+11−5 src/Text/Pandoc/Parsing.hs+29−5 src/Text/Pandoc/Readers/Textile.hs+2−1 src/pandoc.hs+10 tests/rst-reader.native+2−2 tests/testsuite.txt+2−2
README+11−5
@@ -206,11 +206,17 @@ Options
206206
207207`-S`, `--smart`
208208: Produce typographically correct output, converting straight quotes
209− to curly quotes, `---` and `--` to dashes, and `...` to ellipses.
210− Nonbreaking spaces are inserted after certain abbreviations, such
211− as "Mr." (Note: This option is significant only when the input format is
212− `markdown` or `textile`. It is selected automatically when the input
213− format is `textile` or the output format is `latex` or `context`.)
209+ to curly quotes, `---` to em-dashes, `--` to en-dashes, and
210+ `...` to ellipses. Nonbreaking spaces are inserted after certain
211+ abbreviations, such as "Mr." (Note: This option is significant only when
212+ the input format is `markdown` or `textile`. It is selected automatically
213+ when the input format is `textile` or the output format is `latex` or
214+ `context`.)
215+
216+`--old-dashes`
217+: Selects the pandoc <= 1.8.2.1 behavior for parsing smart dashes: `-` before
218+ a numeral is an en-dash, and `--` is an em-dash. This option is selected
219+ automatically for `textile` input.
214220
215221`-5`, `--html5`
216222: Produce HTML5 instead of HTML4. This option has no effect for writers
src/Text/Pandoc/Parsing.hs+29−5
@@ -614,6 +614,9 @@ data ParserState = ParserState
614614 stateDate :: [Inline], -- ^ Date of document
615615 stateStrict :: Bool, -- ^ Use strict markdown syntax?
616616 stateSmart :: Bool, -- ^ Use smart typography?
617+ stateOldDashes :: Bool, -- ^ Use pandoc <= 1.8.2.1 behavior
618+ -- in parsing dashes; -- is em-dash;
619+ -- before numeral is en-dash
617620 stateLiterateHaskell :: Bool, -- ^ Treat input as literate haskell
618621 stateColumns :: Int, -- ^ Number of columns in terminal
619622 stateHeaderTable :: [HeaderType], -- ^ Ordered list of header types used
@@ -642,6 +645,7 @@ defaultParserState =
642645 stateDate = [],
643646 stateStrict = False,
644647 stateSmart = False,
648+ stateOldDashes = False,
645649 stateLiterateHaskell = False,
646650 stateColumns = 80,
647651 stateHeaderTable = [],
@@ -788,17 +792,37 @@ ellipses = do
788792 try (charOrRef "\8230\133") <|> try (string "..." >> return '…')
789793 return (Str "\8230")
790794
791−dash :: GenParser Char st Inline
792−dash = enDash <|> emDash
795+dash :: GenParser Char ParserState Inline
796+dash = do
797+ oldDashes <- stateOldDashes `fmap` getState
798+ if oldDashes
799+ then emDashOld <|> enDashOld
800+ else Str `fmap` (hyphenDash <|> emDash <|> enDash)
793801
794−enDash :: GenParser Char st Inline
802+-- Two hyphens = en-dash, three = em-dash
803+hyphenDash :: GenParser Char st String
804+hyphenDash = do
805+ try $ string "--"
806+ option "\8211" (char '-' >> return "\8212")
807+
808+emDash :: GenParser Char st String
809+emDash = do
810+ try (charOrRef "\8212\151")
811+ return "\8212"
812+
813+enDash :: GenParser Char st String
795814enDash = do
815+ try (charOrRef "\8212\151")
816+ return "\8211"
817+
818+enDashOld :: GenParser Char st Inline
819+enDashOld = do
796820 try (charOrRef "\8211\150") <|>
797821 try (char '-' >> lookAhead (satisfy isDigit) >> return '–')
798822 return (Str "\8211")
799823
800−emDash :: GenParser Char st Inline
801−emDash = do
824+emDashOld :: GenParser Char st Inline
825+emDashOld = do
802826 try (charOrRef "\8212\151") <|> (try $ string "--" >> optional (char '-') >> return '-')
803827 return (Str "\8212")
804828
src/Text/Pandoc/Readers/Textile.hs+2−1
@@ -68,7 +68,8 @@ import Control.Monad ( guard, liftM )
6868readTextile :: ParserState -- ^ Parser state, including options for parser
6969 -> String -- ^ String to parse (assuming @'\n'@ line endings)
7070 -> Pandoc
71−readTextile state s = (readWith parseTextile) state (s ++ "\n\n")
71+readTextile state s =
72+ (readWith parseTextile) state{ stateOldDashes = True } (s ++ "\n\n")
7273
7374
7475--
src/pandoc.hs+10
@@ -103,6 +103,7 @@ data Opt = Opt
103103 , optSelfContained :: Bool -- ^ Make HTML accessible offline
104104 , optXeTeX :: Bool -- ^ Format latex for xetex
105105 , optSmart :: Bool -- ^ Use smart typography
106+ , optOldDashes :: Bool -- ^ Parse dashes like pandoc <=1.8.2.1
106107 , optHtml5 :: Bool -- ^ Produce HTML5 in HTML
107108 , optHighlight :: Bool -- ^ Highlight source code
108109 , optHighlightStyle :: Style -- ^ Style to use for highlighted code
@@ -149,6 +150,7 @@ defaultOpts = Opt
149150 , optSelfContained = False
150151 , optXeTeX = False
151152 , optSmart = False
153+ , optOldDashes = False
152154 , optHtml5 = False
153155 , optHighlight = True
154156 , optHighlightStyle = pygments
@@ -245,6 +247,12 @@ options =
245247 (\opt -> return opt { optSmart = True }))
246248 "" -- "Use smart quotes, dashes, and ellipses"
247249
250+ , Option "" ["old-dashes"]
251+ (NoArg
252+ (\opt -> return opt { optSmart = True
253+ , optOldDashes = True }))
254+ "" -- "Use smart quotes, dashes, and ellipses"
255+
248256 , Option "5" ["html5"]
249257 (NoArg
250258 (\opt -> do
@@ -735,6 +743,7 @@ main = do
735743 , optIncremental = incremental
736744 , optSelfContained = selfContained
737745 , optSmart = smart
746+ , optOldDashes = oldDashes
738747 , optHtml5 = html5
739748 , optHighlight = highlight
740749 , optHighlightStyle = highlightStyle
@@ -858,6 +867,7 @@ main = do
858867 stateCitations = map CSL.refId refs,
859868 stateSmart = smart || writerName' `elem`
860869 ["latex", "context", "latex+lhs", "beamer"],
870+ stateOldDashes = oldDashes,
861871 stateColumns = columns,
862872 stateStrict = strict,
863873 stateIndentedCodeClasses = codeBlockClasses,
tests/rst-reader.native+2−2
@@ -165,14 +165,14 @@ Pandoc (Meta {docTitle = [Str "Pandoc",Space,Str "Test",Space,Str "Suite",Str ":
165165 ,([Str "city"],
166166 [[Para [Emph [Str "Nowhere"],Str ",",Space,Str "MA,",Space,Str "USA"]]])
167167 ,([Str "phone"],
168− [[Para [Str "123",Str "\8211",Str "4567"]]])]]
168+ [[Para [Str "123",Str "-",Str "4567"]]])]]
169169,DefinitionList
170170 [([Str "address"],
171171 [[Para [Str "61",Space,Str "Main",Space,Str "St",Str "."]]])
172172 ,([Str "city"],
173173 [[Para [Emph [Str "Nowhere"],Str ",",Space,Str "MA,",Space,Str "USA"]]])
174174 ,([Str "phone"],
175− [[Para [Str "123",Str "\8211",Str "4567"]]])]
175+ [[Para [Str "123",Str "-",Str "4567"]]])]
176176,Header 1 [Str "HTML",Space,Str "Blocks"]
177177,Para [Str "Simple",Space,Str "block",Space,Str "on",Space,Str "one",Space,Str "line",Str ":"]
178178,RawBlock "html" "<div>foo</div>\n"
tests/testsuite.txt+2−2
@@ -492,9 +492,9 @@ So is 'pine.'
492492
493493Here is some quoted '`code`' and a "[quoted link][1]".
494494
495−Some dashes: one---two --- three--four -- five.
495+Some dashes: one---two --- three---four --- five.
496496
497−Dashes between numbers: 5-7, 255-66, 1987-1999.
497+Dashes between numbers: 5--7, 255--66, 1987--1999.
498498
499499Ellipses...and...and....
500500