925a4c5 · typeable/pandoc

git.iko.soy/typeable/pandoc

Better smart quote parsing.

* Added stateLastStrPos to ParserState. This lets us keep track of whether we're parsing the position immediately after a 'str'. If we encounter a ' in such a location, it must be an apostrophe, and can't be a single quote start. * Set this in the markdown, textile, html, and rst str parsers. * Closes #360.

commit925a4c5164026bfda25bf50b552bacec074fdf3f authorJohn MacFarlane <fiddlosopher@gmail.com> dateThu 29 Dec 2011, 23:44 (15 years ago) parent600c22e

5 files changed+22−4

compare with master

unifiedsplit

src/Text/Pandoc/Parsing.hs+7−1 src/Text/Pandoc/Readers/HTML.hs+6−2 src/Text/Pandoc/Readers/Markdown.hs+2 src/Text/Pandoc/Readers/RST.hs+5−1 src/Text/Pandoc/Readers/Textile.hs+2

src/Text/Pandoc/Parsing.hs+7−1

@@ -603,6 +603,7 @@ data ParserState = ParserState

603603 { stateParseRaw :: Bool, -- ^ Parse raw HTML and LaTeX?

604604 stateParserContext :: ParserContext, -- ^ Inside list?

605605 stateQuoteContext :: QuoteContext, -- ^ Inside quoted environment?

606+ stateLastStrPos :: Maybe SourcePos, -- ^ Position after last str parsed

606607 stateKeys :: KeyTable, -- ^ List of reference keys

607608 stateCitations :: [String], -- ^ List of available citations

608609 stateNotes :: NoteTable, -- ^ List of notes

@@ -630,6 +631,7 @@ defaultParserState =

630631 ParserState { stateParseRaw = False,

631632 stateParserContext = NullState,

632633 stateQuoteContext = NoQuote,

634+ stateLastStrPos = Nothing,

633635 stateKeys = M.empty,

634636 stateCitations = [],

635637 stateNotes = [],

@@ -751,8 +753,12 @@ charOrRef cs =

751753 return c)

752754

753755singleQuoteStart :: GenParser Char ParserState ()

754−singleQuoteStart = do

756+singleQuoteStart = do

755757 failIfInQuoteContext InSingleQuote

758+ pos <- getPosition

759+ st <- getState

760+ -- single quote start can't be right after str

761+ guard $ stateLastStrPos st /= Just pos

756762 try $ do charOrRef "'\8216\145"

757763 notFollowedBy (oneOf ")!],;:-? \t\n")

758764 notFollowedBy (char '.') <|> lookAhead (string "..." >> return ())

src/Text/Pandoc/Readers/HTML.hs+6−2

@@ -421,8 +421,12 @@ pTagContents =

421421 pStr <|> pSpace <|> smartPunctuation pTagContents <|> pSymbol <|> pBad

422422

423423pStr :: GenParser Char ParserState Inline

424−pStr = liftM Str $ many1 $ satisfy $ \c ->

425− not (isSpace c) && not (isSpecial c) && not (isBad c)

424+pStr = do

425+ result <- many1 $ satisfy $ \c ->

426+ not (isSpace c) && not (isSpecial c) && not (isBad c)

427+ pos <- getPosition

428+ updateState $ \s -> s{ stateLastStrPos = Just pos }

429+ return $ Str result

426430

427431isSpecial :: Char -> Bool

428432isSpecial '"' = True

src/Text/Pandoc/Readers/Markdown.hs+2

@@ -1096,6 +1096,8 @@ str = do

10961096 lookAhead alphaNum >> return '\x2019')

10971097 -- for things like l'aide

10981098 else mzero

1099+ pos <- getPosition

1100+ updateState $ \s -> s{ stateLastStrPos = Just pos }

10991101 let result = a:as

11001102 let spacesToNbr = map (\c -> if c == ' ' then '\160' else c)

11011103 if smart

src/Text/Pandoc/Readers/RST.hs+5−1

@@ -791,7 +791,11 @@ whitespace :: GenParser Char ParserState Inline

791791whitespace = many1 spaceChar >> return Space <?> "whitespace"

792792

793793str :: GenParser Char ParserState Inline

794−str = many1 (noneOf (specialChars ++ "\t\n ")) >>= return . Str

794+str = do

795+ result <- many1 (noneOf (specialChars ++ "\t\n "))

796+ pos <- getPosition

797+ updateState $ \s -> s{ stateLastStrPos = Just pos }

798+ return $ Str result

795799

796800-- an endline character that can be treated as a space, not a structural break

797801endline :: GenParser Char ParserState Inline

src/Text/Pandoc/Readers/Textile.hs+2

@@ -436,6 +436,8 @@ str = do

436436 next <- lookAhead letter

437437 guard $ isLetter (last xs) || isLetter next

438438 return $ xs ++ "-"

439+ pos <- getPosition

440+ updateState $ \s -> s{ stateLastStrPos = Just pos }

439441 return $ Str result

440442

441443-- | Textile allows HTML span infos, we discard them