6424e7d · typeable/pandoc

git.iko.soy/typeable/pandoc

Properly handle characters in the 128..159 range.

These aren't valid in HTML, but many HTML files produced by Windows tools contain them. We substitute correct unicode characters.

commit6424e7d02c8800a1964d0ae26a523597b8a365fa authorJohn MacFarlane <jgm@berkeley.edu> dateSat 23 Jul 2011, 12:35 (15 years ago) parent26418b7

2 files changed+48−9

compare with master

unifiedsplit

src/Text/Pandoc/Parsing.hs+7−7 src/Text/Pandoc/Readers/HTML.hs+41−2

src/Text/Pandoc/Parsing.hs+7−7

@@ -758,7 +758,7 @@ charOrRef cs =

758758singleQuoteStart :: GenParser Char ParserState ()

759759singleQuoteStart = do

760760 failIfInQuoteContext InSingleQuote

761− try $ do charOrRef "'\8216"

761+ try $ do charOrRef "'\8216\145"

762762 notFollowedBy (oneOf ")!],.;:-? \t\n")

763763 notFollowedBy (try (oneOfStrings ["s","t","m","ve","ll","re"] >>

764764 satisfy (not . isAlphaNum)))

@@ -767,23 +767,23 @@ singleQuoteStart = do

767767

768768singleQuoteEnd :: GenParser Char st ()

769769singleQuoteEnd = try $ do

770− charOrRef "'\8217"

770+ charOrRef "'\8217\146"

771771 notFollowedBy alphaNum

772772

773773doubleQuoteStart :: GenParser Char ParserState ()

774774doubleQuoteStart = do

775775 failIfInQuoteContext InDoubleQuote

776− try $ do charOrRef "\"\8220"

776+ try $ do charOrRef "\"\8220\147"

777777 notFollowedBy (satisfy (\c -> c == ' ' || c == '\t' || c == '\n'))

778778

779779doubleQuoteEnd :: GenParser Char st ()

780780doubleQuoteEnd = do

781− charOrRef "\"\8221"

781+ charOrRef "\"\8221\148"

782782 return ()

783783

784784ellipses :: GenParser Char st Inline

785785ellipses = do

786− try (charOrRef "…") <|> try (string "..." >> return '…')

786+ try (charOrRef "…\133") <|> try (string "..." >> return '…')

787787 return Ellipses

788788

789789dash :: GenParser Char st Inline

@@ -791,13 +791,13 @@ dash = enDash <|> emDash

791791

792792enDash :: GenParser Char st Inline

793793enDash = do

794− try (charOrRef "–") <|>

794+ try (charOrRef "–\150") <|>

795795 try (char '-' >> lookAhead (satisfy isDigit) >> return '–')

796796 return EnDash

797797

798798emDash :: GenParser Char st Inline

799799emDash = do

800− try (charOrRef "—") <|> (try $ string "--" >> optional (char '-') >> return '—')

800+ try (charOrRef "—\151") <|> (try $ string "--" >> optional (char '-') >> return '—')

801801 return EmDash

802802

803803--

src/Text/Pandoc/Readers/HTML.hs+41−2

@@ -416,10 +416,12 @@ pBlank = try $ do

416416 guard $ all isSpace str

417417

418418pTagContents :: GenParser Char ParserState Inline

419−pTagContents = pStr <|> pSpace <|> smartPunctuation pTagContents <|> pSymbol

419+pTagContents =

420+ pStr <|> pSpace <|> smartPunctuation pTagContents <|> pSymbol <|> pBad

420421

421422pStr :: GenParser Char ParserState Inline

422−pStr = liftM Str $ many1 $ satisfy $ \c -> not (isSpace c) && not (isSpecial c)

423+pStr = liftM Str $ many1 $ satisfy $ \c ->

424+ not (isSpace c) && not (isSpecial c) && not (isBad c)

423425

424426isSpecial :: Char -> Bool

425427isSpecial '"' = True

@@ -435,6 +437,43 @@ isSpecial _ = False

435437pSymbol :: GenParser Char ParserState Inline

436438pSymbol = satisfy isSpecial >>= return . Str . (:[])

437439

440+isBad :: Char -> Bool

441+isBad c = c >= '\128' && c <= '\159' -- not allowed in HTML

442+

443+pBad :: GenParser Char ParserState Inline

444+pBad = do

445+ c <- satisfy isBad

446+ let c' = case c of

447+ '\128' -> '\8364'

448+ '\130' -> '\8218'

449+ '\131' -> '\402'

450+ '\132' -> '\8222'

451+ '\133' -> '\8230'

452+ '\134' -> '\8224'

453+ '\135' -> '\8225'

454+ '\136' -> '\710'

455+ '\137' -> '\8240'

456+ '\138' -> '\352'

457+ '\139' -> '\8249'

458+ '\140' -> '\338'

459+ '\142' -> '\381'

460+ '\145' -> '\8216'

461+ '\146' -> '\8217'

462+ '\147' -> '\8220'

463+ '\148' -> '\8221'

464+ '\149' -> '\8226'

465+ '\150' -> '\8211'

466+ '\151' -> '\8212'

467+ '\152' -> '\732'

468+ '\153' -> '\8482'

469+ '\154' -> '\353'

470+ '\155' -> '\8250'

471+ '\156' -> '\339'

472+ '\158' -> '\382'

473+ '\159' -> '\376'

474+ _ -> '?'

475+ return $ Str [c']

476+

438477pSpace :: GenParser Char ParserState Inline

439478pSpace = many1 (satisfy isSpace) >> return Space

440479