d30791a · typeable/pandoc

git.iko.soy/typeable/pandoc

Fall back to latin1 if UTF-8 decoding fails...

...when handling URL argument served with no charset in the mime type. The assumption is that most pages that don't specify a charset in the mime type are either UTF-8 or latin1. I think that's a good assumption, though I'm not sure.

commitd30791a38166538be60a134196f1d2675275017d authorJohn MacFarlane <jgm@berkeley.edu> dateMon 22 Feb 2021, 14:17 (6 years ago) parent5a73c5d

1 file changed+7−1

compare with master

unifiedsplit

src/Text/Pandoc/App.hs+7−1

src/Text/Pandoc/App.hs+7−1

@@ -1,3 +1,4 @@

1+{-# LANGUAGE LambdaCase #-}

12{-# LANGUAGE OverloadedStrings #-}

23{-# LANGUAGE CPP #-}

34{-# LANGUAGE ScopedTypeVariables #-}

@@ -352,7 +353,12 @@ readURI src = do

352353 Just "UTF-8" -> return $ UTF8.toText bs

353354 Just "ISO-8859-1" -> return $ T.pack $ B8.unpack bs

354355 Just charset -> throwError $ PandocUnsupportedCharsetError charset

355− Nothing -> return $ UTF8.toText bs

356+ Nothing -> liftIO $ -- try first as UTF-8, then as latin1

357+ E.catch (return $! UTF8.toText bs)

358+ (\case

359+ TSE.DecodeError{} ->

360+ return $ T.pack $ B8.unpack bs

361+ e -> E.throwIO e)

356362

357363readFile' :: MonadIO m => FilePath -> m BL.ByteString

358364readFile' "-" = liftIO BL.getContents