git.iko.soy/typeable/pandoc
on master help sign in
When downloading content from URL arguments, be sensitive to...

the character encoding. We can properly handle UTF-8 and latin1 (ISO-8859-1); for others we raise an error. See #5600.

commit5a73c5d3f8136c7fba7429c3ae3a8ae31c58030b authorJohn MacFarlane <jgm@berkeley.edu> dateMon 22 Feb 2021, 14:01 (6 years ago) parentbafccd5
1 file changed+9−1
src/Text/Pandoc/App.hs+9−1
src/Text/Pandoc/App.hs+9−1
@@ -28,6 +28,7 @@ import Control.Monad ( (>=>), when )
2828import Control.Monad.Trans ( MonadIO(..) )
2929import Control.Monad.Except (throwError)
3030import qualified Data.ByteString as BS
31+import qualified Data.ByteString.Char8 as B8
3132import qualified Data.ByteString.Lazy as BL
3233import Data.Char (toLower)
3334import Data.Maybe (fromMaybe, isJust, isNothing)
@@ -45,6 +46,7 @@ import System.FilePath ( takeBaseName, takeExtension )
4546import System.IO (nativeNewline, stdout)
4647import qualified System.IO as IO (Newline (..))
4748import Text.Pandoc
49+import Text.Pandoc.MIME (getCharset)
4850import Text.Pandoc.App.FormatHeuristics (formatFromFilePaths)
4951import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), defaultOpts,
5052 IpynbOutput (..) )
@@ -344,7 +346,13 @@ readSource src = case parseURI src of
344346 _ -> PandocAppError (tshow e))
345347
346348readURI :: FilePath -> PandocIO Text
347−readURI src = UTF8.toText . fst <$> openURL (T.pack src)
349+readURI src = do
350+ (bs, mt) <- openURL (T.pack src)
351+ case mt >>= getCharset of
352+ Just "UTF-8" -> return $ UTF8.toText bs
353+ Just "ISO-8859-1" -> return $ T.pack $ B8.unpack bs
354+ Just charset -> throwError $ PandocUnsupportedCharsetError charset
355+ Nothing -> return $ UTF8.toText bs
348356
349357readFile' :: MonadIO m => FilePath -> m BL.ByteString
350358readFile' "-" = liftIO BL.getContents