git.iko.soy/typeable/pandoc
on master help sign in
UTF8 module: Remove `\r` when reading.
c7f4333John MacFarlane
src/Text/Pandoc/UTF8.hs+7−4
@@ -56,7 +56,6 @@ import qualified Data.Text.Encoding as T
56import qualified Data.Text as T
56import qualified Data.Text as T
57import qualified Data.Text.Lazy as TL
57import qualified Data.Text.Lazy as TL
58import qualified Data.Text.Lazy.Encoding as TL
58import qualified Data.Text.Lazy.Encoding as TL
59−import Data.Text.Encoding.Error
60
59
61readFile :: FilePath -> IO String
60readFile :: FilePath -> IO String
62readFile f = do
61readFile f = do
@@ -82,19 +81,23 @@ hPutStrLn :: Handle -> String -> IO ()
82hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s
81hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s
83
82
84hGetContents :: Handle -> IO String
83hGetContents :: Handle -> IO String
85−hGetContents h = fmap (TL.unpack . TL.decodeUtf8) $ BL.hGetContents h
84+hGetContents = fmap toStringLazy . BL.hGetContents
86-- hGetContents h = hSetEncoding h utf8_bom
85-- hGetContents h = hSetEncoding h utf8_bom
87-- >> hSetNewlineMode h universalNewlineMode
86-- >> hSetNewlineMode h universalNewlineMode
88-- >> IO.hGetContents h
87-- >> IO.hGetContents h
89
88
89+-- | Convert UTF8-encoded ByteString to String, also
90+-- removing '\r' characters.
90toString :: B.ByteString -> String
91toString :: B.ByteString -> String
91−toString = T.unpack . T.decodeUtf8With lenientDecode
92+toString = filter (/='\r') . T.unpack . T.decodeUtf8
92
93
93fromString :: String -> B.ByteString
94fromString :: String -> B.ByteString
94fromString = T.encodeUtf8 . T.pack
95fromString = T.encodeUtf8 . T.pack
95
96
97+-- | Convert UTF8-encoded ByteString to String, also
98+-- removing '\r' characters.
96toStringLazy :: BL.ByteString -> String
99toStringLazy :: BL.ByteString -> String
97−toStringLazy = TL.unpack . TL.decodeUtf8With lenientDecode
100+toStringLazy = filter (/='\r') . TL.unpack . TL.decodeUtf8
98
101
99fromStringLazy :: String -> BL.ByteString
102fromStringLazy :: String -> BL.ByteString
100fromStringLazy = TL.encodeUtf8 . TL.pack
103fromStringLazy = TL.encodeUtf8 . TL.pack