@@ -1,3 +1,4 @@
1+{-# LANGUAGE OverloadedStrings #-}
2Copyright (C) 2010-2017 John MacFarlane <jgm@berkeley.edu>
3Copyright (C) 2010-2017 John MacFarlane <jgm@berkeley.edu>
@@ -41,8 +42,10 @@ module Text.Pandoc.UTF8 ( readFile
41 , hPutStrLn
42 , hPutStrLn
42 , hGetContents
43 , hGetContents
43 , toString
44 , toString
44 , fromString
46 , fromString
45 , toStringLazy
47 , toStringLazy
46 , fromStringLazy
49 , fromStringLazy
47 , encodePath
50 , encodePath
48 , decodeArg
51 , decodeArg
@@ -51,7 +54,7 @@ module Text.Pandoc.UTF8 ( readFile
53import qualified Data.ByteString.Char8 as B
56import qualified Data.ByteString.Char8 as B
54−import qualified Data.ByteString.Lazy as BL
57+import qualified Data.ByteString.Lazy.Char8 as BL
55import qualified Data.Text as T
58import qualified Data.Text as T
56import qualified Data.Text.Encoding as T
59import qualified Data.Text.Encoding as T
57import qualified Data.Text.Lazy as TL
60import qualified Data.Text.Lazy as TL
@@ -110,31 +113,38 @@ hGetContents = fmap toString . B.hGetContents
110-- >> hSetNewlineMode h universalNewlineMode
113-- >> hSetNewlineMode h universalNewlineMode
111-- >> IO.hGetContents h
114-- >> IO.hGetContents h
113−-- | Drop BOM (byte order marker) if present at beginning of string.
116+-- | Convert UTF8-encoded ByteString to Text, also
114−-- Note that Data.Text converts the BOM to code point FEFF, zero-width
117+-- removing '\r' characters.
115−-- no-break space, so if the string begins with this we strip it off.
118+toText :: B.ByteString -> T.Text
116−dropBOM :: String -> String
119+toText = T.decodeUtf8 . filterCRs . dropBOM
117−dropBOM ('\xFEFF':xs) = xs
120+ where dropBOM bs =
118−dropBOM xs = xs
121+ if "\xEF\xBB\xBF" `B.isPrefixOf` bs
119−
122+ then B.drop 3 bs
120−filterCRs :: String -> String
123+ else bs
121−filterCRs ('\r':'\n':xs) = '\n': filterCRs xs
122−filterCRs ('\r':xs) = '\n' : filterCRs xs
123−filterCRs (x:xs) = x : filterCRs xs
124−filterCRs [] = []
124+ filterCRs = B.filter (/='\r')
126-- | Convert UTF8-encoded ByteString to String, also
126-- | Convert UTF8-encoded ByteString to String, also
127-- removing '\r' characters.
127-- removing '\r' characters.
128toString :: B.ByteString -> String
128toString :: B.ByteString -> String
129−toString = filterCRs . dropBOM . T.unpack . T.decodeUtf8
129+toString = T.unpack . toText
131−fromString :: String -> B.ByteString
131+-- | Convert UTF8-encoded ByteString to Text, also
132−fromString = T.encodeUtf8 . T.pack
132+-- removing '\r' characters.
133+toTextLazy :: BL.ByteString -> TL.Text
134+toTextLazy = TL.decodeUtf8 . filterCRs . dropBOM
136+ if "\xEF\xBB\xBF" `BL.isPrefixOf` bs
139+ filterCRs = BL.filter (/='\r')
134-- | Convert UTF8-encoded ByteString to String, also
141-- | Convert UTF8-encoded ByteString to String, also
135-- removing '\r' characters.
142-- removing '\r' characters.
136toStringLazy :: BL.ByteString -> String
143toStringLazy :: BL.ByteString -> String
137−toStringLazy = filterCRs . dropBOM . TL.unpack . TL.decodeUtf8
144+toStringLazy = TL.unpack . toTextLazy
146+fromString :: String -> B.ByteString
147+fromString = T.encodeUtf8 . T.pack
139fromStringLazy :: String -> BL.ByteString
149fromStringLazy :: String -> BL.ByteString
140fromStringLazy = TL.encodeUtf8 . TL.pack
150fromStringLazy = TL.encodeUtf8 . TL.pack