@@ -1,3 +1,4 @@
1+{-# LANGUAGE OverloadedStrings #-}
12{-
23Copyright (C) 2010-2017 John MacFarlane <jgm@berkeley.edu>
34
@@ -41,8 +42,10 @@ module Text.Pandoc.UTF8 ( readFile
4142 , hPutStrLn
4243 , hGetContents
4344 , toString
45+ , toText
4446 , fromString
4547 , toStringLazy
48+ , toTextLazy
4649 , fromStringLazy
4750 , encodePath
4851 , decodeArg
@@ -51,7 +54,7 @@ module Text.Pandoc.UTF8 ( readFile
5154where
5255
5356import qualified Data.ByteString.Char8 as B
54−import qualified Data.ByteString.Lazy as BL
57+import qualified Data.ByteString.Lazy.Char8 as BL
5558import qualified Data.Text as T
5659import qualified Data.Text.Encoding as T
5760import qualified Data.Text.Lazy as TL
@@ -110,31 +113,38 @@ hGetContents = fmap toString . B.hGetContents
110113-- >> hSetNewlineMode h universalNewlineMode
111114-- >> IO.hGetContents h
112115
113−-- | Drop BOM (byte order marker) if present at beginning of string.
114−-- Note that Data.Text converts the BOM to code point FEFF, zero-width
115−-- no-break space, so if the string begins with this we strip it off.
116−dropBOM :: String -> String
117−dropBOM ('\xFEFF':xs) = xs
118−dropBOM xs = xs
119−
120−filterCRs :: String -> String
121−filterCRs ('\r':'\n':xs) = '\n': filterCRs xs
122−filterCRs ('\r':xs) = '\n' : filterCRs xs
123−filterCRs (x:xs) = x : filterCRs xs
124−filterCRs [] = []
116+-- | Convert UTF8-encoded ByteString to Text, also
117+-- removing '\r' characters.
118+toText :: B.ByteString -> T.Text
119+toText = T.decodeUtf8 . filterCRs . dropBOM
120+ where dropBOM bs =
121+ if "\xEF\xBB\xBF" `B.isPrefixOf` bs
122+ then B.drop 3 bs
123+ else bs
124+ filterCRs = B.filter (/='\r')
125125
126126-- | Convert UTF8-encoded ByteString to String, also
127127-- removing '\r' characters.
128128toString :: B.ByteString -> String
129−toString = filterCRs . dropBOM . T.unpack . T.decodeUtf8
129+toString = T.unpack . toText
130130
131−fromString :: String -> B.ByteString
132−fromString = T.encodeUtf8 . T.pack
131+-- | Convert UTF8-encoded ByteString to Text, also
132+-- removing '\r' characters.
133+toTextLazy :: BL.ByteString -> TL.Text
134+toTextLazy = TL.decodeUtf8 . filterCRs . dropBOM
135+ where dropBOM bs =
136+ if "\xEF\xBB\xBF" `BL.isPrefixOf` bs
137+ then BL.drop 3 bs
138+ else bs
139+ filterCRs = BL.filter (/='\r')
133140
134141-- | Convert UTF8-encoded ByteString to String, also
135142-- removing '\r' characters.
136143toStringLazy :: BL.ByteString -> String
137−toStringLazy = filterCRs . dropBOM . TL.unpack . TL.decodeUtf8
144+toStringLazy = TL.unpack . toTextLazy
145+
146+fromString :: String -> B.ByteString
147+fromString = T.encodeUtf8 . T.pack
138148
139149fromStringLazy :: String -> BL.ByteString
140150fromStringLazy = TL.encodeUtf8 . TL.pack