git.iko.soy/typeable/pandoc
on 34412cf help sign in
typeable/pandoc/src/Text/Pandoc/UTF8.hs
121 lines 4.0 kB history raw
1{-
2Copyright (C) 2010-2016 John MacFarlane <jgm@berkeley.edu>
4This program is free software; you can redistribute it and/or modify
5it under the terms of the GNU General Public License as published by
6the Free Software Foundation; either version 2 of the License, or
7(at your option) any later version.
9This program is distributed in the hope that it will be useful,
10but WITHOUT ANY WARRANTY; without even the implied warranty of
11MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12GNU General Public License for more details.
14You should have received a copy of the GNU General Public License
15along with this program; if not, write to the Free Software
16Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17-}
19{- |
20 Module : Text.Pandoc.UTF8
21 Copyright : Copyright (C) 2010-2016 John MacFarlane
22 License : GNU GPL, version 2 or above
24 Maintainer : John MacFarlane <jgm@berkeley.edu>
25 Stability : alpha
26 Portability : portable
28UTF-8 aware string IO functions that will work with GHC 6.10, 6.12, or 7.
29-}
30module Text.Pandoc.UTF8 ( readFile
31 , writeFile
32 , getContents
33 , putStr
34 , putStrLn
35 , hPutStr
36 , hPutStrLn
37 , hGetContents
38 , toString
39 , fromString
40 , toStringLazy
41 , fromStringLazy
42 , encodePath
43 , decodeArg
44 )
46where
48import qualified Data.ByteString.Char8 as B
49import qualified Data.ByteString.Lazy as BL
50import qualified Data.Text as T
51import qualified Data.Text.Encoding as T
52import qualified Data.Text.Lazy as TL
53import qualified Data.Text.Lazy.Encoding as TL
54import Prelude hiding (getContents, putStr, putStrLn, readFile, writeFile)
55import System.IO hiding (getContents, hGetContents, hPutStr, hPutStrLn, putStr,
56 putStrLn, readFile, writeFile)
57import qualified System.IO as IO
59readFile :: FilePath -> IO String
60readFile f = do
61 h <- openFile (encodePath f) ReadMode
62 hGetContents h
64writeFile :: FilePath -> String -> IO ()
65writeFile f s = withFile (encodePath f) WriteMode $ \h -> hPutStr h s
67getContents :: IO String
68getContents = hGetContents stdin
70putStr :: String -> IO ()
71putStr s = hPutStr stdout s
73putStrLn :: String -> IO ()
74putStrLn s = hPutStrLn stdout s
76hPutStr :: Handle -> String -> IO ()
77hPutStr h s = hSetEncoding h utf8 >> IO.hPutStr h s
79hPutStrLn :: Handle -> String -> IO ()
80hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s
82hGetContents :: Handle -> IO String
83hGetContents = fmap toString . B.hGetContents
84-- hGetContents h = hSetEncoding h utf8_bom
85-- >> hSetNewlineMode h universalNewlineMode
86-- >> IO.hGetContents h
88-- | Drop BOM (byte order marker) if present at beginning of string.
89-- Note that Data.Text converts the BOM to code point FEFF, zero-width
90-- no-break space, so if the string begins with this we strip it off.
91dropBOM :: String -> String
92dropBOM ('\xFEFF':xs) = xs
93dropBOM xs = xs
95filterCRs :: String -> String
96filterCRs ('\r':'\n':xs) = '\n': filterCRs xs
97filterCRs ('\r':xs) = '\n' : filterCRs xs
98filterCRs (x:xs) = x : filterCRs xs
99filterCRs [] = []
101-- | Convert UTF8-encoded ByteString to String, also
102-- removing '\r' characters.
103toString :: B.ByteString -> String
104toString = filterCRs . dropBOM . T.unpack . T.decodeUtf8
106fromString :: String -> B.ByteString
107fromString = T.encodeUtf8 . T.pack
109-- | Convert UTF8-encoded ByteString to String, also
110-- removing '\r' characters.
111toStringLazy :: BL.ByteString -> String
112toStringLazy = filterCRs . dropBOM . TL.unpack . TL.decodeUtf8
114fromStringLazy :: String -> BL.ByteString
115fromStringLazy = TL.encodeUtf8 . TL.pack
117encodePath :: FilePath -> FilePath
118encodePath = id
120decodeArg :: String -> String
121decodeArg = id