7272735 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

More intelligent handling of text encodings.

7272735John MacFarlane14 years ago

src/Text/Pandoc/Parsing.hs+1−2

compare with master

unifiedsplit

@@ -146,7 +146,6 @@ where

146import Text.Pandoc.Definition

147import Text.Pandoc.Options

148import Text.Pandoc.Builder (Blocks)

149−import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)

150import Text.Parsec

149import Text.Parsec

151import Text.Parsec.Pos (newPos)

150import Text.Parsec.Pos (newPos)

152import Data.Char ( toLower, toUpper, ord, isAscii, isAlphaNum, isDigit, isPunctuation )

151import Data.Char ( toLower, toUpper, ord, isAscii, isAlphaNum, isDigit, isPunctuation )

@@ -708,7 +707,7 @@ readWith parser state input =

708testStringWith :: (Show a) => Parser [Char] ParserState a

707testStringWith :: (Show a) => Parser [Char] ParserState a

709 -> String

708 -> String

710 -> IO ()

709 -> IO ()

711−testStringWith parser str = UTF8.putStrLn $ show $

710+testStringWith parser str = putStrLn $ show $

712 readWith parser defaultParserState str

711 readWith parser defaultParserState str

713

712

714-- | Parsing options.

713-- | Parsing options.

src/Text/Pandoc/Shared.hs+3−3

@@ -90,7 +90,7 @@ import Paths_pandoc (getDataFileName)

90import Text.Pandoc.Pretty (charWidth)

91import System.Locale (defaultTimeLocale)

92import Data.Time

93−import System.IO (stderr)

93+import System.IO (stderr, hPutStrLn)

94import Text.HTML.TagSoup (renderTagsOptions, RenderOptions(..), Tag(..),

95 renderOptions)

@@ -503,14 +503,14 @@ readDataFile userDir fname = findDataFile userDir fname >>= UTF8.readFile

503err :: Int -> String -> IO a

504err exitCode msg = do

505 name <- getProgName

506− UTF8.hPutStrLn stderr $ name ++ ": " ++ msg

506+ hPutStrLn stderr $ name ++ ": " ++ msg

507 exitWith $ ExitFailure exitCode

508 return undefined

509

510warn :: String -> IO ()

511warn msg = do

512 name <- getProgName

513− UTF8.hPutStrLn stderr $ name ++ ": " ++ msg

513+ hPutStrLn stderr $ name ++ ": " ++ msg

514

515--

516-- Safe read

src/Text/Pandoc/UTF8.hs+16−4

@@ -25,7 +25,11 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

25 Stability : alpha

26 Portability : portable

28−UTF-8 aware string IO functions that will work with GHC 6.10, 6.12, or 7.

28+UTF-8 aware string IO functions that will work with GHC 6.12 or 7.

29+The reading functions first attempt to read UTF-8; if an encoding

30+error is encountered, the local encoding is used instead. This

31+should work well in practice because text in other encodings

32+is usually not valid UTF-8.

29-}

33-}

30module Text.Pandoc.UTF8 ( readFile

34module Text.Pandoc.UTF8 ( readFile

31 , writeFile

35 , writeFile

@@ -45,10 +49,11 @@ where

45#else

49#else

46import Codec.Binary.UTF8.String (encodeString, decodeString)

50import Codec.Binary.UTF8.String (encodeString, decodeString)

47#endif

51#endif

48−

52+import Control.Exception (catch, throwIO)

53+import GHC.IO.Exception (IOException(..), IOErrorType(..))

49import System.IO hiding (readFile, writeFile, getContents,

54import System.IO hiding (readFile, writeFile, getContents,

50 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

55 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

51−import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn )

56+import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn, catch )

52import qualified System.IO as IO

57import qualified System.IO as IO

54readFile :: FilePath -> IO String

59readFile :: FilePath -> IO String

@@ -75,7 +80,14 @@ hPutStrLn :: Handle -> String -> IO ()

75hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

80hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

77hGetContents :: Handle -> IO String

82hGetContents :: Handle -> IO String

83+hGetContents h = do

78−hGetContents h = hSetEncoding h utf8_bom >> IO.hGetContents h

84+ hSetEncoding h utf8_bom

85+ catch (IO.hGetContents h) $ \e ->

86+ case ioe_type e of

87+ InvalidArgument -> do

88+ hSetEncoding h localeEncoding

89+ IO.hGetContents h

90+ _ -> throwIO e

80encodePath :: FilePath -> FilePath

92encodePath :: FilePath -> FilePath

81decodeArg :: String -> String

93decodeArg :: String -> String

src/Text/Pandoc/Writers/Docx.hs+2−3

@@ -33,8 +33,7 @@ import System.FilePath ( (</>) )

33import qualified Data.ByteString.Lazy as B

34import qualified Data.Map as M

35import Data.ByteString.Lazy.UTF8 ( fromString, toString )

36−import Text.Pandoc.UTF8 as UTF8

37−import System.IO ( stderr )

36+import System.IO ( stderr, hPutStrLn )

38import Codec.Archive.Zip

37import Codec.Archive.Zip

39import Data.Time.Clock.POSIX

38import Data.Time.Clock.POSIX

40import Paths_pandoc ( getDataFileName )

39import Paths_pandoc ( getDataFileName )

@@ -663,7 +662,7 @@ inlineToOpenXML opts (Image alt (src, tit)) = do

663 , mknode "wp:docPr" [("descr",tit),("id","1"),("name","Picture")] ()

662 , mknode "wp:docPr" [("descr",tit),("id","1"),("name","Picture")] ()

664 , graphic ] ]

663 , graphic ] ]

665 else do

664 else do

666− liftIO $ UTF8.hPutStrLn stderr $

665+ liftIO $ hPutStrLn stderr $

667 "Could not find image `" ++ src ++ "', skipping..."

666 "Could not find image `" ++ src ++ "', skipping..."

668 inlinesToOpenXML opts alt

667 inlinesToOpenXML opts alt

669

668

src/pandoc.hs+8−8

@@ -46,7 +46,7 @@ import System.Console.GetOpt

46import Data.Char ( toLower )

47import Data.List ( intercalate, isPrefixOf )

48import System.Directory ( getAppUserDataDirectory, doesFileExist, findExecutable )

49−import System.IO ( stdout )

49+import System.IO ( stdout, hPutStr, hPutStrLn )

50import System.IO.Error ( isDoesNotExistError )

51import qualified Control.Exception as E

52import Control.Exception.Extensible ( throwIO )

@@ -312,7 +312,7 @@ options =

312 (\arg _ -> do

313 templ <- getDefaultTemplate Nothing arg

314 case templ of

315− Right t -> UTF8.hPutStr stdout t

315+ Right t -> hPutStr stdout t

316 Left e -> error $ show e

317 exitWith ExitSuccess)

318 "FORMAT")

@@ -663,7 +663,7 @@ options =

663 (NoArg

664 (\_ -> do

665 prg <- getProgName

666− UTF8.hPutStrLn stdout (prg ++ " " ++ pandocVersion ++ compileInfo ++

666+ hPutStrLn stdout (prg ++ " " ++ pandocVersion ++ compileInfo ++

667 copyrightMessage)

668 exitWith ExitSuccess ))

669 "" -- "Print version"

@@ -672,7 +672,7 @@ options =

672 (NoArg

673 (\_ -> do

674 prg <- getProgName

675− UTF8.hPutStr stdout (usageMessage prg options)

675+ hPutStr stdout (usageMessage prg options)

676 exitWith ExitSuccess ))

677 "" -- "Show help"

678

@@ -827,8 +827,8 @@ main = do

827 } = opts

828

829 when dumpArgs $

830− do UTF8.hPutStrLn stdout outputFile

830+ do hPutStrLn stdout outputFile

831− mapM_ (\arg -> UTF8.hPutStrLn stdout arg) args

831+ mapM_ (\arg -> hPutStrLn stdout arg) args

832 exitWith ExitSuccess

833

834 let sources = if ignoreArgs then [] else args

@@ -1026,8 +1026,8 @@ main = do

1026 writeBinary = B.writeFile (UTF8.encodePath outputFile)

1027

1028 let writerFn :: FilePath -> String -> IO ()

1029− writerFn "-" = UTF8.putStr

1029+ writerFn "-" = putStr

1030− writerFn f = UTF8.writeFile f

1030+ writerFn f = writeFile f

1031

1032 case getWriter writerName' of

1033 Left e -> err 9 e