7272735 · typeable/pandoc

git.iko.soy/typeable/pandoc

More intelligent handling of text encodings.

Previously, UTF-8 was enforced for both input and output. The new system: * For input, UTF-8 is tried first; if an error is raised, the locale encoding is tried. * For output, the locale encoding is always used.

commit7272735b3d413a644fd9ab01eeae8ae9cd5a925b authorJohn MacFarlane <fiddlosopher@gmail.com> dateSun 23 Sep 2012, 22:12 (14 years ago) parent3110774

5 files changed+30−20

compare with master

unifiedsplit

src/Text/Pandoc/Parsing.hs+1−2 src/Text/Pandoc/Shared.hs+3−3 src/Text/Pandoc/UTF8.hs+16−4 src/Text/Pandoc/Writers/Docx.hs+2−3 src/pandoc.hs+8−8

src/Text/Pandoc/Parsing.hs+1−2

@@ -146,7 +146,6 @@ where

146146import Text.Pandoc.Definition

147147import Text.Pandoc.Options

148148import Text.Pandoc.Builder (Blocks)

149−import qualified Text.Pandoc.UTF8 as UTF8 (putStrLn)

150149import Text.Parsec

151150import Text.Parsec.Pos (newPos)

152151import Data.Char ( toLower, toUpper, ord, isAscii, isAlphaNum, isDigit, isPunctuation )

@@ -708,7 +707,7 @@ readWith parser state input =

708707testStringWith :: (Show a) => Parser [Char] ParserState a

709708 -> String

710709 -> IO ()

711−testStringWith parser str = UTF8.putStrLn $ show $

710+testStringWith parser str = putStrLn $ show $

712711 readWith parser defaultParserState str

713712

714713-- | Parsing options.

src/Text/Pandoc/Shared.hs+3−3

@@ -90,7 +90,7 @@ import Paths_pandoc (getDataFileName)

9090import Text.Pandoc.Pretty (charWidth)

9191import System.Locale (defaultTimeLocale)

9292import Data.Time

93−import System.IO (stderr)

93+import System.IO (stderr, hPutStrLn)

9494import Text.HTML.TagSoup (renderTagsOptions, RenderOptions(..), Tag(..),

9595 renderOptions)

9696

@@ -503,14 +503,14 @@ readDataFile userDir fname = findDataFile userDir fname >>= UTF8.readFile

503503err :: Int -> String -> IO a

504504err exitCode msg = do

505505 name <- getProgName

506− UTF8.hPutStrLn stderr $ name ++ ": " ++ msg

506+ hPutStrLn stderr $ name ++ ": " ++ msg

507507 exitWith $ ExitFailure exitCode

508508 return undefined

509509

510510warn :: String -> IO ()

511511warn msg = do

512512 name <- getProgName

513− UTF8.hPutStrLn stderr $ name ++ ": " ++ msg

513+ hPutStrLn stderr $ name ++ ": " ++ msg

514514

515515--

516516-- Safe read

src/Text/Pandoc/UTF8.hs+16−4

@@ -25,7 +25,11 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

2525 Stability : alpha

2626 Portability : portable

2727

28−UTF-8 aware string IO functions that will work with GHC 6.10, 6.12, or 7.

28+UTF-8 aware string IO functions that will work with GHC 6.12 or 7.

29+The reading functions first attempt to read UTF-8; if an encoding

30+error is encountered, the local encoding is used instead. This

31+should work well in practice because text in other encodings

32+is usually not valid UTF-8.

2933-}

3034module Text.Pandoc.UTF8 ( readFile

3135 , writeFile

@@ -45,10 +49,11 @@ where

4549#else

4650import Codec.Binary.UTF8.String (encodeString, decodeString)

4751#endif

48−

52+import Control.Exception (catch, throwIO)

53+import GHC.IO.Exception (IOException(..), IOErrorType(..))

4954import System.IO hiding (readFile, writeFile, getContents,

5055 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

51−import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn )

56+import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn, catch )

5257import qualified System.IO as IO

5358

5459readFile :: FilePath -> IO String

@@ -75,7 +80,14 @@ hPutStrLn :: Handle -> String -> IO ()

7580hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

7681

7782hGetContents :: Handle -> IO String

78−hGetContents h = hSetEncoding h utf8_bom >> IO.hGetContents h

83+hGetContents h = do

84+ hSetEncoding h utf8_bom

85+ catch (IO.hGetContents h) $ \e ->

86+ case ioe_type e of

87+ InvalidArgument -> do

88+ hSetEncoding h localeEncoding

89+ IO.hGetContents h

90+ _ -> throwIO e

7991

8092encodePath :: FilePath -> FilePath

8193decodeArg :: String -> String

src/Text/Pandoc/Writers/Docx.hs+2−3

@@ -33,8 +33,7 @@ import System.FilePath ( (</>) )

3333import qualified Data.ByteString.Lazy as B

3434import qualified Data.Map as M

3535import Data.ByteString.Lazy.UTF8 ( fromString, toString )

36−import Text.Pandoc.UTF8 as UTF8

37−import System.IO ( stderr )

36+import System.IO ( stderr, hPutStrLn )

3837import Codec.Archive.Zip

3938import Data.Time.Clock.POSIX

4039import Paths_pandoc ( getDataFileName )

@@ -663,7 +662,7 @@ inlineToOpenXML opts (Image alt (src, tit)) = do

663662 , mknode "wp:docPr" [("descr",tit),("id","1"),("name","Picture")] ()

664663 , graphic ] ]

665664 else do

666− liftIO $ UTF8.hPutStrLn stderr $

665+ liftIO $ hPutStrLn stderr $

667666 "Could not find image `" ++ src ++ "', skipping..."

668667 inlinesToOpenXML opts alt

669668

src/pandoc.hs+8−8

@@ -46,7 +46,7 @@ import System.Console.GetOpt

4646import Data.Char ( toLower )

4747import Data.List ( intercalate, isPrefixOf )

4848import System.Directory ( getAppUserDataDirectory, doesFileExist, findExecutable )

49−import System.IO ( stdout )

49+import System.IO ( stdout, hPutStr, hPutStrLn )

5050import System.IO.Error ( isDoesNotExistError )

5151import qualified Control.Exception as E

5252import Control.Exception.Extensible ( throwIO )

@@ -312,7 +312,7 @@ options =

312312 (\arg _ -> do

313313 templ <- getDefaultTemplate Nothing arg

314314 case templ of

315− Right t -> UTF8.hPutStr stdout t

315+ Right t -> hPutStr stdout t

316316 Left e -> error $ show e

317317 exitWith ExitSuccess)

318318 "FORMAT")

@@ -663,7 +663,7 @@ options =

663663 (NoArg

664664 (\_ -> do

665665 prg <- getProgName

666− UTF8.hPutStrLn stdout (prg ++ " " ++ pandocVersion ++ compileInfo ++

666+ hPutStrLn stdout (prg ++ " " ++ pandocVersion ++ compileInfo ++

667667 copyrightMessage)

668668 exitWith ExitSuccess ))

669669 "" -- "Print version"

@@ -672,7 +672,7 @@ options =

672672 (NoArg

673673 (\_ -> do

674674 prg <- getProgName

675− UTF8.hPutStr stdout (usageMessage prg options)

675+ hPutStr stdout (usageMessage prg options)

676676 exitWith ExitSuccess ))

677677 "" -- "Show help"

678678

@@ -827,8 +827,8 @@ main = do

827827 } = opts

828828

829829 when dumpArgs $

830− do UTF8.hPutStrLn stdout outputFile

831− mapM_ (\arg -> UTF8.hPutStrLn stdout arg) args

830+ do hPutStrLn stdout outputFile

831+ mapM_ (\arg -> hPutStrLn stdout arg) args

832832 exitWith ExitSuccess

833833

834834 let sources = if ignoreArgs then [] else args

@@ -1026,8 +1026,8 @@ main = do

10261026 writeBinary = B.writeFile (UTF8.encodePath outputFile)

10271027

10281028 let writerFn :: FilePath -> String -> IO ()

1029− writerFn "-" = UTF8.putStr

1030− writerFn f = UTF8.writeFile f

1029+ writerFn "-" = putStr

1030+ writerFn f = writeFile f

10311031

10321032 case getWriter writerName' of

10331033 Left e -> err 9 e