6ad7ac1 · typeable/pandoc

pandoc.cabal+4−4

@@ -209,8 +209,8 @@ Library

209209 process >= 1 && < 1.2,

210210 directory >= 1 && < 1.3,

211211 bytestring >= 0.9 && < 1.0,

212+ text >= 0.11 && < 0.12,

212213 zip-archive >= 0.1.1.7 && < 0.2,

213− utf8-string >= 0.3 && < 0.4,

214214 old-locale >= 1 && < 1.1,

215215 time >= 1.2 && < 1.5,

216216 HTTP >= 4000.0.5 && < 4000.3,

@@ -289,11 +289,11 @@ Library

289289 Text.Pandoc.Writers.EPUB,

290290 Text.Pandoc.Writers.FB2,

291291 Text.Pandoc.PDF,

292+ Text.Pandoc.UTF8,

292293 Text.Pandoc.Templates,

293294 Text.Pandoc.Biblio,

294295 Text.Pandoc.SelfContained

295296 Other-Modules: Text.Pandoc.XML,

296− Text.Pandoc.UTF8,

297297 Text.Pandoc.MIME,

298298 Text.Pandoc.UUID,

299299 Text.Pandoc.ImageSize,

@@ -319,8 +319,8 @@ Executable pandoc

319319 process >= 1 && < 1.2,

320320 directory >= 1 && < 1.3,

321321 bytestring >= 0.9 && < 1.0,

322+ text >= 0.11 && < 0.12,

322323 zip-archive >= 0.1.1.7 && < 0.2,

323− utf8-string >= 0.3 && < 0.4,

324324 old-locale >= 1 && < 1.1,

325325 time >= 1.2 && < 1.5,

326326 HTTP >= 4000.0.5 && < 4000.3,

@@ -378,8 +378,8 @@ Test-Suite test-pandoc

378378 syb >= 0.1 && < 0.4,

379379 pandoc,

380380 pandoc-types >= 1.10 && < 1.11,

381− utf8-string >= 0.3 && < 0.4,

382381 bytestring >= 0.9 && < 1.0,

382+ text >= 0.11 && < 0.12,

383383 directory >= 1 && < 1.3,

384384 filepath >= 1.1 && < 1.4,

385385 process >= 1 && < 1.2,

src/Text/Pandoc/SelfContained.hs+1−1

@@ -37,7 +37,6 @@ import Network.HTTP

3737import Data.ByteString.Base64

3838import qualified Data.ByteString.Char8 as B

3939import Data.ByteString (ByteString)

40−import Data.ByteString.UTF8 (toString, fromString)

4140import System.FilePath (takeExtension, dropExtension, takeDirectory, (</>))

4241import Data.Char (toLower, isAscii, isAlphaNum)

4342import Codec.Compression.GZip as Gzip

@@ -45,6 +44,7 @@ import qualified Data.ByteString.Lazy as L

4544import Text.Pandoc.Shared (findDataFile, renderTags')

4645import Text.Pandoc.MIME (getMimeType)

4746import System.Directory (doesFileExist)

47+import Text.Pandoc.UTF8 (toString, fromString)

4848

4949getItem :: Maybe FilePath -> String -> IO (ByteString, Maybe String)

5050getItem userdata f =

src/Text/Pandoc/Templates.hs+3−2

@@ -78,7 +78,8 @@ import Text.Blaze.Internal (preEscapedString)

7878#else

7979import Text.Blaze (preEscapedString, Html)

8080#endif

81−import Data.ByteString.Lazy.UTF8 (ByteString, fromString)

81+import Text.Pandoc.UTF8 (fromStringLazy)

82+import Data.ByteString.Lazy (ByteString)

8283import Text.Pandoc.Shared (readDataFile)

8384import qualified Control.Exception.Extensible as E (try, IOException)

8485

@@ -118,7 +119,7 @@ instance TemplateTarget String where

118119 toTarget = id

119120

120121instance TemplateTarget ByteString where

121− toTarget = fromString

122+ toTarget = fromStringLazy

122123

123124instance TemplateTarget Html where

124125 toTarget = preEscapedString

src/Text/Pandoc/UTF8.hs+27−3

@@ -35,6 +35,10 @@ module Text.Pandoc.UTF8 ( readFile

3535 , hPutStr

3636 , hPutStrLn

3737 , hGetContents

38+ , toString

39+ , fromString

40+ , toStringLazy

41+ , fromStringLazy

3842 , encodePath

3943 , decodeArg

4044 )

@@ -50,6 +54,13 @@ import System.IO hiding (readFile, writeFile, getContents,

5054 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

5155import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn )

5256import qualified System.IO as IO

57+import qualified Data.ByteString.Char8 as B

58+import qualified Data.ByteString.Lazy as BL

59+import qualified Data.Text.Encoding as T

60+import qualified Data.Text as T

61+import qualified Data.Text.Lazy as TL

62+import qualified Data.Text.Lazy.Encoding as TL

63+import Data.Text.Encoding.Error

5364

5465readFile :: FilePath -> IO String

5566readFile f = do

@@ -75,15 +86,28 @@ hPutStrLn :: Handle -> String -> IO ()

7586hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

7687

7788hGetContents :: Handle -> IO String

78−hGetContents h = hSetEncoding h utf8_bom >> hSetNewlineMode h universalNewlineMode

89+hGetContents h = hSetEncoding h utf8_bom

90+ >> hSetNewlineMode h universalNewlineMode

7991 >> IO.hGetContents h

8092

93+toString :: B.ByteString -> String

94+toString = T.unpack . T.decodeUtf8With lenientDecode

95+

96+fromString :: String -> B.ByteString

97+fromString = T.encodeUtf8 . T.pack

98+

99+toStringLazy :: BL.ByteString -> String

100+toStringLazy = TL.unpack . TL.decodeUtf8With lenientDecode

101+

102+fromStringLazy :: String -> BL.ByteString

103+fromStringLazy = TL.encodeUtf8 . TL.pack

104+

81105encodePath :: FilePath -> FilePath

82106decodeArg :: String -> String

83107#if MIN_VERSION_base(4,4,0)

84108encodePath = id

85109decodeArg = id

86110#else

87−encodePath = encodeString

88−decodeArg = decodeString

111+encodePath = B.unpack . fromString

112+decodeArg = toString . B.pack

89113#endif

src/Text/Pandoc/Writers/Docx.hs+10−11

@@ -32,8 +32,7 @@ import Data.List ( intercalate )

3232import System.FilePath ( (</>) )

3333import qualified Data.ByteString.Lazy as B

3434import qualified Data.Map as M

35−import Data.ByteString.Lazy.UTF8 ( fromString, toString )

36−import Text.Pandoc.UTF8 as UTF8

35+import qualified Text.Pandoc.UTF8 as UTF8

3736import System.IO ( stderr )

3837import Codec.Archive.Zip

3938import Data.Time.Clock.POSIX

@@ -126,7 +125,7 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

126125 let newrels = map toImgRel imgs

127126 let relpath = "word/_rels/document.xml.rels"

128127 let reldoc = case findEntryByPath relpath refArchive >>=

129− parseXMLDoc . toString . fromEntry of

128+ parseXMLDoc . UTF8.toStringLazy . fromEntry of

130129 Just d -> d

131130 Nothing -> error $ relpath ++ "missing in reference docx"

132131 let reldoc' = reldoc{ elContent = elContent reldoc ++ map Elem newrels }

@@ -138,21 +137,21 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

138137 let toLinkRel (src,ident) = mknode "Relationship" [("Type","http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"),("Id",ident),("Target",src),("TargetMode","External") ] ()

139138 let newrels' = map toLinkRel $ M.toList $ stExternalLinks st

140139 let reldoc'' = reldoc' { elContent = elContent reldoc' ++ map Elem newrels' }

141− let relEntry = toEntry relpath epochtime $ fromString $ showTopElement' reldoc''

142− let contentEntry = toEntry "word/document.xml" epochtime $ fromString $ showTopElement' newContents

140+ let relEntry = toEntry relpath epochtime $ UTF8.fromStringLazy $ showTopElement' reldoc''

141+ let contentEntry = toEntry "word/document.xml" epochtime $ UTF8.fromStringLazy $ showTopElement' newContents

143142 -- styles

144143 let newstyles = styleToOpenXml $ writerHighlightStyle opts

145144 let stylepath = "word/styles.xml"

146145 let styledoc = case findEntryByPath stylepath refArchive >>=

147− parseXMLDoc . toString . fromEntry of

146+ parseXMLDoc . UTF8.toStringLazy . fromEntry of

148147 Just d -> d

149148 Nothing -> error $ "Unable to parse " ++ stylepath ++

150149 " from reference.docx"

151150 let styledoc' = styledoc{ elContent = elContent styledoc ++ map Elem newstyles }

152− let styleEntry = toEntry stylepath epochtime $ fromString $ showTopElement' styledoc'

151+ let styleEntry = toEntry stylepath epochtime $ UTF8.fromStringLazy $ showTopElement' styledoc'

153152 -- construct word/numbering.xml

154153 let numpath = "word/numbering.xml"

155− let numEntry = toEntry numpath epochtime $ fromString $ showTopElement'

154+ let numEntry = toEntry numpath epochtime $ UTF8.fromStringLazy $ showTopElement'

156155 $ mkNumbering (stNumStyles st) (stLists st)

157156 let docPropsPath = "docProps/core.xml"

158157 let docProps = mknode "cp:coreProperties"

@@ -166,16 +165,16 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

166165 (maybe "" id $ normalizeDate $ stringify date)

167166 : mknode "dcterms:modified" [("xsi:type","dcterms:W3CDTF")] () -- put current time here

168167 : map (mknode "dc:creator" [] . stringify) auths

169− let docPropsEntry = toEntry docPropsPath epochtime $ fromString $ showTopElement' docProps

168+ let docPropsEntry = toEntry docPropsPath epochtime $ UTF8.fromStringLazy $ showTopElement' docProps

170169 let relsPath = "_rels/.rels"

171170 rels <- case findEntryByPath relsPath refArchive of

172− Just e -> return $ toString $ fromEntry e

171+ Just e -> return $ UTF8.toStringLazy $ fromEntry e

173172 Nothing -> err 57 "could not find .rels/_rels in reference docx"

174173 -- fix .rels/_rels, which can get screwed up when reference.docx is edited by Word

175174 let rels' = substitute "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties"

176175 "http://schemas.openxmlformats.org/officedocument/2006/relationships/metadata/core-properties"

177176 rels

178− let relsEntry = toEntry relsPath epochtime $ fromString rels'

177+ let relsEntry = toEntry relsPath epochtime $ UTF8.fromStringLazy rels'

179178 let archive = foldr addEntryToArchive refArchive $

180179 relsEntry : contentEntry : relEntry : numEntry : styleEntry : docPropsEntry : imageEntries

181180 return $ fromArchive archive

src/Text/Pandoc/Writers/EPUB.hs+10−10

@@ -35,7 +35,7 @@ import System.Environment ( getEnv )

3535import Text.Printf (printf)

3636import System.FilePath ( (</>), (<.>), takeBaseName, takeExtension, takeFileName )

3737import qualified Data.ByteString.Lazy as B

38−import Data.ByteString.Lazy.UTF8 ( fromString )

38+import Text.Pandoc.UTF8 ( fromStringLazy )

3939import Codec.Archive.Zip

4040import Data.Time.Clock.POSIX

4141import Text.Pandoc.Shared hiding ( Element )

@@ -82,7 +82,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

8282 Nothing -> return ([],[])

8383 Just img -> do

8484 let coverImage = "cover-image" ++ takeExtension img

85− let cpContent = fromString $ writeHtmlString

85+ let cpContent = fromStringLazy $ writeHtmlString

8686 opts'{writerTemplate = coverImageTemplate,

8787 writerVariables = ("coverimage",coverImage):vars}

8888 (Pandoc meta [])

@@ -91,7 +91,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

9191 , [mkEntry coverImage imgContent] )

9292

9393 -- title page

94− let tpContent = fromString $ writeHtmlString

94+ let tpContent = fromStringLazy $ writeHtmlString

9595 opts'{writerTemplate = titlePageTemplate}

9696 (Pandoc meta [])

9797 let tpEntry = mkEntry "title_page.xhtml" tpContent

@@ -125,7 +125,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

125125 let chapterToEntry :: Int -> Pandoc -> Entry

126126 chapterToEntry num chap = mkEntry

127127 (showChapter num) $

128− fromString $ chapToHtml chap

128+ fromStringLazy $ chapToHtml chap

129129 let chapterEntries = zipWith chapterToEntry [1..] chapters

130130

131131 -- contents.opf

@@ -157,7 +157,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

157157 let plainTitle = plainify $ docTitle meta

158158 let plainAuthors = map plainify $ docAuthors meta

159159 let plainDate = maybe "" id $ normalizeDate $ stringify $ docDate meta

160− let contentsData = fromString $ ppTopElement $

160+ let contentsData = fromStringLazy $ ppTopElement $

161161 unode "package" ! [("version","2.0")

162162 ,("xmlns","http://www.idpf.org/2007/opf")

163163 ,("unique-identifier","BookId")] $

@@ -189,7 +189,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

189189 , unode "content" ! [("src",

190190 eRelativePath ent)] $ ()

191191 ]

192− let tocData = fromString $ ppTopElement $

192+ let tocData = fromStringLazy $ ppTopElement $

193193 unode "ncx" ! [("version","2005-1")

194194 ,("xmlns","http://www.daisy.org/z3986/2005/ncx/")] $

195195 [ unode "head" $

@@ -214,10 +214,10 @@ writeEPUB opts doc@(Pandoc meta _) = do

214214 let tocEntry = mkEntry "toc.ncx" tocData

215215

216216 -- mimetype

217− let mimetypeEntry = mkEntry "mimetype" $ fromString "application/epub+zip"

217+ let mimetypeEntry = mkEntry "mimetype" $ fromStringLazy "application/epub+zip"

218218

219219 -- container.xml

220− let containerData = fromString $ ppTopElement $

220+ let containerData = fromStringLazy $ ppTopElement $

221221 unode "container" ! [("version","1.0")

222222 ,("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")] $

223223 unode "rootfiles" $

@@ -226,7 +226,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

226226 let containerEntry = mkEntry "META-INF/container.xml" containerData

227227

228228 -- com.apple.ibooks.display-options.xml

229− let apple = fromString $ ppTopElement $

229+ let apple = fromStringLazy $ ppTopElement $

230230 unode "display_options" $

231231 unode "platform" ! [("name","*")] $

232232 unode "option" ! [("name","specified-fonts")] $ "true"

@@ -236,7 +236,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

236236 stylesheet <- case writerEpubStylesheet opts of

237237 Just s -> return s

238238 Nothing -> readDataFile (writerUserDataDir opts) "epub.css"

239− let stylesheetEntry = mkEntry "stylesheet.css" $ fromString stylesheet

239+ let stylesheetEntry = mkEntry "stylesheet.css" $ fromStringLazy stylesheet

240240

241241 -- construct archive

242242 let archive = foldr addEntryToArchive emptyArchive

src/Text/Pandoc/Writers/ODT.hs+4−4

@@ -32,7 +32,7 @@ import Data.IORef

3232import Data.List ( isPrefixOf )

3333import System.FilePath ( (</>), takeExtension )

3434import qualified Data.ByteString.Lazy as B

35−import Data.ByteString.Lazy.UTF8 ( fromString )

35+import Text.Pandoc.UTF8 ( fromStringLazy )

3636import Codec.Archive.Zip

3737import Data.Time.Clock.POSIX

3838import Paths_pandoc ( getDataFileName )

@@ -74,7 +74,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

7474 doc' <- bottomUpM (transformPic sourceDir picEntriesRef) doc

7575 let newContents = writeOpenDocument opts{writerWrapText = False} doc'

7676 epochtime <- floor `fmap` getPOSIXTime

77− let contentEntry = toEntry "content.xml" epochtime $ fromString newContents

77+ let contentEntry = toEntry "content.xml" epochtime $ fromStringLazy newContents

7878 picEntries <- readIORef picEntriesRef

7979 let archive = foldr addEntryToArchive refArchive $ contentEntry : picEntries

8080 -- construct META-INF/manifest.xml based on archive

@@ -86,7 +86,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

8686 ]

8787 let files = [ ent | ent <- filesInArchive archive, not ("META-INF" `isPrefixOf` ent) ]

8888 let manifestEntry = toEntry "META-INF/manifest.xml" epochtime

89− $ fromString $ show

89+ $ fromStringLazy $ show

9090 $ text "<?xml version=\"1.0\" encoding=\"utf-8\"?>"

9191 $$

9292 ( inTags True "manifest:manifest"

@@ -100,7 +100,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

100100 )

101101 let archive' = addEntryToArchive manifestEntry archive

102102 let metaEntry = toEntry "meta.xml" epochtime

103− $ fromString $ show

103+ $ fromStringLazy $ show

104104 $ text "<?xml version=\"1.0\" encoding=\"utf-8\"?>"

105105 $$

106106 ( inTags True "office:document-meta"

src/pandoc.hs+2−3

@@ -57,7 +57,6 @@ import Control.Monad (when, unless, liftM)

5757import Network.HTTP (simpleHTTP, mkRequest, getResponseBody, RequestMethod(..))

5858import Network.URI (parseURI, isURI, URI(..))

5959import qualified Data.ByteString.Lazy as B

60−import Data.ByteString.Lazy.UTF8 (toString)

6160import Text.CSL.Reference (Reference(..))

6261

6362copyrightMessage :: String

@@ -988,7 +987,7 @@ main = do

988987 readURI u

989988 _ -> UTF8.readFile src

990989 readURI uri = simpleHTTP (mkRequest GET uri) >>= getResponseBody >>=

991− return . toString -- treat all as UTF8

990+ return . UTF8.toStringLazy -- treat all as UTF8

992991

993992 let convertTabs = tabFilter (if preserveTabs then 0 else tabStop)

994993

@@ -1038,7 +1037,7 @@ main = do

10381037 res <- tex2pdf latexEngine $ f writerOptions doc2

10391038 case res of

10401039 Right pdf -> writeBinary pdf

1041− Left err' -> err 43 $ toString err'

1040+ Left err' -> err 43 $ UTF8.toStringLazy err'

10421041 | otherwise -> selfcontain (f writerOptions doc2 ++

10431042 ['\n' | not standalone'])

10441043 >>= writerFn outputFile . handleEntities

tests/Tests/Old.hs+2−2

@@ -16,11 +16,11 @@ import Text.Pandoc.Writers.Native ( writeNative )

1616import Text.Pandoc.Readers.Native ( readNative )

1717import Prelude hiding ( readFile )

1818import qualified Data.ByteString.Lazy as B

19−import Data.ByteString.Lazy.UTF8 (toString)

19+import Text.Pandoc.UTF8 (toStringLazy)

2020import Text.Printf

2121

2222readFileUTF8 :: FilePath -> IO String

23−readFileUTF8 f = B.readFile f >>= return . toString

23+readFileUTF8 f = B.readFile f >>= return . toStringLazy

2424

2525pandocPath :: FilePath

2626pandocPath = ".." </> "dist" </> "build" </> "pandoc" </> "pandoc"