6ad7ac1 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Removed need for utf8-string package.

6ad7ac1John MacFarlane14 years ago

pandoc.cabal+4−4

compare with master

unifiedsplit

@@ -209,8 +209,8 @@ Library

209 process >= 1 && < 1.2,

210 directory >= 1 && < 1.3,

211 bytestring >= 0.9 && < 1.0,

212+ text >= 0.11 && < 0.12,

212 zip-archive >= 0.1.1.7 && < 0.2,

213 zip-archive >= 0.1.1.7 && < 0.2,

213− utf8-string >= 0.3 && < 0.4,

214 old-locale >= 1 && < 1.1,

215 time >= 1.2 && < 1.5,

216 HTTP >= 4000.0.5 && < 4000.3,

@@ -289,11 +289,11 @@ Library

289 Text.Pandoc.Writers.EPUB,

290 Text.Pandoc.Writers.FB2,

291 Text.Pandoc.PDF,

292+ Text.Pandoc.UTF8,

292 Text.Pandoc.Templates,

293 Text.Pandoc.Templates,

293 Text.Pandoc.Biblio,

294 Text.Pandoc.Biblio,

294 Text.Pandoc.SelfContained

295 Text.Pandoc.SelfContained

295 Other-Modules: Text.Pandoc.XML,

296 Other-Modules: Text.Pandoc.XML,

296− Text.Pandoc.UTF8,

297 Text.Pandoc.MIME,

298 Text.Pandoc.UUID,

299 Text.Pandoc.ImageSize,

@@ -319,8 +319,8 @@ Executable pandoc

319 process >= 1 && < 1.2,

320 directory >= 1 && < 1.3,

321 bytestring >= 0.9 && < 1.0,

322+ text >= 0.11 && < 0.12,

322 zip-archive >= 0.1.1.7 && < 0.2,

323 zip-archive >= 0.1.1.7 && < 0.2,

323− utf8-string >= 0.3 && < 0.4,

324 old-locale >= 1 && < 1.1,

325 time >= 1.2 && < 1.5,

326 HTTP >= 4000.0.5 && < 4000.3,

@@ -378,8 +378,8 @@ Test-Suite test-pandoc

378 syb >= 0.1 && < 0.4,

379 pandoc,

380 pandoc-types >= 1.10 && < 1.11,

381− utf8-string >= 0.3 && < 0.4,

382 bytestring >= 0.9 && < 1.0,

381 bytestring >= 0.9 && < 1.0,

382+ text >= 0.11 && < 0.12,

383 directory >= 1 && < 1.3,

384 filepath >= 1.1 && < 1.4,

385 process >= 1 && < 1.2,

src/Text/Pandoc/SelfContained.hs+1−1

@@ -37,7 +37,6 @@ import Network.HTTP

37import Data.ByteString.Base64

38import qualified Data.ByteString.Char8 as B

39import Data.ByteString (ByteString)

40−import Data.ByteString.UTF8 (toString, fromString)

41import System.FilePath (takeExtension, dropExtension, takeDirectory, (</>))

40import System.FilePath (takeExtension, dropExtension, takeDirectory, (</>))

42import Data.Char (toLower, isAscii, isAlphaNum)

41import Data.Char (toLower, isAscii, isAlphaNum)

43import Codec.Compression.GZip as Gzip

42import Codec.Compression.GZip as Gzip

@@ -45,6 +44,7 @@ import qualified Data.ByteString.Lazy as L

45import Text.Pandoc.Shared (findDataFile, renderTags')

44import Text.Pandoc.Shared (findDataFile, renderTags')

46import Text.Pandoc.MIME (getMimeType)

45import Text.Pandoc.MIME (getMimeType)

47import System.Directory (doesFileExist)

46import System.Directory (doesFileExist)

47+import Text.Pandoc.UTF8 (toString, fromString)

49getItem :: Maybe FilePath -> String -> IO (ByteString, Maybe String)

50getItem userdata f =

src/Text/Pandoc/Templates.hs+3−2

@@ -78,7 +78,8 @@ import Text.Blaze.Internal (preEscapedString)

78#else

79import Text.Blaze (preEscapedString, Html)

80#endif

81+import Text.Pandoc.UTF8 (fromStringLazy)

81−import Data.ByteString.Lazy.UTF8 (ByteString, fromString)

82+import Data.ByteString.Lazy (ByteString)

82import Text.Pandoc.Shared (readDataFile)

83import Text.Pandoc.Shared (readDataFile)

83import qualified Control.Exception.Extensible as E (try, IOException)

84import qualified Control.Exception.Extensible as E (try, IOException)

@@ -118,7 +119,7 @@ instance TemplateTarget String where

118 toTarget = id

119 toTarget = id

119

120

120instance TemplateTarget ByteString where

121instance TemplateTarget ByteString where

121− toTarget = fromString

122+ toTarget = fromStringLazy

122

123

123instance TemplateTarget Html where

124instance TemplateTarget Html where

124 toTarget = preEscapedString

125 toTarget = preEscapedString

src/Text/Pandoc/UTF8.hs+27−3

@@ -35,6 +35,10 @@ module Text.Pandoc.UTF8 ( readFile

35 , hPutStr

36 , hPutStrLn

37 , hGetContents

38+ , toString

39+ , fromString

40+ , toStringLazy

41+ , fromStringLazy

38 , encodePath

42 , encodePath

39 , decodeArg

43 , decodeArg

40 )

44 )

@@ -50,6 +54,13 @@ import System.IO hiding (readFile, writeFile, getContents,

50 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

54 putStr, putStrLn, hPutStr, hPutStrLn, hGetContents)

51import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn )

55import Prelude hiding (readFile, writeFile, getContents, putStr, putStrLn )

52import qualified System.IO as IO

56import qualified System.IO as IO

57+import qualified Data.ByteString.Char8 as B

58+import qualified Data.ByteString.Lazy as BL

59+import qualified Data.Text.Encoding as T

60+import qualified Data.Text as T

61+import qualified Data.Text.Lazy as TL

62+import qualified Data.Text.Lazy.Encoding as TL

63+import Data.Text.Encoding.Error

54readFile :: FilePath -> IO String

65readFile :: FilePath -> IO String

55readFile f = do

66readFile f = do

@@ -75,15 +86,28 @@ hPutStrLn :: Handle -> String -> IO ()

75hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

86hPutStrLn h s = hSetEncoding h utf8 >> IO.hPutStrLn h s

77hGetContents :: Handle -> IO String

88hGetContents :: Handle -> IO String

89+hGetContents h = hSetEncoding h utf8_bom

78−hGetContents h = hSetEncoding h utf8_bom >> hSetNewlineMode h universalNewlineMode

90+ >> hSetNewlineMode h universalNewlineMode

79 >> IO.hGetContents h

91 >> IO.hGetContents h

93+toString :: B.ByteString -> String

94+toString = T.unpack . T.decodeUtf8With lenientDecode

95+

96+fromString :: String -> B.ByteString

97+fromString = T.encodeUtf8 . T.pack

98+

99+toStringLazy :: BL.ByteString -> String

100+toStringLazy = TL.unpack . TL.decodeUtf8With lenientDecode

101+

102+fromStringLazy :: String -> BL.ByteString

103+fromStringLazy = TL.encodeUtf8 . TL.pack

104+

81encodePath :: FilePath -> FilePath

105encodePath :: FilePath -> FilePath

82decodeArg :: String -> String

106decodeArg :: String -> String

83#if MIN_VERSION_base(4,4,0)

107#if MIN_VERSION_base(4,4,0)

84encodePath = id

108encodePath = id

85decodeArg = id

109decodeArg = id

86#else

110#else

87−encodePath = encodeString

111+encodePath = B.unpack . fromString

88−decodeArg = decodeString

112+decodeArg = toString . B.pack

89#endif

113#endif

src/Text/Pandoc/Writers/Docx.hs+10−11

@@ -32,8 +32,7 @@ import Data.List ( intercalate )

32import System.FilePath ( (</>) )

33import qualified Data.ByteString.Lazy as B

34import qualified Data.Map as M

35−import Data.ByteString.Lazy.UTF8 ( fromString, toString )

36−import Text.Pandoc.UTF8 as UTF8

35+import qualified Text.Pandoc.UTF8 as UTF8

37import System.IO ( stderr )

36import System.IO ( stderr )

38import Codec.Archive.Zip

37import Codec.Archive.Zip

39import Data.Time.Clock.POSIX

38import Data.Time.Clock.POSIX

@@ -126,7 +125,7 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

126 let newrels = map toImgRel imgs

125 let newrels = map toImgRel imgs

127 let relpath = "word/_rels/document.xml.rels"

126 let relpath = "word/_rels/document.xml.rels"

128 let reldoc = case findEntryByPath relpath refArchive >>=

127 let reldoc = case findEntryByPath relpath refArchive >>=

129− parseXMLDoc . toString . fromEntry of

128+ parseXMLDoc . UTF8.toStringLazy . fromEntry of

130 Just d -> d

129 Just d -> d

131 Nothing -> error $ relpath ++ "missing in reference docx"

130 Nothing -> error $ relpath ++ "missing in reference docx"

132 let reldoc' = reldoc{ elContent = elContent reldoc ++ map Elem newrels }

131 let reldoc' = reldoc{ elContent = elContent reldoc ++ map Elem newrels }

@@ -138,21 +137,21 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

138 let toLinkRel (src,ident) = mknode "Relationship" [("Type","http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"),("Id",ident),("Target",src),("TargetMode","External") ] ()

137 let toLinkRel (src,ident) = mknode "Relationship" [("Type","http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"),("Id",ident),("Target",src),("TargetMode","External") ] ()

139 let newrels' = map toLinkRel $ M.toList $ stExternalLinks st

138 let newrels' = map toLinkRel $ M.toList $ stExternalLinks st

140 let reldoc'' = reldoc' { elContent = elContent reldoc' ++ map Elem newrels' }

139 let reldoc'' = reldoc' { elContent = elContent reldoc' ++ map Elem newrels' }

141− let relEntry = toEntry relpath epochtime $ fromString $ showTopElement' reldoc''

140+ let relEntry = toEntry relpath epochtime $ UTF8.fromStringLazy $ showTopElement' reldoc''

142− let contentEntry = toEntry "word/document.xml" epochtime $ fromString $ showTopElement' newContents

141+ let contentEntry = toEntry "word/document.xml" epochtime $ UTF8.fromStringLazy $ showTopElement' newContents

143 -- styles

142 -- styles

144 let newstyles = styleToOpenXml $ writerHighlightStyle opts

143 let newstyles = styleToOpenXml $ writerHighlightStyle opts

145 let stylepath = "word/styles.xml"

144 let stylepath = "word/styles.xml"

146 let styledoc = case findEntryByPath stylepath refArchive >>=

145 let styledoc = case findEntryByPath stylepath refArchive >>=

147− parseXMLDoc . toString . fromEntry of

146+ parseXMLDoc . UTF8.toStringLazy . fromEntry of

148 Just d -> d

147 Just d -> d

149 Nothing -> error $ "Unable to parse " ++ stylepath ++

148 Nothing -> error $ "Unable to parse " ++ stylepath ++

150 " from reference.docx"

149 " from reference.docx"

151 let styledoc' = styledoc{ elContent = elContent styledoc ++ map Elem newstyles }

150 let styledoc' = styledoc{ elContent = elContent styledoc ++ map Elem newstyles }

152− let styleEntry = toEntry stylepath epochtime $ fromString $ showTopElement' styledoc'

151+ let styleEntry = toEntry stylepath epochtime $ UTF8.fromStringLazy $ showTopElement' styledoc'

153 -- construct word/numbering.xml

152 -- construct word/numbering.xml

154 let numpath = "word/numbering.xml"

153 let numpath = "word/numbering.xml"

155− let numEntry = toEntry numpath epochtime $ fromString $ showTopElement'

154+ let numEntry = toEntry numpath epochtime $ UTF8.fromStringLazy $ showTopElement'

156 $ mkNumbering (stNumStyles st) (stLists st)

155 $ mkNumbering (stNumStyles st) (stLists st)

157 let docPropsPath = "docProps/core.xml"

156 let docPropsPath = "docProps/core.xml"

158 let docProps = mknode "cp:coreProperties"

157 let docProps = mknode "cp:coreProperties"

@@ -166,16 +165,16 @@ writeDocx opts doc@(Pandoc (Meta tit auths date) _) = do

166 (maybe "" id $ normalizeDate $ stringify date)

165 (maybe "" id $ normalizeDate $ stringify date)

167 : mknode "dcterms:modified" [("xsi:type","dcterms:W3CDTF")] () -- put current time here

166 : mknode "dcterms:modified" [("xsi:type","dcterms:W3CDTF")] () -- put current time here

168 : map (mknode "dc:creator" [] . stringify) auths

167 : map (mknode "dc:creator" [] . stringify) auths

169− let docPropsEntry = toEntry docPropsPath epochtime $ fromString $ showTopElement' docProps

168+ let docPropsEntry = toEntry docPropsPath epochtime $ UTF8.fromStringLazy $ showTopElement' docProps

170 let relsPath = "_rels/.rels"

169 let relsPath = "_rels/.rels"

171 rels <- case findEntryByPath relsPath refArchive of

170 rels <- case findEntryByPath relsPath refArchive of

172− Just e -> return $ toString $ fromEntry e

171+ Just e -> return $ UTF8.toStringLazy $ fromEntry e

173 Nothing -> err 57 "could not find .rels/_rels in reference docx"

172 Nothing -> err 57 "could not find .rels/_rels in reference docx"

174 -- fix .rels/_rels, which can get screwed up when reference.docx is edited by Word

173 -- fix .rels/_rels, which can get screwed up when reference.docx is edited by Word

175 let rels' = substitute "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties"

174 let rels' = substitute "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties"

176 "http://schemas.openxmlformats.org/officedocument/2006/relationships/metadata/core-properties"

175 "http://schemas.openxmlformats.org/officedocument/2006/relationships/metadata/core-properties"

177 rels

176 rels

178− let relsEntry = toEntry relsPath epochtime $ fromString rels'

177+ let relsEntry = toEntry relsPath epochtime $ UTF8.fromStringLazy rels'

179 let archive = foldr addEntryToArchive refArchive $

178 let archive = foldr addEntryToArchive refArchive $

180 relsEntry : contentEntry : relEntry : numEntry : styleEntry : docPropsEntry : imageEntries

179 relsEntry : contentEntry : relEntry : numEntry : styleEntry : docPropsEntry : imageEntries

181 return $ fromArchive archive

180 return $ fromArchive archive

src/Text/Pandoc/Writers/EPUB.hs+10−10

@@ -35,7 +35,7 @@ import System.Environment ( getEnv )

35import Text.Printf (printf)

36import System.FilePath ( (</>), (<.>), takeBaseName, takeExtension, takeFileName )

37import qualified Data.ByteString.Lazy as B

38−import Data.ByteString.Lazy.UTF8 ( fromString )

38+import Text.Pandoc.UTF8 ( fromStringLazy )

39import Codec.Archive.Zip

40import Data.Time.Clock.POSIX

41import Text.Pandoc.Shared hiding ( Element )

@@ -82,7 +82,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

82 Nothing -> return ([],[])

83 Just img -> do

84 let coverImage = "cover-image" ++ takeExtension img

85− let cpContent = fromString $ writeHtmlString

85+ let cpContent = fromStringLazy $ writeHtmlString

86 opts'{writerTemplate = coverImageTemplate,

87 writerVariables = ("coverimage",coverImage):vars}

88 (Pandoc meta [])

@@ -91,7 +91,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

91 , [mkEntry coverImage imgContent] )

93 -- title page

94− let tpContent = fromString $ writeHtmlString

94+ let tpContent = fromStringLazy $ writeHtmlString

95 opts'{writerTemplate = titlePageTemplate}

96 (Pandoc meta [])

97 let tpEntry = mkEntry "title_page.xhtml" tpContent

@@ -125,7 +125,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

125 let chapterToEntry :: Int -> Pandoc -> Entry

126 chapterToEntry num chap = mkEntry

127 (showChapter num) $

128− fromString $ chapToHtml chap

128+ fromStringLazy $ chapToHtml chap

129 let chapterEntries = zipWith chapterToEntry [1..] chapters

130

131 -- contents.opf

@@ -157,7 +157,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

157 let plainTitle = plainify $ docTitle meta

158 let plainAuthors = map plainify $ docAuthors meta

159 let plainDate = maybe "" id $ normalizeDate $ stringify $ docDate meta

160− let contentsData = fromString $ ppTopElement $

160+ let contentsData = fromStringLazy $ ppTopElement $

161 unode "package" ! [("version","2.0")

162 ,("xmlns","http://www.idpf.org/2007/opf")

163 ,("unique-identifier","BookId")] $

@@ -189,7 +189,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

189 , unode "content" ! [("src",

190 eRelativePath ent)] $ ()

191 ]

192− let tocData = fromString $ ppTopElement $

192+ let tocData = fromStringLazy $ ppTopElement $

193 unode "ncx" ! [("version","2005-1")

194 ,("xmlns","http://www.daisy.org/z3986/2005/ncx/")] $

195 [ unode "head" $

@@ -214,10 +214,10 @@ writeEPUB opts doc@(Pandoc meta _) = do

214 let tocEntry = mkEntry "toc.ncx" tocData

215

216 -- mimetype

217− let mimetypeEntry = mkEntry "mimetype" $ fromString "application/epub+zip"

217+ let mimetypeEntry = mkEntry "mimetype" $ fromStringLazy "application/epub+zip"

218

219 -- container.xml

220− let containerData = fromString $ ppTopElement $

220+ let containerData = fromStringLazy $ ppTopElement $

221 unode "container" ! [("version","1.0")

222 ,("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")] $

223 unode "rootfiles" $

@@ -226,7 +226,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

226 let containerEntry = mkEntry "META-INF/container.xml" containerData

227

228 -- com.apple.ibooks.display-options.xml

229− let apple = fromString $ ppTopElement $

229+ let apple = fromStringLazy $ ppTopElement $

230 unode "display_options" $

231 unode "platform" ! [("name","*")] $

232 unode "option" ! [("name","specified-fonts")] $ "true"

@@ -236,7 +236,7 @@ writeEPUB opts doc@(Pandoc meta _) = do

236 stylesheet <- case writerEpubStylesheet opts of

237 Just s -> return s

238 Nothing -> readDataFile (writerUserDataDir opts) "epub.css"

239− let stylesheetEntry = mkEntry "stylesheet.css" $ fromString stylesheet

239+ let stylesheetEntry = mkEntry "stylesheet.css" $ fromStringLazy stylesheet

240

241 -- construct archive

242 let archive = foldr addEntryToArchive emptyArchive

src/Text/Pandoc/Writers/ODT.hs+4−4

@@ -32,7 +32,7 @@ import Data.IORef

32import Data.List ( isPrefixOf )

33import System.FilePath ( (</>), takeExtension )

34import qualified Data.ByteString.Lazy as B

35−import Data.ByteString.Lazy.UTF8 ( fromString )

35+import Text.Pandoc.UTF8 ( fromStringLazy )

36import Codec.Archive.Zip

37import Data.Time.Clock.POSIX

38import Paths_pandoc ( getDataFileName )

@@ -74,7 +74,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

74 doc' <- bottomUpM (transformPic sourceDir picEntriesRef) doc

75 let newContents = writeOpenDocument opts{writerWrapText = False} doc'

76 epochtime <- floor `fmap` getPOSIXTime

77− let contentEntry = toEntry "content.xml" epochtime $ fromString newContents

77+ let contentEntry = toEntry "content.xml" epochtime $ fromStringLazy newContents

78 picEntries <- readIORef picEntriesRef

79 let archive = foldr addEntryToArchive refArchive $ contentEntry : picEntries

80 -- construct META-INF/manifest.xml based on archive

@@ -86,7 +86,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

86 ]

87 let files = [ ent | ent <- filesInArchive archive, not ("META-INF" `isPrefixOf` ent) ]

88 let manifestEntry = toEntry "META-INF/manifest.xml" epochtime

89− $ fromString $ show

89+ $ fromStringLazy $ show

90 $ text "<?xml version=\"1.0\" encoding=\"utf-8\"?>"

91 $$

92 ( inTags True "manifest:manifest"

@@ -100,7 +100,7 @@ writeODT opts doc@(Pandoc (Meta title _ _) _) = do

100 )

101 let archive' = addEntryToArchive manifestEntry archive

102 let metaEntry = toEntry "meta.xml" epochtime

103− $ fromString $ show

103+ $ fromStringLazy $ show

104 $ text "<?xml version=\"1.0\" encoding=\"utf-8\"?>"

105 $$

106 ( inTags True "office:document-meta"

src/pandoc.hs+2−3

@@ -57,7 +57,6 @@ import Control.Monad (when, unless, liftM)

57import Network.HTTP (simpleHTTP, mkRequest, getResponseBody, RequestMethod(..))

58import Network.URI (parseURI, isURI, URI(..))

59import qualified Data.ByteString.Lazy as B

60−import Data.ByteString.Lazy.UTF8 (toString)

61import Text.CSL.Reference (Reference(..))

60import Text.CSL.Reference (Reference(..))

63copyrightMessage :: String

62copyrightMessage :: String

@@ -988,7 +987,7 @@ main = do

988 readURI u

987 readURI u

989 _ -> UTF8.readFile src

988 _ -> UTF8.readFile src

990 readURI uri = simpleHTTP (mkRequest GET uri) >>= getResponseBody >>=

989 readURI uri = simpleHTTP (mkRequest GET uri) >>= getResponseBody >>=

991− return . toString -- treat all as UTF8

990+ return . UTF8.toStringLazy -- treat all as UTF8

992

991

993 let convertTabs = tabFilter (if preserveTabs then 0 else tabStop)

992 let convertTabs = tabFilter (if preserveTabs then 0 else tabStop)

994

993

@@ -1038,7 +1037,7 @@ main = do

1038 res <- tex2pdf latexEngine $ f writerOptions doc2

1037 res <- tex2pdf latexEngine $ f writerOptions doc2

1039 case res of

1038 case res of

1040 Right pdf -> writeBinary pdf

1039 Right pdf -> writeBinary pdf

1041− Left err' -> err 43 $ toString err'

1040+ Left err' -> err 43 $ UTF8.toStringLazy err'

1042 | otherwise -> selfcontain (f writerOptions doc2 ++

1041 | otherwise -> selfcontain (f writerOptions doc2 ++

1043 ['\n' | not standalone'])

1042 ['\n' | not standalone'])

1044 >>= writerFn outputFile . handleEntities

1043 >>= writerFn outputFile . handleEntities

tests/Tests/Old.hs+2−2

@@ -16,11 +16,11 @@ import Text.Pandoc.Writers.Native ( writeNative )

16import Text.Pandoc.Readers.Native ( readNative )

17import Prelude hiding ( readFile )

18import qualified Data.ByteString.Lazy as B

19−import Data.ByteString.Lazy.UTF8 (toString)

19+import Text.Pandoc.UTF8 (toStringLazy)

20import Text.Printf

22readFileUTF8 :: FilePath -> IO String

23−readFileUTF8 f = B.readFile f >>= return . toString

23+readFileUTF8 f = B.readFile f >>= return . toStringLazy

25pandocPath :: FilePath

26pandocPath = ".." </> "dist" </> "build" </> "pandoc" </> "pandoc"