90e436d · typeable/pandoc

benchmark/benchmark-pandoc.hs+12−10

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE TupleSections #-}

2+{-# LANGUAGE OverloadedStrings #-}

3+{-# LANGUAGE TupleSections #-}

34{-

56

@@ -24,6 +25,7 @@ import Text.Pandoc.Error (PandocError(..))

2425import Control.Monad.Except (throwError)

2526import qualified Text.Pandoc.UTF8 as UTF8

2627import qualified Data.ByteString as B

28+import qualified Data.Text as T

2729import Criterion.Main

2830import Criterion.Types (Config(..))

2931import Data.List (intersect)

@@ -32,12 +34,12 @@ import System.Environment (getArgs)

3234import qualified Data.ByteString.Lazy as BL

3335

3436readerBench :: Pandoc

35− -> String

37+ -> T.Text

3638 -> Maybe Benchmark

3739readerBench doc name =

3840 case res of

3941 Right (readerFun, inp) ->

40− Just $ bench (name ++ " reader")

42+ Just $ bench (T.unpack $ name <> " reader")

4143 $ nf (\i -> either (error . show) id $ runPure (readerFun i))

4244 inp

4345 Left _ -> Nothing

@@ -51,7 +53,7 @@ readerBench doc name =

5153 , writerExtensions = wexts } doc

5254 return $ (r def{ readerExtensions = rexts }, inp)

5355 _ -> throwError $ PandocSomeError $ "not a text format: "

54− ++ name

56+ <> name

5557

5658getImages :: IO [(FilePath, MimeType, BL.ByteString)]

5759getImages = do

@@ -61,13 +63,13 @@ getImages = do

6163 ,("movie.jpg", "image/jpg", mv)]

6264

6365writerBench :: Pandoc

64− -> String

66+ -> T.Text

6567 -> Maybe Benchmark

6668writerBench doc name =

6769 case res of

6870 Right writerFun ->

6971 Just $ env getImages $ \imgs ->

70− bench (name ++ " writer")

72+ bench (T.unpack $ name <> " writer")

7173 $ nf (\d -> either (error . show) id $

7274 runPure (do mapM_

7375 (\(fp, mt, bs) ->

@@ -81,11 +83,11 @@ writerBench doc name =

8183 TextWriter w ->

8284 return $ w def{ writerExtensions = wexts }

8385 _ -> throwError $ PandocSomeError

84− $ "could not get text writer for " ++ name

86+ $ "could not get text writer for " <> name

8587

8688main :: IO ()

8789main = do

88− args <- filter (\x -> take 1 x /= "-") <$> getArgs

90+ args <- filter (\x -> T.take 1 x /= "-") . fmap T.pack <$> getArgs

8991 print args

9092 let matchReader (n, TextReader _) =

9193 null args || ("reader" `elem` args && n `elem` args)

@@ -94,9 +96,9 @@ main = do

9496 null args || ("writer" `elem` args && n `elem` args)

9597 matchWriter _ = False

9698 let matchedReaders = map fst $ (filter matchReader readers

97− :: [(String, Reader PandocPure)])

99+ :: [(T.Text, Reader PandocPure)])

98100 let matchedWriters = map fst $ (filter matchWriter writers

99− :: [(String, Writer PandocPure)])

101+ :: [(T.Text, Writer PandocPure)])

100102 inp <- UTF8.toText <$> B.readFile "test/testsuite.txt"

101103 let opts = def

102104 let doc = either (error . show) id $ runPure $ readMarkdown opts inp

benchmark/weigh-pandoc.hs+4−3

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Main

@@ -13,7 +14,7 @@ Benchmarks to determine resource use of readers and writers.

1314import Prelude

1415import Weigh

1516import Text.Pandoc

16−import Data.Text (Text)

17+import Data.Text (Text, unpack)

1718

1819main :: IO ()

1920main = do

@@ -40,12 +41,12 @@ main = do

4041weighWriter :: Pandoc -> String -> (Pandoc -> Text) -> Weigh ()

4142weighWriter doc name writer = func (name ++ " writer") writer doc

4243

43−weighReader :: Pandoc -> String -> (Text -> Pandoc) -> Weigh ()

44+weighReader :: Pandoc -> Text -> (Text -> Pandoc) -> Weigh ()

4445weighReader doc name reader = do

4546 case lookup name writers of

4647 Just (TextWriter writer) ->

4748 let inp = either (error . show) id $ runPure $ writer def{ writerWrapText = WrapAuto} doc

48− in func (name ++ " reader") reader inp

49+ in func (unpack $ name <> " reader") reader inp

4950 _ -> return () -- no writer for reader

5051

5152

cabal.project+1−6

@@ -11,9 +11,4 @@ package pandoc-citeproc

1111source-repository-package

1212 type: git

1313 location: https://github.com/jgm/pandoc-citeproc

14− tag: 0.16.3.1

15−

16−source-repository-package

17− type: git

18− location: https://github.com/jgm/pandoc-types

19− tag: 00f7bb79e79d7cfd3523880dbc64ba3ea46c3da2

14+ tag: dc09b028d6876df81cd76b731e58886f77f269b1

pandoc.cabal+3−3

@@ -381,11 +381,11 @@ library

381381 safe >= 0.3 && < 0.4,

382382 zip-archive >= 0.2.3.4 && < 0.5,

383383 HTTP >= 4000.0.5 && < 4000.4,

384− texmath >= 0.11.3 && < 0.12,

384+ texmath >= 0.12 && < 0.13,

385385 xml >= 1.3.12 && < 1.4,

386386 split >= 0.2 && < 0.3,

387387 random >= 1 && < 1.2,

388− pandoc-types >= 1.17.6 && < 1.18,

388+ pandoc-types >= 1.20 && < 1.21,

389389 aeson >= 0.7 && < 1.5,

390390 scientific >= 0.3 && < 0.4,

391391 aeson-pretty >= 0.8.5 && < 0.9,

@@ -705,7 +705,7 @@ test-suite test-pandoc

705705 hs-source-dirs: test

706706 build-depends: base >= 4.8 && < 5,

707707 pandoc,

708− pandoc-types >= 1.17.6 && < 1.18,

708+ pandoc-types >= 1.20 && < 1.21,

709709 mtl >= 2.2 && < 2.3,

710710 bytestring >= 0.9 && < 0.11,

711711 base64-bytestring >= 0.1 && < 1.1,

src/Text/Pandoc/App.hs+21−23

@@ -1,4 +1,5 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{-# LANGUAGE CPP #-}

34{-# LANGUAGE ScopedTypeVariables #-}

45{- |

@@ -58,7 +59,7 @@ import Text.Pandoc.Readers.Markdown (yamlToMeta)

5859import Text.Pandoc.SelfContained (makeDataURI, makeSelfContained)

5960import Text.Pandoc.Shared (eastAsianLineBreakFilter, stripEmptyParagraphs,

6061 headerShift, isURI, tabFilter, uriPathToPath, filterIpynbOutput,

61− defaultUserDataDirs)

62+ defaultUserDataDirs, tshow)

6263import Text.Pandoc.Writers.Shared (lookupMetaString)

6364import qualified Text.Pandoc.UTF8 as UTF8

6465#ifndef _WINDOWS

@@ -66,7 +67,6 @@ import System.Posix.IO (stdOutput)

6667import System.Posix.Terminal (queryTerminal)

6768#endif

6869

69−

7070convertWithOpts :: Opt -> IO ()

7171convertWithOpts opts = do

7272 let outputFile = fromMaybe "-" (optOutputFile opts)

@@ -141,17 +141,17 @@ convertWithOpts opts = do

141141 Nothing -> case formatFromFilePaths sources of

142142 Just f' -> return f'

143143 Nothing | sources == ["-"] -> return "markdown"

144− | any isURI sources -> return "html"

144+ | any (isURI . T.pack) sources -> return "html"

145145 | otherwise -> do

146146 report $ CouldNotDeduceFormat

147− (map takeExtension sources) "markdown"

147+ (map (T.pack . takeExtension) sources) "markdown"

148148 return "markdown"

149149

150150 let pdfOutput = map toLower (takeExtension outputFile) == ".pdf"

151151

152152 when (pdfOutput && readerName == "latex") $

153153 case (optInputFiles opts) of

154− (inputFile:_) -> report $ UnusualConversion $

154+ (inputFile:_) -> report $ UnusualConversion $ T.pack $

155155 "to convert a .tex file to PDF, you get better results by using pdflatex "

156156 <> "(or lualatex or xelatex) directly, try `pdflatex " <> inputFile

157157 <> "` instead of `pandoc " <> inputFile <> " -o " <> outputFile <> "`."

@@ -187,15 +187,15 @@ convertWithOpts opts = do

187187 when ((pdfOutput || not (isTextFormat format)) &&

188188 istty && isNothing ( optOutputFile opts)) $

189189 throwError $ PandocAppError $

190− "Cannot write " ++ format ++ " output to terminal.\n" ++

191− "Specify an output file using the -o option, or " ++

190+ "Cannot write " <> format <> " output to terminal.\n" <>

191+ "Specify an output file using the -o option, or " <>

192192 "use '-o -' to force output to stdout."

193193

194194

195− abbrevs <- Set.fromList . filter (not . null) . lines <$>

195+ abbrevs <- Set.fromList . filter (not . T.null) . T.lines <$>

196196 case optAbbreviations opts of

197− Nothing -> UTF8.toString <$> readDataFile "abbreviations"

198− Just f -> UTF8.toString <$> readFileStrict f

197+ Nothing -> UTF8.toText <$> readDataFile "abbreviations"

198+ Just f -> UTF8.toText <$> readFileStrict f

199199

200200 metadata <- if format == "jats" &&

201201 isNothing (lookupMeta "csl" (optMetadata opts)) &&

@@ -285,7 +285,7 @@ convertWithOpts opts = do

285285 >=> return . adjustMetadata (metadataFromFile <>)

286286 >=> return . adjustMetadata (<> metadata)

287287 >=> applyTransforms transforms

288− >=> applyFilters readerOpts filters' [format]

288+ >=> applyFilters readerOpts filters' [T.unpack format]

289289 >=> maybe return extractMedia (optExtractMedia opts)

290290 )

291291

@@ -298,7 +298,7 @@ convertWithOpts opts = do

298298 case res of

299299 Right pdf -> writeFnBinary outputFile pdf

300300 Left err' -> throwError $ PandocPDFError $

301− TL.unpack (TE.decodeUtf8With TE.lenientDecode err')

301+ TL.toStrict (TE.decodeUtf8With TE.lenientDecode err')

302302

303303 Nothing -> do

304304 let ensureNl t

@@ -308,18 +308,16 @@ convertWithOpts opts = do

308308 output <- ensureNl <$> f writerOptions doc

309309 writerFn eol outputFile =<<

310310 if optSelfContained opts && htmlFormat format

311− -- TODO not maximally efficient; change type

312− -- of makeSelfContained so it works w/ Text

313− then T.pack <$> makeSelfContained (T.unpack output)

311+ then makeSelfContained output

314312 else return output

315313

316314type Transform = Pandoc -> Pandoc

317315

318−htmlFormat :: String -> Bool

316+htmlFormat :: Text -> Bool

319317htmlFormat = (`elem` ["html","html4","html5","s5","slidy",

320318 "slideous","dzslides","revealjs"])

321319

322−isTextFormat :: String -> Bool

320+isTextFormat :: Text -> Bool

323321isTextFormat s = s `notElem` ["odt","docx","epub2","epub3","epub","pptx"]

324322

325323adjustMetadata :: (Meta -> Meta) -> Pandoc -> Pandoc

@@ -335,7 +333,7 @@ readSource src = case parseURI src of

335333 Just u | uriScheme u `elem` ["http:","https:"] ->

336334 readURI src

337335 | uriScheme u == "file:" -> liftIO $

338− readTextFile (uriPathToPath $ uriPath u)

336+ readTextFile (uriPathToPath $ T.pack $ uriPath u)

339337 _ -> liftIO $ readTextFile src

340338 where readTextFile :: FilePath -> IO Text

341339 readTextFile fp = do

@@ -347,12 +345,12 @@ readSource src = case parseURI src of

347345 TSE.DecodeError _ (Just w) -> do

348346 case BS.elemIndex w bs of

349347 Just offset -> E.throwIO $

350− PandocUTF8DecodingError fp offset w

351− _ -> E.throwIO $ PandocUTF8DecodingError fp 0 w

352− _ -> E.throwIO $ PandocAppError (show e))

348+ PandocUTF8DecodingError (T.pack fp) offset w

349+ _ -> E.throwIO $ PandocUTF8DecodingError (T.pack fp) 0 w

350+ _ -> E.throwIO $ PandocAppError (tshow e))

353351

354352readURI :: FilePath -> PandocIO Text

355−readURI src = UTF8.toText . fst <$> openURL src

353+readURI src = UTF8.toText . fst <$> openURL (T.pack src)

356354

357355readFile' :: MonadIO m => FilePath -> m BL.ByteString

358356readFile' "-" = liftIO BL.getContents

src/Text/Pandoc/App/CommandLineOptions.hs+52−49

@@ -2,6 +2,7 @@

22{-# LANGUAGE CPP #-}

33{-# LANGUAGE ScopedTypeVariables #-}

44{-# LANGUAGE TupleSections #-}

5+{-# LANGUAGE OverloadedStrings #-}

56{- |

67 Module : Text.Pandoc.App.CommandLineOptions

@@ -45,7 +46,7 @@ import Text.Pandoc.Builder (setMeta)

4546import Text.Pandoc.App.Opt (Opt (..), LineEnding (..), IpynbOutput (..))

4647import Text.Pandoc.Filter (Filter (..))

4748import Text.Pandoc.Highlighting (highlightingStyles)

48−import Text.Pandoc.Shared (ordNub, safeRead, defaultUserDataDirs)

49+import Text.Pandoc.Shared (ordNub, elemText, safeStrRead, defaultUserDataDirs)

4950import Text.Printf

5051

5152#ifdef EMBED_DATA_FILES

@@ -78,7 +79,7 @@ parseOptions options' defaults = do

7879 unrecognizedOpts

7980

8081 unless (null errors && null unknownOptionErrors) $

81− E.throwIO $ PandocOptionError $

82+ E.throwIO $ PandocOptionError $ T.pack $

8283 concat errors ++ unlines unknownOptionErrors ++

8384 ("Try " ++ prg ++ " --help for more information.")

8485

@@ -92,7 +93,7 @@ latexEngines = ["pdflatex", "lualatex", "xelatex", "latexmk", "tectonic"]

9293htmlEngines :: [String]

9394htmlEngines = ["wkhtmltopdf", "weasyprint", "prince"]

9495

95−engines :: [(String, String)]

96+engines :: [(Text, String)]

9697engines = map ("html",) htmlEngines ++

9798 map ("html5",) htmlEngines ++

9899 map ("latex",) latexEngines ++

@@ -119,13 +120,13 @@ options =

119120 [ Option "fr" ["from","read"]

120121 (ReqArg

121122 (\arg opt -> return opt { optFrom =

122− Just (map toLower arg) })

123+ Just (T.toLower $ T.pack arg) })

123124 "FORMAT")

124125 ""

125126

126127 , Option "tw" ["to","write"]

127128 (ReqArg

128− (\arg opt -> return opt { optTo = Just arg })

129+ (\arg opt -> return opt { optTo = Just $ T.pack arg })

129130 "FORMAT")

130131 ""

131132

@@ -218,7 +219,7 @@ options =

218219 , Option "" ["toc-depth"]

219220 (ReqArg

220221 (\arg opt ->

221− case safeRead arg of

222+ case safeStrRead arg of

222223 Just t | t >= 1 && t <= 6 ->

223224 return opt { optTOCDepth = t }

224225 _ -> E.throwIO $ PandocOptionError

@@ -234,7 +235,7 @@ options =

234235 , Option "" ["number-offset"]

235236 (ReqArg

236237 (\arg opt ->

237− case safeRead ('[':arg ++ "]") of

238+ case safeStrRead ("[" <> arg <> "]") of

238239 Just ns -> return opt { optNumberOffset = ns,

239240 optNumberSections = True }

240241 _ -> E.throwIO $ PandocOptionError

@@ -255,7 +256,7 @@ options =

255256 "default" -> return opt{ optTopLevelDivision =

256257 TopLevelDefault }

257258 _ -> E.throwIO $ PandocOptionError $

258− "Top-level division must be " ++

259+ "Top-level division must be " <>

259260 "section, chapter, part, or default" )

260261 "section|chapter|part")

261262 "" -- "Use top-level division type in LaTeX, ConTeXt, DocBook"

@@ -307,7 +308,7 @@ options =

307308 , Option "" ["highlight-style"]

308309 (ReqArg

309310 (\arg opt ->

310− return opt{ optHighlightStyle = Just arg })

311+ return opt{ optHighlightStyle = Just $ T.pack arg })

311312 "STYLE|FILE")

312313 "" -- "Style for highlighted code"

313314

@@ -328,7 +329,7 @@ options =

328329 , Option "" ["dpi"]

329330 (ReqArg

330331 (\arg opt ->

331− case safeRead arg of

332+ case safeStrRead arg of

332333 Just t | t > 0 -> return opt { optDpi = t }

333334 _ -> E.throwIO $ PandocOptionError

334335 "dpi must be a number greater than 0")

@@ -351,7 +352,7 @@ options =

351352 , Option "" ["columns"]

352353 (ReqArg

353354 (\arg opt ->

354− case safeRead arg of

355+ case safeStrRead arg of

355356 Just t | t > 0 -> return opt { optColumns = t }

356357 _ -> E.throwIO $ PandocOptionError

357358 "columns must be a number greater than 0")

@@ -366,7 +367,7 @@ options =

366367 , Option "" ["tab-stop"]

367368 (ReqArg

368369 (\arg opt ->

369− case safeRead arg of

370+ case safeStrRead arg of

370371 Just t | t > 0 -> return opt { optTabStop = t }

371372 _ -> E.throwIO $ PandocOptionError

372373 "tab-stop must be a number greater than 0")

@@ -379,7 +380,7 @@ options =

379380 let b = takeBaseName arg

380381 if b `elem` pdfEngines

381382 then return opt { optPdfEngine = Just arg }

382− else E.throwIO $ PandocOptionError $ "pdf-engine must be one of "

383+ else E.throwIO $ PandocOptionError $ T.pack $ "pdf-engine must be one of "

383384 ++ intercalate ", " pdfEngines)

384385 "PROGRAM")

385386 "" -- "Name of program to use in generating PDF"

@@ -410,7 +411,7 @@ options =

410411 (\arg opt -> do

411412 let (key, val) = splitField arg

412413 return opt{ optRequestHeaders =

413− (key, val) : optRequestHeaders opt })

414+ (T.pack key, T.pack val) : optRequestHeaders opt })

414415 "NAME:VALUE")

415416 ""

416417

@@ -422,14 +423,15 @@ options =

422423

423424 , Option "" ["indented-code-classes"]

424425 (ReqArg

425− (\arg opt -> return opt { optIndentedCodeClasses = words $

426− map (\c -> if c == ',' then ' ' else c) arg })

426+ (\arg opt -> return opt { optIndentedCodeClasses = T.words $

427+ T.map (\c -> if c == ',' then ' ' else c) $

428+ T.pack arg })

427429 "STRING")

428430 "" -- "Classes (whitespace- or comma-separated) to use for indented code-blocks"

429431

430432 , Option "" ["default-image-extension"]

431433 (ReqArg

432− (\arg opt -> return opt { optDefaultImageExtension = arg })

434+ (\arg opt -> return opt { optDefaultImageExtension = T.pack arg })

433435 "extension")

434436 "" -- "Default extension for extensionless images"

435437

@@ -450,7 +452,7 @@ options =

450452 , Option "" ["shift-heading-level-by"]

451453 (ReqArg

452454 (\arg opt ->

453− case safeRead arg of

455+ case safeStrRead arg of

454456 Just t ->

455457 return opt{ optShiftHeadingLevelBy = t }

456458 _ -> E.throwIO $ PandocOptionError

@@ -463,7 +465,7 @@ options =

463465 (\arg opt -> do

464466 deprecatedOption "--base-header-level"

465467 "Use --shift-heading-level-by instead."

466− case safeRead arg of

468+ case safeStrRead arg of

467469 Just t | t > 0 && t < 6 ->

468470 return opt{ optShiftHeadingLevelBy = t - 1 }

469471 _ -> E.throwIO $ PandocOptionError

@@ -486,7 +488,7 @@ options =

486488 "accept" -> return AcceptChanges

487489 "reject" -> return RejectChanges

488490 "all" -> return AllChanges

489− _ -> E.throwIO $ PandocOptionError

491+ _ -> E.throwIO $ PandocOptionError $ T.pack

490492 ("Unknown option for track-changes: " ++ arg)

491493 return opt { optTrackChanges = action })

492494 "accept|reject|all")

@@ -509,7 +511,7 @@ options =

509511 "block" -> return EndOfBlock

510512 "section" -> return EndOfSection

511513 "document" -> return EndOfDocument

512− _ -> E.throwIO $ PandocOptionError

514+ _ -> E.throwIO $ PandocOptionError $ T.pack

513515 ("Unknown option for reference-location: " ++ arg)

514516 return opt { optReferenceLocation = action })

515517 "block|section|document")

@@ -533,7 +535,7 @@ options =

533535 , Option "" ["slide-level"]

534536 (ReqArg

535537 (\arg opt ->

536− case safeRead arg of

538+ case safeStrRead arg of

537539 Just t | t >= 1 && t <= 6 ->

538540 return opt { optSlideLevel = Just t }

539541 _ -> E.throwIO $ PandocOptionError

@@ -559,7 +561,7 @@ options =

559561 "references" -> return ReferenceObfuscation

560562 "javascript" -> return JavascriptObfuscation

561563 "none" -> return NoObfuscation

562− _ -> E.throwIO $ PandocOptionError

564+ _ -> E.throwIO $ PandocOptionError $ T.pack

563565 ("Unknown obfuscation method: " ++ arg)

564566 return opt { optEmailObfuscation = method })

565567 "none|javascript|references")

@@ -567,7 +569,7 @@ options =

567569

568570 , Option "" ["id-prefix"]

569571 (ReqArg

570− (\arg opt -> return opt { optIdentifierPrefix = arg })

572+ (\arg opt -> return opt { optIdentifierPrefix = T.pack arg })

571573 "STRING")

572574 "" -- "Prefix to add to automatically generated HTML identifiers"

573575

@@ -620,7 +622,7 @@ options =

620622 , Option "" ["epub-chapter-level"]

621623 (ReqArg

622624 (\arg opt ->

623− case safeRead arg of

625+ case safeStrRead arg of

624626 Just t | t >= 1 && t <= 6 ->

625627 return opt { optEpubChapterLevel = t }

626628 _ -> E.throwIO $ PandocOptionError

@@ -685,15 +687,15 @@ options =

685687 (OptArg

686688 (\arg opt -> do

687689 let url' = fromMaybe "https://latex.codecogs.com/png.latex?" arg

688− return opt { optHTMLMathMethod = WebTeX url' })

690+ return opt { optHTMLMathMethod = WebTeX $ T.pack url' })

689691 "URL")

690692 "" -- "Use web service for HTML math"

691693

692694 , Option "" ["mathjax"]

693695 (OptArg

694696 (\arg opt -> do

695− let url' = fromMaybe (defaultMathJaxURL ++

696− "tex-mml-chtml.js") arg

697+ let url' = maybe (defaultMathJaxURL <>

698+ "tex-mml-chtml.js") T.pack arg

697699 return opt { optHTMLMathMethod = MathJax url'})

698700 "URL")

699701 "" -- "Use MathJax for HTML math"

@@ -703,7 +705,7 @@ options =

703705 (\arg opt ->

704706 return opt

705707 { optHTMLMathMethod = KaTeX $

706− fromMaybe defaultKaTeXURL arg })

708+ maybe defaultKaTeXURL T.pack arg })

707709 "URL")

708710 "" -- Use KaTeX for HTML Math

709711

@@ -763,7 +765,7 @@ options =

763765 UTF8.hPutStrLn stdout $ printf tpl allopts

764766 (unwords readersNames)

765767 (unwords writersNames)

766− (unwords $ map fst highlightingStyles)

768+ (unwords $ map (T.unpack . fst) highlightingStyles)

767769 (unwords datafiles)

768770 exitSuccess ))

769771 "" -- "Print bash completion script"

@@ -790,12 +792,12 @@ options =

790792 let allExts =

791793 case arg of

792794 Nothing -> extensionsFromList extList

793− Just fmt -> getAllExtensions fmt

795+ Just fmt -> getAllExtensions $ T.pack fmt

794796 let defExts =

795797 case arg of

796798 Nothing -> getDefaultExtensions

797799 "markdown"

798− Just fmt -> getDefaultExtensions fmt

800+ Just fmt -> getDefaultExtensions $ T.pack fmt

799801 let showExt x =

800802 (if extensionEnabled x defExts

801803 then '+'

@@ -823,7 +825,7 @@ options =

823825 , Option "" ["list-highlight-styles"]

824826 (NoArg

825827 (\_ -> do

826− mapM_ (UTF8.hPutStrLn stdout . fst) highlightingStyles

828+ mapM_ (UTF8.hPutStrLn stdout . T.unpack . fst) highlightingStyles

827829 exitSuccess ))

828830 ""

829831

@@ -835,11 +837,11 @@ options =

835837 Nothing -> UTF8.hPutStr stdout

836838 templ <- runIO $ do

837839 setUserDataDir Nothing

838− getDefaultTemplate arg

840+ getDefaultTemplate (T.pack arg)

839841 case templ of

840842 Right t

841843 | T.null t -> -- e.g. for docx, odt, json:

842− E.throwIO $ PandocCouldNotFindDataFileError

844+ E.throwIO $ PandocCouldNotFindDataFileError $ T.pack

843845 ("templates/default." ++ arg)

844846 | otherwise -> write . T.unpack $ t

845847 Left e -> E.throwIO e

@@ -890,7 +892,7 @@ options =

890892 (\_ -> do

891893 prg <- getProgName

892894 defaultDatadirs <- defaultUserDataDirs

893− UTF8.hPutStrLn stdout (prg ++ " " ++ pandocVersion ++

895+ UTF8.hPutStrLn stdout (prg ++ " " ++ T.unpack pandocVersion ++

894896 compileInfo ++ "\nDefault user data directory: " ++

895897 intercalate " or " defaultDatadirs ++

896898 ('\n':copyrightMessage))

@@ -963,14 +965,14 @@ handleUnrecognizedOption x =

963965 (("Unknown option " ++ x ++ ".") :)

964966

965967readersNames :: [String]

966−readersNames = sort (map fst (readers :: [(String, Reader PandocIO)]))

968+readersNames = sort (map (T.unpack . fst) (readers :: [(Text, Reader PandocIO)]))

967969

968970writersNames :: [String]

969−writersNames = sort (map fst (writers :: [(String, Writer PandocIO)]))

971+writersNames = sort (map (T.unpack . fst) (writers :: [(Text, Writer PandocIO)]))

970972

971973splitField :: String -> (String, String)

972974splitField s =

973− case break (`elem` ":=") s of

975+ case break (`elemText` ":=") s of

974976 (k,_:v) -> (k,v)

975977 (k,[]) -> (k,"true")

976978

@@ -991,7 +993,7 @@ applyDefaults opt file = runIOorExplode $ do

991993 case Y.decode1 inp of

992994 Right (f :: Opt -> Opt) -> return $ f opt

993995 Left (errpos, errmsg) -> throwError $

994− PandocParseError $

996+ PandocParseError $ T.pack $

995997 "Error parsing " ++ fp' ++ " line " ++

996998 show (Y.posLine errpos) ++ " column " ++

997999 show (Y.posColumn errpos) ++ ":\n" ++ errmsg

@@ -1001,18 +1003,18 @@ lookupHighlightStyle s

10011003 | takeExtension s == ".theme" = -- attempt to load KDE theme

10021004 do contents <- readFileLazy s

10031005 case parseTheme contents of

1004− Left _ -> throwError $ PandocOptionError $

1006+ Left _ -> throwError $ PandocOptionError $ T.pack $

10051007 "Could not read highlighting theme " ++ s

10061008 Right sty -> return sty

10071009 | otherwise =

1008− case lookup (map toLower s) highlightingStyles of

1010+ case lookup (T.toLower $ T.pack s) highlightingStyles of

10091011 Just sty -> return sty

1010− Nothing -> throwError $ PandocOptionError $

1012+ Nothing -> throwError $ PandocOptionError $ T.pack $

10111013 "Unknown highlight-style " ++ s

10121014

10131015deprecatedOption :: String -> String -> IO ()

10141016deprecatedOption o msg =

1015− runIO (report $ Deprecated o msg) >>=

1017+ runIO (report $ Deprecated (T.pack o) (T.pack msg)) >>=

10161018 \r -> case r of

10171019 Right () -> return ()

10181020 Left e -> E.throwIO e

@@ -1024,13 +1026,14 @@ setVariable key val (Context ctx) =

10241026

10251027addMeta :: String -> String -> Meta -> Meta

10261028addMeta k v meta =

1027− case lookupMeta k meta of

1028− Nothing -> setMeta k v' meta

1029+ case lookupMeta k' meta of

1030+ Nothing -> setMeta k' v' meta

10291031 Just (MetaList xs) ->

1030− setMeta k (MetaList (xs ++ [v'])) meta

1031− Just x -> setMeta k (MetaList [x, v']) meta

1032+ setMeta k' (MetaList (xs ++ [v'])) meta

1033+ Just x -> setMeta k' (MetaList [x, v']) meta

10321034 where

10331035 v' = readMetaValue v

1036+ k' = T.pack k

10341037

10351038readMetaValue :: String -> MetaValue

10361039readMetaValue s

@@ -1040,7 +1043,7 @@ readMetaValue s

10401043 | s == "false" = MetaBool False

10411044 | s == "False" = MetaBool False

10421045 | s == "FALSE" = MetaBool False

1043− | otherwise = MetaString s

1046+ | otherwise = MetaString $ T.pack s

10441047

10451048-- On Windows with ghc 8.6+, we need to rewrite paths

10461049-- beginning with \\ to \\?\UNC\. -- See #5127.

src/Text/Pandoc/App/FormatHeuristics.hs+4−2

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.App.FormatHeuristics

@@ -16,10 +17,11 @@ module Text.Pandoc.App.FormatHeuristics

1617

1718import Prelude

1819import Data.Char (toLower)

20+import Data.Text (Text)

1921import System.FilePath (takeExtension)

2022

2123-- Determine default format based on file extensions.

22−formatFromFilePaths :: [FilePath] -> Maybe String

24+formatFromFilePaths :: [FilePath] -> Maybe Text

2325formatFromFilePaths [] = Nothing

2426formatFromFilePaths (x:xs) =

2527 case formatFromFilePath x of

@@ -27,7 +29,7 @@ formatFromFilePaths (x:xs) =

2729 Nothing -> formatFromFilePaths xs

2830

2931-- Determine format based on file extension

30−formatFromFilePath :: FilePath -> Maybe String

32+formatFromFilePath :: FilePath -> Maybe Text

3133formatFromFilePath x =

3234 case takeExtension (map toLower x) of

3335 ".adoc" -> Just "asciidoc"

src/Text/Pandoc/App/Opt.hs+23−25

@@ -33,7 +33,7 @@ import Text.Pandoc.Options (TopLevelDivision (TopLevelDefault),

3333 ReferenceLocation (EndOfDocument),

3434 ObfuscationMethod (NoObfuscation),

3535 CiteMethod (Citeproc))

36−import Text.Pandoc.Shared (camelCaseToHyphenated)

36+import Text.Pandoc.Shared (camelCaseStrToHyphenated)

3737import Text.DocLayout (render)

3838import Text.DocTemplates (Context(..), Val(..))

3939import Data.Text (Text, unpack)

@@ -76,8 +76,8 @@ data Opt = Opt

7676 { optTabStop :: Int -- ^ Number of spaces per tab

7777 , optPreserveTabs :: Bool -- ^ Preserve tabs instead of converting to spaces

7878 , optStandalone :: Bool -- ^ Include header, footer

79− , optFrom :: Maybe String -- ^ Reader format

80− , optTo :: Maybe String -- ^ Writer format

79+ , optFrom :: Maybe Text -- ^ Reader format

80+ , optTo :: Maybe Text -- ^ Writer format

8181 , optTableOfContents :: Bool -- ^ Include table of contents

8282 , optShiftHeadingLevelBy :: Int -- ^ Shift heading level by

8383 , optTemplate :: Maybe FilePath -- ^ Custom template

@@ -92,7 +92,7 @@ data Opt = Opt

9292 , optIncremental :: Bool -- ^ Use incremental lists in Slidy/Slideous/S5

9393 , optSelfContained :: Bool -- ^ Make HTML accessible offline

9494 , optHtmlQTags :: Bool -- ^ Use <q> tags in HTML

95− , optHighlightStyle :: Maybe String -- ^ Style to use for highlighted code

95+ , optHighlightStyle :: Maybe Text -- ^ Style to use for highlighted code

9696 , optSyntaxDefinitions :: [FilePath] -- ^ xml syntax defs to load

9797 , optTopLevelDivision :: TopLevelDivision -- ^ Type of the top-level divisions

9898 , optHTMLMathMethod :: HTMLMathMethod -- ^ Method to print HTML math

@@ -117,9 +117,9 @@ data Opt = Opt

117117 , optColumns :: Int -- ^ Line length in characters

118118 , optFilters :: [Filter] -- ^ Filters to apply

119119 , optEmailObfuscation :: ObfuscationMethod

120− , optIdentifierPrefix :: String

120+ , optIdentifierPrefix :: Text

121121 , optStripEmptyParagraphs :: Bool -- ^ Strip empty paragraphs

122− , optIndentedCodeClasses :: [String] -- ^ Default classes for indented code blocks

122+ , optIndentedCodeClasses :: [Text] -- ^ Default classes for indented code blocks

123123 , optDataDir :: Maybe FilePath

124124 , optCiteMethod :: CiteMethod -- ^ Method to output cites

125125 , optListings :: Bool -- ^ Use listings package for code blocks

@@ -128,18 +128,18 @@ data Opt = Opt

128128 , optSlideLevel :: Maybe Int -- ^ Header level that creates slides

129129 , optSetextHeaders :: Bool -- ^ Use atx headers for markdown level 1-2

130130 , optAscii :: Bool -- ^ Prefer ascii output

131− , optDefaultImageExtension :: String -- ^ Default image extension

131+ , optDefaultImageExtension :: Text -- ^ Default image extension

132132 , optExtractMedia :: Maybe FilePath -- ^ Path to extract embedded media

133133 , optTrackChanges :: TrackChanges -- ^ Accept or reject MS Word track-changes.

134134 , optFileScope :: Bool -- ^ Parse input files before combining

135− , optTitlePrefix :: Maybe String -- ^ Prefix for title

135+ , optTitlePrefix :: Maybe Text -- ^ Prefix for title

136136 , optCss :: [FilePath] -- ^ CSS files to link to

137137 , optIpynbOutput :: IpynbOutput -- ^ How to treat ipynb output blocks

138138 , optIncludeBeforeBody :: [FilePath] -- ^ Files to include before

139139 , optIncludeAfterBody :: [FilePath] -- ^ Files to include after body

140140 , optIncludeInHeader :: [FilePath] -- ^ Files to include in header

141141 , optResourcePath :: [FilePath] -- ^ Path to search for images etc

142− , optRequestHeaders :: [(String, String)] -- ^ Headers for HTTP requests

142+ , optRequestHeaders :: [(Text, Text)] -- ^ Headers for HTTP requests

143143 , optEol :: LineEnding -- ^ Style of line-endings to use

144144 , optStripComments :: Bool -- ^ Skip HTML comments

145145 } deriving (Generic, Show)

@@ -167,13 +167,13 @@ doOpt (k',v) = do

167167 "toc" ->

168168 parseYAML v >>= \x -> return (\o -> o{ optTableOfContents = x })

169169 "from" ->

170− parseYAML v >>= \x -> return (\o -> o{ optFrom = unpack <$> x })

170+ parseYAML v >>= \x -> return (\o -> o{ optFrom = x })

171171 "reader" ->

172− parseYAML v >>= \x -> return (\o -> o{ optFrom = unpack <$> x })

172+ parseYAML v >>= \x -> return (\o -> o{ optFrom = x })

173173 "to" ->

174− parseYAML v >>= \x -> return (\o -> o{ optTo = unpack <$> x })

174+ parseYAML v >>= \x -> return (\o -> o{ optTo = x })

175175 "writer" ->

176− parseYAML v >>= \x -> return (\o -> o{ optTo = unpack <$> x })

176+ parseYAML v >>= \x -> return (\o -> o{ optTo = x })

177177 "shift-heading-level-by" ->

178178 parseYAML v >>= \x -> return (\o -> o{ optShiftHeadingLevelBy = x })

179179 "template" ->

@@ -211,7 +211,7 @@ doOpt (k',v) = do

211211 "html-q-tags" ->

212212 parseYAML v >>= \x -> return (\o -> o{ optHtmlQTags = x })

213213 "highlight-style" ->

214− parseYAML v >>= \x -> return (\o -> o{ optHighlightStyle = unpack <$> x })

214+ parseYAML v >>= \x -> return (\o -> o{ optHighlightStyle = x })

215215 "syntax-definition" ->

216216 (parseYAML v >>= \x ->

217217 return (\o -> o{ optSyntaxDefinitions = map unpack x }))

@@ -274,12 +274,12 @@ doOpt (k',v) = do

274274 parseYAML v >>= \x -> return (\o -> o{ optEmailObfuscation = x })

275275 "identifier-prefix" ->

276276 parseYAML v >>= \x ->

277− return (\o -> o{ optIdentifierPrefix = unpack x })

277+ return (\o -> o{ optIdentifierPrefix = x })

278278 "strip-empty-paragraphs" ->

279279 parseYAML v >>= \x -> return (\o -> o{ optStripEmptyParagraphs = x })

280280 "indented-code-classes" ->

281281 parseYAML v >>= \x ->

282− return (\o -> o{ optIndentedCodeClasses = map unpack x })

282+ return (\o -> o{ optIndentedCodeClasses = x })

283283 "data-dir" ->

284284 parseYAML v >>= \x -> return (\o -> o{ optDataDir = unpack <$> x })

285285 "cite-method" ->

@@ -305,7 +305,7 @@ doOpt (k',v) = do

305305 parseYAML v >>= \x -> return (\o -> o{ optAscii = x })

306306 "default-image-extension" ->

307307 parseYAML v >>= \x ->

308− return (\o -> o{ optDefaultImageExtension = unpack x })

308+ return (\o -> o{ optDefaultImageExtension = x })

309309 "extract-media" ->

310310 parseYAML v >>= \x ->

311311 return (\o -> o{ optExtractMedia = unpack <$> x })

@@ -314,7 +314,7 @@ doOpt (k',v) = do

314314 "file-scope" ->

315315 parseYAML v >>= \x -> return (\o -> o{ optFileScope = x })

316316 "title-prefix" ->

317− parseYAML v >>= \x -> return (\o -> o{ optTitlePrefix = unpack <$> x })

317+ parseYAML v >>= \x -> return (\o -> o{ optTitlePrefix = x })

318318 "css" ->

319319 (parseYAML v >>= \x -> return (\o -> o{ optCss = map unpack x }))

320320 <|>

@@ -344,9 +344,7 @@ doOpt (k',v) = do

344344 return (\o -> o{ optResourcePath = map unpack x })

345345 "request-headers" ->

346346 parseYAML v >>= \x ->

347− return (\o -> o{ optRequestHeaders =

348− map (\(key,val) ->

349− (unpack key, unpack val)) x })

347+ return (\o -> o{ optRequestHeaders = x })

350348 "eol" ->

351349 parseYAML v >>= \x -> return (\o -> o{ optEol = x })

352350 "strip-comments" ->

@@ -429,13 +427,13 @@ defaultOpts = Opt

429427

430428contextToMeta :: Context Text -> Meta

431429contextToMeta (Context m) =

432− Meta . M.mapKeys unpack . M.map valToMetaVal $ m

430+ Meta . M.map valToMetaVal $ m

433431

434432valToMetaVal :: Val Text -> MetaValue

435433valToMetaVal (MapVal (Context m)) =

436− MetaMap . M.mapKeys unpack . M.map valToMetaVal $ m

434+ MetaMap . M.map valToMetaVal $ m

437435valToMetaVal (ListVal xs) = MetaList $ map valToMetaVal xs

438−valToMetaVal (SimpleVal d) = MetaString (unpack $ render Nothing d)

436+valToMetaVal (SimpleVal d) = MetaString $ render Nothing d

439437valToMetaVal NullVal = MetaString ""

440438

441439-- see https://github.com/jgm/pandoc/pull/4083

@@ -446,5 +444,5 @@ $(deriveJSON

446444 defaultOptions{ fieldLabelModifier = map toLower } ''LineEnding)

447445$(deriveJSON

448446 defaultOptions{ fieldLabelModifier =

449− camelCaseToHyphenated . dropWhile isLower

447+ camelCaseStrToHyphenated . dropWhile isLower

450448 } ''Opt)

src/Text/Pandoc/App/OutputSettings.hs+32−31

@@ -1,6 +1,7 @@

11{-# LANGUAGE CPP #-}

22{-# LANGUAGE FlexibleContexts #-}

33{-# LANGUAGE NoImplicitPrelude #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{-# LANGUAGE ScopedTypeVariables #-}

56{-# LANGUAGE TupleSections #-}

67{- |

@@ -27,7 +28,7 @@ import Control.Monad

2728import Control.Monad.Except (catchError, throwError)

2829import Control.Monad.Trans

2930import Data.Char (toLower)

30−import Data.List (find, isPrefixOf, isSuffixOf)

31+import Data.List (find, isPrefixOf)

3132import Data.Maybe (fromMaybe)

3233import Skylighting (defaultSyntaxMap)

3334import Skylighting.Parser (addSyntaxDefinition, parseSyntaxDefinition)

@@ -42,18 +43,18 @@ import Text.Pandoc.App.CommandLineOptions (engines, lookupHighlightStyle,

4243 setVariable)

4344import qualified Text.Pandoc.UTF8 as UTF8

4445

46+readUtf8File :: PandocMonad m => FilePath -> m T.Text

47+readUtf8File = fmap UTF8.toText . readFileStrict

48+

4549-- | Settings specifying how document output should be produced.

4650data OutputSettings = OutputSettings

47− { outputFormat :: String

51+ { outputFormat :: T.Text

4852 , outputWriter :: Writer PandocIO

49− , outputWriterName :: String

53+ , outputWriterName :: T.Text

5054 , outputWriterOptions :: WriterOptions

5155 , outputPdfProgram :: Maybe String

5256 }

5357

54−readUtf8File :: PandocMonad m => FilePath -> m String

55−readUtf8File = fmap UTF8.toString . readFileStrict

56−

5758-- | Get output settings from command line options.

5859optToOutputSettings :: Opt -> PandocIO OutputSettings

5960optToOutputSettings opts = do

@@ -85,33 +86,33 @@ optToOutputSettings opts = do

8586 case formatFromFilePaths [outputFile] of

8687 Nothing -> do

8788 report $ CouldNotDeduceFormat

88− [takeExtension outputFile] "html"

89+ [T.pack $ takeExtension outputFile] "html"

8990 return ("html", Nothing)

9091 Just f -> return (f, Nothing)

9192

92− let format = if ".lua" `isSuffixOf` writerName

93+ let format = if ".lua" `T.isSuffixOf` writerName

9394 then writerName

94− else map toLower $ baseWriterName writerName

95+ else T.toLower $ baseWriterName writerName

9596

9697 (writer :: Writer PandocIO, writerExts) <-

97− if ".lua" `isSuffixOf` format

98+ if ".lua" `T.isSuffixOf` format

9899 then return (TextWriter

99− (\o d -> writeCustom writerName o d)

100+ (\o d -> writeCustom (T.unpack writerName) o d)

100101 :: Writer PandocIO, mempty)

101− else getWriter (map toLower writerName)

102+ else getWriter (T.toLower writerName)

102103

103104 let standalone = optStandalone opts || not (isTextFormat format) || pdfOutput

104105

105106 let addSyntaxMap existingmap f = do

106107 res <- liftIO (parseSyntaxDefinition f)

107108 case res of

108− Left errstr -> throwError $ PandocSyntaxMapError errstr

109+ Left errstr -> throwError $ PandocSyntaxMapError $ T.pack errstr

109110 Right syn -> return $ addSyntaxDefinition syn existingmap

110111

111112 syntaxMap <- foldM addSyntaxMap defaultSyntaxMap

112113 (optSyntaxDefinitions opts)

113114

114− hlStyle <- maybe (return Nothing) (fmap Just . lookupHighlightStyle)

115+ hlStyle <- maybe (return Nothing) (fmap Just . lookupHighlightStyle . T.unpack)

115116 (optHighlightStyle opts)

116117

117118 let setVariableM k v = return . setVariable k v

@@ -135,15 +136,15 @@ optToOutputSettings opts = do

135136 >>=

136137 setVariableM "outputfile" outputFile

137138 >>=

138− setFilesVariableM "include-before" (optIncludeBeforeBody opts)

139+ setFilesVariableM "include-before" (T.pack <$> optIncludeBeforeBody opts)

139140 >>=

140− setFilesVariableM "include-after" (optIncludeAfterBody opts)

141+ setFilesVariableM "include-after" (T.pack <$> optIncludeAfterBody opts)

141142 >>=

142− setFilesVariableM "header-includes" (optIncludeInHeader opts)

143+ setFilesVariableM "header-includes" (T.pack <$> optIncludeInHeader opts)

143144 >>=

144145 setListVariableM "css" (optCss opts)

145146 >>=

146− maybe return (setVariableM "title-prefix")

147+ maybe return (setVariableM "title-prefix" . T.unpack)

147148 (optTitlePrefix opts)

148149 >>=

149150 maybe return (setVariableM "epub-cover-image")

@@ -168,7 +169,7 @@ optToOutputSettings opts = do

168169 Just tp -> do

169170 -- strip off extensions

170171 let tp' = case takeExtension tp of

171− "" -> tp <.> format

172+ "" -> tp <.> T.unpack format

172173 _ -> tp

173174 Just . UTF8.toText <$>

174175 ((do surl <- stSourceURL <$> getCommonState

@@ -176,7 +177,7 @@ optToOutputSettings opts = do

176177 -- unless the full URL is specified:

177178 modifyCommonState $ \st -> st{

178179 stSourceURL = Nothing }

179− (bs, _) <- fetchItem tp'

180+ (bs, _) <- fetchItem $ T.pack tp'

180181 modifyCommonState $ \st -> st{

181182 stSourceURL = surl }

182183 return bs)

@@ -194,7 +195,7 @@ optToOutputSettings opts = do

194195 Just ts -> do

195196 res <- compileTemplate templatePath ts

196197 case res of

197− Left e -> throwError $ PandocTemplateError e

198+ Left e -> throwError $ PandocTemplateError $ T.pack e

198199 Right t -> return $ Just t

199200

200201 let writerOpts = def {

@@ -222,7 +223,7 @@ optToOutputSettings opts = do

222223 , writerSlideLevel = optSlideLevel opts

223224 , writerHighlightStyle = hlStyle

224225 , writerSetextHeaders = optSetextHeaders opts

225− , writerEpubSubdirectory = optEpubSubdirectory opts

226+ , writerEpubSubdirectory = T.pack $ optEpubSubdirectory opts

226227 , writerEpubMetadata = epubMetadata

227228 , writerEpubFonts = optEpubFonts opts

228229 , writerEpubChapterLevel = optEpubChapterLevel opts

@@ -239,12 +240,12 @@ optToOutputSettings opts = do

239240 , outputPdfProgram = maybePdfProg

240241 }

241242

242−baseWriterName :: String -> String

243−baseWriterName = takeWhile (\c -> c /= '+' && c /= '-')

243+baseWriterName :: T.Text -> T.Text

244+baseWriterName = T.takeWhile (\c -> c /= '+' && c /= '-')

244245

245−pdfWriterAndProg :: Maybe String -- ^ user-specified writer name

246+pdfWriterAndProg :: Maybe T.Text -- ^ user-specified writer name

246247 -> Maybe String -- ^ user-specified pdf-engine

247− -> IO (String, Maybe String) -- ^ IO (writerName, maybePdfEngineProg)

248+ -> IO (T.Text, Maybe String) -- ^ IO (writerName, maybePdfEngineProg)

248249pdfWriterAndProg mWriter mEngine =

249250 case go mWriter mEngine of

250251 Right (writ, prog) -> return (writ, Just prog)

@@ -256,20 +257,20 @@ pdfWriterAndProg mWriter mEngine =

256257 go (Just writer) (Just engine) =

257258 case find (== (baseWriterName writer, takeBaseName engine)) engines of

258259 Just _ -> Right (writer, engine)

259− Nothing -> Left $ "pdf-engine " ++ engine ++

260− " is not compatible with output format " ++ writer

260+ Nothing -> Left $ "pdf-engine " <> T.pack engine <>

261+ " is not compatible with output format " <> writer

261262

262263 writerForEngine eng = case [f | (f,e) <- engines, e == eng] of

263264 fmt : _ -> Right fmt

264265 [] -> Left $

265− "pdf-engine " ++ eng ++ " not known"

266+ "pdf-engine " <> T.pack eng <> " not known"

266267

267268 engineForWriter "pdf" = Left "pdf writer"

268269 engineForWriter w = case [e | (f,e) <- engines, f == baseWriterName w] of

269270 eng : _ -> Right eng

270271 [] -> Left $

271− "cannot produce pdf output from " ++ w

272+ "cannot produce pdf output from " <> w

272273

273−isTextFormat :: String -> Bool

274+isTextFormat :: T.Text -> Bool

274275isTextFormat s =

275276 s `notElem` ["odt","docx","epub2","epub3","epub","pptx","pdf"]

src/Text/Pandoc/BCP47.hs+17−19

@@ -20,9 +20,7 @@ module Text.Pandoc.BCP47 (

2020where

2121import Prelude

2222import Control.Monad (guard)

23−import Data.Char (isAlphaNum, isAscii, isLetter, isLower, isUpper, toLower,

24− toUpper)

25−import Data.List (intercalate)

23+import Data.Char (isAlphaNum, isAscii, isLetter, isLower, isUpper)

2624import Text.Pandoc.Definition

2725import Text.Pandoc.Options

2826import Text.DocTemplates (FromContext(..))

@@ -30,22 +28,22 @@ import qualified Data.Text as T

3028import qualified Text.Parsec as P

3129

3230-- | Represents BCP 47 language/country code.

33−data Lang = Lang{ langLanguage :: String

34− , langScript :: String

35− , langRegion :: String

36− , langVariants :: [String] }

31+data Lang = Lang{ langLanguage :: T.Text

32+ , langScript :: T.Text

33+ , langRegion :: T.Text

34+ , langVariants :: [T.Text] }

3735 deriving (Eq, Ord, Show)

3836

3937-- | Render a Lang as BCP 47.

40−renderLang :: Lang -> String

41−renderLang lang = intercalate "-" (langLanguage lang : filter (not . null)

38+renderLang :: Lang -> T.Text

39+renderLang lang = T.intercalate "-" (langLanguage lang : filter (not . T.null)

4240 ([langScript lang, langRegion lang] ++ langVariants lang))

4341

4442-- | Get the contents of the `lang` metadata field or variable.

45−getLang :: WriterOptions -> Meta -> Maybe String

43+getLang :: WriterOptions -> Meta -> Maybe T.Text

4644getLang opts meta =

4745 case lookupContext "lang" (writerVariables opts) of

48− Just s -> Just $ T.unpack s

46+ Just s -> Just s

4947 _ ->

5048 case lookupMeta "lang" meta of

5149 Just (MetaInlines [Str s]) -> Just s

@@ -55,11 +53,11 @@ getLang opts meta =

5553-- | Parse a BCP 47 string as a Lang. Currently we parse

5654-- extensions and private-use fields as "variants," even

5755-- though officially they aren't.

58−parseBCP47 :: String -> Either String Lang

56+parseBCP47 :: T.Text -> Either T.Text Lang

5957parseBCP47 lang =

6058 case P.parse bcp47 "lang" lang of

6159 Right r -> Right r

62− Left e -> Left $ show e

60+ Left e -> Left $ T.pack $ show e

6361 where bcp47 = do

6462 language <- pLanguage

6563 script <- P.option "" pScript

@@ -75,19 +73,19 @@ parseBCP47 lang =

7573 cs <- P.many1 asciiLetter

7674 let lcs = length cs

7775 guard $ lcs == 2 || lcs == 3

78− return $ map toLower cs

76+ return $ T.toLower $ T.pack $ cs

7977 pScript = P.try $ do

8078 P.char '-'

8179 x <- P.satisfy (\c -> isAscii c && isLetter c && isUpper c)

8280 xs <- P.count 3

8381 (P.satisfy (\c -> isAscii c && isLetter c && isLower c))

84− return $ map toLower (x:xs)

82+ return $ T.toLower $ T.pack (x:xs)

8583 pRegion = P.try $ do

8684 P.char '-'

8785 cs <- P.many1 asciiLetter

8886 let lcs = length cs

8987 guard $ lcs == 2 || lcs == 3

90− return $ map toUpper cs

88+ return $ T.toUpper $ T.pack cs

9189 pVariant = P.try $ do

9290 P.char '-'

9391 ds <- P.option "" (P.count 1 P.digit)

@@ -96,12 +94,12 @@ parseBCP47 lang =

9694 guard $ if null ds

9795 then length var >= 5 && length var <= 8

9896 else length var == 4

99− return $ map toLower var

97+ return $ T.toLower $ T.pack var

10098 pExtension = P.try $ do

10199 P.char '-'

102100 cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

103101 guard $ length cs >= 2 && length cs <= 8

104− return $ map toLower cs

102+ return $ T.toLower $ T.pack cs

105103 pPrivateUse = P.try $ do

106104 P.char '-'

107105 P.char 'x'

@@ -109,4 +107,4 @@ parseBCP47 lang =

109107 cs <- P.many1 $ P.satisfy (\c -> isAscii c && isAlphaNum c)

110108 guard $ not (null cs) && length cs <= 8

111109 let var = "x-" ++ cs

112− return $ map toLower var

110+ return $ T.toLower $ T.pack var

src/Text/Pandoc/CSS.hs+7−6

@@ -19,17 +19,18 @@ module Text.Pandoc.CSS ( foldOrElse

1919where

2020

2121import Prelude

22+import qualified Data.Text as T

2223import Text.Pandoc.Shared (trim)

2324import Text.Parsec

24−import Text.Parsec.String

25+import Text.Parsec.Text

2526

26−ruleParser :: Parser (String, String)

27+ruleParser :: Parser (T.Text, T.Text)

2728ruleParser = do

2829 p <- many1 (noneOf ":") <* char ':'

2930 v <- many1 (noneOf ":;") <* optional (char ';') <* spaces

30− return (trim p, trim v)

31+ return (trim $ T.pack p, trim $ T.pack v)

3132

32−styleAttrParser :: Parser [(String, String)]

33+styleAttrParser :: Parser [(T.Text, T.Text)]

3334styleAttrParser = many1 ruleParser

3435

3536orElse :: Eq a => a -> a -> a -> a

@@ -44,7 +45,7 @@ eitherToMaybe _ = Nothing

4445

4546-- | takes a list of keys/properties and a CSS string and

4647-- returns the corresponding key-value-pairs.

47−pickStylesToKVs :: [String] -> String -> [(String, String)]

48+pickStylesToKVs :: [T.Text] -> T.Text -> [(T.Text, T.Text)]

4849pickStylesToKVs props styleAttr =

4950 case parse styleAttrParser "" styleAttr of

5051 Left _ -> []

@@ -52,7 +53,7 @@ pickStylesToKVs props styleAttr =

5253

5354-- | takes a list of key/property synonyms and a CSS string and maybe

5455-- returns the value of the first match (in order of the supplied list)

55−pickStyleAttrProps :: [String] -> String -> Maybe String

56+pickStyleAttrProps :: [T.Text] -> T.Text -> Maybe T.Text

5657pickStyleAttrProps lookupProps styleAttr = do

5758 styles <- eitherToMaybe $ parse styleAttrParser "" styleAttr

5859 foldOrElse Nothing $ map (`lookup` styles) lookupProps

src/Text/Pandoc/Class.hs+68−66

@@ -9,6 +9,8 @@

99{-# LANGUAGE FlexibleContexts #-}

1010{-# LANGUAGE UndecidableInstances #-}

1111{-# LANGUAGE ScopedTypeVariables #-}

12+{-# LANGUAGE OverloadedStrings #-}

13+{-# LANGUAGE ViewPatterns #-}

1214{- |

1315 Module : Text.Pandoc.Class

@@ -79,7 +81,6 @@ import qualified System.Random as IO (newStdGen)

7981import Codec.Archive.Zip

8082import qualified Data.CaseInsensitive as CI

8183import Data.Unique (hashUnique)

82−import Data.List (stripPrefix)

8384import qualified Data.Unique as IO (newUnique)

8485import qualified Text.Pandoc.UTF8 as UTF8

8586import qualified System.Directory as Directory

@@ -134,6 +135,7 @@ import Data.Default

134135import System.IO.Error

135136import System.IO (stderr)

136137import qualified Data.Map as M

138+import qualified Data.Text as T

137139import Text.Pandoc.Error

138140import Text.Pandoc.BCP47 (Lang(..), parseBCP47, renderLang)

139141import Text.Pandoc.Translations (Term(..), Translations, lookupTerm,

@@ -153,7 +155,7 @@ import qualified Paths_pandoc as Paths

153155class (Functor m, Applicative m, Monad m, MonadError PandocError m)

154156 => PandocMonad m where

155157 -- | Lookup an environment variable.

156− lookupEnv :: String -> m (Maybe String)

158+ lookupEnv :: T.Text -> m (Maybe T.Text)

157159 -- | Get the current (UTC) time.

158160 getCurrentTime :: m UTCTime

159161 -- | Get the locale's time zone.

@@ -164,7 +166,7 @@ class (Functor m, Applicative m, Monad m, MonadError PandocError m)

164166 newUniqueHash :: m Int

165167 -- | Retrieve contents and mime type from a URL, raising

166168 -- an error on failure.

167− openURL :: String -> m (B.ByteString, Maybe MimeType)

169+ openURL :: T.Text -> m (B.ByteString, Maybe MimeType)

168170 -- | Read the lazy ByteString contents from a file path,

169171 -- raising an error on failure.

170172 readFileLazy :: FilePath -> m BL.ByteString

@@ -199,10 +201,10 @@ class (Functor m, Applicative m, Monad m, MonadError PandocError m)

199201 -- Output a debug message to sterr, using 'Debug.Trace.trace',

200202 -- if tracing is enabled. Note: this writes to stderr even in

201203 -- pure instances.

202− trace :: String -> m ()

204+ trace :: T.Text -> m ()

203205 trace msg = do

204206 tracing <- getsCommonState stTrace

205− when tracing $ Debug.Trace.trace ("[trace] " ++ msg) (return ())

207+ when tracing $ Debug.Trace.trace ("[trace] " ++ T.unpack msg) (return ())

206208

207209-- * Functions defined for all PandocMonad instances

208210

@@ -238,8 +240,8 @@ setTrace useTracing = modifyCommonState $ \st -> st{stTrace = useTracing}

238240

239241-- | Set request header to use in HTTP requests.

240242setRequestHeader :: PandocMonad m

241− => String -- ^ Header name

242− -> String -- ^ Value

243+ => T.Text -- ^ Header name

244+ -> T.Text -- ^ Value

243245 -> m ()

244246setRequestHeader name val = modifyCommonState $ \st ->

245247 st{ stRequestHeaders =

@@ -277,7 +279,7 @@ setInputFiles fs = do

277279 _ -> Nothing

278280

279281 modifyCommonState $ \st -> st{ stInputFiles = fs

280− , stSourceURL = sourceURL }

282+ , stSourceURL = T.pack <$> sourceURL }

281283

282284-- Retrieve the output filename.

283285getOutputFile :: PandocMonad m => m (Maybe FilePath)

@@ -307,10 +309,10 @@ getZonedTime = do

307309 return $ utcToZonedTime tz t

308310

309311-- | Read file, checking in any number of directories.

310−readFileFromDirs :: PandocMonad m => [FilePath] -> FilePath -> m (Maybe String)

312+readFileFromDirs :: PandocMonad m => [FilePath] -> FilePath -> m (Maybe T.Text)

311313readFileFromDirs [] _ = return Nothing

312314readFileFromDirs (d:ds) f = catchError

313− ((Just . UTF8.toStringLazy) <$> readFileLazy (d </> f))

315+ ((Just . T.pack . UTF8.toStringLazy) <$> readFileLazy (d </> f))

314316 (\_ -> readFileFromDirs ds f)

315317

316318instance TemplateMonad PandocIO where

@@ -331,9 +333,9 @@ data CommonState = CommonState { stLog :: [LogMessage]

331333 -- ^ A list of log messages in reverse order

332334 , stUserDataDir :: Maybe FilePath

333335 -- ^ Directory to search for data files

334− , stSourceURL :: Maybe String

336+ , stSourceURL :: Maybe T.Text

335337 -- ^ Absolute URL + dir of 1st source file

336− , stRequestHeaders :: [(String, String)]

338+ , stRequestHeaders :: [(T.Text, T.Text)]

337339 -- ^ Headers to add for HTTP requests

338340 , stMediaBag :: MediaBag

339341 -- ^ Media parsed from binary containers

@@ -370,7 +372,7 @@ instance Default CommonState where

370372

371373-- | Convert BCP47 string to a Lang, issuing warning

372374-- if there are problems.

373−toLang :: PandocMonad m => Maybe String -> m (Maybe Lang)

375+toLang :: PandocMonad m => Maybe T.Text -> m (Maybe Lang)

374376toLang Nothing = return Nothing

375377toLang (Just s) =

376378 case parseBCP47 s of

@@ -395,14 +397,14 @@ getTranslations = do

395397 Nothing -> return mempty -- no language defined

396398 Just (_, Just t) -> return t

397399 Just (lang, Nothing) -> do -- read from file

398− let translationFile = "translations/" ++ renderLang lang ++ ".yaml"

399− let fallbackFile = "translations/" ++ langLanguage lang ++ ".yaml"

400+ let translationFile = "translations/" <> renderLang lang <> ".yaml"

401+ let fallbackFile = "translations/" <> langLanguage lang <> ".yaml"

400402 let getTrans fp = do

401403 bs <- readDataFile fp

402− case readTranslations (UTF8.toString bs) of

404+ case readTranslations (UTF8.toText bs) of

403405 Left e -> do

404406 report $ CouldNotLoadTranslations (renderLang lang)

405− (fp ++ ": " ++ e)

407+ (T.pack fp <> ": " <> e)

406408 -- make sure we don't try again...

407409 modifyCommonState $ \st ->

408410 st{ stTranslations = Nothing }

@@ -411,14 +413,14 @@ getTranslations = do

411413 modifyCommonState $ \st ->

412414 st{ stTranslations = Just (lang, Just t) }

413415 return t

414− catchError (getTrans translationFile)

416+ catchError (getTrans $ T.unpack translationFile)

415417 (\_ ->

416− catchError (getTrans fallbackFile)

418+ catchError (getTrans $ T.unpack fallbackFile)

417419 (\e -> do

418420 report $ CouldNotLoadTranslations (renderLang lang)

419421 $ case e of

420422 PandocCouldNotFindDataFileError _ ->

421− "data file " ++ fallbackFile ++ " not found"

423+ "data file " <> fallbackFile <> " not found"

422424 _ -> ""

423425 -- make sure we don't try again...

424426 modifyCommonState $ \st -> st{ stTranslations = Nothing }

@@ -426,13 +428,13 @@ getTranslations = do

426428

427429-- | Get a translation from the current term map.

428430-- Issue a warning if the term is not defined.

429−translateTerm :: PandocMonad m => Term -> m String

431+translateTerm :: PandocMonad m => Term -> m T.Text

430432translateTerm term = do

431433 translations <- getTranslations

432434 case lookupTerm term translations of

433435 Just s -> return s

434436 Nothing -> do

435− report $ NoTranslation (show term)

437+ report $ NoTranslation $ T.pack $ show term

436438 return ""

437439

438440-- | Evaluate a 'PandocIO' operation.

@@ -458,7 +460,7 @@ liftIOError :: (String -> IO a) -> String -> PandocIO a

458460liftIOError f u = do

459461 res <- liftIO $ tryIOError $ f u

460462 case res of

461− Left e -> throwError $ PandocIOError u e

463+ Left e -> throwError $ PandocIOError (T.pack u) e

462464 Right r -> return r

463465

464466-- | Show potential IO errors to the user continuing execution anyway

@@ -466,24 +468,24 @@ logIOError :: IO () -> PandocIO ()

466468logIOError f = do

467469 res <- liftIO $ tryIOError f

468470 case res of

469− Left e -> report $ IgnoredIOError (E.displayException e)

471+ Left e -> report $ IgnoredIOError $ T.pack $ E.displayException e

470472 Right _ -> pure ()

471473

472474instance PandocMonad PandocIO where

473− lookupEnv = liftIO . IO.lookupEnv

475+ lookupEnv = fmap (fmap T.pack) . liftIO . IO.lookupEnv . T.unpack

474476 getCurrentTime = liftIO IO.getCurrentTime

475477 getCurrentTimeZone = liftIO IO.getCurrentTimeZone

476478 newStdGen = liftIO IO.newStdGen

477479 newUniqueHash = hashUnique <$> liftIO IO.newUnique

478480

479481 openURL u

480− | Just u'' <- stripPrefix "data:" u = do

481− let mime = takeWhile (/=',') u''

482+ | Just u'' <- T.stripPrefix "data:" u = do

483+ let mime = T.takeWhile (/=',') u''

482484 let contents = UTF8.fromString $

483− unEscapeString $ drop 1 $ dropWhile (/=',') u''

485+ unEscapeString $ T.unpack $ T.drop 1 $ T.dropWhile (/=',') u''

484486 return (decodeLenient contents, Just mime)

485487 | otherwise = do

486− let toReqHeader (n, v) = (CI.mk (UTF8.fromString n), UTF8.fromString v)

488+ let toReqHeader (n, v) = (CI.mk (UTF8.fromText n), UTF8.fromText v)

487489 customHeaders <- map toReqHeader <$> getsCommonState stRequestHeaders

488490 report $ Fetching u

489491 res <- liftIO $ E.try $ withSocketsDo $ do

@@ -493,11 +495,11 @@ instance PandocMonad PandocIO where

493495 Left _ -> return x

494496 Right pr -> parseReq pr >>= \r ->

495497 return (addProxy (host r) (port r) x)

496− req <- parseReq u >>= addProxy'

498+ req <- parseReq (T.unpack u) >>= addProxy'

497499 let req' = req{requestHeaders = customHeaders ++ requestHeaders req}

498500 resp <- newManager tlsManagerSettings >>= httpLbs req'

499501 return (B.concat $ toChunks $ responseBody resp,

500− UTF8.toString `fmap` lookup hContentType (responseHeaders resp))

502+ UTF8.toText `fmap` lookup hContentType (responseHeaders resp))

501503

502504 case res of

503505 Right r -> return r

@@ -519,22 +521,22 @@ instance PandocMonad PandocIO where

519521 logOutput msg = liftIO $ do

520522 UTF8.hPutStr stderr $

521523 "[" ++ show (messageVerbosity msg) ++ "] "

522− alertIndent $ lines $ showLogMessage msg

524+ alertIndent $ T.lines $ showLogMessage msg

523525

524−alertIndent :: [String] -> IO ()

526+alertIndent :: [T.Text] -> IO ()

525527alertIndent [] = return ()

526528alertIndent (l:ls) = do

527− UTF8.hPutStrLn stderr l

529+ UTF8.hPutStrLn stderr $ T.unpack l

528530 mapM_ go ls

529531 where go l' = do UTF8.hPutStr stderr " "

530− UTF8.hPutStrLn stderr l'

532+ UTF8.hPutStrLn stderr $ T.unpack l'

531533

532534-- | Specialized version of parseURIReference that disallows

533535-- single-letter schemes. Reason: these are usually windows absolute

534536-- paths.

535−parseURIReference' :: String -> Maybe URI

537+parseURIReference' :: T.Text -> Maybe URI

536538parseURIReference' s =

537− case parseURIReference s of

539+ case parseURIReference (T.unpack s) of

538540 Just u

539541 | length (uriScheme u) > 2 -> Just u

540542 | null (uriScheme u) -> Just u -- protocol-relative

@@ -554,16 +556,16 @@ getUserDataDir = getsCommonState stUserDataDir

554556-- | Fetch an image or other item from the local filesystem or the net.

555557-- Returns raw content and maybe mime type.

556558fetchItem :: PandocMonad m

557− => String

559+ => T.Text

558560 -> m (B.ByteString, Maybe MimeType)

559561fetchItem s = do

560562 mediabag <- getMediaBag

561− case lookupMedia s mediabag of

563+ case lookupMedia (T.unpack s) mediabag of

562564 Just (mime, bs) -> return (BL.toStrict bs, Just mime)

563565 Nothing -> downloadOrRead s

564566

565567downloadOrRead :: PandocMonad m

566− => String

568+ => T.Text

567569 -> m (B.ByteString, Maybe MimeType)

568570downloadOrRead s = do

569571 sourceURL <- getsCommonState stSourceURL

@@ -571,19 +573,19 @@ downloadOrRead s = do

571573 ensureEscaped, ensureEscaped s) of

572574 (Just u, s') -> -- try fetching from relative path at source

573575 case parseURIReference' s' of

574− Just u' -> openURL $ show $ u' `nonStrictRelativeTo` u

576+ Just u' -> openURL $ T.pack $ show $ u' `nonStrictRelativeTo` u

575577 Nothing -> openURL s' -- will throw error

576− (Nothing, s'@('/':'/':c:_)) | c /= '?' -> -- protocol-relative URI

578+ (Nothing, s'@(T.unpack -> ('/':'/':c:_))) | c /= '?' -> -- protocol-relative URI

577579 -- we exclude //? because of //?UNC/ on Windows

578580 case parseURIReference' s' of

579− Just u' -> openURL $ show $ u' `nonStrictRelativeTo` httpcolon

581+ Just u' -> openURL $ T.pack $ show $ u' `nonStrictRelativeTo` httpcolon

580582 Nothing -> openURL s' -- will throw error

581583 (Nothing, s') ->

582− case parseURI s' of -- requires absolute URI

584+ case parseURI (T.unpack s') of -- requires absolute URI

583585 Just u' | uriScheme u' == "file:" ->

584− readLocalFile $ uriPathToPath (uriPath u')

586+ readLocalFile $ uriPathToPath (T.pack $ uriPath u')

585587 -- We don't want to treat C:/ as a scheme:

586− Just u' | length (uriScheme u') > 2 -> openURL (show u')

588+ Just u' | length (uriScheme u') > 2 -> openURL (T.pack $ show u')

587589 _ -> readLocalFile fp -- get from local file system

588590 where readLocalFile f = do

589591 resourcePath <- getResourcePath

@@ -596,13 +598,13 @@ downloadOrRead s = do

596598 uriPath = "",

597599 uriQuery = "",

598600 uriFragment = "" }

599− dropFragmentAndQuery = takeWhile (\c -> c /= '?' && c /= '#')

600− fp = unEscapeString $ dropFragmentAndQuery s

601+ dropFragmentAndQuery = T.takeWhile (\c -> c /= '?' && c /= '#')

602+ fp = unEscapeString $ T.unpack $ dropFragmentAndQuery s

601603 mime = case takeExtension fp of

602604 ".gz" -> getMimeType $ dropExtension fp

603605 ".svgz" -> getMimeType $ dropExtension fp ++ ".svg"

604606 x -> getMimeType x

605− ensureEscaped = escapeURIString isAllowedInURI . map convertSlash

607+ ensureEscaped = T.pack . escapeURIString isAllowedInURI . T.unpack . T.map convertSlash

606608 convertSlash '\\' = '/'

607609 convertSlash x = x

608610

@@ -770,7 +772,7 @@ readDefaultDataFile "reference.odt" =

770772readDefaultDataFile fname =

771773#ifdef EMBED_DATA_FILES

772774 case lookup (makeCanonical fname) dataFiles of

773− Nothing -> throwError $ PandocCouldNotFindDataFileError fname

775+ Nothing -> throwError $ PandocCouldNotFindDataFileError $ T.pack fname

774776 Just contents -> return contents

775777#else

776778 getDataFileName fname' >>= checkExistence >>= readFileStrict

@@ -781,7 +783,7 @@ checkExistence fn = do

781783 exists <- fileExists fn

782784 if exists

783785 then return fn

784− else throwError $ PandocCouldNotFindDataFileError fn

786+ else throwError $ PandocCouldNotFindDataFileError $ T.pack fn

785787#endif

786788

787789makeCanonical :: FilePath -> FilePath

@@ -792,7 +794,7 @@ makeCanonical = Posix.joinPath . transformPathParts . splitDirectories

792794 go as x = x : as

793795

794796withPaths :: PandocMonad m => [FilePath] -> (FilePath -> m a) -> FilePath -> m a

795−withPaths [] _ fp = throwError $ PandocResourceNotFound fp

797+withPaths [] _ fp = throwError $ PandocResourceNotFound $ T.pack fp

796798withPaths (p:ps) action fp =

797799 catchError (action (p </> fp))

798800 (\_ -> withPaths ps action fp)

@@ -800,14 +802,14 @@ withPaths (p:ps) action fp =

800802-- | Fetch local or remote resource (like an image) and provide data suitable

801803-- for adding it to the MediaBag.

802804fetchMediaResource :: PandocMonad m

803− => String -> m (FilePath, Maybe MimeType, BL.ByteString)

805+ => T.Text -> m (FilePath, Maybe MimeType, BL.ByteString)

804806fetchMediaResource src = do

805807 (bs, mt) <- downloadOrRead src

806− let ext = fromMaybe (takeExtension src)

808+ let ext = fromMaybe (T.pack $ takeExtension $ T.unpack src)

807809 (mt >>= extensionFromMimeType)

808810 let bs' = BL.fromChunks [bs]

809811 let basename = showDigest $ sha1 bs'

810− let fname = basename <.> ext

812+ let fname = basename <.> T.unpack ext

811813 return (fname, mt, bs')

812814

813815-- | Traverse tree, filling media bag for any images that

@@ -817,12 +819,12 @@ fillMediaBag d = walkM handleImage d

817819 where handleImage :: PandocMonad m => Inline -> m Inline

818820 handleImage (Image attr lab (src, tit)) = catchError

819821 (do mediabag <- getMediaBag

820− case lookupMedia src mediabag of

822+ case lookupMedia (T.unpack src) mediabag of

821823 Just (_, _) -> return $ Image attr lab (src, tit)

822824 Nothing -> do

823825 (fname, mt, bs) <- fetchMediaResource src

824826 insertMedia fname mt bs

825− return $ Image attr lab (fname, tit))

827+ return $ Image attr lab (T.pack fname, tit))

826828 (\e ->

827829 case e of

828830 PandocResourceNotFound _ -> do

@@ -832,7 +834,7 @@ fillMediaBag d = walkM handleImage d

832834 return $ Span ("",["image"],[]) lab

833835 PandocHttpError u er -> do

834836 report $ CouldNotFetchResource u

835− (show er ++ "\rReplacing image with description.")

837+ (T.pack $ show er ++ "\rReplacing image with description.")

836838 -- emit alt text

837839 return $ Span ("",["image"],[]) lab

838840 _ -> throwError e)

@@ -856,15 +858,15 @@ writeMedia dir mediabag subpath = do

856858 let fullpath = dir </> unEscapeString (normalise subpath)

857859 let mbcontents = lookupMedia subpath mediabag

858860 case mbcontents of

859− Nothing -> throwError $ PandocResourceNotFound subpath

861+ Nothing -> throwError $ PandocResourceNotFound $ T.pack subpath

860862 Just (_, bs) -> do

861− report $ Extracting fullpath

863+ report $ Extracting $ T.pack fullpath

862864 liftIOError (createDirectoryIfMissing True) (takeDirectory fullpath)

863865 logIOError $ BL.writeFile fullpath bs

864866

865867adjustImagePath :: FilePath -> [FilePath] -> Inline -> Inline

866868adjustImagePath dir paths (Image attr lab (src, tit))

867− | src `elem` paths = Image attr lab (dir ++ "/" ++ src, tit)

869+ | T.unpack src `elem` paths = Image attr lab (T.pack dir <> "/" <> src, tit)

868870adjustImagePath _ _ x = x

869871

870872-- | The 'PureState' contains ersatz representations

@@ -878,7 +880,7 @@ data PureState = PureState { stStdGen :: StdGen

878880 -- contain every

879881 -- element at most

880882 -- once, e.g. [1..]

881− , stEnv :: [(String, String)]

883+ , stEnv :: [(T.Text, T.Text)]

882884 , stTime :: UTCTime

883885 , stTimeZone :: TimeZone

884886 , stReferenceDocx :: Archive

@@ -996,12 +998,12 @@ instance PandocMonad PandocPure where

996998 fps <- getsPureState stFiles

997999 case infoFileContents <$> getFileInfo fp fps of

9981000 Just bs -> return (BL.fromStrict bs)

999− Nothing -> throwError $ PandocResourceNotFound fp

1001+ Nothing -> throwError $ PandocResourceNotFound $ T.pack fp

10001002 readFileStrict fp = do

10011003 fps <- getsPureState stFiles

10021004 case infoFileContents <$> getFileInfo fp fps of

10031005 Just bs -> return bs

1004− Nothing -> throwError $ PandocResourceNotFound fp

1006+ Nothing -> throwError $ PandocResourceNotFound $ T.pack fp

10051007

10061008 glob s = do

10071009 FileTree ftmap <- getsPureState stFiles

@@ -1019,7 +1021,7 @@ instance PandocMonad PandocPure where

10191021 fps <- getsPureState stFiles

10201022 case infoFileMTime <$> getFileInfo fp fps of

10211023 Just tm -> return tm

1022− Nothing -> throwError $ PandocIOError fp

1024+ Nothing -> throwError $ PandocIOError (T.pack fp)

10231025 (userError "Can't get modification time")

10241026

10251027 getCommonState = PandocPure $ lift get

@@ -1070,7 +1072,7 @@ instance {-# OVERLAPS #-} PandocMonad m => PandocMonad (ParsecT s st m) where

10701072 when tracing $ do

10711073 pos <- getPosition

10721074 Debug.Trace.trace

1073− ("[trace] Parsed " ++ msg ++ " at line " ++

1075+ ("[trace] Parsed " ++ T.unpack msg ++ " at line " ++

10741076 show (sourceLine pos) ++

10751077 if sourceName pos == "chunk"

10761078 then " of chunk"

src/Text/Pandoc/Emoji.hs+4−2

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Emoji

@@ -13,9 +14,10 @@ Emoji symbol lookup from canonical string identifier.

1314module Text.Pandoc.Emoji ( emojis, emojiToInline ) where

1415import Prelude

1516import qualified Data.Map as M

17+import qualified Data.Text as T

1618import Text.Pandoc.Definition (Inline (Span, Str))

1719

18−emojis :: M.Map String String

20+emojis :: M.Map T.Text T.Text

1921emojis = M.fromList

2022 [("+1","\128077")

2123 ,("-1","\128078")

@@ -1810,6 +1812,6 @@ emojis = M.fromList

18101812 ,("zzz","\128164")

18111813 ]

18121814

1813−emojiToInline :: String -> Maybe Inline

1815+emojiToInline :: T.Text -> Maybe Inline

18141816emojiToInline emojikey = makeSpan <$> M.lookup emojikey emojis

18151817 where makeSpan = Span ("", ["emoji"], [("data-emoji", emojikey)]) . (:[]) . Str

src/Text/Pandoc/Error.hs+61−55

@@ -1,6 +1,7 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE DeriveDataTypeable #-}

33{-# LANGUAGE DeriveGeneric #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Error

@@ -22,6 +23,8 @@ import Prelude

2223import Control.Exception (Exception)

2324import Data.Typeable (Typeable)

2425import Data.Word (Word8)

26+import Data.Text (Text)

27+import qualified Data.Text as T

2528import GHC.Generics (Generic)

2629import Network.HTTP.Client (HttpException)

2730import System.Exit (ExitCode (..), exitWith)

@@ -31,32 +34,32 @@ import Text.Printf (printf)

3134import Text.Parsec.Error

3235import Text.Parsec.Pos hiding (Line)

3336

34−type Input = String

37+type Input = Text

3538

36−data PandocError = PandocIOError String IOError

37− | PandocHttpError String HttpException

38− | PandocShouldNeverHappenError String

39− | PandocSomeError String

40− | PandocParseError String

39+data PandocError = PandocIOError Text IOError

40+ | PandocHttpError Text HttpException

41+ | PandocShouldNeverHappenError Text

42+ | PandocSomeError Text

43+ | PandocParseError Text

4144 | PandocParsecError Input ParseError

42− | PandocMakePDFError String

43− | PandocOptionError String

44− | PandocSyntaxMapError String

45+ | PandocMakePDFError Text

46+ | PandocOptionError Text

47+ | PandocSyntaxMapError Text

4548 | PandocFailOnWarningError

46− | PandocPDFProgramNotFoundError String

47− | PandocPDFError String

48− | PandocFilterError String String

49− | PandocCouldNotFindDataFileError String

50− | PandocResourceNotFound String

51− | PandocTemplateError String

52− | PandocAppError String

53− | PandocEpubSubdirectoryError String

54− | PandocMacroLoop String

55− | PandocUTF8DecodingError String Int Word8

56− | PandocIpynbDecodingError String

57− | PandocUnknownReaderError String

58− | PandocUnknownWriterError String

59− | PandocUnsupportedExtensionError String String

49+ | PandocPDFProgramNotFoundError Text

50+ | PandocPDFError Text

51+ | PandocFilterError Text Text

52+ | PandocCouldNotFindDataFileError Text

53+ | PandocResourceNotFound Text

54+ | PandocTemplateError Text

55+ | PandocAppError Text

56+ | PandocEpubSubdirectoryError Text

57+ | PandocMacroLoop Text

58+ | PandocUTF8DecodingError Text Int Word8

59+ | PandocIpynbDecodingError Text

60+ | PandocUnknownReaderError Text

61+ | PandocUnknownWriterError Text

62+ | PandocUnsupportedExtensionError Text Text

6063 deriving (Show, Typeable, Generic)

6164

6265instance Exception PandocError

@@ -68,23 +71,23 @@ handleError (Left e) =

6871 case e of

6972 PandocIOError _ err' -> ioError err'

7073 PandocHttpError u err' -> err 61 $

71− "Could not fetch " ++ u ++ "\n" ++ show err'

74+ "Could not fetch " <> u <> "\n" <> tshow err'

7275 PandocShouldNeverHappenError s -> err 62 $

73− "Something we thought was impossible happened!\n" ++

74− "Please report this to pandoc's developers: " ++ s

76+ "Something we thought was impossible happened!\n" <>

77+ "Please report this to pandoc's developers: " <> s

7578 PandocSomeError s -> err 63 s

7679 PandocParseError s -> err 64 s

7780 PandocParsecError input err' ->

7881 let errPos = errorPos err'

7982 errLine = sourceLine errPos

8083 errColumn = sourceColumn errPos

81− ls = lines input ++ [""]

84+ ls = T.lines input <> [""]

8285 errorInFile = if length ls > errLine - 1

83− then concat ["\n", ls !! (errLine - 1)

84− ,"\n", replicate (errColumn - 1) ' '

85− ,"^"]

86+ then T.concat ["\n", ls !! (errLine - 1)

87+ ,"\n", T.replicate (errColumn - 1) " "

88+ ,"^"]

8689 else ""

87− in err 65 $ "\nError at " ++ show err' ++

90+ in err 65 $ "\nError at " <> tshow err' <>

8891 -- if error comes from a chunk or included file,

8992 -- then we won't get the right text this way:

9093 if sourceName errPos == "source"

@@ -95,49 +98,52 @@ handleError (Left e) =

9598 PandocSyntaxMapError s -> err 67 s

9699 PandocFailOnWarningError -> err 3 "Failing because there were warnings."

97100 PandocPDFProgramNotFoundError pdfprog -> err 47 $

98− pdfprog ++ " not found. Please select a different --pdf-engine or install " ++ pdfprog

99− PandocPDFError logmsg -> err 43 $ "Error producing PDF.\n" ++ logmsg

100− PandocFilterError filtername msg -> err 83 $ "Error running filter " ++

101− filtername ++ ":\n" ++ msg

101+ pdfprog <> " not found. Please select a different --pdf-engine or install " <> pdfprog

102+ PandocPDFError logmsg -> err 43 $ "Error producing PDF.\n" <> logmsg

103+ PandocFilterError filtername msg -> err 83 $ "Error running filter " <>

104+ filtername <> ":\n" <> msg

102105 PandocCouldNotFindDataFileError fn -> err 97 $

103− "Could not find data file " ++ fn

106+ "Could not find data file " <> fn

104107 PandocResourceNotFound fn -> err 99 $

105− "File " ++ fn ++ " not found in resource path"

106− PandocTemplateError s -> err 5 $ "Error compiling template " ++ s

108+ "File " <> fn <> " not found in resource path"

109+ PandocTemplateError s -> err 5 $ "Error compiling template " <> s

107110 PandocAppError s -> err 4 s

108111 PandocEpubSubdirectoryError s -> err 31 $

109− "EPUB subdirectory name '" ++ s ++ "' contains illegal characters"

112+ "EPUB subdirectory name '" <> s <> "' contains illegal characters"

110113 PandocMacroLoop s -> err 91 $

111− "Loop encountered in expanding macro " ++ s

114+ "Loop encountered in expanding macro " <> s

112115 PandocUTF8DecodingError f offset w -> err 92 $

113− "UTF-8 decoding error in " ++ f ++ " at byte offset " ++ show offset ++

114− " (" ++ printf "%2x" w ++ ").\n" ++

116+ "UTF-8 decoding error in " <> f <> " at byte offset " <> tshow offset <>

117+ " (" <> T.pack (printf "%2x" w) <> ").\n" <>

115118 "The input must be a UTF-8 encoded text."

116119 PandocIpynbDecodingError w -> err 93 $

117− "ipynb decoding error: " ++ w

120+ "ipynb decoding error: " <> w

118121 PandocUnknownReaderError r -> err 21 $

119− "Unknown input format " ++ r ++

122+ "Unknown input format " <> r <>

120123 case r of

121− "doc" -> "\nPandoc can convert from DOCX, but not from DOC." ++

122− "\nTry using Word to save your DOC file as DOCX," ++

124+ "doc" -> "\nPandoc can convert from DOCX, but not from DOC." <>

125+ "\nTry using Word to save your DOC file as DOCX," <>

123126 " and convert that with pandoc."

124127 "pdf" -> "\nPandoc can convert to PDF, but not from PDF."

125128 _ -> ""

126129 PandocUnknownWriterError w -> err 22 $

127− "Unknown output format " ++ w ++

130+ "Unknown output format " <> w <>

128131 case w of

129− "pdf" -> "To create a pdf using pandoc, use" ++

130− " -t latex|beamer|context|ms|html5" ++

131− "\nand specify an output file with " ++

132+ "pdf" -> "To create a pdf using pandoc, use" <>

133+ " -t latex|beamer|context|ms|html5" <>

134+ "\nand specify an output file with " <>

132135 ".pdf extension (-o filename.pdf)."

133136 "doc" -> "\nPandoc can convert to DOCX, but not from DOC."

134137 _ -> ""

135138 PandocUnsupportedExtensionError ext f -> err 23 $

136− "The extension " ++ ext ++ " is not supported " ++

137− "for " ++ f

139+ "The extension " <> ext <> " is not supported " <>

140+ "for " <> f

138141

139−err :: Int -> String -> IO a

142+err :: Int -> Text -> IO a

140143err exitCode msg = do

141− UTF8.hPutStrLn stderr msg

144+ UTF8.hPutStrLn stderr (T.unpack msg)

142145 exitWith $ ExitFailure exitCode

143146 return undefined

147+

148+tshow :: Show a => a -> Text

149+tshow = T.pack . show

src/Text/Pandoc/Extensions.hs+7−5

@@ -4,6 +4,7 @@

44{-# LANGUAGE DeriveGeneric #-}

55{-# LANGUAGE GeneralizedNewtypeDeriving #-}

66{-# LANGUAGE TemplateHaskell #-}

7+{-# LANGUAGE OverloadedStrings #-}

78{- |

89 Module : Text.Pandoc.Extensions

@@ -35,6 +36,7 @@ where

3536import Prelude

3637import Data.Bits (clearBit, setBit, testBit, (.|.))

3738import Data.Data (Data)

39+import qualified Data.Text as T

3840import Data.Typeable (Typeable)

3941import GHC.Generics (Generic)

4042import Safe (readMay)

@@ -304,7 +306,7 @@ strictExtensions = extensionsFromList

304306 ]

305307

306308-- | Default extensions from format-describing string.

307−getDefaultExtensions :: String -> Extensions

309+getDefaultExtensions :: T.Text -> Extensions

308310getDefaultExtensions "markdown_strict" = strictExtensions

309311getDefaultExtensions "markdown_phpextra" = phpMarkdownExtraExtensions

310312getDefaultExtensions "markdown_mmd" = multimarkdownExtensions

@@ -402,7 +404,7 @@ allMarkdownExtensions =

402404

403405-- | Get all valid extensions for a format. This is used

404406-- mainly in checking format specifications for validity.

405−getAllExtensions :: String -> Extensions

407+getAllExtensions :: T.Text -> Extensions

406408getAllExtensions f = universalExtensions <> getAll f

407409 where

408410 autoIdExtensions = extensionsFromList

@@ -507,14 +509,14 @@ getAllExtensions f = universalExtensions <> getAll f

507509

508510-- | Parse a format-specifying string into a markup format,

509511-- a set of extensions to enable, and a set of extensions to disable.

510−parseFormatSpec :: String

511− -> Either ParseError (String, [Extension], [Extension])

512+parseFormatSpec :: T.Text

513+ -> Either ParseError (T.Text, [Extension], [Extension])

512514parseFormatSpec = parse formatSpec ""

513515 where formatSpec = do

514516 name <- formatName

515517 (extsToEnable, extsToDisable) <- foldl (flip ($)) ([],[]) <$>

516518 many extMod

517− return (name, reverse extsToEnable, reverse extsToDisable)

519+ return (T.pack name, reverse extsToEnable, reverse extsToDisable)

518520 formatName = many1 $ noneOf "-+"

519521 extMod = do

520522 polarity <- oneOf "-+"

src/Text/Pandoc/Filter/JSON.hs+12−8

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Filter

@@ -18,6 +19,7 @@ import Control.Monad.Trans (MonadIO (liftIO))

1819import Data.Aeson (eitherDecode', encode)

1920import Data.Char (toLower)

2021import Data.Maybe (isNothing)

22+import qualified Data.Text as T

2123import System.Directory (executable, doesFileExist, findExecutable,

2224 getPermissions)

2325import System.Environment (getEnvironment)

@@ -28,7 +30,7 @@ import Text.Pandoc.Error (PandocError (PandocFilterError))

2830import Text.Pandoc.Definition (Pandoc)

2931import Text.Pandoc.Options (ReaderOptions)

3032import Text.Pandoc.Process (pipeProcess)

31−import Text.Pandoc.Shared (pandocVersion)

33+import Text.Pandoc.Shared (pandocVersion, tshow)

3234import qualified Control.Exception as E

3335import qualified Text.Pandoc.UTF8 as UTF8

3436

@@ -61,18 +63,20 @@ externalFilter ropts f args' d = liftIO $ do

6163 unless (exists && isExecutable) $ do

6264 mbExe <- findExecutable f'

6365 when (isNothing mbExe) $

64− E.throwIO $ PandocFilterError f ("Could not find executable " ++ f')

66+ E.throwIO $ PandocFilterError fText (T.pack $ "Could not find executable " <> f')

6567 env <- getEnvironment

6668 let env' = Just

67− ( ("PANDOC_VERSION", pandocVersion)

69+ ( ("PANDOC_VERSION", T.unpack pandocVersion)

6870 : ("PANDOC_READER_OPTIONS", UTF8.toStringLazy (encode ropts))

6971 : env )

7072 (exitcode, outbs) <- E.handle filterException $

7173 pipeProcess env' f' args'' $ encode d

7274 case exitcode of

73− ExitSuccess -> either (E.throwIO . PandocFilterError f)

75+ ExitSuccess -> either (E.throwIO . PandocFilterError fText . T.pack)

7476 return $ eitherDecode' outbs

75− ExitFailure ec -> E.throwIO $ PandocFilterError f

76− ("Filter returned error status " ++ show ec)

77− where filterException :: E.SomeException -> IO a

78− filterException e = E.throwIO $ PandocFilterError f (show e)

77+ ExitFailure ec -> E.throwIO $ PandocFilterError fText

78+ ("Filter returned error status " <> tshow ec)

79+ where fText = T.pack f

80+

81+ filterException :: E.SomeException -> IO a

82+ filterException e = E.throwIO $ PandocFilterError fText $ tshow e

src/Text/Pandoc/Filter/Lua.hs+3−2

@@ -15,6 +15,7 @@ module Text.Pandoc.Filter.Lua (apply) where

1515import Prelude

1616import Control.Exception (throw)

1717import Control.Monad ((>=>))

18+import qualified Data.Text as T

1819import Text.Pandoc.Class (PandocIO)

1920import Text.Pandoc.Definition (Pandoc)

2021import Text.Pandoc.Error (PandocError (PandocFilterError))

@@ -35,7 +36,7 @@ apply ropts args fp doc = do

3536 (x:_) -> x

3637 _ -> error "Format not supplied for Lua filter"

3738 runLua >=> forceResult fp $ do

38− setGlobals [ FORMAT format

39+ setGlobals [ FORMAT $ T.pack format

3940 , PANDOC_READER_OPTIONS ropts

4041 , PANDOC_SCRIPT_FILE fp

4142 ]

@@ -44,4 +45,4 @@ apply ropts args fp doc = do

4445forceResult :: FilePath -> Either LuaException Pandoc -> PandocIO Pandoc

4546forceResult fp eitherResult = case eitherResult of

4647 Right x -> return x

47− Left (LuaException s) -> throw (PandocFilterError fp s)

48+ Left (LuaException s) -> throw (PandocFilterError (T.pack fp) s)

src/Text/Pandoc/Highlighting.hs+22−24

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Highlighting

@@ -35,7 +36,6 @@ module Text.Pandoc.Highlighting ( highlightingStyles

3536 ) where

3637import Prelude

3738import Control.Monad

38−import Data.Char (toLower)

3939import qualified Data.Map as M

4040import Data.Maybe (fromMaybe)

4141import qualified Data.Text as T

@@ -43,7 +43,7 @@ import Skylighting

4343import Text.Pandoc.Definition

4444import Text.Pandoc.Shared (safeRead)

4545

46−highlightingStyles :: [(String, Style)]

46+highlightingStyles :: [(T.Text, Style)]

4747highlightingStyles =

4848 [("pygments", pygments),

4949 ("tango", tango),

@@ -54,18 +54,18 @@ highlightingStyles =

5454 ("breezedark", breezeDark),

5555 ("haddock", haddock)]

5656

57−languages :: [String]

58−languages = [T.unpack (T.toLower (sName s)) | s <- M.elems defaultSyntaxMap]

57+languages :: [T.Text]

58+languages = [T.toLower (sName s) | s <- M.elems defaultSyntaxMap]

5959

60−languagesByExtension :: String -> [String]

60+languagesByExtension :: T.Text -> [T.Text]

6161languagesByExtension ext =

62− [T.unpack (T.toLower (sName s)) | s <- syntaxesByExtension defaultSyntaxMap ext]

62+ [T.toLower (sName s) | s <- syntaxesByExtension defaultSyntaxMap (T.unpack ext)]

6363

6464highlight :: SyntaxMap

6565 -> (FormatOptions -> [SourceLine] -> a) -- ^ Formatter

6666 -> Attr -- ^ Attributes of the CodeBlock

67− -> String -- ^ Raw contents of the CodeBlock

68− -> Either String a

67+ -> T.Text -- ^ Raw contents of the CodeBlock

68+ -> Either T.Text a

6969highlight syntaxmap formatter (ident, classes, keyvals) rawCode =

7070 let firstNum = fromMaybe 1 (safeRead (fromMaybe "1" $ lookup "startFrom" keyvals))

7171 fmtOpts = defaultFormatOpts{

@@ -74,38 +74,36 @@ highlight syntaxmap formatter (ident, classes, keyvals) rawCode =

7474 ["line-anchors", "lineAnchors"]) classes,

7575 numberLines = any (`elem`

7676 ["number","numberLines", "number-lines"]) classes,

77− lineIdPrefix = if null ident

77+ lineIdPrefix = if T.null ident

7878 then mempty

79− else T.pack (ident ++ "-") }

79+ else ident <> "-" }

8080 tokenizeOpts = TokenizerConfig{ syntaxMap = syntaxmap

8181 , traceOutput = False }

82− classes' = map T.pack classes

83− rawCode' = T.pack rawCode

84− in case msum (map (`lookupSyntax` syntaxmap) classes') of

82+ in case msum (map (`lookupSyntax` syntaxmap) classes) of

8583 Nothing

8684 | numberLines fmtOpts -> Right

8785 $ formatter fmtOpts{ codeClasses = [],

88− containerClasses = classes' }

86+ containerClasses = classes }

8987 $ map (\ln -> [(NormalTok, ln)])

90− $ T.lines rawCode'

88+ $ T.lines rawCode

9189 | otherwise -> Left ""

92− Just syntax ->

90+ Just syntax -> either (Left . T.pack) Right $

9391 formatter fmtOpts{ codeClasses =

9492 [T.toLower (sShortname syntax)],

95− containerClasses = classes' } <$>

96− tokenize tokenizeOpts syntax rawCode'

93+ containerClasses = classes } <$>

94+ tokenize tokenizeOpts syntax rawCode

9795

9896-- Functions for correlating latex listings package's language names

9997-- with skylighting language names:

10098

101−langToListingsMap :: M.Map String String

99+langToListingsMap :: M.Map T.Text T.Text

102100langToListingsMap = M.fromList langsList

103101

104−listingsToLangMap :: M.Map String String

102+listingsToLangMap :: M.Map T.Text T.Text

105103listingsToLangMap = M.fromList $ map switch langsList

106104 where switch (a,b) = (b,a)

107105

108−langsList :: [(String, String)]

106+langsList :: [(T.Text, T.Text)]

109107langsList =

110108 [("abap","ABAP"),

111109 ("acm","ACM"),

@@ -212,9 +210,9 @@ langsList =

212210 ("xslt","XSLT")]

213211

214212-- | Determine listings language name from skylighting language name.

215−toListingsLanguage :: String -> Maybe String

216−toListingsLanguage lang = M.lookup (map toLower lang) langToListingsMap

213+toListingsLanguage :: T.Text -> Maybe T.Text

214+toListingsLanguage lang = M.lookup (T.toLower lang) langToListingsMap

217215

218216-- | Determine skylighting language name from listings language name.

219−fromListingsLanguage :: String -> Maybe String

217+fromListingsLanguage :: T.Text -> Maybe T.Text

220218fromListingsLanguage lang = M.lookup lang listingsToLangMap

src/Text/Pandoc/ImageSize.hs+38−37

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE OverloadedStrings, ScopedTypeVariables, CPP #-}

3+{-# LANGUAGE ViewPatterns #-}

34{-# OPTIONS_GHC -fno-warn-type-defaults #-}

45{- |

56Module : Text.Pandoc.ImageSize

@@ -49,6 +50,8 @@ import Text.Pandoc.Options

4950import qualified Text.Pandoc.UTF8 as UTF8

5051import qualified Text.XML.Light as Xml

5152import qualified Data.Map as M

53+import qualified Data.Text as T

54+import qualified Data.Text.Encoding as TE

5255import Control.Monad.Except

5356import Control.Applicative

5457import Data.Maybe (fromMaybe)

@@ -72,12 +75,12 @@ data Dimension = Pixel Integer

7275 deriving Eq

7376

7477instance Show Dimension where

75− show (Pixel a) = show a ++ "px"

76− show (Centimeter a) = showFl a ++ "cm"

77− show (Millimeter a) = showFl a ++ "mm"

78− show (Inch a) = showFl a ++ "in"

79− show (Percent a) = show a ++ "%"

80− show (Em a) = showFl a ++ "em"

78+ show (Pixel a) = show a ++ "px"

79+ show (Centimeter a) = T.unpack (showFl a) ++ "cm"

80+ show (Millimeter a) = T.unpack (showFl a) ++ "mm"

81+ show (Inch a) = T.unpack (showFl a) ++ "in"

82+ show (Percent a) = show a ++ "%"

83+ show (Em a) = T.unpack (showFl a) ++ "em"

8184

8285data ImageSize = ImageSize{

8386 pxX :: Integer

@@ -88,14 +91,13 @@ data ImageSize = ImageSize{

8891instance Default ImageSize where

8992 def = ImageSize 300 200 72 72

9093

91−showFl :: (RealFloat a) => a -> String

92−showFl a = removeExtra0s $ showFFloat (Just 5) a ""

94+showFl :: (RealFloat a) => a -> T.Text

95+showFl a = removeExtra0s $ T.pack $ showFFloat (Just 5) a ""

9396

94−removeExtra0s :: String -> String

95−removeExtra0s s =

96− case dropWhile (=='0') $ reverse s of

97− '.':xs -> reverse xs

98− xs -> reverse xs

97+removeExtra0s :: T.Text -> T.Text

98+removeExtra0s s = case T.dropWhileEnd (=='0') s of

99+ (T.unsnoc -> Just (xs, '.')) -> xs

100+ xs -> xs

99101

100102imageType :: ByteString -> Maybe ImageType

101103imageType img = case B.take 4 img of

@@ -119,7 +121,7 @@ imageType img = case B.take 4 img of

119121findSvgTag :: ByteString -> Bool

120122findSvgTag img = "<svg" `B.isInfixOf` img || "<SVG" `B.isInfixOf` img

121123

122−imageSize :: WriterOptions -> ByteString -> Either String ImageSize

124+imageSize :: WriterOptions -> ByteString -> Either T.Text ImageSize

123125imageSize opts img =

124126 case imageType img of

125127 Just Png -> mbToEither "could not determine PNG size" $ pngSize img

@@ -194,22 +196,22 @@ inPixel opts dim =

194196 where

195197 dpi = fromIntegral $ writerDpi opts

196198

197−-- | Convert a Dimension to a String denoting its equivalent in inches, for example "2.00000".

199+-- | Convert a Dimension to Text denoting its equivalent in inches, for example "2.00000".

198200-- Note: Dimensions in percentages are converted to the empty string.

199−showInInch :: WriterOptions -> Dimension -> String

201+showInInch :: WriterOptions -> Dimension -> T.Text

200202showInInch _ (Percent _) = ""

201203showInInch opts dim = showFl $ inInch opts dim

202204

203−-- | Convert a Dimension to a String denoting its equivalent in pixels, for example "600".

205+-- | Convert a Dimension to Text denoting its equivalent in pixels, for example "600".

204206-- Note: Dimensions in percentages are converted to the empty string.

205−showInPixel :: WriterOptions -> Dimension -> String

207+showInPixel :: WriterOptions -> Dimension -> T.Text

206208showInPixel _ (Percent _) = ""

207−showInPixel opts dim = show $ inPixel opts dim

209+showInPixel opts dim = T.pack $ show $ inPixel opts dim

208210

209211-- | Maybe split a string into a leading number and trailing unit, e.g. "3cm" to Just (3.0, "cm")

210−numUnit :: String -> Maybe (Double, String)

212+numUnit :: T.Text -> Maybe (Double, T.Text)

211213numUnit s =

212− let (nums, unit) = span (\c -> isDigit c || ('.'==c)) s

214+ let (nums, unit) = T.span (\c -> isDigit c || ('.'==c)) s

213215 in case safeRead nums of

214216 Just n -> Just (n, unit)

215217 Nothing -> Nothing

@@ -235,7 +237,7 @@ dimension dir (_, _, kvs) =

235237 where

236238 extractDim key = lookup key kvs >>= lengthToDim

237239

238−lengthToDim :: String -> Maybe Dimension

240+lengthToDim :: T.Text -> Maybe Dimension

239241lengthToDim s = numUnit s >>= uncurry toDim

240242 where

241243 toDim a "cm" = Just $ Centimeter a

@@ -258,8 +260,8 @@ epsSize img = do

258260 [] -> mzero

259261 (x:_) -> case B.words x of

260262 [_, _, _, ux, uy] -> do

261− ux' <- safeRead $ B.unpack ux

262− uy' <- safeRead $ B.unpack uy

263+ ux' <- safeRead $ TE.decodeUtf8 ux

264+ uy' <- safeRead $ TE.decodeUtf8 uy

263265 return ImageSize{

264266 pxX = ux'

265267 , pxY = uy'

@@ -284,7 +286,7 @@ pPdfSize = do

284286 [x1,y1,x2,y2] <- A.count 4 $ do

285287 A.skipSpace

286288 raw <- A.many1 $ A.satisfy (\c -> isDigit c || c == '.')

287− case safeRead raw of

289+ case safeRead $ T.pack raw of

288290 Just (r :: Double) -> return $ floor r

289291 Nothing -> mzero

290292 A.skipSpace

@@ -345,7 +347,7 @@ svgSize opts img = do

345347 doc <- Xml.parseXMLDoc $ UTF8.toString img

346348 let dpi = fromIntegral $ writerDpi opts

347349 let dirToInt dir = do

348− dim <- Xml.findAttrBy (== Xml.QName dir Nothing Nothing) doc >>= lengthToDim

350+ dim <- Xml.findAttrBy (== Xml.QName dir Nothing Nothing) doc >>= lengthToDim . T.pack

349351 return $ inPixel opts dim

350352 w <- dirToInt "width"

351353 h <- dirToInt "height"

@@ -388,7 +390,7 @@ emfSize img =

388390 Right (_, _, size) -> Just size

389391

390392

391−jpegSize :: ByteString -> Either String ImageSize

393+jpegSize :: ByteString -> Either T.Text ImageSize

392394jpegSize img =

393395 let (hdr, rest) = B.splitAt 4 img

394396 in if B.length rest < 14

@@ -398,7 +400,7 @@ jpegSize img =

398400 "\xff\xd8\xff\xe1" -> exifSize rest

399401 _ -> Left "unable to determine JPEG size"

400402

401−jfifSize :: ByteString -> Either String ImageSize

403+jfifSize :: ByteString -> Either T.Text ImageSize

402404jfifSize rest =

403405 case map fromIntegral $ unpack $ B.take 5 $ B.drop 9 rest of

404406 [dpiDensity,dpix1,dpix2,dpiy1,dpiy2] ->

@@ -416,7 +418,7 @@ jfifSize rest =

416418 , dpiY = dpiy }

417419 _ -> Left "unable to determine JFIF size"

418420

419−findJfifSize :: ByteString -> Either String (Integer,Integer)

421+findJfifSize :: ByteString -> Either T.Text (Integer,Integer)

420422findJfifSize bs =

421423 let bs' = B.dropWhile (=='\xff') $ B.dropWhile (/='\xff') bs

422424 in case B.uncons bs' of

@@ -433,19 +435,18 @@ findJfifSize bs =

433435 _ -> Left "JFIF parse error"

434436 Nothing -> Left "Did not find JFIF length record"

435437

436−runGet' :: Get (Either String a) -> BL.ByteString -> Either String a

438+runGet' :: Get (Either T.Text a) -> BL.ByteString -> Either T.Text a

437439runGet' p bl =

438440#if MIN_VERSION_binary(0,7,0)

439441 case runGetOrFail p bl of

440− Left (_,_,msg) -> Left msg

442+ Left (_,_,msg) -> Left $ T.pack msg

441443 Right (_,_,x) -> x

442444#else

443445 runGet p bl

444446#endif

445447

446−

447−exifSize :: ByteString -> Either String ImageSize

448−exifSize bs =runGet' header bl

448+exifSize :: ByteString -> Either T.Text ImageSize

449+exifSize bs = runGet' header bl

449450 where bl = BL.fromChunks [bs]

450451 header = runExceptT $ exifHeader bl

451452-- NOTE: It would be nicer to do

@@ -454,7 +455,7 @@ exifSize bs =runGet' header bl

454455-- be parsed. But we only get an Alternative instance for Get in binary 0.6,

455456-- and binary 0.5 ships with ghc 7.6.

456457

457−exifHeader :: BL.ByteString -> ExceptT String Get ImageSize

458+exifHeader :: BL.ByteString -> ExceptT T.Text Get ImageSize

458459exifHeader hdr = do

459460 _app1DataSize <- lift getWord16be

460461 exifHdr <- lift getWord32be

@@ -479,7 +480,7 @@ exifHeader hdr = do

479480 ifdOffset <- lift getWord32

480481 lift $ skip (fromIntegral ifdOffset - 8) -- skip to IDF

481482 numentries <- lift getWord16

482− let ifdEntry :: ExceptT String Get (TagType, DataFormat)

483+ let ifdEntry :: ExceptT T.Text Get (TagType, DataFormat)

483484 ifdEntry = do

484485 tag <- fromMaybe UnknownTagType . flip M.lookup tagTypeTable

485486 <$> lift getWord16

@@ -502,7 +503,7 @@ exifHeader hdr = do

502503 10 -> return (SignedRational <$> getRational, 8)

503504 11 -> return (SingleFloat <$> getWord32 {- TODO -}, 4)

504505 12 -> return (DoubleFloat <$> getWord64 {- TODO -}, 8)

505− _ -> throwError $ "Unknown data format " ++ show dataFormat

506+ _ -> throwError $ "Unknown data format " <> T.pack (show dataFormat)

506507 let totalBytes = fromIntegral $ numComponents * bytesPerComponent

507508 payload <- if totalBytes <= 4 -- data is right here

508509 then lift $ fmt <* skip (4 - totalBytes)

src/Text/Pandoc/Logging.hs+179−180

@@ -31,7 +31,6 @@ import Data.Aeson.Encode.Pretty (Config (..), defConfig, encodePretty',

3131 keyOrder)

3232import qualified Data.ByteString.Lazy as BL

3333import Data.Data (Data, toConstr)

34−import Data.List (isSuffixOf, intercalate)

3534import qualified Data.Text as Text

3635import Data.Typeable (Typeable)

3736import GHC.Generics (Generic)

@@ -62,43 +61,43 @@ instance FromYAML Verbosity where

6261 _ -> mzero

6362

6463data LogMessage =

65− SkippedContent String SourcePos

66− | IgnoredElement String

67− | CouldNotParseYamlMetadata String SourcePos

68− | DuplicateLinkReference String SourcePos

69− | DuplicateNoteReference String SourcePos

70− | NoteDefinedButNotUsed String SourcePos

71− | DuplicateIdentifier String SourcePos

72− | ReferenceNotFound String SourcePos

73− | CircularReference String SourcePos

74− | UndefinedToggle String SourcePos

75− | ParsingUnescaped String SourcePos

76− | CouldNotLoadIncludeFile String SourcePos

77− | MacroAlreadyDefined String SourcePos

64+ SkippedContent Text.Text SourcePos

65+ | IgnoredElement Text.Text

66+ | CouldNotParseYamlMetadata Text.Text SourcePos

67+ | DuplicateLinkReference Text.Text SourcePos

68+ | DuplicateNoteReference Text.Text SourcePos

69+ | NoteDefinedButNotUsed Text.Text SourcePos

70+ | DuplicateIdentifier Text.Text SourcePos

71+ | ReferenceNotFound Text.Text SourcePos

72+ | CircularReference Text.Text SourcePos

73+ | UndefinedToggle Text.Text SourcePos

74+ | ParsingUnescaped Text.Text SourcePos

75+ | CouldNotLoadIncludeFile Text.Text SourcePos

76+ | MacroAlreadyDefined Text.Text SourcePos

7877 | InlineNotRendered Inline

7978 | BlockNotRendered Block

80− | DocxParserWarning String

81− | IgnoredIOError String

82− | CouldNotFetchResource String String

83− | CouldNotDetermineImageSize String String

84− | CouldNotConvertImage String String

85− | CouldNotDetermineMimeType String

86− | CouldNotConvertTeXMath String String

87− | CouldNotParseCSS String

88− | Fetching String

89− | Extracting String

90− | NoTitleElement String

79+ | DocxParserWarning Text.Text

80+ | IgnoredIOError Text.Text

81+ | CouldNotFetchResource Text.Text Text.Text

82+ | CouldNotDetermineImageSize Text.Text Text.Text

83+ | CouldNotConvertImage Text.Text Text.Text

84+ | CouldNotDetermineMimeType Text.Text

85+ | CouldNotConvertTeXMath Text.Text Text.Text

86+ | CouldNotParseCSS Text.Text

87+ | Fetching Text.Text

88+ | Extracting Text.Text

89+ | NoTitleElement Text.Text

9190 | NoLangSpecified

92− | InvalidLang String

93− | CouldNotHighlight String

94− | MissingCharacter String

95− | Deprecated String String

96− | NoTranslation String

97− | CouldNotLoadTranslations String String

98− | UnusualConversion String

99− | UnexpectedXmlElement String String

100− | UnknownOrgExportOption String

101− | CouldNotDeduceFormat [String] String

91+ | InvalidLang Text.Text

92+ | CouldNotHighlight Text.Text

93+ | MissingCharacter Text.Text

94+ | Deprecated Text.Text Text.Text

95+ | NoTranslation Text.Text

96+ | CouldNotLoadTranslations Text.Text Text.Text

97+ | UnusualConversion Text.Text

98+ | UnexpectedXmlElement Text.Text Text.Text

99+ | UnknownOrgExportOption Text.Text

100+ | CouldNotDeduceFormat [Text.Text] Text.Text

102101 deriving (Show, Eq, Data, Ord, Typeable, Generic)

103102

104103instance ToJSON LogMessage where

@@ -107,65 +106,65 @@ instance ToJSON LogMessage where

107106 "type" .= toJSON (show $ toConstr x) :

108107 case x of

109108 SkippedContent s pos ->

110− ["contents" .= Text.pack s,

111− "source" .= Text.pack (sourceName pos),

109+ ["contents" .= s,

110+ "source" .= sourceName pos,

112111 "line" .= sourceLine pos,

113112 "column" .= sourceColumn pos]

114113 IgnoredElement s ->

115− ["contents" .= Text.pack s]

114+ ["contents" .= s]

116115 CouldNotParseYamlMetadata s pos ->

117− ["message" .= Text.pack s,

118− "source" .= Text.pack (sourceName pos),

116+ ["message" .= s,

117+ "source" .= sourceName pos,

119118 "line" .= toJSON (sourceLine pos),

120119 "column" .= toJSON (sourceColumn pos)]

121120 DuplicateLinkReference s pos ->

122− ["contents" .= Text.pack s,

123− "source" .= Text.pack (sourceName pos),

121+ ["contents" .= s,

122+ "source" .= sourceName pos,

124123 "line" .= toJSON (sourceLine pos),

125124 "column" .= toJSON (sourceColumn pos)]

126125 NoteDefinedButNotUsed s pos ->

127− ["key" .= Text.pack s,

128− "source" .= Text.pack (sourceName pos),

126+ ["key" .= s,

127+ "source" .= sourceName pos,

129128 "line" .= toJSON (sourceLine pos),

130129 "column" .= toJSON (sourceColumn pos)]

131130 DuplicateNoteReference s pos ->

132− ["contents" .= Text.pack s,

133− "source" .= Text.pack (sourceName pos),

131+ ["contents" .= s,

132+ "source" .= sourceName pos,

134133 "line" .= toJSON (sourceLine pos),

135134 "column" .= toJSON (sourceColumn pos)]

136135 DuplicateIdentifier s pos ->

137− ["contents" .= Text.pack s,

138− "source" .= Text.pack (sourceName pos),

136+ ["contents" .= s,

137+ "source" .= sourceName pos,

139138 "line" .= toJSON (sourceLine pos),

140139 "column" .= toJSON (sourceColumn pos)]

141140 ReferenceNotFound s pos ->

142− ["contents" .= Text.pack s,

143− "source" .= Text.pack (sourceName pos),

141+ ["contents" .= s,

142+ "source" .= sourceName pos,

144143 "line" .= toJSON (sourceLine pos),

145144 "column" .= toJSON (sourceColumn pos)]

146145 CircularReference s pos ->

147− ["contents" .= Text.pack s,

148− "source" .= Text.pack (sourceName pos),

146+ ["contents" .= s,

147+ "source" .= sourceName pos,

149148 "line" .= toJSON (sourceLine pos),

150149 "column" .= toJSON (sourceColumn pos)]

151150 UndefinedToggle s pos ->

152− ["contents" .= Text.pack s,

153− "source" .= Text.pack (sourceName pos),

151+ ["contents" .= s,

152+ "source" .= sourceName pos,

154153 "line" .= toJSON (sourceLine pos),

155154 "column" .= toJSON (sourceColumn pos)]

156155 ParsingUnescaped s pos ->

157− ["contents" .= Text.pack s,

158− "source" .= Text.pack (sourceName pos),

156+ ["contents" .= s,

157+ "source" .= sourceName pos,

159158 "line" .= toJSON (sourceLine pos),

160159 "column" .= toJSON (sourceColumn pos)]

161160 CouldNotLoadIncludeFile fp pos ->

162− ["path" .= Text.pack fp,

163− "source" .= Text.pack (sourceName pos),

161+ ["path" .= fp,

162+ "source" .= sourceName pos,

164163 "line" .= toJSON (sourceLine pos),

165164 "column" .= toJSON (sourceColumn pos)]

166165 MacroAlreadyDefined name pos ->

167− ["name" .= Text.pack name,

168− "source" .= Text.pack (sourceName pos),

166+ ["name" .= name,

167+ "source" .= sourceName pos,

169168 "line" .= toJSON (sourceLine pos),

170169 "column" .= toJSON (sourceColumn pos)]

171170 InlineNotRendered il ->

@@ -173,60 +172,60 @@ instance ToJSON LogMessage where

173172 BlockNotRendered bl ->

174173 ["contents" .= toJSON bl]

175174 DocxParserWarning s ->

176− ["contents" .= Text.pack s]

175+ ["contents" .= s]

177176 IgnoredIOError s ->

178− ["contents" .= Text.pack s]

177+ ["contents" .= s]

179178 CouldNotFetchResource fp s ->

180− ["path" .= Text.pack fp,

181− "message" .= Text.pack s]

179+ ["path" .= fp,

180+ "message" .= s]

182181 CouldNotDetermineImageSize fp s ->

183− ["path" .= Text.pack fp,

184− "message" .= Text.pack s]

182+ ["path" .= fp,

183+ "message" .= s]

185184 CouldNotConvertImage fp s ->

186− ["path" .= Text.pack fp,

187− "message" .= Text.pack s]

185+ ["path" .= fp,

186+ "message" .= s]

188187 CouldNotDetermineMimeType fp ->

189− ["path" .= Text.pack fp]

188+ ["path" .= fp]

190189 CouldNotConvertTeXMath s msg ->

191− ["contents" .= Text.pack s,

192− "message" .= Text.pack msg]

190+ ["contents" .= s,

191+ "message" .= msg]

193192 CouldNotParseCSS msg ->

194− ["message" .= Text.pack msg]

193+ ["message" .= msg]

195194 Fetching fp ->

196− ["path" .= Text.pack fp]

195+ ["path" .= fp]

197196 Extracting fp ->

198− ["path" .= Text.pack fp]

197+ ["path" .= fp]

199198 NoTitleElement fallback ->

200− ["fallback" .= Text.pack fallback]

199+ ["fallback" .= fallback]

201200 NoLangSpecified -> []

202201 InvalidLang s ->

203− ["lang" .= Text.pack s]

202+ ["lang" .= s]

204203 CouldNotHighlight msg ->

205− ["message" .= Text.pack msg]

204+ ["message" .= msg]

206205 MissingCharacter msg ->

207− ["message" .= Text.pack msg]

206+ ["message" .= msg]

208207 Deprecated thing msg ->

209− ["thing" .= Text.pack thing,

210− "message" .= Text.pack msg]

208+ ["thing" .= thing,

209+ "message" .= msg]

211210 NoTranslation term ->

212− ["term" .= Text.pack term]

211+ ["term" .= term]

213212 CouldNotLoadTranslations lang msg ->

214− ["lang" .= Text.pack lang,

215− "message" .= Text.pack msg]

213+ ["lang" .= lang,

214+ "message" .= msg]

216215 UnusualConversion msg ->

217− ["message" .= Text.pack msg]

216+ ["message" .= msg]

218217 UnexpectedXmlElement element parent ->

219− ["element" .= Text.pack element,

220− "parent" .= Text.pack parent]

218+ ["element" .= element,

219+ "parent" .= parent]

221220 UnknownOrgExportOption option ->

222− ["option" .= Text.pack option]

221+ ["option" .= option]

223222 CouldNotDeduceFormat exts format ->

224− ["extensions" .= map Text.pack exts

225− ,"format" .= Text.pack format]

223+ ["extensions" .= exts

224+ ,"format" .= format]

226225

227226

228−showPos :: SourcePos -> String

229−showPos pos = sn ++ "line " ++

227+showPos :: SourcePos -> Text.Text

228+showPos pos = Text.pack $ sn ++ "line " ++

230229 show (sourceLine pos) ++ " column " ++ show (sourceColumn pos)

231230 where sn = if sourceName pos == "source" || sourceName pos == ""

232231 then ""

@@ -238,140 +237,140 @@ encodeLogMessages ms =

238237 keyOrder [ "type", "verbosity", "contents", "message", "path",

239238 "source", "line", "column" ] } ms

240239

241−showLogMessage :: LogMessage -> String

240+showLogMessage :: LogMessage -> Text.Text

242241showLogMessage msg =

243242 case msg of

244243 SkippedContent s pos ->

245− "Skipped '" ++ s ++ "' at " ++ showPos pos

244+ "Skipped '" <> s <> "' at " <> showPos pos

246245 IgnoredElement s ->

247− "Ignored element " ++ s

246+ "Ignored element " <> s

248247 CouldNotParseYamlMetadata s pos ->

249− "Could not parse YAML metadata at " ++ showPos pos ++

250− if null s then "" else ": " ++ s

248+ "Could not parse YAML metadata at " <> showPos pos <>

249+ if Text.null s then "" else ": " <> s

251250 DuplicateLinkReference s pos ->

252− "Duplicate link reference '" ++ s ++ "' at " ++ showPos pos

251+ "Duplicate link reference '" <> s <> "' at " <> showPos pos

253252 DuplicateNoteReference s pos ->

254− "Duplicate note reference '" ++ s ++ "' at " ++ showPos pos

253+ "Duplicate note reference '" <> s <> "' at " <> showPos pos

255254 NoteDefinedButNotUsed s pos ->

256− "Note with key '" ++ s ++ "' defined at " ++ showPos pos ++

255+ "Note with key '" <> s <> "' defined at " <> showPos pos <>

257256 " but not used."

258257 DuplicateIdentifier s pos ->

259− "Duplicate identifier '" ++ s ++ "' at " ++ showPos pos

258+ "Duplicate identifier '" <> s <> "' at " <> showPos pos

260259 ReferenceNotFound s pos ->

261− "Reference not found for '" ++ s ++ "' at " ++ showPos pos

260+ "Reference not found for '" <> s <> "' at " <> showPos pos

262261 CircularReference s pos ->

263− "Circular reference '" ++ s ++ "' at " ++ showPos pos

262+ "Circular reference '" <> s <> "' at " <> showPos pos

264263 UndefinedToggle s pos ->

265− "Undefined toggle '" ++ s ++ "' at " ++ showPos pos

264+ "Undefined toggle '" <> s <> "' at " <> showPos pos

266265 ParsingUnescaped s pos ->

267− "Parsing unescaped '" ++ s ++ "' at " ++ showPos pos

266+ "Parsing unescaped '" <> s <> "' at " <> showPos pos

268267 CouldNotLoadIncludeFile fp pos ->

269− "Could not load include file '" ++ fp ++ "' at " ++ showPos pos

268+ "Could not load include file '" <> fp <> "' at " <> showPos pos

270269 MacroAlreadyDefined name pos ->

271− "Macro '" ++ name ++ "' already defined, ignoring at " ++ showPos pos

270+ "Macro '" <> name <> "' already defined, ignoring at " <> showPos pos

272271 InlineNotRendered il ->

273− "Not rendering " ++ show il

272+ "Not rendering " <> Text.pack (show il)

274273 BlockNotRendered bl ->

275− "Not rendering " ++ show bl

274+ "Not rendering " <> Text.pack (show bl)

276275 DocxParserWarning s ->

277− "Docx parser warning: " ++ s

276+ "Docx parser warning: " <> s

278277 IgnoredIOError s ->

279− "IO Error (ignored): " ++ s

278+ "IO Error (ignored): " <> s

280279 CouldNotFetchResource fp s ->

281− "Could not fetch resource '" ++ fp ++ "'" ++

282− if null s then "" else ": " ++ s

280+ "Could not fetch resource '" <> fp <> "'" <>

281+ if Text.null s then "" else ": " <> s

283282 CouldNotDetermineImageSize fp s ->

284− "Could not determine image size for '" ++ fp ++ "'" ++

285− if null s then "" else ": " ++ s

283+ "Could not determine image size for '" <> fp <> "'" <>

284+ if Text.null s then "" else ": " <> s

286285 CouldNotConvertImage fp s ->

287− "Could not convert image '" ++ fp ++ "'" ++

288− if null s then "" else ": " ++ s

286+ "Could not convert image '" <> fp <> "'" <>

287+ if Text.null s then "" else ": " <> s

289288 CouldNotDetermineMimeType fp ->

290− "Could not determine mime type for '" ++ fp ++ "'"

289+ "Could not determine mime type for '" <> fp <> "'"

291290 CouldNotConvertTeXMath s m ->

292− "Could not convert TeX math '" ++ s ++ "', rendering as TeX" ++

293− if null m then "" else ':' : '\n' : m

291+ "Could not convert TeX math '" <> s <> "', rendering as TeX" <>

292+ if Text.null m then "" else ":\n" <> m

294293 CouldNotParseCSS m ->

295− "Could not parse CSS" ++ if null m then "" else ':' : '\n' : m

294+ "Could not parse CSS" <> if Text.null m then "" else ":\n" <> m

296295 Fetching fp ->

297− "Fetching " ++ fp ++ "..."

296+ "Fetching " <> fp <> "..."

298297 Extracting fp ->

299− "Extracting " ++ fp ++ "..."

298+ "Extracting " <> fp <> "..."

300299 NoTitleElement fallback ->

301− "This document format requires a nonempty <title> element.\n" ++

302− "Defaulting to '" ++ fallback ++ "' as the title.\n" ++

303− "To specify a title, use 'title' in metadata or " ++

300+ "This document format requires a nonempty <title> element.\n" <>

301+ "Defaulting to '" <> fallback <> "' as the title.\n" <>

302+ "To specify a title, use 'title' in metadata or " <>

304303 "--metadata title=\"...\"."

305304 NoLangSpecified ->

306− "No value for 'lang' was specified in the metadata.\n" ++

305+ "No value for 'lang' was specified in the metadata.\n" <>

307306 "It is recommended that lang be specified for this format."

308307 InvalidLang s ->

309− "Invalid 'lang' value '" ++ s ++ "'.\n" ++

308+ "Invalid 'lang' value '" <> s <> "'.\n" <>

310309 "Use an IETF language tag like 'en-US'."

311310 CouldNotHighlight m ->

312− "Could not highlight code block:\n" ++ m

311+ "Could not highlight code block:\n" <> m

313312 MissingCharacter m ->

314− "Missing character: " ++ m

313+ "Missing character: " <> m

315314 Deprecated t m ->

316− "Deprecated: " ++ t ++

317− if null m

315+ "Deprecated: " <> t <>

316+ if Text.null m

318317 then ""

319− else ". " ++ m

318+ else ". " <> m

320319 NoTranslation t ->

321− "The term " ++ t ++ " has no translation defined."

320+ "The term " <> t <> " has no translation defined."

322321 CouldNotLoadTranslations lang m ->

323− "Could not load translations for " ++ lang ++

324− if null m then "" else '\n' : m

322+ "Could not load translations for " <> lang <>

323+ if Text.null m then "" else "\n" <> m

325324 UnusualConversion m ->

326− "Unusual conversion: " ++ m

325+ "Unusual conversion: " <> m

327326 UnexpectedXmlElement element parent ->

328− "Unexpected XML element " ++ element ++ " in " ++ parent

327+ "Unexpected XML element " <> element <> " in " <> parent

329328 UnknownOrgExportOption option ->

330− "Ignoring unknown Org export option: " ++ option

329+ "Ignoring unknown Org export option: " <> option

331330 CouldNotDeduceFormat exts format ->

332− "Could not deduce format from file extension " ++

333− intercalate " or " exts ++ "\n" ++

334− "Defaulting to " ++ format

331+ "Could not deduce format from file extension " <>

332+ Text.intercalate " or " exts <> "\n" <>

333+ "Defaulting to " <> format

335334

336−messageVerbosity:: LogMessage -> Verbosity

335+messageVerbosity :: LogMessage -> Verbosity

337336messageVerbosity msg =

338337 case msg of

339− SkippedContent{} -> INFO

340− IgnoredElement{} -> INFO

341− CouldNotParseYamlMetadata{} -> WARNING

342− DuplicateLinkReference{} -> WARNING

343− DuplicateNoteReference{} -> WARNING

344− NoteDefinedButNotUsed{} -> WARNING

345− DuplicateIdentifier{} -> WARNING

346− ReferenceNotFound{} -> WARNING

347− CircularReference{} -> WARNING

348− UndefinedToggle{} -> WARNING

338+ SkippedContent{} -> INFO

339+ IgnoredElement{} -> INFO

340+ CouldNotParseYamlMetadata{} -> WARNING

341+ DuplicateLinkReference{} -> WARNING

342+ DuplicateNoteReference{} -> WARNING

343+ NoteDefinedButNotUsed{} -> WARNING

344+ DuplicateIdentifier{} -> WARNING

345+ ReferenceNotFound{} -> WARNING

346+ CircularReference{} -> WARNING

347+ UndefinedToggle{} -> WARNING

349348 CouldNotLoadIncludeFile f _

350− | ".sty" `isSuffixOf` f -> INFO

351− | otherwise -> WARNING

352− MacroAlreadyDefined{} -> WARNING

353− ParsingUnescaped{} -> INFO

354− InlineNotRendered{} -> INFO

355− BlockNotRendered{} -> INFO

356− DocxParserWarning{} -> INFO

357− IgnoredIOError{} -> WARNING

358− CouldNotFetchResource{} -> WARNING

359− CouldNotDetermineImageSize{} -> WARNING

360− CouldNotConvertImage{} -> WARNING

361− CouldNotDetermineMimeType{} -> WARNING

362− CouldNotConvertTeXMath{} -> WARNING

363− CouldNotParseCSS{} -> WARNING

364− Fetching{} -> INFO

365− Extracting{} -> INFO

366− NoTitleElement{} -> WARNING

367− NoLangSpecified -> INFO

368− InvalidLang{} -> WARNING

369− CouldNotHighlight{} -> WARNING

370− MissingCharacter{} -> WARNING

371− Deprecated{} -> WARNING

372− NoTranslation{} -> WARNING

373− CouldNotLoadTranslations{} -> WARNING

374− UnusualConversion {} -> WARNING

375− UnexpectedXmlElement {} -> WARNING

376− UnknownOrgExportOption {} -> WARNING

377− CouldNotDeduceFormat{} -> WARNING

349+ | ".sty" `Text.isSuffixOf` f -> INFO

350+ | otherwise -> WARNING

351+ MacroAlreadyDefined{} -> WARNING

352+ ParsingUnescaped{} -> INFO

353+ InlineNotRendered{} -> INFO

354+ BlockNotRendered{} -> INFO

355+ DocxParserWarning{} -> INFO

356+ IgnoredIOError{} -> WARNING

357+ CouldNotFetchResource{} -> WARNING

358+ CouldNotDetermineImageSize{} -> WARNING

359+ CouldNotConvertImage{} -> WARNING

360+ CouldNotDetermineMimeType{} -> WARNING

361+ CouldNotConvertTeXMath{} -> WARNING

362+ CouldNotParseCSS{} -> WARNING

363+ Fetching{} -> INFO

364+ Extracting{} -> INFO

365+ NoTitleElement{} -> WARNING

366+ NoLangSpecified -> INFO

367+ InvalidLang{} -> WARNING

368+ CouldNotHighlight{} -> WARNING

369+ MissingCharacter{} -> WARNING

370+ Deprecated{} -> WARNING

371+ NoTranslation{} -> WARNING

372+ CouldNotLoadTranslations{} -> WARNING

373+ UnusualConversion {} -> WARNING

374+ UnexpectedXmlElement {} -> WARNING

375+ UnknownOrgExportOption {} -> WARNING

376+ CouldNotDeduceFormat{} -> WARNING

src/Text/Pandoc/Lua/Filter.hs+2−2

@@ -1,4 +1,4 @@

1−{-# LANGUAGE FlexibleContexts #-}

1+{-# LANGUAGE FlexibleContexts #-}

22{-# LANGUAGE NoImplicitPrelude #-}

33{- |

44Module : Text.Pandoc.Lua.Filter

@@ -180,7 +180,7 @@ constructorsFor :: DataType -> [String]

180180constructorsFor x = map show (dataTypeConstrs x)

181181

182182inlineElementNames :: [String]

183−inlineElementNames = "Inline" : constructorsFor (dataTypeOf (Str []))

183+inlineElementNames = "Inline" : constructorsFor (dataTypeOf (Str mempty))

184184

185185blockElementNames :: [String]

186186blockElementNames = "Block" : constructorsFor (dataTypeOf (Para []))

src/Text/Pandoc/Lua/Global.hs+2−1

@@ -27,11 +27,12 @@ import Text.Pandoc.Lua.Marshaling ()

2727import Text.Pandoc.Lua.Util (addFunction)

2828import Text.Pandoc.Options (ReaderOptions)

2929

30+import qualified Data.Text as Text

3031import qualified Foreign.Lua as Lua

3132

3233-- | Permissible global Lua variables.

3334data Global =

34− FORMAT String

35+ FORMAT Text.Text

3536 | PANDOC_API_VERSION

3637 | PANDOC_DOCUMENT Pandoc

3738 | PANDOC_READER_OPTIONS ReaderOptions

src/Text/Pandoc/Lua/Init.hs+3−2

@@ -28,13 +28,14 @@ import Text.Pandoc.Lua.Packages (LuaPackageParams (..),

2828 installPandocPackageSearcher)

2929import Text.Pandoc.Lua.Util (loadScriptFromDataDir)

3030

31+import qualified Data.Text as Text

3132import qualified Foreign.Lua as Lua

3233import qualified Foreign.Lua.Module.Text as Lua

3334import qualified Text.Pandoc.Definition as Pandoc

3435import qualified Text.Pandoc.Lua.Module.Pandoc as ModulePandoc

3536

3637-- | Lua error message

37−newtype LuaException = LuaException String deriving (Show)

38+newtype LuaException = LuaException Text.Text deriving (Show)

3839

3940-- | Run the lua interpreter, using pandoc's default way of environment

4041-- initialization.

@@ -56,7 +57,7 @@ runLua luaOp = do

5657 return (opResult, st)

5758 liftIO $ setForeignEncoding enc

5859 case res of

59− Left (Lua.Exception msg) -> return $ Left (LuaException msg)

60+ Left (Lua.Exception msg) -> return $ Left (LuaException $ Text.pack msg)

6061 Right (x, newState) -> do

6162 putCommonState newState

6263 return $ Right x

src/Text/Pandoc/Lua/Marshaling/CommonState.hs+5−3

@@ -1,6 +1,7 @@

11{-# OPTIONS_GHC -fno-warn-orphans #-}

22{-# LANGUAGE LambdaCase #-}

33{-# LANGUAGE NoImplicitPrelude #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Lua.Marshaling.CommonState

@@ -23,6 +24,7 @@ import Text.Pandoc.Logging (LogMessage, showLogMessage)

2324import Text.Pandoc.Lua.Marshaling.AnyValue (AnyValue (..))

2425

2526import qualified Data.Map as Map

27+import qualified Data.Text as Text

2628import qualified Foreign.Lua as Lua

2729import qualified Text.Pandoc.Lua.Util as LuaUtil

2830

@@ -46,7 +48,7 @@ indexCommonState st (AnyValue idx) = Lua.ltype idx >>= \case

4648 Lua.TypeString -> 1 <$ (Lua.peek idx >>= pushField)

4749 _ -> 1 <$ Lua.pushnil

4850 where

49− pushField :: String -> Lua ()

51+ pushField :: Text.Text -> Lua ()

5052 pushField name = case lookup name commonStateFields of

5153 Just pushValue -> pushValue st

5254 Nothing -> Lua.pushnil

@@ -71,7 +73,7 @@ pairsCommonState st = do

7173 (nextKey, pushValue):_ -> 2 <$ (Lua.push nextKey *> pushValue st)

7274 _ -> 2 <$ (Lua.pushnil *> Lua.pushnil)

7375

74−commonStateFields :: [(String, CommonState -> Lua ())]

76+commonStateFields :: [(Text.Text, CommonState -> Lua ())]

7577commonStateFields =

7678 [ ("input_files", Lua.push . stInputFiles)

7779 , ("output_file", Lua.push . Lua.Optional . stOutputFile)

@@ -98,5 +100,5 @@ instance Pushable LogMessage where

98100 pushLogMessageMetatable = ensureUserdataMetatable logMessageTypeName $

99101 LuaUtil.addFunction "__tostring" tostringLogMessage

100102

101−tostringLogMessage :: LogMessage -> Lua String

103+tostringLogMessage :: LogMessage -> Lua Text.Text

102104tostringLogMessage = return . showLogMessage

src/Text/Pandoc/Lua/Marshaling/ReaderOptions.hs+4−3

@@ -25,6 +25,7 @@ import Text.Pandoc.Lua.Marshaling.CommonState ()

2525import Text.Pandoc.Options (ReaderOptions (..), TrackChanges)

2626

2727import qualified Data.Set as Set

28+import qualified Data.Text as Text

2829import qualified Foreign.Lua as Lua

2930import qualified Text.Pandoc.Lua.Util as LuaUtil

3031

@@ -44,9 +45,9 @@ instance Pushable ReaderOptions where

4445 (standalone :: Bool)

4546 (columns :: Int)

4647 (tabStop :: Int)

47− (indentedCodeClasses :: [String])

48− (abbreviations :: Set.Set String)

49− (defaultImageExtension :: String)

48+ (indentedCodeClasses :: [Text.Text])

49+ (abbreviations :: Set.Set Text.Text)

50+ (defaultImageExtension :: Text.Text)

5051 (trackChanges :: TrackChanges)

5152 (stripComments :: Bool)

5253 = ro

src/Text/Pandoc/Lua/Module/MediaBag.hs+3−3

@@ -15,7 +15,6 @@ module Text.Pandoc.Lua.Module.MediaBag

1515

1616import Prelude

1717import Control.Monad (zipWithM_)

18−import Data.Maybe (fromMaybe)

1918import Foreign.Lua (Lua, NumResults, Optional, liftIO)

2019import Text.Pandoc.Class (CommonState (..), fetchItem, putCommonState,

2120 runIOorExplode, setMediaBag)

@@ -25,6 +24,7 @@ import Text.Pandoc.Lua.Util (addFunction)

2524import Text.Pandoc.MIME (MimeType)

2625

2726import qualified Data.ByteString.Lazy as BL

27+import qualified Data.Text as T

2828import qualified Foreign.Lua as Lua

2929import qualified Text.Pandoc.MediaBag as MB

3030

@@ -113,7 +113,7 @@ mediaDirectoryFn = do

113113 Lua.push "length" *> Lua.push contentLength *> Lua.rawset (-3)

114114 Lua.rawseti (-2) idx

115115

116−fetch :: String

116+fetch :: T.Text

117117 -> Lua NumResults

118118fetch src = do

119119 commonState <- getCommonState

@@ -122,6 +122,6 @@ fetch src = do

122122 putCommonState commonState

123123 setMediaBag mediaBag

124124 fetchItem src

125− Lua.push $ fromMaybe "" mimeType

125+ Lua.push $ maybe "" T.unpack mimeType

126126 Lua.push bs

127127 return 2 -- returns 2 values: contents, mimetype

src/Text/Pandoc/Lua/Module/Pandoc.hs+10−9

@@ -1,5 +1,6 @@

1−{-# LANGUAGE FlexibleContexts #-}

1+{-# LANGUAGE FlexibleContexts #-}

22{-# LANGUAGE NoImplicitPrelude #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Lua.Module.Pandoc

@@ -19,7 +20,6 @@ import Control.Monad (when)

1920import Control.Monad.Except (throwError)

2021import Data.Default (Default (..))

2122import Data.Maybe (fromMaybe)

22−import Data.Text (pack)

2323import Foreign.Lua (Lua, NumResults, Optional, Peekable, Pushable)

2424import System.Exit (ExitCode (..))

2525import Text.Pandoc.Class (runIO)

@@ -33,6 +33,7 @@ import Text.Pandoc.Readers (Reader (..), getReader)

3333

3434import qualified Data.ByteString.Lazy as BL

3535import qualified Data.ByteString.Lazy.Char8 as BSL

36+import qualified Data.Text as T

3637import qualified Foreign.Lua as Lua

3738import qualified Text.Pandoc.Lua.Util as LuaUtil

3839import Text.Pandoc.Error

@@ -59,22 +60,22 @@ walkInline = walkElement

5960walkBlock :: Block -> LuaFilter -> Lua Block

6061walkBlock = walkElement

6162

62−readDoc :: String -> Optional String -> Lua NumResults

63+readDoc :: T.Text -> Optional T.Text -> Lua NumResults

6364readDoc content formatSpecOrNil = do

6465 let formatSpec = fromMaybe "markdown" (Lua.fromOptional formatSpecOrNil)

6566 res <- Lua.liftIO . runIO $

6667 getReader formatSpec >>= \(rdr,es) ->

6768 case rdr of

6869 TextReader r ->

69− r def{ readerExtensions = es } (pack content)

70+ r def{ readerExtensions = es } content

7071 _ -> throwError $ PandocSomeError $

7172 "Only textual formats are supported"

7273 case res of

7374 Right pd -> (1 :: NumResults) <$ Lua.push pd -- success, push Pandoc

7475 Left (PandocUnknownReaderError f) -> Lua.raiseError $

75− "Unknown reader: " ++ f

76+ "Unknown reader: " <> f

7677 Left (PandocUnsupportedExtensionError e f) -> Lua.raiseError $

77− "Extension " ++ e ++ " not supported for " ++ f

78+ "Extension " <> e <> " not supported for " <> f

7879 Left e -> Lua.raiseError $ show e

7980

8081-- | Pipes input through a command.

@@ -86,10 +87,10 @@ pipeFn command args input = do

8687 (ec, output) <- Lua.liftIO $ pipeProcess Nothing command args input

8788 case ec of

8889 ExitSuccess -> 1 <$ Lua.push output

89− ExitFailure n -> Lua.raiseError (PipeError command n output)

90+ ExitFailure n -> Lua.raiseError (PipeError (T.pack command) n output)

9091

9192data PipeError = PipeError

92− { pipeErrorCommand :: String

93+ { pipeErrorCommand :: T.Text

9394 , pipeErrorCode :: Int

9495 , pipeErrorOutput :: BL.ByteString

9596 }

@@ -118,7 +119,7 @@ instance Pushable PipeError where

118119 pipeErrorMessage :: PipeError -> Lua BL.ByteString

119120 pipeErrorMessage (PipeError cmd errorCode output) = return $ mconcat

120121 [ BSL.pack "Error running "

121− , BSL.pack cmd

122+ , BSL.pack $ T.unpack cmd

122123 , BSL.pack " (error code "

123124 , BSL.pack $ show errorCode

124125 , BSL.pack "): "

src/Text/Pandoc/Lua/Module/Utils.hs+9−9

@@ -15,7 +15,6 @@ module Text.Pandoc.Lua.Module.Utils

1515

1616import Prelude

1717import Control.Applicative ((<|>))

18−import Data.Char (toLower)

1918import Data.Default (def)

2019import Data.Version (Version)

2120import Foreign.Lua (Peekable, Lua, NumResults)

@@ -27,6 +26,7 @@ import Text.Pandoc.Lua.Util (addFunction)

2726

2827import qualified Data.Digest.Pure.SHA as SHA

2928import qualified Data.ByteString.Lazy as BSL

29+import qualified Data.Text as T

3030import qualified Foreign.Lua as Lua

3131import qualified Text.Pandoc.Builder as B

3232import qualified Text.Pandoc.Filter.JSON as JSONFilter

@@ -64,7 +64,7 @@ makeSections number baselevel =

6464-- limit years to the range 1601-9999 (ISO 8601 accepts greater than

6565-- or equal to 1583, but MS Word only accepts dates starting 1601).

6666-- Returns nil instead of a string if the conversion failed.

67−normalizeDate :: String -> Lua (Lua.Optional String)

67+normalizeDate :: T.Text -> Lua (Lua.Optional T.Text)

6868normalizeDate = return . Lua.Optional . Shared.normalizeDate

6969

7070-- | Run a JSON filter on the given document.

@@ -88,13 +88,13 @@ runJSONFilter mbDatadir doc filterFile optArgs = do

8888

8989-- | Calculate the hash of the given contents.

9090sha1 :: BSL.ByteString

91− -> Lua String

92−sha1 = return . SHA.showDigest . SHA.sha1

91+ -> Lua T.Text

92+sha1 = return . T.pack . SHA.showDigest . SHA.sha1

9393

9494-- | Convert pandoc structure to a string with formatting removed.

9595-- Footnotes are skipped (since we don't want their contents in link

9696-- labels).

97−stringify :: AstElement -> Lua String

97+stringify :: AstElement -> Lua T.Text

9898stringify el = return $ case el of

9999 PandocElement pd -> Shared.stringify pd

100100 InlineElement i -> Shared.stringify i

@@ -102,11 +102,11 @@ stringify el = return $ case el of

102102 MetaElement m -> Shared.stringify m

103103 CitationElement c -> Shared.stringify c

104104 MetaValueElement m -> stringifyMetaValue m

105− _ -> ""

105+ _ -> mempty

106106

107−stringifyMetaValue :: MetaValue -> String

107+stringifyMetaValue :: MetaValue -> T.Text

108108stringifyMetaValue mv = case mv of

109− MetaBool b -> map toLower (show b)

109+ MetaBool b -> T.toLower $ T.pack (show b)

110110 MetaString s -> s

111111 _ -> Shared.stringify mv

112112

@@ -139,5 +139,5 @@ instance Peekable AstElement where

139139 "Expected an AST element, but could not parse value as such."

140140

141141-- | Convert a number < 4000 to uppercase roman numeral.

142−toRomanNumeral :: Lua.Integer -> Lua String

142+toRomanNumeral :: Lua.Integer -> Lua T.Text

143143toRomanNumeral = return . Shared.toRomanNumeral . fromIntegral

src/Text/Pandoc/MIME.hs+11−11

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.MIME

@@ -13,14 +14,13 @@ Mime type lookup for ODT writer.

1314module Text.Pandoc.MIME ( MimeType, getMimeType, getMimeTypeDef,

1415 extensionFromMimeType, mediaCategory ) where

1516import Prelude

16−import Data.Char (toLower)

1717import Data.List (isPrefixOf, isSuffixOf)

18−import Data.List.Split (splitOn)

1918import qualified Data.Map as M

19+import qualified Data.Text as T

2020import Data.Maybe (fromMaybe, listToMaybe)

2121import System.FilePath

2222

23−type MimeType = String

23+type MimeType = T.Text

2424

2525-- | Determine mime type appropriate for file path.

2626getMimeType :: FilePath -> Maybe MimeType

@@ -31,34 +31,34 @@ getMimeType fp

3131 | "Formula-" `isPrefixOf` fp && "/" `isSuffixOf` fp =

3232 Just "application/vnd.oasis.opendocument.formula"

3333 -- generic

34− | otherwise = M.lookup (map toLower $ drop 1 $ takeExtension fp) mimeTypes

34+ | otherwise = M.lookup (T.toLower $ T.drop 1 $ T.pack $ takeExtension fp) mimeTypes

3535

3636-- | Determime mime type appropriate for file path, defaulting to

3737-- “application/octet-stream” if nothing else fits.

3838getMimeTypeDef :: FilePath -> MimeType

3939getMimeTypeDef = fromMaybe "application/octet-stream" . getMimeType

4040

41−extensionFromMimeType :: MimeType -> Maybe String

41+extensionFromMimeType :: MimeType -> Maybe T.Text

4242extensionFromMimeType mimetype =

43− M.lookup (takeWhile (/=';') mimetype) reverseMimeTypes

43+ M.lookup (T.takeWhile (/=';') mimetype) reverseMimeTypes

4444 -- note: we just look up the basic mime type, dropping the content-encoding etc.

4545

4646-- | Determine general media category for file path, e.g.

4747--

4848-- prop> mediaCategory "foo.jpg" = Just "image"

49−mediaCategory :: FilePath -> Maybe String

50−mediaCategory fp = getMimeType fp >>= listToMaybe . splitOn "/"

49+mediaCategory :: FilePath -> Maybe T.Text

50+mediaCategory fp = getMimeType fp >>= listToMaybe . T.splitOn "/"

5151

52−reverseMimeTypes :: M.Map MimeType String

52+reverseMimeTypes :: M.Map MimeType T.Text

5353reverseMimeTypes = M.fromList $ map (\(k,v) -> (v,k)) mimeTypesList

5454

55−mimeTypes :: M.Map String MimeType

55+mimeTypes :: M.Map T.Text MimeType

5656mimeTypes = M.fromList mimeTypesList

5757

5858-- | Collection of common mime types.

5959-- Except for first entry, list borrowed from

6060-- <https://github.com/Happstack/happstack-server/blob/master/src/Happstack/Server/FileServe/BuildingBlocks.hs happstack-server>

61−mimeTypesList :: [(String, MimeType)]

61+mimeTypesList :: [(T.Text, MimeType)]

6262mimeTypesList =

6363 [("cpt","image/x-corelphotopaint")

6464 ,("gz","application/x-gzip")

src/Text/Pandoc/MediaBag.hs+3−3

@@ -36,7 +36,7 @@ import Text.Pandoc.MIME (MimeType, getMimeTypeDef)

3636-- mime types. Note that a 'MediaBag' is a Monoid, so 'mempty'

3737-- can be used for an empty 'MediaBag', and '<>' can be used to append

3838-- two 'MediaBag's.

39−newtype MediaBag = MediaBag (M.Map [String] (MimeType, BL.ByteString))

39+newtype MediaBag = MediaBag (M.Map [FilePath] (MimeType, BL.ByteString))

4040 deriving (Semigroup, Monoid, Data, Typeable)

4141

4242instance Show MediaBag where

@@ -72,12 +72,12 @@ lookupMedia fp (MediaBag mediamap) = M.lookup (splitDirectories fp) mediamap

7272

7373-- | Get a list of the file paths stored in a 'MediaBag', with

7474-- their corresponding mime types and the lengths in bytes of the contents.

75−mediaDirectory :: MediaBag -> [(String, MimeType, Int)]

75+mediaDirectory :: MediaBag -> [(FilePath, MimeType, Int)]

7676mediaDirectory (MediaBag mediamap) =

7777 M.foldrWithKey (\fp (mime,contents) ->

7878 ((Posix.joinPath fp, mime, fromIntegral $ BL.length contents):)) [] mediamap

7979

80−mediaItems :: MediaBag -> [(String, MimeType, BL.ByteString)]

80+mediaItems :: MediaBag -> [(FilePath, MimeType, BL.ByteString)]

8181mediaItems (MediaBag mediamap) =

8282 M.foldrWithKey (\fp (mime,contents) ->

8383 ((Posix.joinPath fp, mime, contents):)) [] mediamap

src/Text/Pandoc/Options.hs+25−24

@@ -36,9 +36,10 @@ module Text.Pandoc.Options ( module Text.Pandoc.Extensions

3636import Prelude

3737import Control.Applicative ((<|>))

3838import Data.Char (toLower)

39+import Data.Maybe (fromMaybe)

3940import Data.Data (Data)

4041import Data.Default

41−import Data.Text (Text, unpack)

42+import Data.Text (Text)

4243import Text.DocTemplates (Context(..))

4344import qualified Data.Set as Set

4445import Data.Typeable (Typeable)

@@ -46,7 +47,7 @@ import GHC.Generics (Generic)

4647import Skylighting (SyntaxMap, defaultSyntaxMap)

4748import Text.Pandoc.Extensions

4849import Text.Pandoc.Highlighting (Style, pygments)

49−import Text.Pandoc.Shared (camelCaseToHyphenated)

50+import Text.Pandoc.Shared (camelCaseStrToHyphenated)

5051import Text.DocTemplates (Template)

5152import Data.Aeson.TH (deriveJSON, defaultOptions, Options(..),

5253 SumEncoding(..))

@@ -60,10 +61,10 @@ data ReaderOptions = ReaderOptions{

6061 , readerStandalone :: Bool -- ^ Standalone document with header

6162 , readerColumns :: Int -- ^ Number of columns in terminal

6263 , readerTabStop :: Int -- ^ Tab stop

63− , readerIndentedCodeClasses :: [String] -- ^ Default classes for

64+ , readerIndentedCodeClasses :: [Text] -- ^ Default classes for

6465 -- indented code blocks

65− , readerAbbreviations :: Set.Set String -- ^ Strings to treat as abbreviations

66− , readerDefaultImageExtension :: String -- ^ Default extension for images

66+ , readerAbbreviations :: Set.Set Text -- ^ Strings to treat as abbreviations

67+ , readerDefaultImageExtension :: Text -- ^ Default extension for images

6768 , readerTrackChanges :: TrackChanges -- ^ Track changes setting for docx

6869 , readerStripComments :: Bool -- ^ Strip HTML comments instead of parsing as raw HTML

6970} deriving (Show, Read, Data, Typeable, Generic)

@@ -84,7 +85,7 @@ instance Default ReaderOptions

8485 , readerStripComments = False

8586 }

8687

87−defaultAbbrevs :: Set.Set String

88+defaultAbbrevs :: Set.Set Text

8889defaultAbbrevs = Set.fromList

8990 [ "Mr.", "Mrs.", "Ms.", "Capt.", "Dr.", "Prof.",

9091 "Gen.", "Gov.", "e.g.", "i.e.", "Sgt.", "St.",

@@ -99,11 +100,11 @@ defaultAbbrevs = Set.fromList

99100data EPUBVersion = EPUB2 | EPUB3 deriving (Eq, Show, Read, Data, Typeable, Generic)

100101

101102data HTMLMathMethod = PlainMath

102− | WebTeX String -- url of TeX->image script.

103+ | WebTeX Text -- url of TeX->image script.

103104 | GladTeX

104105 | MathML

105− | MathJax String -- url of MathJax.js

106− | KaTeX String -- url of KaTeX files

106+ | MathJax Text -- url of MathJax.js

107+ | KaTeX Text -- url of KaTeX files

107108 deriving (Show, Read, Eq, Data, Typeable, Generic)

108109

109110instance FromYAML HTMLMathMethod where

@@ -111,18 +112,18 @@ instance FromYAML HTMLMathMethod where

111112 (withMap "HTMLMathMethod" $ \m -> do

112113 method <- m .: "method"

113114 mburl <- m .:? "url"

114− case unpack method of

115+ case method :: Text of

115116 "plain" -> return PlainMath

116− "webtex" -> return $ WebTeX $ maybe "" unpack mburl

117+ "webtex" -> return $ WebTeX $ fromMaybe "" mburl

117118 "gladtex" -> return GladTeX

118119 "mathml" -> return MathML

119120 "mathjax" -> return $ MathJax $

120− maybe defaultMathJaxURL unpack mburl

121+ fromMaybe defaultMathJaxURL mburl

121122 "katex" -> return $ KaTeX $

122− maybe defaultKaTeXURL unpack mburl

123+ fromMaybe defaultKaTeXURL mburl

123124 _ -> fail $ "Unknown HTML math method " ++ show method) node

124125 <|> (withStr "HTMLMathMethod" $ \method ->

125− case unpack method of

126+ case method of

126127 "plain" -> return PlainMath

127128 "webtex" -> return $ WebTeX ""

128129 "gladtex" -> return GladTeX

@@ -246,7 +247,7 @@ data WriterOptions = WriterOptions

246247 , writerWrapText :: WrapOption -- ^ Option for wrapping text

247248 , writerColumns :: Int -- ^ Characters in a line (for text wrapping)

248249 , writerEmailObfuscation :: ObfuscationMethod -- ^ How to obfuscate emails

249− , writerIdentifierPrefix :: String -- ^ Prefix for section & note ids in HTML

250+ , writerIdentifierPrefix :: Text -- ^ Prefix for section & note ids in HTML

250251 -- and for footnote marks in markdown

251252 , writerCiteMethod :: CiteMethod -- ^ How to print cites

252253 , writerHtmlQTags :: Bool -- ^ Use @<q>@ tags for quotes in HTML

@@ -256,8 +257,8 @@ data WriterOptions = WriterOptions

256257 , writerHighlightStyle :: Maybe Style -- ^ Style to use for highlighting

257258 -- (Nothing = no highlighting)

258259 , writerSetextHeaders :: Bool -- ^ Use setext headers for levels 1-2 in markdown

259− , writerEpubSubdirectory :: String -- ^ Subdir for epub in OCF

260− , writerEpubMetadata :: Maybe String -- ^ Metadata to include in EPUB

260+ , writerEpubSubdirectory :: Text -- ^ Subdir for epub in OCF

261+ , writerEpubMetadata :: Maybe Text -- ^ Metadata to include in EPUB

261262 , writerEpubFonts :: [FilePath] -- ^ Paths to fonts to embed

262263 , writerEpubChapterLevel :: Int -- ^ Header level for chapters (separate files)

263264 , writerTOCDepth :: Int -- ^ Number of levels to include in TOC

@@ -309,10 +310,10 @@ instance HasSyntaxExtensions WriterOptions where

309310isEnabled :: HasSyntaxExtensions a => Extension -> a -> Bool

310311isEnabled ext opts = ext `extensionEnabled` getExtensions opts

311312

312−defaultMathJaxURL :: String

313+defaultMathJaxURL :: Text

313314defaultMathJaxURL = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/"

314315

315−defaultKaTeXURL :: String

316+defaultKaTeXURL :: Text

316317defaultKaTeXURL = "https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.11.1/"

317318

318319$(deriveJSON defaultOptions ''ReaderOptions)

@@ -325,7 +326,7 @@ $(deriveJSON defaultOptions{

325326 } ''HTMLMathMethod)

326327

327328$(deriveJSON defaultOptions{ constructorTagModifier =

328− camelCaseToHyphenated

329+ camelCaseStrToHyphenated

329330 } ''CiteMethod)

330331

331332$(deriveJSON defaultOptions{ constructorTagModifier =

@@ -339,17 +340,17 @@ $(deriveJSON defaultOptions{ constructorTagModifier =

339340$(deriveJSON defaultOptions ''HTMLSlideVariant)

340341

341342$(deriveJSON defaultOptions{ constructorTagModifier =

342− camelCaseToHyphenated

343+ camelCaseStrToHyphenated

343344 } ''TrackChanges)

344345

345346$(deriveJSON defaultOptions{ constructorTagModifier =

346− camelCaseToHyphenated

347+ camelCaseStrToHyphenated

347348 } ''WrapOption)

348349

349350$(deriveJSON defaultOptions{ constructorTagModifier =

350− camelCaseToHyphenated . drop 8

351+ camelCaseStrToHyphenated . drop 8

351352 } ''TopLevelDivision)

352353

353354$(deriveJSON defaultOptions{ constructorTagModifier =

354− camelCaseToHyphenated

355+ camelCaseStrToHyphenated

355356 } ''ReferenceLocation)

src/Text/Pandoc/PDF.hs+16−17

@@ -1,4 +1,4 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE CPP #-}

33{-# LANGUAGE OverloadedStrings #-}

44{-# LANGUAGE ScopedTypeVariables #-}

@@ -46,7 +46,7 @@ import Text.Pandoc.MIME (getMimeType)

4646import Text.Pandoc.Options (HTMLMathMethod (..), WriterOptions (..))

4747import Text.Pandoc.Process (pipeProcess)

4848import System.Process (readProcessWithExitCode)

49−import Text.Pandoc.Shared (inDirectory, stringify)

49+import Text.Pandoc.Shared (inDirectory, stringify, tshow)

5050import qualified Text.Pandoc.UTF8 as UTF8

5151import Text.Pandoc.Walk (walkM)

5252import Text.Pandoc.Writers.Shared (getField, metaToContext)

@@ -141,7 +141,7 @@ makeWithWkhtmltopdf program pdfargs writer opts doc@(Pandoc meta _) = do

141141 (return . literal . stringify)

142142 (return . literal . stringify)

143143 meta

144− let toArgs (f, mbd) = maybe [] (\d -> ['-':'-':f, d]) mbd

144+ let toArgs (f, mbd) = maybe [] (\d -> ["--" <> f, T.unpack d]) mbd

145145 let args = pdfargs ++ mathArgs ++ concatMap toArgs

146146 [("page-size", getField "papersize" meta')

147147 ,("title", getField "title" meta')

@@ -173,19 +173,19 @@ handleImages opts tmpdir doc =

173173

174174convertImages :: WriterOptions -> FilePath -> Inline -> PandocIO Inline

175175convertImages opts tmpdir (Image attr ils (src, tit)) = do

176− img <- liftIO $ convertImage opts tmpdir src

176+ img <- liftIO $ convertImage opts tmpdir $ T.unpack src

177177 newPath <-

178178 case img of

179179 Left e -> do

180180 report $ CouldNotConvertImage src e

181181 return src

182− Right fp -> return fp

182+ Right fp -> return $ T.pack fp

183183 return (Image attr ils (newPath, tit))

184184convertImages _ _ x = return x

185185

186186-- Convert formats which do not work well in pdf to png

187187convertImage :: WriterOptions -> FilePath -> FilePath

188− -> IO (Either String FilePath)

188+ -> IO (Either Text FilePath)

189189convertImage opts tmpdir fname = do

190190 let dpi = show $ writerDpi opts

191191 case mime of

@@ -202,14 +202,14 @@ convertImage opts tmpdir fname = do

202202 then return $ Right pdfOut

203203 else return $ Left "conversion from SVG failed")

204204 (\(e :: E.SomeException) -> return $ Left $

205− "check that rsvg-convert is in path.\n" ++

206− show e)

205+ "check that rsvg-convert is in path.\n" <>

206+ tshow e)

207207 _ -> JP.readImage fname >>= \res ->

208208 case res of

209− Left e -> return $ Left e

209+ Left e -> return $ Left $ T.pack e

210210 Right img ->

211211 E.catch (Right pngOut <$ JP.savePngImage pngOut img) $

212− \(e :: E.SomeException) -> return (Left (show e))

212+ \(e :: E.SomeException) -> return (Left (tshow e))

213213 where

214214 pngOut = replaceDirectory (replaceExtension fname ".png") tmpdir

215215 pdfOut = replaceDirectory (replaceExtension fname ".pdf") tmpdir

@@ -262,12 +262,11 @@ missingCharacterWarnings :: Verbosity -> ByteString -> PandocIO ()

262262missingCharacterWarnings verbosity log' = do

263263 let ls = BC.lines log'

264264 let isMissingCharacterWarning = BC.isPrefixOf "Missing character: "

265− let addCodePoint [] = []

266− addCodePoint (c:cs)

267− | isAscii c = c : addCodePoint cs

268− | otherwise = c : " (U+" ++ printf "%04X" (ord c) ++ ")" ++

269− addCodePoint cs

270− let warnings = [ addCodePoint (utf8ToString (BC.drop 19 l))

265+ let toCodePoint c

266+ | isAscii c = T.singleton c

267+ | otherwise = T.pack $ c : " (U+" ++ printf "%04X" (ord c) ++ ")"

268+ let addCodePoint = T.concatMap toCodePoint

269+ let warnings = [ addCodePoint (T.pack $ utf8ToString (BC.drop 19 l))

271270 | l <- ls

272271 , isMissingCharacterWarning l

273272 ]

@@ -513,7 +512,7 @@ showVerboseInfo mbTmpDir program programArgs env source = do

513512handlePDFProgramNotFound :: String -> IE.IOError -> IO a

514513handlePDFProgramNotFound program e

515514 | IE.isDoesNotExistError e =

516− E.throwIO $ PandocPDFProgramNotFoundError program

515+ E.throwIO $ PandocPDFProgramNotFoundError $ T.pack program

517516 | otherwise = E.throwIO e

518517

519518utf8ToString :: ByteString -> String

src/Text/Pandoc/Parsing.hs+226−150

@@ -1,4 +1,4 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE CPP #-}

33{-# LANGUAGE ExplicitForAll #-}

44{-# LANGUAGE FlexibleContexts #-}

@@ -7,6 +7,8 @@

77{-# LANGUAGE IncoherentInstances #-}

88{-# LANGUAGE MultiParamTypeClasses #-}

99{-# LANGUAGE TypeSynonymInstances #-}

10+{-# LANGUAGE ViewPatterns #-}

11+{-# LANGUAGE OverloadedStrings #-}

1012{- |

1113 Module : Text.Pandoc.Parsing

@@ -18,13 +20,21 @@

1820

1921A utility library with parsers used in pandoc readers.

2022-}

23+

2124module Text.Pandoc.Parsing ( takeWhileP,

2225 takeP,

26+ countChar,

27+ textStr,

2328 anyLine,

2429 anyLineNewline,

2530 indentWith,

31+ manyChar,

32+ many1Char,

33+ manyTillChar,

34+ many1TillChar,

2635 many1Till,

2736 manyUntil,

37+ manyUntilChar,

2838 sepBy1',

2939 notFollowedBy',

3040 oneOfStrings,

@@ -183,12 +193,14 @@ import Control.Monad.Reader

183193import Data.Char (chr, isAlphaNum, isAscii, isAsciiUpper,

184194 isPunctuation, isSpace, ord, toLower, toUpper)

185195import Data.Default

186−import Data.List (intercalate, isSuffixOf, transpose)

196+import Data.Functor (($>))

197+import Data.List (intercalate, transpose)

187198import qualified Data.Map as M

188199import Data.Maybe (mapMaybe, fromMaybe)

189200import qualified Data.Set as Set

190201import Data.String

191202import Data.Text (Text)

203+import qualified Data.Text as T

192204import Text.HTML.TagSoup.Entity (lookupEntity)

193205import Text.Pandoc.Asciify (toAsciiChar)

194206import Text.Pandoc.Builder (Blocks, HasMeta (..), Inlines, trimInlines)

@@ -240,45 +252,56 @@ instance (Semigroup a, Monoid a) => Monoid (Future s a) where

240252 mempty = return mempty

241253 mappend = (<>)

242254

255+-- | Like @count@, but packs its result

256+countChar :: (Stream s m Char, Monad m)

257+ => Int

258+ -> ParsecT s st m Char

259+ -> ParsecT s st m Text

260+countChar n = fmap T.pack . count n

261+

262+-- | Like @string@, but uses @Text@.

263+textStr :: Stream s m Char => Text -> ParsecT s u m Text

264+textStr t = string (T.unpack t) $> t

265+

243266-- | Parse characters while a predicate is true.

244267takeWhileP :: Monad m

245268 => (Char -> Bool)

246− -> ParserT [Char] st m [Char]

269+ -> ParserT Text st m Text

247270takeWhileP f = do

248271 -- faster than 'many (satisfy f)'

249272 inp <- getInput

250273 pos <- getPosition

251− let (xs, rest) = span f inp

274+ let (xs, rest) = T.span f inp

252275 -- needed to persuade parsec that this won't match an empty string:

253276 anyChar

254277 setInput rest

255− setPosition $ updatePosString pos xs

278+ setPosition $ updatePosString pos $ T.unpack xs

256279 return xs

257280

258281-- Parse n characters of input (or the rest of the input if

259282-- there aren't n characters).

260−takeP :: Monad m => Int -> ParserT [Char] st m [Char]

283+takeP :: Monad m => Int -> ParserT Text st m Text

261284takeP n = do

262285 guard (n > 0)

263286 -- faster than 'count n anyChar'

264287 inp <- getInput

265288 pos <- getPosition

266− let (xs, rest) = splitAt n inp

289+ let (xs, rest) = T.splitAt n inp

267290 -- needed to persuade parsec that this won't match an empty string:

268291 anyChar

269292 setInput rest

270− setPosition $ updatePosString pos xs

293+ setPosition $ updatePosString pos $ T.unpack xs

271294 return xs

272295

273296-- | Parse any line of text

274−anyLine :: Monad m => ParserT [Char] st m [Char]

297+anyLine :: Monad m => ParserT Text st m Text

275298anyLine = do

276299 -- This is much faster than:

277300 -- manyTill anyChar newline

278301 inp <- getInput

279302 pos <- getPosition

280− case break (=='\n') inp of

281− (this, '\n':rest) -> do

303+ case T.break (=='\n') inp of

304+ (this, T.uncons -> Just ('\n', rest)) -> do

282305 -- needed to persuade parsec that this won't match an empty string:

283306 anyChar

284307 setInput rest

@@ -287,20 +310,39 @@ anyLine = do

287310 _ -> mzero

288311

289312-- | Parse any line, include the final newline in the output

290−anyLineNewline :: Monad m => ParserT [Char] st m [Char]

291−anyLineNewline = (++ "\n") <$> anyLine

313+anyLineNewline :: Monad m => ParserT Text st m Text

314+anyLineNewline = (<> "\n") <$> anyLine

292315

293316-- | Parse indent by specified number of spaces (or equiv. tabs)

294317indentWith :: Stream s m Char

295318 => HasReaderOptions st

296− => Int -> ParserT s st m [Char]

319+ => Int -> ParserT s st m Text

297320indentWith num = do

298321 tabStop <- getOption readerTabStop

299322 if num < tabStop

300− then count num (char ' ')

301− else choice [ try (count num (char ' '))

323+ then countChar num (char ' ')

324+ else choice [ try (countChar num (char ' '))

302325 , try (char '\t' >> indentWith (num - tabStop)) ]

303326

327+-- | Like @many@, but packs its result.

328+manyChar :: Stream s m t

329+ => ParserT s st m Char

330+ -> ParserT s st m Text

331+manyChar = fmap T.pack . many

332+

333+-- | Like @many1@, but packs its result.

334+many1Char :: Stream s m t

335+ => ParserT s st m Char

336+ -> ParserT s st m Text

337+many1Char = fmap T.pack . many1

338+

339+-- | Like @manyTill@, but packs its result.

340+manyTillChar :: Stream s m t

341+ => ParserT s st m Char

342+ -> ParserT s st m a

343+ -> ParserT s st m Text

344+manyTillChar p = fmap T.pack . manyTill p

345+

304346-- | Like @manyTill@, but reads at least one item.

305347many1Till :: (Show end, Stream s m t)

306348 => ParserT s st m a

@@ -312,6 +354,13 @@ many1Till p end = do

312354 rest <- manyTill p end

313355 return (first:rest)

314356

357+-- | Like @many1Till@, but packs its result

358+many1TillChar :: (Show end, Stream s m t)

359+ => ParserT s st m Char

360+ -> ParserT s st m end

361+ -> ParserT s st m Text

362+many1TillChar p = fmap T.pack . many1Till p

363+

315364-- | Like @manyTill@, but also returns the result of end parser.

316365manyUntil :: ParserT s u m a

317366 -> ParserT s u m b

@@ -325,6 +374,14 @@ manyUntil p end = scan

325374 (xs, e) <- scan

326375 return (x:xs, e))

327376

377+-- | Like @manyUntil@, but also packs its result.

378+manyUntilChar :: ParserT s u m Char

379+ -> ParserT s u m b

380+ -> ParserT s u m (Text, b)

381+manyUntilChar p = fmap go . manyUntil p

382+ where

383+ go (x, y) = (T.pack x, y)

384+

328385-- | Like @sepBy1@ from Parsec,

329386-- but does not fail if it @sep@ succeeds and @p@ fails.

330387sepBy1' :: ParsecT s u m a

@@ -342,14 +399,18 @@ notFollowedBy' p = try $ join $ do a <- try p

342399 return (return ())

343400-- (This version due to Andrew Pimlott on the Haskell mailing list.)

344401

345−oneOfStrings' :: Stream s m Char => (Char -> Char -> Bool) -> [String] -> ParserT s st m String

346−oneOfStrings' _ [] = Prelude.fail "no strings"

347−oneOfStrings' matches strs = try $ do

402+oneOfStrings' :: Stream s m Char => (Char -> Char -> Bool) -> [Text] -> ParserT s st m Text

403+oneOfStrings' f = fmap T.pack . oneOfStrings'' f . fmap T.unpack

404+

405+-- TODO: This should be re-implemented in a Text-aware way

406+oneOfStrings'' :: Stream s m Char => (Char -> Char -> Bool) -> [String] -> ParserT s st m String

407+oneOfStrings'' _ [] = Prelude.fail "no strings"

408+oneOfStrings'' matches strs = try $ do

348409 c <- anyChar

349410 let strs' = [xs | (x:xs) <- strs, x `matches` c]

350411 case strs' of

351412 [] -> Prelude.fail "not found"

352− _ -> (c:) <$> oneOfStrings' matches strs'

413+ _ -> (c:) <$> oneOfStrings'' matches strs'

353414 <|> if "" `elem` strs'

354415 then return [c]

355416 else Prelude.fail "not found"

@@ -357,11 +418,14 @@ oneOfStrings' matches strs = try $ do

357418-- | Parses one of a list of strings. If the list contains

358419-- two strings one of which is a prefix of the other, the longer

359420-- string will be matched if possible.

360−oneOfStrings :: Stream s m Char => [String] -> ParserT s st m String

421+oneOfStrings :: Stream s m Char => [Text] -> ParserT s st m Text

361422oneOfStrings = oneOfStrings' (==)

362423

363424-- | Parses one of a list of strings (tried in order), case insensitive.

364−oneOfStringsCI :: Stream s m Char => [String] -> ParserT s st m String

425+

426+-- TODO: This will not be accurate with general Unicode (neither

427+-- Text.toLower nor Text.toCaseFold can be implemented with a map)

428+oneOfStringsCI :: Stream s m Char => [Text] -> ParserT s st m Text

365429oneOfStringsCI = oneOfStrings' ciMatch

366430 where ciMatch x y = toLower' x == toLower' y

367431 -- this optimizes toLower by checking common ASCII case

@@ -388,13 +452,13 @@ blankline :: Stream s m Char => ParserT s st m Char

388452blankline = try $ skipSpaces >> newline

389453

390454-- | Parses one or more blank lines and returns a string of newlines.

391−blanklines :: Stream s m Char => ParserT s st m [Char]

392−blanklines = many1 blankline

455+blanklines :: Stream s m Char => ParserT s st m Text

456+blanklines = T.pack <$> many1 blankline

393457

394458-- | Gobble n spaces; if tabs are encountered, expand them

395459-- and gobble some or all of their spaces, leaving the rest.

396460gobbleSpaces :: (HasReaderOptions st, Monad m)

397− => Int -> ParserT [Char] st m ()

461+ => Int -> ParserT Text st m ()

398462gobbleSpaces 0 = return ()

399463gobbleSpaces n

400464 | n < 0 = error "gobbleSpaces called with negative number"

@@ -402,18 +466,18 @@ gobbleSpaces n

402466 char ' ' <|> eatOneSpaceOfTab

403467 gobbleSpaces (n - 1)

404468

405−eatOneSpaceOfTab :: (HasReaderOptions st, Monad m) => ParserT [Char] st m Char

469+eatOneSpaceOfTab :: (HasReaderOptions st, Monad m) => ParserT Text st m Char

406470eatOneSpaceOfTab = do

407471 char '\t'

408472 tabstop <- getOption readerTabStop

409473 inp <- getInput

410− setInput $ replicate (tabstop - 1) ' ' ++ inp

474+ setInput $ T.replicate (tabstop - 1) " " <> inp

411475 return ' '

412476

413477-- | Gobble up to n spaces; if tabs are encountered, expand them

414478-- and gobble some or all of their spaces, leaving the rest.

415479gobbleAtMostSpaces :: (HasReaderOptions st, Monad m)

416− => Int -> ParserT [Char] st m Int

480+ => Int -> ParserT Text st m Int

417481gobbleAtMostSpaces 0 = return 0

418482gobbleAtMostSpaces n

419483 | n < 0 = error "gobbleAtMostSpaces called with negative number"

@@ -430,23 +494,26 @@ enclosed start end parser = try $

430494 start >> notFollowedBy space >> many1Till parser end

431495

432496-- | Parse string, case insensitive.

433−stringAnyCase :: Stream s m Char => [Char] -> ParserT s st m String

434−stringAnyCase [] = string ""

435−stringAnyCase (x:xs) = do

497+stringAnyCase :: Stream s m Char => Text -> ParserT s st m Text

498+stringAnyCase = fmap T.pack . stringAnyCase' . T.unpack

499+

500+stringAnyCase' :: Stream s m Char => String -> ParserT s st m String

501+stringAnyCase' [] = string ""

502+stringAnyCase' (x:xs) = do

436503 firstChar <- char (toUpper x) <|> char (toLower x)

437− rest <- stringAnyCase xs

504+ rest <- stringAnyCase' xs

438505 return (firstChar:rest)

439506

440507-- | Parse contents of 'str' using 'parser' and return result.

441508parseFromString :: (Stream s m Char, IsString s)

442509 => ParserT s st m r

443− -> String

510+ -> Text

444511 -> ParserT s st m r

445512parseFromString parser str = do

446513 oldPos <- getPosition

447514 setPosition $ initialPos "chunk"

448515 oldInput <- getInput

449− setInput $ fromString str

516+ setInput $ fromString $ T.unpack str

450517 result <- parser

451518 spaces

452519 eof

@@ -458,7 +525,7 @@ parseFromString parser str = do

458525-- This resets 'stateLastStrPos', which is almost always what we want.

459526parseFromString' :: (Stream s m Char, IsString s, HasLastStrPosition u)

460527 => ParserT s u m a

461− -> String

528+ -> Text

462529 -> ParserT s u m a

463530parseFromString' parser str = do

464531 oldLastStrPos <- getLastStrPos <$> getState

@@ -468,9 +535,9 @@ parseFromString' parser str = do

468535 return res

469536

470537-- | Parse raw line block up to and including blank lines.

471−lineClump :: Monad m => ParserT [Char] st m String

538+lineClump :: Monad m => ParserT Text st m Text

472539lineClump = blanklines

473− <|> (unlines <$> many1 (notFollowedBy blankline >> anyLine))

540+ <|> (T.unlines <$> many1 (notFollowedBy blankline >> anyLine))

474541

475542-- | Parse a string of characters between an open character

476543-- and a close character, including text between balanced

@@ -478,15 +545,15 @@ lineClump = blanklines

478545-- @charsInBalanced '(' ')' anyChar@ will parse "(hello (there))"

479546-- and return "hello (there)".

480547charsInBalanced :: Stream s m Char => Char -> Char -> ParserT s st m Char

481− -> ParserT s st m String

548+ -> ParserT s st m Text

482549charsInBalanced open close parser = try $ do

483550 char open

484551 let isDelim c = c == open || c == close

485− raw <- many $ many1 (notFollowedBy (satisfy isDelim) >> parser)

552+ raw <- many $ T.pack <$> many1 (notFollowedBy (satisfy isDelim) >> parser)

486553 <|> (do res <- charsInBalanced open close parser

487− return $ [open] ++ res ++ [close])

554+ return $ T.singleton open <> res <> T.singleton close)

488555 char close

489− return $ concat raw

556+ return $ T.concat raw

490557

491558-- old charsInBalanced would be:

492559-- charsInBalanced open close (noneOf "\n" <|> char '\n' >> notFollowedBy blankline)

@@ -532,10 +599,10 @@ romanNumeral upperCase = do

532599

533600-- | Parses an email address; returns original and corresponding

534601-- escaped mailto: URI.

535−emailAddress :: Stream s m Char => ParserT s st m (String, String)

602+emailAddress :: Stream s m Char => ParserT s st m (Text, Text)

536603emailAddress = try $ toResult <$> mailbox <*> (char '@' *> domain)

537− where toResult mbox dom = let full = fromEntities $ mbox ++ '@':dom

538− in (full, escapeURI $ "mailto:" ++ full)

604+ where toResult mbox dom = let full = fromEntities $ T.pack $ mbox ++ '@':dom

605+ in (full, escapeURI $ "mailto:" <> full)

539606 mailbox = intercalate "." <$> (emailWord `sepBy1'` dot)

540607 domain = intercalate "." <$> (subdomain `sepBy1'` dot)

541608 dot = char '.'

@@ -553,14 +620,14 @@ emailAddress = try $ toResult <$> mailbox <*> (char '@' *> domain)

553620 xs <- many (satisfy isEmailChar)

554621 return (x:xs)

555622 isEmailChar c = isAlphaNum c || isEmailPunct c

556− isEmailPunct c = c `elem` "!\"#$%&'*+-/=?^_{|}~;"

623+ isEmailPunct c = T.any (== c) "!\"#$%&'*+-/=?^_{|}~;"

557624

558625

559−uriScheme :: Stream s m Char => ParserT s st m String

626+uriScheme :: Stream s m Char => ParserT s st m Text

560627uriScheme = oneOfStringsCI (Set.toList schemes)

561628

562629-- | Parses a URI. Returns pair of original and URI-escaped version.

563−uri :: Stream s m Char => ParserT s st m (String, String)

630+uri :: Stream s m Char => ParserT s st m (Text, Text)

564631uri = try $ do

565632 scheme <- uriScheme

566633 char ':'

@@ -571,12 +638,12 @@ uri = try $ do

571638 -- http://en.wikipedia.org/wiki/State_of_emergency_(disambiguation)

572639 -- as a URL, while NOT picking up the closing paren in

573640 -- (http://wikipedia.org). So we include balanced parens in the URL.

574− str <- concat <$> many1 (uriChunkBetween '(' ')'

575− <|> uriChunkBetween '{' '}'

576− <|> uriChunkBetween '[' ']'

577− <|> uriChunk)

578− str' <- option str $ char '/' >> return (str ++ "/")

579− let uri' = scheme ++ ":" ++ fromEntities str'

641+ str <- T.concat <$> many1 (uriChunkBetween '(' ')'

642+ <|> uriChunkBetween '{' '}'

643+ <|> uriChunkBetween '[' ']'

644+ <|> T.pack <$> uriChunk)

645+ str' <- option str $ char '/' >> return (str <> "/")

646+ let uri' = scheme <> ":" <> fromEntities str'

580647 return (uri', escapeURI uri')

581648 where

582649 wordChar = alphaNum <|> oneOf "#$%+/@\\_-&="

@@ -588,51 +655,54 @@ uri = try $ do

588655 <|> entity

589656 <|> try (punct <* lookAhead (void wordChar <|> void percentEscaped))

590657 uriChunkBetween l r = try $ do chunk <- between (char l) (char r) uriChunk

591− return ([l] ++ chunk ++ [r])

658+ return (T.pack $ [l] ++ chunk ++ [r])

592659

593−mathInlineWith :: Stream s m Char => String -> String -> ParserT s st m String

660+mathInlineWith :: Stream s m Char => Text -> Text -> ParserT s st m Text

594661mathInlineWith op cl = try $ do

595− string op

662+ textStr op

596663 when (op == "$") $ notFollowedBy space

597− words' <- many1Till (count 1 (noneOf " \t\n\\")

664+ words' <- many1Till (countChar 1 (noneOf " \t\n\\")

598665 <|> (char '\\' >>

599666 -- This next clause is needed because \text{..} can

600667 -- contain $, , etc.

601668 (try (string "text" >>

602− (("\\text" ++) <$> inBalancedBraces 0 ""))

603− <|> (\c -> ['\\',c]) <$> anyChar))

669+ (("\\text" <>) <$> inBalancedBraces 0 ""))

670+ <|> (\c -> T.pack ['\\',c]) <$> anyChar))

604671 <|> do (blankline <* notFollowedBy' blankline) <|>

605672 (oneOf " \t" <* skipMany (oneOf " \t"))

606673 notFollowedBy (char '$')

607674 return " "

608− ) (try $ string cl)

675+ ) (try $ textStr cl)

609676 notFollowedBy digit -- to prevent capture of $5

610− return $ trimMath $ concat words'

677+ return $ trimMath $ T.concat words'

611678 where

612− inBalancedBraces :: Stream s m Char => Int -> String -> ParserT s st m String

613− inBalancedBraces 0 "" = do

679+ inBalancedBraces :: Stream s m Char => Int -> Text -> ParserT s st m Text

680+ inBalancedBraces n = fmap T.pack . inBalancedBraces' n . T.unpack

681+

682+ inBalancedBraces' :: Stream s m Char => Int -> String -> ParserT s st m String

683+ inBalancedBraces' 0 "" = do

614684 c <- anyChar

615685 if c == '{'

616− then inBalancedBraces 1 "{"

686+ then inBalancedBraces' 1 "{"

617687 else mzero

618− inBalancedBraces 0 s = return $ reverse s

619− inBalancedBraces numOpen ('\\':xs) = do

688+ inBalancedBraces' 0 s = return $ reverse s

689+ inBalancedBraces' numOpen ('\\':xs) = do

620690 c <- anyChar

621− inBalancedBraces numOpen (c:'\\':xs)

622− inBalancedBraces numOpen xs = do

691+ inBalancedBraces' numOpen (c:'\\':xs)

692+ inBalancedBraces' numOpen xs = do

623693 c <- anyChar

624694 case c of

625− '}' -> inBalancedBraces (numOpen - 1) (c:xs)

626− '{' -> inBalancedBraces (numOpen + 1) (c:xs)

627− _ -> inBalancedBraces numOpen (c:xs)

695+ '}' -> inBalancedBraces' (numOpen - 1) (c:xs)

696+ '{' -> inBalancedBraces' (numOpen + 1) (c:xs)

697+ _ -> inBalancedBraces' numOpen (c:xs)

628698

629−mathDisplayWith :: Stream s m Char => String -> String -> ParserT s st m String

630−mathDisplayWith op cl = try $ do

631− string op

632− many1Till (noneOf "\n" <|> (newline <* notFollowedBy' blankline)) (try $ string cl)

699+mathDisplayWith :: Stream s m Char => Text -> Text -> ParserT s st m Text

700+mathDisplayWith op cl = try $ fmap T.pack $ do

701+ textStr op

702+ many1Till (noneOf "\n" <|> (newline <* notFollowedBy' blankline)) (try $ textStr cl)

633703

634704mathDisplay :: (HasReaderOptions st, Stream s m Char)

635− => ParserT s st m String

705+ => ParserT s st m Text

636706mathDisplay =

637707 (guardEnabled Ext_tex_math_dollars >> mathDisplayWith "$$" "$$")

638708 <|> (guardEnabled Ext_tex_math_single_backslash >>

@@ -641,7 +711,7 @@ mathDisplay =

641711 mathDisplayWith "\\\\[" "\\\\]")

642712

643713mathInline :: (HasReaderOptions st , Stream s m Char)

644− => ParserT s st m String

714+ => ParserT s st m Text

645715mathInline =

646716 (guardEnabled Ext_tex_math_dollars >> mathInlineWith "$" "$")

647717 <|> (guardEnabled Ext_tex_math_single_backslash >>

@@ -665,8 +735,8 @@ withHorizDisplacement parser = do

665735-- | Applies a parser and returns the raw string that was parsed,

666736-- along with the value produced by the parser.

667737withRaw :: Monad m

668− => ParsecT [Char] st m a

669− -> ParsecT [Char] st m (a, [Char])

738+ => ParsecT Text st m a

739+ -> ParsecT Text st m (a, Text)

670740withRaw parser = do

671741 pos1 <- getPosition

672742 inp <- getInput

@@ -674,11 +744,11 @@ withRaw parser = do

674744 pos2 <- getPosition

675745 let (l1,c1) = (sourceLine pos1, sourceColumn pos1)

676746 let (l2,c2) = (sourceLine pos2, sourceColumn pos2)

677− let inplines = take ((l2 - l1) + 1) $ lines inp

747+ let inplines = take ((l2 - l1) + 1) $ T.lines inp

678748 let raw = case inplines of

679749 [] -> ""

680− [l] -> take (c2 - c1) l

681− ls -> unlines (init ls) ++ take (c2 - 1) (last ls)

750+ [l] -> T.take (c2 - c1) l

751+ ls -> T.unlines (init ls) <> T.take (c2 - 1) (last ls)

682752 return (result, raw)

683753

684754-- | Parses backslash, then applies character parser.

@@ -716,7 +786,7 @@ lowerRoman = do

716786decimal :: Stream s m Char => ParserT s st m (ListNumberStyle, Int)

717787decimal = do

718788 num <- many1 digit

719− return (Decimal, fromMaybe 1 $ safeRead num)

789+ return (Decimal, fromMaybe 1 $ safeRead $ T.pack num)

720790

721791-- | Parses a '@' and optional label and

722792-- returns (DefaultStyle, [next example number]). The next

@@ -726,10 +796,10 @@ exampleNum :: Stream s m Char

726796 => ParserT s ParserState m (ListNumberStyle, Int)

727797exampleNum = do

728798 char '@'

729− lab <- many (alphaNum <|> satisfy (\c -> c == '_' || c == '-'))

799+ lab <- T.pack <$> many (alphaNum <|> satisfy (\c -> c == '_' || c == '-'))

730800 st <- getState

731801 let num = stateNextExample st

732− let newlabels = if null lab

802+ let newlabels = if T.null lab

733803 then stateExamples st

734804 else M.insert lab num $ stateExamples st

735805 updateState $ \s -> s{ stateNextExample = num + 1

@@ -825,25 +895,25 @@ orderedListMarker style delim = do

825895charRef :: Stream s m Char => ParserT s st m Inline

826896charRef = do

827897 c <- characterReference

828− return $ Str [c]

898+ return $ Str $ T.singleton c

829899

830−lineBlockLine :: Monad m => ParserT [Char] st m String

900+lineBlockLine :: Monad m => ParserT Text st m Text

831901lineBlockLine = try $ do

832902 char '|'

833903 char ' '

834− white <- many (spaceChar >> return '\160')

904+ white <- T.pack <$> many (spaceChar >> return '\160')

835905 notFollowedBy newline

836906 line <- anyLine

837907 continuations <- many (try $ char ' ' >> anyLine)

838− return $ white ++ unwords (line : continuations)

908+ return $ white <> T.unwords (line : continuations)

839909

840910blankLineBlockLine :: Stream s m Char => ParserT s st m Char

841911blankLineBlockLine = try (char '|' >> blankline)

842912

843913-- | Parses an RST-style line block and returns a list of strings.

844−lineBlockLines :: Monad m => ParserT [Char] st m [String]

914+lineBlockLines :: Monad m => ParserT Text st m [Text]

845915lineBlockLines = try $ do

846− lines' <- many1 (lineBlockLine <|> ((:[]) <$> blankLineBlockLine))

916+ lines' <- many1 (lineBlockLine <|> (T.singleton <$> blankLineBlockLine))

847917 skipMany blankline

848918 return lines'

849919

@@ -927,9 +997,9 @@ gridTableWith' blocks headless =

927997 tableWith' (gridTableHeader headless blocks) (gridTableRow blocks)

928998 (gridTableSep '-') gridTableFooter

929999

930−gridTableSplitLine :: [Int] -> String -> [String]

1000+gridTableSplitLine :: [Int] -> Text -> [Text]

9311001gridTableSplitLine indices line = map removeFinalBar $ tail $

932− splitStringByIndices (init indices) $ trimr line

1002+ splitTextByIndices (init indices) $ trimr line

9331003

9341004gridPart :: Stream s m Char => Char -> ParserT s st m ((Int, Int), Alignment)

9351005gridPart ch = do

@@ -949,9 +1019,10 @@ gridPart ch = do

9491019gridDashedLines :: Stream s m Char => Char -> ParserT s st m [((Int, Int), Alignment)]

9501020gridDashedLines ch = try $ char '+' >> many1 (gridPart ch) <* blankline

9511021

952−removeFinalBar :: String -> String

953−removeFinalBar =

954− reverse . dropWhile (`elem` " \t") . dropWhile (=='|') . reverse

1022+removeFinalBar :: Text -> Text

1023+removeFinalBar = T.dropWhileEnd go . T.dropWhileEnd (=='|')

1024+ where

1025+ go c = T.any (== c) " \t"

9551026

9561027-- | Separator between rows of grid table.

9571028gridTableSep :: Stream s m Char => Char -> ParserT s st m Char

@@ -969,7 +1040,7 @@ gridTableHeader headless blocks = try $ do

9691040 then return $ repeat ""

9701041 else many1

9711042 (notFollowedBy (gridTableSep '=') >> char '|' >>

972− many1Till anyChar newline)

1043+ T.pack <$> many1Till anyChar newline)

9731044 underDashes <- if headless

9741045 then return dashes

9751046 else gridDashedLines '='

@@ -979,16 +1050,16 @@ gridTableHeader headless blocks = try $ do

9791050 let aligns = map snd underDashes

9801051 let rawHeads = if headless

9811052 then replicate (length underDashes) ""

982− else map (unlines . map trim) $ transpose

1053+ else map (T.unlines . map trim) $ transpose

9831054 $ map (gridTableSplitLine indices) rawContent

9841055 heads <- sequence <$> mapM (parseFromString' blocks . trim) rawHeads

9851056 return (heads, aligns, indices)

9861057

987−gridTableRawLine :: Stream s m Char => [Int] -> ParserT s st m [String]

1058+gridTableRawLine :: Stream s m Char => [Int] -> ParserT s st m [Text]

9881059gridTableRawLine indices = do

9891060 char '|'

9901061 line <- many1Till anyChar newline

991− return (gridTableSplitLine indices line)

1062+ return (gridTableSplitLine indices $ T.pack line)

9921063

9931064-- | Parse row of grid table.

9941065gridTableRow :: (Stream s m Char, Monad mf, IsString s, HasLastStrPosition st)

@@ -997,7 +1068,7 @@ gridTableRow :: (Stream s m Char, Monad mf, IsString s, HasLastStrPosition st)

9971068 -> ParserT s st m (mf [Blocks])

9981069gridTableRow blocks indices = do

9991070 colLines <- many1 (gridTableRawLine indices)

1000− let cols = map ((++ "\n") . unlines . removeOneLeadingSpace) $

1071+ let cols = map ((<> "\n") . T.unlines . removeOneLeadingSpace) $

10011072 transpose colLines

10021073 compactifyCell bs = case compactify [bs] of

10031074 [] -> mempty

@@ -1005,40 +1076,41 @@ gridTableRow blocks indices = do

10051076 cells <- sequence <$> mapM (parseFromString' blocks) cols

10061077 return $ fmap (map compactifyCell) cells

10071078

1008−removeOneLeadingSpace :: [String] -> [String]

1079+removeOneLeadingSpace :: [Text] -> [Text]

10091080removeOneLeadingSpace xs =

10101081 if all startsWithSpace xs

1011− then map (drop 1) xs

1082+ then map (T.drop 1) xs

10121083 else xs

1013− where startsWithSpace "" = True

1014− startsWithSpace (y:_) = y == ' '

1084+ where startsWithSpace t = case T.uncons t of

1085+ Nothing -> True

1086+ Just (c, _) -> c == ' '

10151087

10161088-- | Parse footer for a grid table.

1017−gridTableFooter :: Stream s m Char => ParserT s st m [Char]

1089+gridTableFooter :: Stream s m Char => ParserT s st m Text

10181090gridTableFooter = blanklines

10191091

10201092---

10211093

10221094-- | Removes the ParsecT layer from the monad transformer stack

1023−readWithM :: (Stream s m Char, ToString s)

1095+readWithM :: (Stream s m Char, ToText s)

10241096 => ParserT s st m a -- ^ parser

10251097 -> st -- ^ initial state

10261098 -> s -- ^ input

10271099 -> m (Either PandocError a)

10281100readWithM parser state input =

1029− mapLeft (PandocParsecError $ toString input) `liftM` runParserT parser state "source" input

1101+ mapLeft (PandocParsecError $ toText input) `liftM` runParserT parser state "source" input

10301102

10311103-- | Parse a string with a given parser and state

1032−readWith :: Parser [Char] st a

1104+readWith :: Parser Text st a

10331105 -> st

1034− -> String

1106+ -> Text

10351107 -> Either PandocError a

10361108readWith p t inp = runIdentity $ readWithM p t inp

10371109

10381110-- | Parse a string with @parser@ (for testing).

10391111testStringWith :: Show a

1040− => ParserT [Char] ParserState Identity a

1041− -> [Char]

1112+ => ParserT Text ParserState Identity a

1113+ -> Text

10421114 -> IO ()

10431115testStringWith parser str = UTF8.putStrLn $ show $

10441116 readWith parser defaultParserState str

@@ -1057,23 +1129,23 @@ data ParserState = ParserState

10571129 stateSubstitutions :: SubstTable, -- ^ List of substitution references

10581130 stateNotes :: NoteTable, -- ^ List of notes (raw bodies)

10591131 stateNotes' :: NoteTable', -- ^ List of notes (parsed bodies)

1060− stateNoteRefs :: Set.Set String, -- ^ List of note references used

1132+ stateNoteRefs :: Set.Set Text, -- ^ List of note references used

10611133 stateMeta :: Meta, -- ^ Document metadata

10621134 stateMeta' :: F Meta, -- ^ Document metadata

1063− stateCitations :: M.Map String String, -- ^ RST-style citations

1135+ stateCitations :: M.Map Text Text, -- ^ RST-style citations

10641136 stateHeaderTable :: [HeaderType], -- ^ Ordered list of header types used

1065− stateIdentifiers :: Set.Set String, -- ^ Header identifiers used

1137+ stateIdentifiers :: Set.Set Text, -- ^ Header identifiers used

10661138 stateNextExample :: Int, -- ^ Number of next example

1067− stateExamples :: M.Map String Int, -- ^ Map from example labels to numbers

1139+ stateExamples :: M.Map Text Int, -- ^ Map from example labels to numbers

10681140 stateMacros :: M.Map Text Macro, -- ^ Table of macros defined so far

1069− stateRstDefaultRole :: String, -- ^ Current rST default interpreted text role

1070− stateRstCustomRoles :: M.Map String (String, Maybe String, Attr), -- ^ Current rST custom text roles

1141+ stateRstDefaultRole :: Text, -- ^ Current rST default interpreted text role

1142+ stateRstCustomRoles :: M.Map Text (Text, Maybe Text, Attr), -- ^ Current rST custom text roles

10711143 -- Triple represents: 1) Base role, 2) Optional format (only for :raw:

10721144 -- roles), 3) Additional classes (rest of Attr is unused)).

10731145 stateCaption :: Maybe Inlines, -- ^ Caption in current environment

1074− stateInHtmlBlock :: Maybe String, -- ^ Tag type of HTML block being parsed

1146+ stateInHtmlBlock :: Maybe Text, -- ^ Tag type of HTML block being parsed

10751147 stateFencedDivLevel :: Int, -- ^ Depth of fenced div

1076− stateContainers :: [String], -- ^ parent include files

1148+ stateContainers :: [Text], -- ^ parent include files

10771149 stateLogMessages :: [LogMessage], -- ^ log messages

10781150 stateMarkdownAttribute :: Bool -- ^ True if in markdown=1 context

10791151 }

@@ -1112,8 +1184,8 @@ instance Monad m => HasQuoteContext ParserState m where

11121184 return result

11131185

11141186class HasIdentifierList st where

1115− extractIdentifierList :: st -> Set.Set String

1116− updateIdentifierList :: (Set.Set String -> Set.Set String) -> st -> st

1187+ extractIdentifierList :: st -> Set.Set Text

1188+ updateIdentifierList :: (Set.Set Text -> Set.Set Text) -> st -> st

11171189

11181190instance HasIdentifierList ParserState where

11191191 extractIdentifierList = stateIdentifiers

@@ -1144,8 +1216,8 @@ instance HasLogMessages ParserState where

11441216 getLogMessages st = reverse $ stateLogMessages st

11451217

11461218class HasIncludeFiles st where

1147− getIncludeFiles :: st -> [String]

1148− addIncludeFile :: String -> st -> st

1219+ getIncludeFiles :: st -> [Text]

1220+ addIncludeFile :: Text -> st -> st

11491221 dropLatestIncludeFile :: st -> st

11501222

11511223instance HasIncludeFiles ParserState where

@@ -1232,17 +1304,21 @@ data QuoteContext

12321304 | NoQuote -- ^ Used when not parsing inside quotes

12331305 deriving (Eq, Show)

12341306

1235−type NoteTable = [(String, String)]

1307+type NoteTable = [(Text, Text)]

12361308

1237−type NoteTable' = M.Map String (SourcePos, F Blocks)

1309+type NoteTable' = M.Map Text (SourcePos, F Blocks)

12381310-- used in markdown reader

12391311

1240−newtype Key = Key String deriving (Show, Read, Eq, Ord)

1312+newtype Key = Key Text deriving (Show, Read, Eq, Ord)

12411313

1242−toKey :: String -> Key

1243−toKey = Key . map toLower . unwords . words . unbracket

1244− where unbracket ('[':xs) | "]" `isSuffixOf` xs = take (length xs - 1) xs

1245− unbracket xs = xs

1314+toKey :: Text -> Key

1315+toKey = Key . T.toLower . T.unwords . T.words . unbracket

1316+ where unbracket t

1317+ | Just ('[', t') <- T.uncons t

1318+ , Just (t'', ']') <- T.unsnoc t'

1319+ = t''

1320+ | otherwise

1321+ = t

12461322

12471323type KeyTable = M.Map Key (Target, Attr)

12481324

@@ -1261,17 +1337,17 @@ registerHeader :: (Stream s m a, HasReaderOptions st,

12611337registerHeader (ident,classes,kvs) header' = do

12621338 ids <- extractIdentifierList <$> getState

12631339 exts <- getOption readerExtensions

1264− if null ident && Ext_auto_identifiers `extensionEnabled` exts

1340+ if T.null ident && Ext_auto_identifiers `extensionEnabled` exts

12651341 then do

12661342 let id' = uniqueIdent exts (B.toList header') ids

12671343 let id'' = if Ext_ascii_identifiers `extensionEnabled` exts

1268− then mapMaybe toAsciiChar id'

1344+ then T.pack $ mapMaybe toAsciiChar $ T.unpack id'

12691345 else id'

12701346 updateState $ updateIdentifierList $ Set.insert id'

12711347 updateState $ updateIdentifierList $ Set.insert id''

12721348 return (id'',classes,kvs)

12731349 else do

1274− unless (null ident) $ do

1350+ unless (T.null ident) $ do

12751351 when (ident `Set.member` ids) $ do

12761352 pos <- getPosition

12771353 logMessage $ DuplicateIdentifier ident pos

@@ -1314,7 +1390,7 @@ failIfInQuoteContext context = do

13141390 context' <- getQuoteContext

13151391 when (context' == context) $ Prelude.fail "already inside quotes"

13161392

1317−charOrRef :: Stream s m Char => String -> ParserT s st m Char

1393+charOrRef :: Stream s m Char => [Char] -> ParserT s st m Char

13181394charOrRef cs =

13191395 oneOf cs <|> try (do c <- characterReference

13201396 guard (c `elem` cs)

@@ -1379,7 +1455,7 @@ nested p = do

13791455 return res

13801456

13811457citeKey :: (Stream s m Char, HasLastStrPosition st)

1382− => ParserT s st m (Bool, String)

1458+ => ParserT s st m (Bool, Text)

13831459citeKey = try $ do

13841460 guard =<< notAfterString

13851461 suppress_author <- option False (True <$ char '-')

@@ -1390,15 +1466,15 @@ citeKey = try $ do

13901466 rest <- many $ regchar <|> internal (oneOf ":.#$%&-+?<>~/") <|>

13911467 try (oneOf ":/" <* lookAhead (char '/'))

13921468 let key = firstChar:rest

1393− return (suppress_author, key)

1469+ return (suppress_author, T.pack key)

13941470

13951471

13961472token :: (Stream s m t)

1397− => (t -> String)

1473+ => (t -> Text)

13981474 -> (t -> SourcePos)

13991475 -> (t -> Maybe a)

14001476 -> ParsecT s st m a

1401−token pp pos match = tokenPrim pp (\_ t _ -> pos t) match

1477+token pp pos match = tokenPrim (T.unpack . pp) (\_ t _ -> pos t) match

14021478

14031479infixr 5 <+?>

14041480(<+?>) :: (Monoid a) => ParserT s st m a -> ParserT s st m a -> ParserT s st m a

@@ -1409,27 +1485,27 @@ extractIdClass (ident, cls, kvs) = (ident', cls', kvs')

14091485 where

14101486 ident' = fromMaybe ident (lookup "id" kvs)

14111487 cls' = case lookup "class" kvs of

1412− Just cl -> words cl

1488+ Just cl -> T.words cl

14131489 Nothing -> cls

14141490 kvs' = filter (\(k,_) -> k /= "id" || k /= "class") kvs

14151491

14161492insertIncludedFile' :: (PandocMonad m, HasIncludeFiles st)

1417− => ParserT [a] st m (mf Blocks)

1418− -> (String -> [a])

1493+ => ParserT a st m (mf Blocks)

1494+ -> (Text -> a)

14191495 -> [FilePath] -> FilePath

1420− -> ParserT [a] st m (mf Blocks)

1496+ -> ParserT a st m (mf Blocks)

14211497insertIncludedFile' blocks totoks dirs f = do

14221498 oldPos <- getPosition

14231499 oldInput <- getInput

14241500 containers <- getIncludeFiles <$> getState

1425− when (f `elem` containers) $

1426− throwError $ PandocParseError $ "Include file loop at " ++ show oldPos

1427− updateState $ addIncludeFile f

1501+ when (T.pack f `elem` containers) $

1502+ throwError $ PandocParseError $ T.pack $ "Include file loop at " ++ show oldPos

1503+ updateState $ addIncludeFile $ T.pack f

14281504 mbcontents <- readFileFromDirs dirs f

14291505 contents <- case mbcontents of

14301506 Just s -> return s

14311507 Nothing -> do

1432− report $ CouldNotLoadIncludeFile f oldPos

1508+ report $ CouldNotLoadIncludeFile (T.pack f) oldPos

14331509 return ""

14341510 setPosition $ newPos f 1 1

14351511 setInput $ totoks contents

@@ -1443,7 +1519,7 @@ insertIncludedFile' blocks totoks dirs f = do

14431519-- @PandocParseError@.

14441520insertIncludedFile :: (PandocMonad m, HasIncludeFiles st)

14451521 => ParserT [a] st m Blocks

1446− -> (String -> [a])

1522+ -> (Text -> [a])

14471523 -> [FilePath] -> FilePath

14481524 -> ParserT [a] st m Blocks

14491525insertIncludedFile blocks totoks dirs f =

@@ -1452,7 +1528,7 @@ insertIncludedFile blocks totoks dirs f =

14521528-- | Parse content of include file as future blocks. Circular includes result in

14531529-- an @PandocParseError@.

14541530insertIncludedFileF :: (PandocMonad m, HasIncludeFiles st)

1455− => ParserT String st m (Future st Blocks)

1531+ => ParserT Text st m (Future st Blocks)

14561532 -> [FilePath] -> FilePath

1457− -> ParserT String st m (Future st Blocks)

1533+ -> ParserT Text st m (Future st Blocks)

14581534insertIncludedFileF p = insertIncludedFile' p id

src/Text/Pandoc/Readers.hs+6−5

@@ -1,6 +1,7 @@

11{-# LANGUAGE MonoLocalBinds #-}

22{-# LANGUAGE NoImplicitPrelude #-}

33{-# LANGUAGE ScopedTypeVariables #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers

@@ -59,8 +60,8 @@ import Control.Monad (unless)

5960import Control.Monad.Except (throwError)

6061import Data.Aeson

6162import qualified Data.ByteString.Lazy as BL

62−import Data.List (intercalate)

6363import Data.Text (Text)

64+import qualified Data.Text as T

6465import Text.Pandoc.Class

6566import Text.Pandoc.Definition

6667import Text.Pandoc.Error

@@ -99,7 +100,7 @@ data Reader m = TextReader (ReaderOptions -> Text -> m Pandoc)

99100 | ByteStringReader (ReaderOptions -> BL.ByteString -> m Pandoc)

100101

101102-- | Association list of formats and readers.

102−readers :: PandocMonad m => [(String, Reader m)]

103+readers :: PandocMonad m => [(Text, Reader m)]

103104readers = [ ("native" , TextReader readNative)

104105 ,("json" , TextReader readJSON)

105106 ,("markdown" , TextReader readMarkdown)

@@ -135,11 +136,11 @@ readers = [ ("native" , TextReader readNative)

135136 ]

136137

137138-- | Retrieve reader, extensions based on formatSpec (format+extensions).

138−getReader :: PandocMonad m => String -> m (Reader m, Extensions)

139+getReader :: PandocMonad m => Text -> m (Reader m, Extensions)

139140getReader s =

140141 case parseFormatSpec s of

141142 Left e -> throwError $ PandocAppError

142− $ intercalate "\n" [m | Message m <- errorMessages e]

143+ $ T.intercalate "\n" [T.pack m | Message m <- errorMessages e]

143144 Right (readerName, extsToEnable, extsToDisable) ->

144145 case lookup readerName readers of

145146 Nothing -> throwError $ PandocUnknownReaderError

@@ -154,7 +155,7 @@ getReader s =

154155 unless (extensionEnabled ext allExts) $

155156 throwError $

156157 PandocUnsupportedExtensionError

157− (drop 4 $ show ext) readerName)

158+ (T.drop 4 $ T.pack $ show ext) readerName)

158159 (extsToEnable ++ extsToDisable)

159160 return (r, exts)

160161

src/Text/Pandoc/Readers/CommonMark.hs+23−22

@@ -1,4 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

3+{-# LANGUAGE ViewPatterns #-}

24{- |

35 Module : Text.Pandoc.Readers.CommonMark

@@ -18,9 +20,9 @@ where

1820import Prelude

1921import CMarkGFM

2022import Control.Monad.State

21−import Data.List (groupBy)

2223import qualified Data.Set as Set

23−import Data.Text (Text, unpack)

24+import Data.Text (Text)

25+import qualified Data.Text as T

2426import Text.Pandoc.Class (PandocMonad)

2527import Text.Pandoc.Definition

2628import Text.Pandoc.Emoji (emojiToInline)

@@ -40,24 +42,24 @@ readCommonMark opts s = return $

4042 [ extTable | isEnabled Ext_pipe_tables opts ] ++

4143 [ extAutolink | isEnabled Ext_autolink_bare_uris opts ]

4244

43−convertEmojis :: String -> [Inline]

44−convertEmojis s@(':':xs) =

45− case break (==':') xs of

46− (ys,':':zs) ->

45+convertEmojis :: Text -> [Inline]

46+convertEmojis s@(T.uncons -> Just (':',xs)) =

47+ case T.break (==':') xs of

48+ (ys, T.uncons -> Just (':',zs)) ->

4749 case emojiToInline ys of

4850 Just em -> em : convertEmojis zs

49− Nothing -> Str (':' : ys) : convertEmojis (':':zs)

51+ Nothing -> Str (":" <> ys) : convertEmojis (":" <> zs)

5052 _ -> [Str s]

5153convertEmojis s =

52− case break (==':') s of

54+ case T.break (==':') s of

5355 ("","") -> []

5456 (_,"") -> [Str s]

55− (xs,ys) -> Str xs:convertEmojis ys

57+ (xs,ys) -> Str xs : convertEmojis ys

5658

5759addHeaderIdentifiers :: ReaderOptions -> Pandoc -> Pandoc

5860addHeaderIdentifiers opts doc = evalState (walkM (addHeaderId opts) doc) mempty

5961

60−addHeaderId :: ReaderOptions -> Block -> State (Set.Set String) Block

62+addHeaderId :: ReaderOptions -> Block -> State (Set.Set Text) Block

6163addHeaderId opts (Header lev (_,classes,kvs) ils) = do

6264 ids <- get

6365 let ident = uniqueIdent (readerExtensions opts) ils ids

@@ -82,14 +84,14 @@ addBlock _ (Node _ THEMATIC_BREAK _) =

8284addBlock opts (Node _ BLOCK_QUOTE nodes) =

8385 (BlockQuote (addBlocks opts nodes) :)

8486addBlock opts (Node _ (HTML_BLOCK t) _)

85− | isEnabled Ext_raw_html opts = (RawBlock (Format "html") (unpack t) :)

87+ | isEnabled Ext_raw_html opts = (RawBlock (Format "html") t :)

8688 | otherwise = id

8789-- Note: the cmark parser will never generate CUSTOM_BLOCK,

8890-- so we don't need to handle it:

8991addBlock _ (Node _ (CUSTOM_BLOCK _onEnter _onExit) _nodes) =

9092 id

9193addBlock _ (Node _ (CODE_BLOCK info t) _) =

92− (CodeBlock ("", take 1 (words (unpack info)), []) (unpack t) :)

94+ (CodeBlock ("", take 1 (T.words info), []) t :)

9395addBlock opts (Node _ (HEADING lev) nodes) =

9496 (Header lev ("",[],[]) (addInlines opts nodes) :)

9597addBlock opts (Node _ (LIST listAttrs) nodes) =

@@ -176,29 +178,28 @@ addInlines opts = foldr (addInline opts) []

176178

177179addInline :: ReaderOptions -> Node -> [Inline] -> [Inline]

178180addInline opts (Node _ (TEXT t) _) = (foldr ((++) . toinl) [] clumps ++)

179− where raw = unpack t

180− clumps = groupBy samekind raw

181+ where clumps = T.groupBy samekind t

181182 samekind ' ' ' ' = True

182183 samekind ' ' _ = False

183184 samekind _ ' ' = False

184185 samekind _ _ = True

185− toinl (' ':_) = [Space]

186− toinl xs = if isEnabled Ext_emoji opts

187− then convertEmojis xs

188− else [Str xs]

186+ toinl (T.uncons -> Just (' ', _)) = [Space]

187+ toinl xs = if isEnabled Ext_emoji opts

188+ then convertEmojis xs

189+ else [Str xs]

189190addInline _ (Node _ LINEBREAK _) = (LineBreak :)

190191addInline opts (Node _ SOFTBREAK _)

191192 | isEnabled Ext_hard_line_breaks opts = (LineBreak :)

192193 | otherwise = (SoftBreak :)

193194addInline opts (Node _ (HTML_INLINE t) _)

194− | isEnabled Ext_raw_html opts = (RawInline (Format "html") (unpack t) :)

195+ | isEnabled Ext_raw_html opts = (RawInline (Format "html") t :)

195196 | otherwise = id

196197-- Note: the cmark parser will never generate CUSTOM_BLOCK,

197198-- so we don't need to handle it:

198199addInline _ (Node _ (CUSTOM_INLINE _onEnter _onExit) _nodes) =

199200 id

200201addInline _ (Node _ (CODE t) _) =

201− (Code ("",[],[]) (unpack t) :)

202+ (Code ("",[],[]) t :)

202203addInline opts (Node _ EMPH nodes) =

203204 (Emph (addInlines opts nodes) :)

204205addInline opts (Node _ STRONG nodes) =

@@ -206,7 +207,7 @@ addInline opts (Node _ STRONG nodes) =

206207addInline opts (Node _ STRIKETHROUGH nodes) =

207208 (Strikeout (addInlines opts nodes) :)

208209addInline opts (Node _ (LINK url title) nodes) =

209− (Link nullAttr (addInlines opts nodes) (unpack url, unpack title) :)

210+ (Link nullAttr (addInlines opts nodes) (url, title) :)

210211addInline opts (Node _ (IMAGE url title) nodes) =

211− (Image nullAttr (addInlines opts nodes) (unpack url, unpack title) :)

212+ (Image nullAttr (addInlines opts nodes) (url, title) :)

212213addInline _ _ = id

src/Text/Pandoc/Readers/Creole.hs+14−14

@@ -19,6 +19,7 @@ import Control.Monad.Except (guard, liftM2, throwError)

1919import qualified Data.Foldable as F

2020import Data.Maybe (fromMaybe)

2121import Data.Text (Text)

22+import qualified Data.Text as T

2223import qualified Text.Pandoc.Builder as B

2324import Text.Pandoc.Class (PandocMonad (..))

2425import Text.Pandoc.Definition

@@ -70,7 +71,6 @@ parseCreole = do

7071 eof

7172 return $ B.doc bs

7273

73−

7474--

7575-- block parsers

7676--

@@ -92,9 +92,9 @@ nowiki = try $ fmap (B.codeBlock . mconcat) (nowikiStart

9292 >> manyTill content nowikiEnd)

9393 where

9494 content = brackets <|> line

95− brackets = try $ option "" ((:[]) <$> newline)

96− <+> (char ' ' >> (many (char ' ') <+> string "}}}") <* eol)

97− line = option "" ((:[]) <$> newline) <+> manyTill anyChar eol

95+ brackets = try $ option "" (T.singleton <$> newline)

96+ <+> (char ' ' >> (manyChar (char ' ') <+> textStr "}}}") <* eol)

97+ line = option "" (T.singleton <$> newline) <+> manyTillChar anyChar eol

9898 eol = lookAhead $ try $ nowikiEnd <|> newline

9999 nowikiStart = optional newline >> string "{{{" >> skipMany spaceChar >> newline

100100 nowikiEnd = try $ linebreak >> string "}}}" >> skipMany spaceChar >> newline

@@ -106,7 +106,7 @@ header = try $ do

106106 fmap length (many1 (char '='))

107107 guard $ level <= 6

108108 skipSpaces

109− content <- B.str <$> manyTill (noneOf "\n") headerEnd

109+ content <- B.str <$> manyTillChar (noneOf "\n") headerEnd

110110 return $ B.header level content

111111 where

112112 headerEnd = try $ skipSpaces >> many (char '=') >> skipSpaces >> newline

@@ -204,7 +204,7 @@ inline = choice [ whitespace

204204

205205escapedChar :: PandocMonad m => CRLParser m B.Inlines

206206escapedChar =

207− fmap (B.str . (:[])) (try $ char '~' >> noneOf "\t\n ")

207+ fmap (B.str . T.singleton) (try $ char '~' >> noneOf "\t\n ")

208208

209209escapedLink :: PandocMonad m => CRLParser m B.Inlines

210210escapedLink = try $ do

@@ -217,8 +217,8 @@ image = try $ do

217217 (orig, src) <- wikiImg

218218 return $ B.image src "" (B.str orig)

219219 where

220− linkSrc = many $ noneOf "|}\n\r\t"

221− linkDsc = char '|' >> many (noneOf "}\n\r\t")

220+ linkSrc = manyChar $ noneOf "|}\n\r\t"

221+ linkDsc = char '|' >> manyChar (noneOf "}\n\r\t")

222222 wikiImg = try $ do

223223 string "{{"

224224 src <- linkSrc

@@ -231,11 +231,11 @@ link = try $ do

231231 (orig, src) <- uriLink <|> wikiLink

232232 return $ B.link src "" orig

233233 where

234− linkSrc = many $ noneOf "|]\n\r\t"

235− linkDsc :: PandocMonad m => String -> CRLParser m B.Inlines

234+ linkSrc = manyChar $ noneOf "|]\n\r\t"

235+ linkDsc :: PandocMonad m => Text -> CRLParser m B.Inlines

236236 linkDsc otxt = B.str

237237 <$> try (option otxt

238− (char '|' >> many (noneOf "]\n\r\t")))

238+ (char '|' >> manyChar (noneOf "]\n\r\t")))

239239 linkImg = try $ char '|' >> image

240240 wikiLink = try $ do

241241 string "[["

@@ -248,7 +248,7 @@ link = try $ do

248248 return (B.str orig, src)

249249

250250inlineNowiki :: PandocMonad m => CRLParser m B.Inlines

251−inlineNowiki = B.code <$> (start >> manyTill (noneOf "\n\r") end)

251+inlineNowiki = B.code <$> (start >> manyTillChar (noneOf "\n\r") end)

252252 where

253253 start = try $ string "{{{"

254254 end = try $ string "}}}" >> lookAhead (noneOf "}")

@@ -271,11 +271,11 @@ linebreak = newline >> notFollowedBy newline >> (lastNewline <|> innerNewline)

271271 innerNewline = return B.space

272272

273273symbol :: PandocMonad m => CRLParser m B.Inlines

274−symbol = fmap (B.str . (:[])) (oneOf specialChars)

274+symbol = fmap (B.str . T.singleton) (oneOf specialChars)

275275

276276str :: PandocMonad m => CRLParser m B.Inlines

277277str = let strChar = noneOf ("\t\n " ++ specialChars) in

278− fmap B.str (many1 strChar)

278+ fmap B.str (many1Char strChar)

279279

280280bold :: PandocMonad m => CRLParser m B.Inlines

281281bold = B.strong . mconcat <$>

src/Text/Pandoc/Readers/DocBook.hs+52−50

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.DocBook

@@ -536,20 +537,22 @@ instance Default DBState where

536537

537538readDocBook :: PandocMonad m => ReaderOptions -> Text -> m Pandoc

538539readDocBook _ inp = do

539− let tree = normalizeTree . parseXML . handleInstructions

540− $ T.unpack $ crFilter inp

540+ let tree = normalizeTree . parseXML . handleInstructions $ crFilter inp

541541 (bs, st') <- flip runStateT (def{ dbContent = tree }) $ mapM parseBlock tree

542542 return $ Pandoc (dbMeta st') (toList . mconcat $ bs)

543543

544544-- We treat <?asciidoc-br?> specially (issue #1236), converting it

545545-- to , since xml-light doesn't parse the instruction correctly.

546546-- Other xml instructions are simply removed from the input stream.

547−handleInstructions :: String -> String

548−handleInstructions ('<':'?':'a':'s':'c':'i':'i':'d':'o':'c':'-':'b':'r':'?':'>':xs) = '<':'b':'r':'/':'>': handleInstructions xs

549−handleInstructions xs = case break (=='<') xs of

547+handleInstructions :: Text -> Text

548+handleInstructions = T.pack . handleInstructions' . T.unpack

549+

550+handleInstructions' :: String -> String

551+handleInstructions' ('<':'?':'a':'s':'c':'i':'i':'d':'o':'c':'-':'b':'r':'?':'>':xs) = '<':'b':'r':'/':'>': handleInstructions' xs

552+handleInstructions' xs = case break (=='<') xs of

550553 (ys, []) -> ys

551− ([], '<':zs) -> '<' : handleInstructions zs

552− (ys, zs) -> ys ++ handleInstructions zs

554+ ([], '<':zs) -> '<' : handleInstructions' zs

555+ (ys, zs) -> ys ++ handleInstructions' zs

553556

554557getFigure :: PandocMonad m => Element -> DB m Blocks

555558getFigure e = do

@@ -580,13 +583,13 @@ convertEntity :: String -> String

580583convertEntity e = Data.Maybe.fromMaybe (map toUpper e) (lookupEntity e)

581584

582585-- convenience function to get an attribute value, defaulting to ""

583−attrValue :: String -> Element -> String

586+attrValue :: String -> Element -> Text

584587attrValue attr elt =

585− fromMaybe "" (lookupAttrBy (\x -> qName x == attr) (elAttribs elt))

588+ maybe "" T.pack (lookupAttrBy (\x -> qName x == attr) (elAttribs elt))

586589

587590-- convenience function

588−named :: String -> Element -> Bool

589−named s e = qName (elName e) == s

591+named :: Text -> Element -> Bool

592+named s e = qName (elName e) == T.unpack s

590593

591594--

592595

@@ -611,7 +614,7 @@ addMetadataFromElement e = do

611614 [z] -> getInlines z >>= addMeta fieldname

612615 zs -> mapM getInlines zs >>= addMeta fieldname

613616

614−addMeta :: PandocMonad m => ToMetaValue a => String -> a -> DB m ()

617+addMeta :: PandocMonad m => ToMetaValue a => Text -> a -> DB m ()

615618addMeta field val = modify (setMeta field val)

616619

617620instance HasMeta DBState where

@@ -638,10 +641,8 @@ admonitionTags :: [String]

638641admonitionTags = ["important","caution","note","tip","warning"]

639642

640643-- Trim leading and trailing newline characters

641−trimNl :: String -> String

642−trimNl = reverse . go . reverse . go

643− where go ('\n':xs) = xs

644− go xs = xs

644+trimNl :: Text -> Text

645+trimNl = T.dropAround (== '\n')

645646

646647-- meld text into beginning of first paragraph of Blocks.

647648-- assumes Blocks start with a Para; if not, does nothing.

@@ -668,7 +669,7 @@ getMediaobject e = do

668669 h = case atVal "depth" of

669670 "" -> []

670671 d -> [("height", d)]

671− atr = (atVal "id", words $ atVal "role", w ++ h)

672+ atr = (atVal "id", T.words $ atVal "role", w ++ h)

672673 in return (atVal "fileref", atr)

673674 let getCaption el = case filterChild (\x -> named "caption" x

674675 || named "textobject" x

@@ -691,8 +692,8 @@ parseBlock :: PandocMonad m => Content -> DB m Blocks

691692parseBlock (Text (CData CDataRaw _ _)) = return mempty -- DOCTYPE

692693parseBlock (Text (CData _ s _)) = if all isSpace s

693694 then return mempty

694− else return $ plain $ trimInlines $ text s

695−parseBlock (CRef x) = return $ plain $ str $ map toUpper x

695+ else return $ plain $ trimInlines $ text $ T.pack s

696+parseBlock (CRef x) = return $ plain $ str $ T.toUpper $ T.pack x

696697parseBlock (Elem e) =

697698 case qName (elName e) of

698699 "toc" -> skip -- skip TOC, since in pandoc it's autogenerated

@@ -740,7 +741,7 @@ parseBlock (Elem e) =

740741 "refsect2" -> sect 2

741742 "refsect3" -> sect 3

742743 "refsection" -> gets dbSectionLevel >>= sect . (+1)

743− l@_ | l `elem` admonitionTags -> parseAdmonition l

744+ l@_ | l `elem` admonitionTags -> parseAdmonition $ T.pack l

744745 "area" -> skip

745746 "areaset" -> skip

746747 "areaspec" -> skip

@@ -800,7 +801,7 @@ parseBlock (Elem e) =

800801 "subtitle" -> return mempty -- handled in parent element

801802 _ -> skip >> getBlocks e

802803 where skip = do

803− lift $ report $ IgnoredElement $ qName (elName e)

804+ lift $ report $ IgnoredElement $ T.pack $ qName (elName e)

804805 return mempty

805806

806807 parseMixed container conts = do

@@ -818,7 +819,7 @@ parseBlock (Elem e) =

818819 "" -> []

819820 x -> [x]

820821 return $ codeBlockWith (attrValue "id" e, classes', [])

821− $ trimNl $ strContentRecursive e

822+ $ trimNl $ T.pack $ strContentRecursive e

822823 parseBlockquote = do

823824 attrib <- case filterChild (named "attribution") e of

824825 Nothing -> return mempty

@@ -871,9 +872,9 @@ parseBlock (Elem e) =

871872 _ -> AlignDefault

872873 let toWidth c = case findAttr (unqual "colwidth") c of

873874 Just w -> fromMaybe 0

874− $ safeRead $ '0': filter (\x ->

875+ $ safeRead $ "0" <> T.filter (\x ->

875876 (x >= '0' && x <= '9')

876− || x == '.') w

877+ || x == '.') (T.pack w)

877878 Nothing -> 0 :: Double

878879 let numrows = case bodyrows of

879880 [] -> 0

@@ -938,9 +939,9 @@ elementToStr (Elem e') = Text $ CData CDataText (strContentRecursive e') Nothing

938939elementToStr x = x

939940

940941parseInline :: PandocMonad m => Content -> DB m Inlines

941−parseInline (Text (CData _ s _)) = return $ text s

942+parseInline (Text (CData _ s _)) = return $ text $ T.pack s

942943parseInline (CRef ref) =

943− return $ maybe (text $ map toUpper ref) text $ lookupEntity ref

944+ return $ maybe (text $ T.toUpper $ T.pack ref) (text . T.pack) $ lookupEntity ref

944945parseInline (Elem e) =

945946 case qName (elName e) of

946947 "equation" -> equation e displayMath

@@ -980,7 +981,7 @@ parseInline (Elem e) =

980981 "constant" -> codeWithLang

981982 "userinput" -> codeWithLang

982983 "varargs" -> return $ code "(...)"

983− "keycap" -> return (str $ strContent e)

984+ "keycap" -> return (str $ T.pack $ strContent e)

984985 "keycombo" -> keycombo <$>

985986 mapM parseInline (elContent e)

986987 "menuchoice" -> menuchoice <$>

@@ -992,20 +993,20 @@ parseInline (Elem e) =

992993 let title = case attrValue "endterm" e of

993994 "" -> maybe "???" xrefTitleByElem

994995 (findElementById linkend content)

995− endterm -> maybe "???" strContent

996+ endterm -> maybe "???" (T.pack . strContent)

996997 (findElementById endterm content)

997− return $ link ('#' : linkend) "" (text title)

998− "email" -> return $ link ("mailto:" ++ strContent e) ""

999− $ str $ strContent e

1000− "uri" -> return $ link (strContent e) "" $ str $ strContent e

998+ return $ link ("#" <> linkend) "" (text title)

999+ "email" -> return $ link ("mailto:" <> T.pack (strContent e)) ""

1000+ $ str $ T.pack $ strContent e

1001+ "uri" -> return $ link (T.pack $ strContent e) "" $ str $ T.pack $ strContent e

10011002 "ulink" -> link (attrValue "url" e) "" <$> innerInlines

10021003 "link" -> do

10031004 ils <- innerInlines

10041005 let href = case findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e of

1005− Just h -> h

1006− _ -> '#' : attrValue "linkend" e

1006+ Just h -> T.pack h

1007+ _ -> "#" <> attrValue "linkend" e

10071008 let ils' = if ils == mempty then str href else ils

1008− let attr = (attrValue "id" e, words $ attrValue "role" e, [])

1009+ let attr = (attrValue "id" e, T.words $ attrValue "role" e, [])

10091010 return $ linkWith attr href "" ils'

10101011 "foreignphrase" -> emph <$> innerInlines

10111012 "emphasis" -> case attrValue "role" e of

@@ -1023,7 +1024,7 @@ parseInline (Elem e) =

10231024 "br" -> return linebreak

10241025 _ -> skip >> innerInlines

10251026 where skip = do

1026− lift $ report $ IgnoredElement $ qName (elName e)

1027+ lift $ report $ IgnoredElement $ T.pack $ qName (elName e)

10271028 return mempty

10281029

10291030 innerInlines = (trimInlines . mconcat) <$>

@@ -1032,7 +1033,7 @@ parseInline (Elem e) =

10321033 let classes' = case attrValue "language" e of

10331034 "" -> []

10341035 l -> [l]

1035− return $ codeWith (attrValue "id" e,classes',[]) $ strContentRecursive e

1036+ return $ codeWith (attrValue "id" e,classes',[]) $ T.pack $ strContentRecursive e

10361037 simpleList = (mconcat . intersperse (str "," <> space)) <$> mapM getInlines

10371038 (filterChildren (named "member") e)

10381039 segmentedList = do

@@ -1062,8 +1063,8 @@ parseInline (Elem e) =

10621063 -- if there's no such attribute, employ some heuristics based on what

10631064 -- docbook-xsl does.

10641065 xrefTitleByElem el

1065− | not (null xrefLabel) = xrefLabel

1066− | otherwise = case qName (elName el) of

1066+ | not (T.null xrefLabel) = xrefLabel

1067+ | otherwise = case qName (elName el) of

10671068 "chapter" -> descendantContent "title" el

10681069 "section" -> descendantContent "title" el

10691070 "sect1" -> descendantContent "title" el

@@ -1073,10 +1074,10 @@ parseInline (Elem e) =

10731074 "sect5" -> descendantContent "title" el

10741075 "cmdsynopsis" -> descendantContent "command" el

10751076 "funcsynopsis" -> descendantContent "function" el

1076− _ -> qName (elName el) ++ "_title"

1077+ _ -> T.pack $ qName (elName el) ++ "_title"

10771078 where

10781079 xrefLabel = attrValue "xreflabel" el

1079− descendantContent name = maybe "???" strContent

1080+ descendantContent name = maybe "???" (T.pack . strContent)

10801081 . filterElementName (\n -> qName n == name)

10811082

10821083-- | Extract a math equation from an element

@@ -1088,20 +1089,20 @@ equation

10881089 :: Monad m

10891090 => Element

10901091 -- ^ The element from which to extract a mathematical equation

1091− -> (String -> Inlines)

1092+ -> (Text -> Inlines)

10921093 -- ^ A constructor for some Inlines, taking the TeX code as input

10931094 -> m Inlines

10941095equation e constructor =

1095− return $ mconcat $ map constructor $ mathMLEquations ++ latexEquations

1096+ return $ mconcat $ map constructor $ mathMLEquations <> latexEquations

10961097 where

1097− mathMLEquations :: [String]

1098+ mathMLEquations :: [Text]

10981099 mathMLEquations = map writeTeX $ rights $ readMath

10991100 (\x -> qName (elName x) == "math" && qPrefix (elName x) == Just "mml")

1100− (readMathML . showElement)

1101+ (readMathML . T.pack . showElement)

11011102

1102− latexEquations :: [String]

1103+ latexEquations :: [Text]

11031104 latexEquations = readMath (\x -> qName (elName x) == "mathphrase")

1104− (concat . fmap showVerbatimCData . elContent)

1105+ (T.concat . fmap showVerbatimCData . elContent)

11051106

11061107 readMath :: (Element -> Bool) -> (Element -> b) -> [b]

11071108 readMath childPredicate fromElement =

@@ -1111,9 +1112,10 @@ equation e constructor =

11111112

11121113-- | Get the actual text stored in a CData block. 'showContent'

11131114-- returns the text still surrounded by the [[CDATA]] tags.

1114−showVerbatimCData :: Content -> String

1115−showVerbatimCData (Text (CData _ d _)) = d

1116−showVerbatimCData c = showContent c

1115+showVerbatimCData :: Content -> Text

1116+showVerbatimCData (Text (CData _ d _)) = T.pack d

1117+showVerbatimCData c = T.pack $ showContent c

1118+

11171119

11181120-- | Set the prefix of a name to 'Nothing'

11191121removePrefix :: QName -> QName

src/Text/Pandoc/Readers/Docx.hs+39−37

@@ -4,6 +4,7 @@

44{-# LANGUAGE PatternGuards #-}

55{-# LANGUAGE MultiWayIf #-}

66{-# LANGUAGE FlexibleContexts #-}

7+{-# LANGUAGE ViewPatterns #-}

78{- |

89 Module : Text.Pandoc.Readers.Docx

@@ -68,12 +69,12 @@ import Data.Default (Default)

6869import Data.List (delete, intersect)

6970import Data.Char (isSpace)

7071import qualified Data.Map as M

72+import qualified Data.Text as T

7173import Data.Maybe (isJust, fromMaybe)

7274import Data.Sequence (ViewL (..), viewl)

7375import qualified Data.Sequence as Seq

7476import qualified Data.Set as Set

7577import Text.Pandoc.Builder

76−-- import Text.Pandoc.Definition

7778import Text.Pandoc.MediaBag (MediaBag)

7879import Text.Pandoc.Options

7980import Text.Pandoc.Readers.Docx.Combine

@@ -101,14 +102,14 @@ readDocx opts bytes

101102readDocx _ _ =

102103 throwError $ PandocSomeError "couldn't parse docx file"

103104

104−data DState = DState { docxAnchorMap :: M.Map String String

105− , docxAnchorSet :: Set.Set String

106− , docxImmedPrevAnchor :: Maybe String

105+data DState = DState { docxAnchorMap :: M.Map T.Text T.Text

106+ , docxAnchorSet :: Set.Set T.Text

107+ , docxImmedPrevAnchor :: Maybe T.Text

107108 , docxMediaBag :: MediaBag

108109 , docxDropCap :: Inlines

109110 -- keep track of (numId, lvl) values for

110111 -- restarting

111− , docxListState :: M.Map (String, String) Integer

112+ , docxListState :: M.Map (T.Text, T.Text) Integer

112113 , docxPrevPara :: Inlines

113114 }

114115

@@ -142,7 +143,7 @@ spansToKeep = []

142143divsToKeep :: [ParaStyleName]

143144divsToKeep = ["Definition", "Definition Term"]

144145

145−metaStyles :: M.Map ParaStyleName String

146+metaStyles :: M.Map ParaStyleName T.Text

146147metaStyles = M.fromList [ ("Title", "title")

147148 , ("Subtitle", "subtitle")

148149 , ("Author", "author")

@@ -167,7 +168,7 @@ isEmptyPar (Paragraph _ parParts) =

167168 isEmptyElem _ = True

168169isEmptyPar _ = False

169170

170−bodyPartsToMeta' :: PandocMonad m => [BodyPart] -> DocxContext m (M.Map String MetaValue)

171+bodyPartsToMeta' :: PandocMonad m => [BodyPart] -> DocxContext m (M.Map T.Text MetaValue)

171172bodyPartsToMeta' [] = return M.empty

172173bodyPartsToMeta' (bp : bps)

173174 | (Paragraph pPr parParts) <- bp

@@ -232,22 +233,22 @@ runElemToInlines Tab = space

232233runElemToInlines SoftHyphen = text "\xad"

233234runElemToInlines NoBreakHyphen = text "\x2011"

234235

235−runElemToString :: RunElem -> String

236−runElemToString (TextRun s) = s

237−runElemToString LnBrk = ['\n']

238−runElemToString Tab = ['\t']

239−runElemToString SoftHyphen = ['\xad']

240−runElemToString NoBreakHyphen = ['\x2011']

236+runElemToText :: RunElem -> T.Text

237+runElemToText (TextRun s) = s

238+runElemToText LnBrk = T.singleton '\n'

239+runElemToText Tab = T.singleton '\t'

240+runElemToText SoftHyphen = T.singleton '\xad'

241+runElemToText NoBreakHyphen = T.singleton '\x2011'

241242

242−runToString :: Run -> String

243−runToString (Run _ runElems) = concatMap runElemToString runElems

244−runToString _ = ""

243+runToText :: Run -> T.Text

244+runToText (Run _ runElems) = T.concat $ map runElemToText runElems

245+runToText _ = ""

245246

246−parPartToString :: ParPart -> String

247−parPartToString (PlainRun run) = runToString run

248−parPartToString (InternalHyperLink _ runs) = concatMap runToString runs

249−parPartToString (ExternalHyperLink _ runs) = concatMap runToString runs

250−parPartToString _ = ""

247+parPartToText :: ParPart -> T.Text

248+parPartToText (PlainRun run) = runToText run

249+parPartToText (InternalHyperLink _ runs) = T.concat $ map runToText runs

250+parPartToText (ExternalHyperLink _ runs) = T.concat $ map runToText runs

251+parPartToText _ = ""

251252

252253blacklistedCharStyles :: [CharStyleName]

253254blacklistedCharStyles = ["Hyperlink"]

@@ -310,7 +311,7 @@ runToInlines :: PandocMonad m => Run -> DocxContext m Inlines

310311runToInlines (Run rs runElems)

311312 | maybe False isCodeCharStyle $ rParentStyle rs = do

312313 rPr <- resolveDependentRunStyle rs

313− let codeString = code $ concatMap runElemToString runElems

314+ let codeString = code $ T.concat $ map runElemToText runElems

314315 return $ case rVertAlign rPr of

315316 Just SupScrpt -> superscript codeString

316317 Just SubScrpt -> subscript codeString

@@ -328,17 +329,17 @@ runToInlines (Endnote bps) = do

328329 return $ note blksList

329330runToInlines (InlineDrawing fp title alt bs ext) = do

330331 (lift . lift) $ P.insertMedia fp Nothing bs

331− return $ imageWith (extentToAttr ext) fp title $ text alt

332+ return $ imageWith (extentToAttr ext) (T.pack fp) title $ text alt

332333runToInlines InlineChart = return $ spanWith ("", ["chart"], []) $ text "[CHART]"

333334

334335extentToAttr :: Extent -> Attr

335336extentToAttr (Just (w, h)) =

336337 ("", [], [("width", showDim w), ("height", showDim h)] )

337338 where

338− showDim d = show (d / 914400) ++ "in"

339+ showDim d = tshow (d / 914400) <> "in"

339340extentToAttr _ = nullAttr

340341

341−blocksToInlinesWarn :: PandocMonad m => String -> Blocks -> DocxContext m Inlines

342+blocksToInlinesWarn :: PandocMonad m => T.Text -> Blocks -> DocxContext m Inlines

342343blocksToInlinesWarn cmtId blks = do

343344 let blkList = toList blks

344345 notParaOrPlain :: Block -> Bool

@@ -347,7 +348,7 @@ blocksToInlinesWarn cmtId blks = do

347348 notParaOrPlain _ = True

348349 unless ( not (any notParaOrPlain blkList)) $

349350 lift $ P.report $ DocxParserWarning $

350− "Docx comment " ++ cmtId ++ " will not retain formatting"

351+ "Docx comment " <> cmtId <> " will not retain formatting"

351352 return $ blocksToInlines' blkList

352353

353354-- The majority of work in this function is done in the primed

@@ -440,12 +441,12 @@ parPartToInlines' (BookMark _ anchor) =

440441 return $ spanWith (newAnchor, ["anchor"], []) mempty

441442parPartToInlines' (Drawing fp title alt bs ext) = do

442443 (lift . lift) $ P.insertMedia fp Nothing bs

443− return $ imageWith (extentToAttr ext) fp title $ text alt

444+ return $ imageWith (extentToAttr ext) (T.pack fp) title $ text alt

444445parPartToInlines' Chart =

445446 return $ spanWith ("", ["chart"], []) $ text "[CHART]"

446447parPartToInlines' (InternalHyperLink anchor runs) = do

447448 ils <- smushInlines <$> mapM runToInlines runs

448− return $ link ('#' : anchor) "" ils

449+ return $ link ("#" <> anchor) "" ils

449450parPartToInlines' (ExternalHyperLink target runs) = do

450451 ils <- smushInlines <$> mapM runToInlines runs

451452 return $ link target "" ils

@@ -463,7 +464,7 @@ isAnchorSpan (Span (_, classes, kvs) _) =

463464 null kvs

464465isAnchorSpan _ = False

465466

466−dummyAnchors :: [String]

467+dummyAnchors :: [T.Text]

467468dummyAnchors = ["_GoBack"]

468469

469470makeHeaderAnchor :: PandocMonad m => Blocks -> DocxContext m Blocks

@@ -477,7 +478,7 @@ makeHeaderAnchor' (Header n (ident, classes, kvs) ils)

477478 , (Span (anchIdent, ["anchor"], _) cIls) <- c = do

478479 hdrIDMap <- gets docxAnchorMap

479480 exts <- readerExtensions <$> asks docxOptions

480− let newIdent = if null ident

481+ let newIdent = if T.null ident

481482 then uniqueIdent exts ils (Set.fromList $ M.elems hdrIDMap)

482483 else ident

483484 newIls = concatMap f ils where f il | il == c = cIls

@@ -490,7 +491,7 @@ makeHeaderAnchor' (Header n (ident, classes, kvs) ils) =

490491 do

491492 hdrIDMap <- gets docxAnchorMap

492493 exts <- readerExtensions <$> asks docxOptions

493− let newIdent = if null ident

494+ let newIdent = if T.null ident

494495 then uniqueIdent exts ils (Set.fromList $ M.elems hdrIDMap)

495496 else ident

496497 modify $ \s -> s {docxAnchorMap = M.insert newIdent newIdent hdrIDMap}

@@ -558,8 +559,8 @@ parStyleToTransform pPr

558559 else transform

559560parStyleToTransform _ = return id

560561

561−normalizeToClassName :: (FromStyleName a) => a -> String

562−normalizeToClassName = map go . fromStyleName

562+normalizeToClassName :: (FromStyleName a) => a -> T.Text

563+normalizeToClassName = T.map go . fromStyleName

563564 where go c | isSpace c = '-'

564565 | otherwise = c

565566

@@ -574,7 +575,8 @@ bodyPartToBlocks (Paragraph pPr parparts)

574575 return $

575576 transform $

576577 codeBlock $

577− concatMap parPartToString parparts

578+ T.concat $

579+ map parPartToText parparts

578580 | Just (style, n) <- pHeading pPr = do

579581 ils <-local (\s-> s{docxInHeaderBlock=True})

580582 (smushInlines <$> mapM parPartToInlines parparts)

@@ -646,7 +648,7 @@ bodyPartToBlocks (ListItem pPr numId lvl (Just levelInfo) parparts) = do

646648 , ("num-id", numId)

647649 , ("format", fmt)

648650 , ("text", txt)

649− , ("start", show start)

651+ , ("start", tshow start)

650652 ]

651653 modify $ \st -> st{ docxListState =

652654 -- expire all the continuation data for lists of level > this one:

@@ -705,12 +707,12 @@ bodyPartToBlocks (OMathPara e) =

705707

706708-- replace targets with generated anchors.

707709rewriteLink' :: PandocMonad m => Inline -> DocxContext m Inline

708−rewriteLink' l@(Link attr ils ('#':target, title)) = do

710+rewriteLink' l@(Link attr ils (T.uncons -> Just ('#',target), title)) = do

709711 anchorMap <- gets docxAnchorMap

710712 case M.lookup target anchorMap of

711713 Just newTarget -> do

712714 modify $ \s -> s{docxAnchorSet = Set.insert newTarget (docxAnchorSet s)}

713− return $ Link attr ils ('#':newTarget, title)

715+ return $ Link attr ils ("#" <> newTarget, title)

714716 Nothing -> do

715717 modify $ \s -> s{docxAnchorSet = Set.insert target (docxAnchorSet s)}

716718 return l

src/Text/Pandoc/Readers/Docx/Combine.hs+2−1

@@ -1,7 +1,8 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleInstances #-}

33{-# LANGUAGE PatternGuards #-}

44{-# LANGUAGE TypeSynonymInstances #-}

5+{-# LANGUAGE OverloadedStrings #-}

56{- |

67 Module : Text.Pandoc.Readers.Docx.Combine

src/Text/Pandoc/Readers/Docx/Fields.hs+18−15

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Docx.Fields

@@ -16,16 +17,18 @@ module Text.Pandoc.Readers.Docx.Fields ( FieldInfo(..)

1617 ) where

1718

1819import Prelude

20+import Data.Functor (($>))

21+import qualified Data.Text as T

1922import Text.Parsec

20−import Text.Parsec.String (Parser)

23+import Text.Parsec.Text (Parser)

2124

22−type URL = String

25+type URL = T.Text

2326

2427data FieldInfo = HyperlinkField URL

2528 | UnknownField

2629 deriving (Show)

2730

28−parseFieldInfo :: String -> Either ParseError FieldInfo

31+parseFieldInfo :: T.Text -> Either ParseError FieldInfo

2932parseFieldInfo = parse fieldInfo ""

3033

3134fieldInfo :: Parser FieldInfo

@@ -34,31 +37,31 @@ fieldInfo =

3437 <|>

3538 return UnknownField

3639

37−escapedQuote :: Parser String

38−escapedQuote = string "\\\""

40+escapedQuote :: Parser T.Text

41+escapedQuote = string "\\\"" $> "\\\""

3942

40−inQuotes :: Parser String

43+inQuotes :: Parser T.Text

4144inQuotes =

42− (try escapedQuote) <|> (anyChar >>= (\c -> return [c]))

45+ (try escapedQuote) <|> (anyChar >>= (\c -> return $ T.singleton c))

4346

44−quotedString :: Parser String

47+quotedString :: Parser T.Text

4548quotedString = do

4649 char '"'

47− concat <$> manyTill inQuotes (try (char '"'))

50+ T.concat <$> manyTill inQuotes (try (char '"'))

4851

49−unquotedString :: Parser String

50−unquotedString = manyTill anyChar (try $ lookAhead space *> return () <|> eof)

52+unquotedString :: Parser T.Text

53+unquotedString = T.pack <$> manyTill anyChar (try $ lookAhead space *> return () <|> eof)

5154

52−fieldArgument :: Parser String

55+fieldArgument :: Parser T.Text

5356fieldArgument = quotedString <|> unquotedString

5457

5558-- there are other switches, but this is the only one I've seen in the wild so far, so it's the first one I'll implement. See §17.16.5.25

56−hyperlinkSwitch :: Parser (String, String)

59+hyperlinkSwitch :: Parser (T.Text, T.Text)

5760hyperlinkSwitch = do

5861 sw <- string "\\l"

5962 spaces

6063 farg <- fieldArgument

61− return (sw, farg)

64+ return (T.pack sw, farg)

6265

6366hyperlink :: Parser URL

6467hyperlink = do

@@ -68,6 +71,6 @@ hyperlink = do

6871 farg <- fieldArgument

6972 switches <- spaces *> many hyperlinkSwitch

7073 let url = case switches of

71− ("\\l", s) : _ -> farg ++ ('#': s)

74+ ("\\l", s) : _ -> farg <> "#" <> s

7275 _ -> farg

7376 return url

src/Text/Pandoc/Readers/Docx/Lists.hs+6−5

@@ -22,6 +22,7 @@ import Prelude

2222import Data.List

2323import Data.Maybe

2424import Data.String (fromString)

25+import qualified Data.Text as T

2526import Text.Pandoc.Generic (bottomUp)

2627import Text.Pandoc.JSON

2728import Text.Pandoc.Readers.Docx.Parse (ParaStyleName)

@@ -45,20 +46,20 @@ getNumId _ = Nothing

4546getNumIdN :: Block -> Integer

4647getNumIdN b = fromMaybe (-1) (getNumId b)

4748

48−getText :: Block -> Maybe String

49+getText :: Block -> Maybe T.Text

4950getText (Div (_, _, kvs) _) = lookup "text" kvs

5051getText _ = Nothing

5152

5253data ListType = Itemized | Enumerated ListAttributes

5354

54−listStyleMap :: [(String, ListNumberStyle)]

55+listStyleMap :: [(T.Text, ListNumberStyle)]

5556listStyleMap = [("upperLetter", UpperAlpha),

5657 ("lowerLetter", LowerAlpha),

5758 ("upperRoman", UpperRoman),

5859 ("lowerRoman", LowerRoman),

5960 ("decimal", Decimal)]

6061

61−listDelimMap :: [(String, ListNumberDelim)]

62+listDelimMap :: [(T.Text, ListNumberDelim)]

6263listDelimMap = [("%1)", OneParen),

6364 ("(%1)", TwoParens),

6465 ("%1.", Period)]

@@ -82,11 +83,11 @@ getListType b@(Div (_, _, kvs) _) | isListItem b =

8283 _ -> Nothing

8384getListType _ = Nothing

8485

85−listParagraphDivs :: [String]

86+listParagraphDivs :: [T.Text]

8687listParagraphDivs = ["list-paragraph"]

8788

8889listParagraphStyles :: [ParaStyleName]

89−listParagraphStyles = map fromString listParagraphDivs

90+listParagraphStyles = map (fromString . T.unpack) listParagraphDivs

9091

9192-- This is a first stab at going through and attaching meaning to list

9293-- paragraphs, without an item marker, following a list item. We

src/Text/Pandoc/Readers/Docx/Parse.hs+86−85

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE ViewPatterns #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Readers.Docx.Parse

@@ -63,6 +64,7 @@ import qualified Data.ByteString.Lazy as B

6364import Data.Char (chr, ord, readLitChar)

6465import Data.List

6566import qualified Data.Map as M

67+import qualified Data.Text as T

6668import Data.Maybe

6769import System.FilePath

6870import Text.Pandoc.Readers.Docx.Util

@@ -71,7 +73,7 @@ import Text.Pandoc.Shared (filteredFilesFromArchive, safeRead)

7173import qualified Text.Pandoc.UTF8 as UTF8

7274import Text.TeXMath (Exp)

7375import Text.TeXMath.Readers.OMML (readOMML)

74−import Text.TeXMath.Unicode.Fonts (Font (..), getUnicode, stringToFont)

76+import Text.TeXMath.Unicode.Fonts (Font (..), getUnicode, textToFont)

7577import Text.XML.Light

7678import qualified Text.XML.Light.Cursor as XMLC

7779

@@ -88,7 +90,7 @@ data ReaderEnv = ReaderEnv { envNotes :: Notes

8890 }

8991 deriving Show

9092

91−data ReaderState = ReaderState { stateWarnings :: [String]

93+data ReaderState = ReaderState { stateWarnings :: [T.Text]

9294 , stateFldCharState :: FldCharState

9395 }

9496 deriving Show

@@ -119,7 +121,6 @@ eitherToD (Left _) = throwError DocxError

119121concatMapM :: (Monad m) => (a -> m [b]) -> [a] -> m [b]

120122concatMapM f xs = liftM concat (mapM f xs)

121123

122−

123124-- This is similar to `mapMaybe`: it maps a function returning the D

124125-- monad over a list, and only keeps the non-erroring return values.

125126mapD :: (a -> D b) -> [a] -> D [b]

@@ -178,18 +179,18 @@ type ParStyleMap = M.Map ParaStyleId ParStyle

178179data Numbering = Numbering NameSpaces [Numb] [AbstractNumb]

179180 deriving Show

180181

181−data Numb = Numb String String [LevelOverride]

182+data Numb = Numb T.Text T.Text [LevelOverride]

182183 deriving Show

183184

184185-- ilvl startOverride lvl

185−data LevelOverride = LevelOverride String (Maybe Integer) (Maybe Level)

186+data LevelOverride = LevelOverride T.Text (Maybe Integer) (Maybe Level)

186187 deriving Show

187188

188−data AbstractNumb = AbstractNumb String [Level]

189+data AbstractNumb = AbstractNumb T.Text [Level]

189190 deriving Show

190191

191192-- ilvl format string start

192−data Level = Level String String String (Maybe Integer)

193+data Level = Level T.Text T.Text T.Text (Maybe Integer)

193194 deriving Show

194195

195196data DocumentLocation = InDocument | InFootnote | InEndnote

@@ -199,11 +200,11 @@ data Relationship = Relationship DocumentLocation RelId Target

199200 deriving Show

200201

201202data Notes = Notes NameSpaces

202− (Maybe (M.Map String Element))

203− (Maybe (M.Map String Element))

203+ (Maybe (M.Map T.Text Element))

204+ (Maybe (M.Map T.Text Element))

204205 deriving Show

205206

206−data Comments = Comments NameSpaces (M.Map String Element)

207+data Comments = Comments NameSpaces (M.Map T.Text Element)

207208 deriving Show

208209

209210data ParIndentation = ParIndentation { leftParIndent :: Maybe Integer

@@ -238,8 +239,8 @@ defaultParagraphStyle = ParagraphStyle { pStyle = []

238239

239240

240241data BodyPart = Paragraph ParagraphStyle [ParPart]

241− | ListItem ParagraphStyle String String (Maybe Level) [ParPart]

242− | Tbl String TblGrid TblLook [Row]

242+ | ListItem ParagraphStyle T.Text T.Text (Maybe Level) [ParPart]

243+ | Tbl T.Text TblGrid TblLook [Row]

243244 | OMathPara [Exp]

244245 deriving Show

245246

@@ -279,7 +280,7 @@ data ParPart = PlainRun Run

279280 | BookMark BookMarkId Anchor

280281 | InternalHyperLink Anchor [Run]

281282 | ExternalHyperLink URL [Run]

282− | Drawing FilePath String String B.ByteString Extent -- title, alt

283+ | Drawing FilePath T.Text T.Text B.ByteString Extent -- title, alt

283284 | Chart -- placeholder for now

284285 | PlainOMath [Exp]

285286 | Field FieldInfo [Run]

@@ -290,28 +291,28 @@ data ParPart = PlainRun Run

290291data Run = Run RunStyle [RunElem]

291292 | Footnote [BodyPart]

292293 | Endnote [BodyPart]

293− | InlineDrawing FilePath String String B.ByteString Extent -- title, alt

294+ | InlineDrawing FilePath T.Text T.Text B.ByteString Extent -- title, alt

294295 | InlineChart -- placeholder

295296 deriving Show

296297

297−data RunElem = TextRun String | LnBrk | Tab | SoftHyphen | NoBreakHyphen

298+data RunElem = TextRun T.Text | LnBrk | Tab | SoftHyphen | NoBreakHyphen

298299 deriving Show

299300

300−type Target = String

301−type Anchor = String

302−type URL = String

303−type BookMarkId = String

304−type RelId = String

305−type ChangeId = String

306−type CommentId = String

307−type Author = String

308−type ChangeDate = String

309−type CommentDate = String

301+type Target = T.Text

302+type Anchor = T.Text

303+type URL = T.Text

304+type BookMarkId = T.Text

305+type RelId = T.Text

306+type ChangeId = T.Text

307+type CommentId = T.Text

308+type Author = T.Text

309+type ChangeDate = T.Text

310+type CommentDate = T.Text

310311

311312archiveToDocx :: Archive -> Either DocxError Docx

312313archiveToDocx archive = fst <$> archiveToDocxWithWarnings archive

313314

314−archiveToDocxWithWarnings :: Archive -> Either DocxError (Docx, [String])

315+archiveToDocxWithWarnings :: Archive -> Either DocxError (Docx, [T.Text])

315316archiveToDocxWithWarnings archive = do

316317 docXmlPath <- case getDocumentXmlPath archive of

317318 Just fp -> Right fp

@@ -341,7 +342,7 @@ archiveToDocxWithWarnings archive = do

341342 Right doc -> Right (Docx doc, stateWarnings st)

342343 Left e -> Left e

343344

344−getDocumentXmlPath :: Archive -> Maybe String

345+getDocumentXmlPath :: Archive -> Maybe FilePath

345346getDocumentXmlPath zf = do

346347 entry <- findEntryByPath "_rels/.rels" zf

347348 relsElem <- (parseXMLDoc . UTF8.toStringLazy . fromEntry) entry

@@ -394,7 +395,7 @@ constructBogusParStyleData stName = ParStyle

394395 , numInfo = Nothing

395396 , psParentStyle = Nothing

396397 , pStyleName = stName

397− , pStyleId = ParaStyleId . filter (/=' ') . fromStyleName $ stName

398+ , pStyleId = ParaStyleId . T.filter (/=' ') . fromStyleName $ stName

398399 }

399400

400401archiveToNotes :: Archive -> Notes

@@ -441,8 +442,8 @@ filePathToRelType path docXmlPath =

441442relElemToRelationship :: DocumentLocation -> Element -> Maybe Relationship

442443relElemToRelationship relType element | qName (elName element) == "Relationship" =

443444 do

444− relId <- findAttr (QName "Id" Nothing Nothing) element

445− target <- findAttr (QName "Target" Nothing Nothing) element

445+ relId <- findAttrText (QName "Id" Nothing Nothing) element

446+ target <- findAttrText (QName "Target" Nothing Nothing) element

446447 return $ Relationship relType relId target

447448relElemToRelationship _ _ = Nothing

448449

@@ -464,7 +465,7 @@ filePathIsMedia fp =

464465 in

465466 (dir == "word/media/")

466467

467−lookupLevel :: String -> String -> Numbering -> Maybe Level

468+lookupLevel :: T.Text -> T.Text -> Numbering -> Maybe Level

468469lookupLevel numId ilvl (Numbering _ numbs absNumbs) = do

469470 (absNumId, ovrrides) <- lookup numId $

470471 map (\(Numb nid absnumid ovrRides) -> (nid, (absnumid, ovrRides))) numbs

@@ -483,7 +484,7 @@ lookupLevel numId ilvl (Numbering _ numbs absNumbs) = do

483484loElemToLevelOverride :: NameSpaces -> Element -> Maybe LevelOverride

484485loElemToLevelOverride ns element

485486 | isElem ns "w" "lvlOverride" element = do

486− ilvl <- findAttrByName ns "w" "ilvl" element

487+ ilvl <- findAttrTextByName ns "w" "ilvl" element

487488 let startOverride = findChildByName ns "w" "startOverride" element

488489 >>= findAttrByName ns "w" "val"

489490 >>= (\s -> listToMaybe (map fst (reads s :: [(Integer, String)])))

@@ -495,9 +496,9 @@ loElemToLevelOverride _ _ = Nothing

495496numElemToNum :: NameSpaces -> Element -> Maybe Numb

496497numElemToNum ns element

497498 | isElem ns "w" "num" element = do

498− numId <- findAttrByName ns "w" "numId" element

499+ numId <- findAttrTextByName ns "w" "numId" element

499500 absNumId <- findChildByName ns "w" "abstractNumId" element

500− >>= findAttrByName ns "w" "val"

501+ >>= findAttrTextByName ns "w" "val"

501502 let lvlOverrides = mapMaybe

502503 (loElemToLevelOverride ns)

503504 (findChildrenByName ns "w" "lvlOverride" element)

@@ -507,7 +508,7 @@ numElemToNum _ _ = Nothing

507508absNumElemToAbsNum :: NameSpaces -> Element -> Maybe AbstractNumb

508509absNumElemToAbsNum ns element

509510 | isElem ns "w" "abstractNum" element = do

510− absNumId <- findAttrByName ns "w" "abstractNumId" element

511+ absNumId <- findAttrTextByName ns "w" "abstractNumId" element

511512 let levelElems = findChildrenByName ns "w" "lvl" element

512513 levels = mapMaybe (levelElemToLevel ns) levelElems

513514 return $ AbstractNumb absNumId levels

@@ -516,11 +517,11 @@ absNumElemToAbsNum _ _ = Nothing

516517levelElemToLevel :: NameSpaces -> Element -> Maybe Level

517518levelElemToLevel ns element

518519 | isElem ns "w" "lvl" element = do

519− ilvl <- findAttrByName ns "w" "ilvl" element

520+ ilvl <- findAttrTextByName ns "w" "ilvl" element

520521 fmt <- findChildByName ns "w" "numFmt" element

521− >>= findAttrByName ns "w" "val"

522+ >>= findAttrTextByName ns "w" "val"

522523 txt <- findChildByName ns "w" "lvlText" element

523− >>= findAttrByName ns "w" "val"

524+ >>= findAttrTextByName ns "w" "val"

524525 let start = findChildByName ns "w" "start" element

525526 >>= findAttrByName ns "w" "val"

526527 >>= (\s -> listToMaybe (map fst (reads s :: [(Integer, String)])))

@@ -544,11 +545,11 @@ archiveToNumbering :: Archive -> Numbering

544545archiveToNumbering archive =

545546 fromMaybe (Numbering [] [] []) (archiveToNumbering' archive)

546547

547−elemToNotes :: NameSpaces -> String -> Element -> Maybe (M.Map String Element)

548+elemToNotes :: NameSpaces -> String -> Element -> Maybe (M.Map T.Text Element)

548549elemToNotes ns notetype element

549− | isElem ns "w" (notetype ++ "s") element =

550+ | isElem ns "w" (notetype <> "s") element =

550551 let pairs = mapMaybe

551− (\e -> findAttrByName ns "w" "id" e >>=

552+ (\e -> findAttrTextByName ns "w" "id" e >>=

552553 (\a -> Just (a, e)))

553554 (findChildrenByName ns "w" notetype element)

554555 in

@@ -556,11 +557,11 @@ elemToNotes ns notetype element

556557 M.fromList pairs

557558elemToNotes _ _ _ = Nothing

558559

559−elemToComments :: NameSpaces -> Element -> M.Map String Element

560+elemToComments :: NameSpaces -> Element -> M.Map T.Text Element

560561elemToComments ns element

561562 | isElem ns "w" "comments" element =

562563 let pairs = mapMaybe

563− (\e -> findAttrByName ns "w" "id" e >>=

564+ (\e -> findAttrTextByName ns "w" "id" e >>=

564565 (\a -> Just (a, e)))

565566 (findChildrenByName ns "w" "comment" element)

566567 in

@@ -632,7 +633,7 @@ testBitMask bitMaskS n =

632633pHeading :: ParagraphStyle -> Maybe (ParaStyleName, Int)

633634pHeading = getParStyleField headingLev . pStyle

634635

635−pNumInfo :: ParagraphStyle -> Maybe (String, String)

636+pNumInfo :: ParagraphStyle -> Maybe (T.Text, T.Text)

636637pNumInfo = getParStyleField numInfo . pStyle

637638

638639elemToBodyPart :: NameSpaces -> Element -> D BodyPart

@@ -640,7 +641,7 @@ elemToBodyPart ns element

640641 | isElem ns "w" "p" element

641642 , (c:_) <- findChildrenByName ns "m" "oMathPara" element =

642643 do

643− expsLst <- eitherToD $ readOMML $ showElement c

644+ expsLst <- eitherToD $ readOMML $ T.pack $ showElement c

644645 return $ OMathPara expsLst

645646elemToBodyPart ns element

646647 | isElem ns "w" "p" element

@@ -664,7 +665,7 @@ elemToBodyPart ns element

664665 | isElem ns "w" "tbl" element = do

665666 let caption' = findChildByName ns "w" "tblPr" element

666667 >>= findChildByName ns "w" "tblCaption"

667− >>= findAttrByName ns "w" "val"

668+ >>= findAttrTextByName ns "w" "val"

668669 caption = fromMaybe "" caption'

669670 grid' = case findChildByName ns "w" "tblGrid" element of

670671 Just g -> elemToTblGrid ns g

@@ -687,10 +688,10 @@ lookupRelationship docLocation relid rels =

687688 where

688689 pairs = map (\(Relationship loc relid' target) -> ((loc, relid'), target)) rels

689690

690−expandDrawingId :: String -> D (FilePath, B.ByteString)

691+expandDrawingId :: T.Text -> D (FilePath, B.ByteString)

691692expandDrawingId s = do

692693 location <- asks envLocation

693− target <- asks (lookupRelationship location s . envRelationships)

694+ target <- asks (fmap T.unpack . lookupRelationship location s . envRelationships)

694695 case target of

695696 Just filepath -> do

696697 bytes <- asks (lookup ("word/" ++ filepath) . envMedia)

@@ -699,12 +700,12 @@ expandDrawingId s = do

699700 Nothing -> throwError DocxError

700701 Nothing -> throwError DocxError

701702

702−getTitleAndAlt :: NameSpaces -> Element -> (String, String)

703+getTitleAndAlt :: NameSpaces -> Element -> (T.Text, T.Text)

703704getTitleAndAlt ns element =

704705 let mbDocPr = findChildByName ns "wp" "inline" element >>=

705706 findChildByName ns "wp" "docPr"

706− title = fromMaybe "" (mbDocPr >>= findAttrByName ns "" "title")

707− alt = fromMaybe "" (mbDocPr >>= findAttrByName ns "" "descr")

707+ title = fromMaybe "" (mbDocPr >>= findAttrTextByName ns "" "title")

708+ alt = fromMaybe "" (mbDocPr >>= findAttrTextByName ns "" "descr")

708709 in (title, alt)

709710

710711elemToParPart :: NameSpaces -> Element -> D ParPart

@@ -716,7 +717,7 @@ elemToParPart ns element

716717 = let (title, alt) = getTitleAndAlt ns drawingElem

717718 a_ns = "http://schemas.openxmlformats.org/drawingml/2006/main"

718719 drawing = findElement (QName "blip" (Just a_ns) (Just "a")) picElem

719− >>= findAttrByName ns "r" "embed"

720+ >>= findAttrTextByName ns "r" "embed"

720721 in

721722 case drawing of

722723 Just s -> expandDrawingId s >>= (\(fp, bs) -> return $ Drawing fp title alt bs $ elemToExtent drawingElem)

@@ -726,7 +727,7 @@ elemToParPart ns element

726727 | isElem ns "w" "r" element

727728 , Just _ <- findChildByName ns "w" "pict" element =

728729 let drawing = findElement (elemName ns "v" "imagedata") element

729− >>= findAttrByName ns "r" "id"

730+ >>= findAttrTextByName ns "r" "id"

730731 in

731732 case drawing of

732733 -- Todo: check out title and attr for deprecated format.

@@ -795,7 +796,7 @@ elemToParPart ns element

795796 fldCharState <- gets stateFldCharState

796797 case fldCharState of

797798 FldCharOpen -> do

798− info <- eitherToD $ parseFieldInfo $ strContent instrText

799+ info <- eitherToD $ parseFieldInfo $ T.pack $ strContent instrText

799800 modify $ \st -> st{stateFldCharState = FldCharFieldInfo info}

800801 return NullParPart

801802 _ -> return NullParPart

@@ -816,56 +817,56 @@ elemToParPart ns element

816817 return $ ChangedRuns change runs

817818elemToParPart ns element

818819 | isElem ns "w" "bookmarkStart" element

819− , Just bmId <- findAttrByName ns "w" "id" element

820− , Just bmName <- findAttrByName ns "w" "name" element =

820+ , Just bmId <- findAttrTextByName ns "w" "id" element

821+ , Just bmName <- findAttrTextByName ns "w" "name" element =

821822 return $ BookMark bmId bmName

822823elemToParPart ns element

823824 | isElem ns "w" "hyperlink" element

824− , Just relId <- findAttrByName ns "r" "id" element = do

825+ , Just relId <- findAttrTextByName ns "r" "id" element = do

825826 location <- asks envLocation

826827 runs <- mapD (elemToRun ns) (elChildren element)

827828 rels <- asks envRelationships

828829 case lookupRelationship location relId rels of

829830 Just target ->

830− case findAttrByName ns "w" "anchor" element of

831− Just anchor -> return $ ExternalHyperLink (target ++ '#':anchor) runs

831+ case findAttrTextByName ns "w" "anchor" element of

832+ Just anchor -> return $ ExternalHyperLink (target <> "#" <> anchor) runs

832833 Nothing -> return $ ExternalHyperLink target runs

833834 Nothing -> return $ ExternalHyperLink "" runs

834835elemToParPart ns element

835836 | isElem ns "w" "hyperlink" element

836− , Just anchor <- findAttrByName ns "w" "anchor" element = do

837+ , Just anchor <- findAttrTextByName ns "w" "anchor" element = do

837838 runs <- mapD (elemToRun ns) (elChildren element)

838839 return $ InternalHyperLink anchor runs

839840elemToParPart ns element

840841 | isElem ns "w" "commentRangeStart" element

841− , Just cmtId <- findAttrByName ns "w" "id" element = do

842+ , Just cmtId <- findAttrTextByName ns "w" "id" element = do

842843 (Comments _ commentMap) <- asks envComments

843844 case M.lookup cmtId commentMap of

844845 Just cmtElem -> elemToCommentStart ns cmtElem

845846 Nothing -> throwError WrongElem

846847elemToParPart ns element

847848 | isElem ns "w" "commentRangeEnd" element

848− , Just cmtId <- findAttrByName ns "w" "id" element =

849+ , Just cmtId <- findAttrTextByName ns "w" "id" element =

849850 return $ CommentEnd cmtId

850851elemToParPart ns element

851852 | isElem ns "m" "oMath" element =

852− fmap PlainOMath (eitherToD $ readOMML $ showElement element)

853+ fmap PlainOMath (eitherToD $ readOMML $ T.pack $ showElement element)

853854elemToParPart _ _ = throwError WrongElem

854855

855856elemToCommentStart :: NameSpaces -> Element -> D ParPart

856857elemToCommentStart ns element

857858 | isElem ns "w" "comment" element

858− , Just cmtId <- findAttrByName ns "w" "id" element

859− , Just cmtAuthor <- findAttrByName ns "w" "author" element

860− , Just cmtDate <- findAttrByName ns "w" "date" element = do

859+ , Just cmtId <- findAttrTextByName ns "w" "id" element

860+ , Just cmtAuthor <- findAttrTextByName ns "w" "author" element

861+ , Just cmtDate <- findAttrTextByName ns "w" "date" element = do

861862 bps <- mapD (elemToBodyPart ns) (elChildren element)

862863 return $ CommentStart cmtId cmtAuthor cmtDate bps

863864elemToCommentStart _ _ = throwError WrongElem

864865

865−lookupFootnote :: String -> Notes -> Maybe Element

866+lookupFootnote :: T.Text -> Notes -> Maybe Element

866867lookupFootnote s (Notes _ fns _) = fns >>= M.lookup s

867868

868−lookupEndnote :: String -> Notes -> Maybe Element

869+lookupEndnote :: T.Text -> Notes -> Maybe Element

869870lookupEndnote s (Notes _ _ ens) = ens >>= M.lookup s

870871

871872elemToExtent :: Element -> Extent

@@ -876,7 +877,7 @@ elemToExtent drawingElem =

876877 where

877878 wp_ns = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"

878879 getDim at = findElement (QName "extent" (Just wp_ns) (Just "wp")) drawingElem

879− >>= findAttr (QName at Nothing Nothing) >>= safeRead

880+ >>= findAttr (QName at Nothing Nothing) >>= safeRead . T.pack

880881

881882

882883childElemToRun :: NameSpaces -> Element -> D Run

@@ -887,7 +888,7 @@ childElemToRun ns element

887888 = let (title, alt) = getTitleAndAlt ns element

888889 a_ns = "http://schemas.openxmlformats.org/drawingml/2006/main"

889890 drawing = findElement (QName "blip" (Just a_ns) (Just "a")) picElem

890− >>= findAttr (QName "embed" (lookup "r" ns) (Just "r"))

891+ >>= findAttrText (QName "embed" (lookup "r" ns) (Just "r"))

891892 in

892893 case drawing of

893894 Just s -> expandDrawingId s >>=

@@ -900,7 +901,7 @@ childElemToRun ns element

900901 = return InlineChart

901902childElemToRun ns element

902903 | isElem ns "w" "footnoteReference" element

903− , Just fnId <- findAttrByName ns "w" "id" element = do

904+ , Just fnId <- findAttrTextByName ns "w" "id" element = do

904905 notes <- asks envNotes

905906 case lookupFootnote fnId notes of

906907 Just e -> do bps <- local (\r -> r {envLocation=InFootnote}) $ mapD (elemToBodyPart ns) (elChildren e)

@@ -908,7 +909,7 @@ childElemToRun ns element

908909 Nothing -> return $ Footnote []

909910childElemToRun ns element

910911 | isElem ns "w" "endnoteReference" element

911− , Just enId <- findAttrByName ns "w" "id" element = do

912+ , Just enId <- findAttrTextByName ns "w" "id" element = do

912913 notes <- asks envNotes

913914 case lookupEndnote enId notes of

914915 Just e -> do bps <- local (\r -> r {envLocation=InEndnote}) $ mapD (elemToBodyPart ns) (elChildren e)

@@ -961,15 +962,15 @@ getParStyleField _ _ = Nothing

961962getTrackedChange :: NameSpaces -> Element -> Maybe TrackedChange

962963getTrackedChange ns element

963964 | isElem ns "w" "ins" element || isElem ns "w" "moveTo" element

964− , Just cId <- findAttrByName ns "w" "id" element

965− , Just cAuthor <- findAttrByName ns "w" "author" element

966− , Just cDate <- findAttrByName ns "w" "date" element =

965+ , Just cId <- findAttrTextByName ns "w" "id" element

966+ , Just cAuthor <- findAttrTextByName ns "w" "author" element

967+ , Just cDate <- findAttrTextByName ns "w" "date" element =

967968 Just $ TrackedChange Insertion (ChangeInfo cId cAuthor cDate)

968969getTrackedChange ns element

969970 | isElem ns "w" "del" element || isElem ns "w" "moveFrom" element

970− , Just cId <- findAttrByName ns "w" "id" element

971− , Just cAuthor <- findAttrByName ns "w" "author" element

972− , Just cDate <- findAttrByName ns "w" "date" element =

971+ , Just cId <- findAttrTextByName ns "w" "id" element

972+ , Just cAuthor <- findAttrTextByName ns "w" "author" element

973+ , Just cDate <- findAttrTextByName ns "w" "date" element =

973974 Just $ TrackedChange Deletion (ChangeInfo cId cAuthor cDate)

974975getTrackedChange _ _ = Nothing

975976

@@ -978,7 +979,7 @@ elemToParagraphStyle ns element sty

978979 | Just pPr <- findChildByName ns "w" "pPr" element =

979980 let style =

980981 mapMaybe

981− (fmap ParaStyleId . findAttrByName ns "w" "val")

982+ (fmap ParaStyleId . findAttrTextByName ns "w" "val")

982983 (findChildrenByName ns "w" "pStyle" pPr)

983984 in ParagraphStyle

984985 {pStyle = mapMaybe (`M.lookup` sty) style

@@ -1010,7 +1011,7 @@ elemToRunStyleD ns element

10101011 charStyles <- asks envCharStyles

10111012 let parentSty =

10121013 findChildByName ns "w" "rStyle" rPr >>=

1013− findAttrByName ns "w" "val" >>=

1014+ findAttrTextByName ns "w" "val" >>=

10141015 flip M.lookup charStyles . CharStyleId

10151016 return $ elemToRunStyle ns element parentSty

10161017elemToRunStyleD _ _ = return defaultRunStyle

@@ -1020,12 +1021,12 @@ elemToRunElem ns element

10201021 | isElem ns "w" "t" element

10211022 || isElem ns "w" "delText" element

10221023 || isElem ns "m" "t" element = do

1023− let str = strContent element

1024+ let str = T.pack $ strContent element

10241025 font <- asks envFont

10251026 case font of

10261027 Nothing -> return $ TextRun str

10271028 Just f -> return . TextRun $

1028− map (\x -> fromMaybe x . getUnicode f . lowerFromPrivate $ x) str

1029+ T.map (\x -> fromMaybe x . getUnicode f . lowerFromPrivate $ x) str

10291030 | isElem ns "w" "br" element = return LnBrk

10301031 | isElem ns "w" "tab" element = return Tab

10311032 | isElem ns "w" "softHyphen" element = return SoftHyphen

@@ -1043,11 +1044,11 @@ getSymChar ns element

10431044 | Just s <- lowerFromPrivate <$> getCodepoint

10441045 , Just font <- getFont =

10451046 case readLitChar ("\\x" ++ s) of

1046− [(char, _)] -> TextRun . maybe "" (:[]) $ getUnicode font char

1047+ [(char, _)] -> TextRun . maybe "" T.singleton $ getUnicode font char

10471048 _ -> TextRun ""

10481049 where

10491050 getCodepoint = findAttrByName ns "w" "char" element

1050− getFont = stringToFont =<< findAttrByName ns "w" "font" element

1051+ getFont = textToFont . T.pack =<< findAttrByName ns "w" "font" element

10511052 lowerFromPrivate ('F':xs) = '0':xs

10521053 lowerFromPrivate xs = xs

10531054getSymChar _ _ = TextRun ""

@@ -1059,7 +1060,7 @@ elemToRunElems ns element

10591060 let qualName = elemName ns "w"

10601061 let font = do

10611062 fontElem <- findElement (qualName "rFonts") element

1062− stringToFont =<<

1063+ textToFont . T.pack =<<

10631064 foldr ((<|>) . (flip findAttr fontElem . qualName)) Nothing ["ascii", "hAnsi"]

10641065 local (setFont font) (mapD (elemToRunElem ns) (elChildren element))

10651066elemToRunElems _ _ = throwError WrongElem

src/Text/Pandoc/Readers/Docx/Parse/Styles.hs+27−21

@@ -3,6 +3,7 @@

33{-# LANGUAGE FlexibleContexts #-}

44{-# LANGUAGE TypeFamilies #-}

55{-# LANGUAGE GeneralizedNewtypeDeriving #-}

6+{-# LANGUAGE OverloadedStrings #-}

67{- |

78 Module : Text.Pandoc.Readers.Docx.Parse.Styles

@@ -46,20 +47,19 @@ import Prelude

4647import Codec.Archive.Zip

4748import Control.Applicative ((<|>))

4849import Control.Monad.Except

49−import Data.Char (toLower)

50−import Data.List

5150import Data.Function (on)

5251import Data.String (IsString(..))

5352import qualified Data.Map as M

53+import qualified Data.Text as T

5454import Data.Maybe

5555import Data.Coerce

5656import Text.Pandoc.Readers.Docx.Util

5757import qualified Text.Pandoc.UTF8 as UTF8

5858import Text.XML.Light

5959

60−newtype CharStyleId = CharStyleId String

60+newtype CharStyleId = CharStyleId T.Text

6161 deriving (Show, Eq, Ord, IsString, FromStyleId)

62−newtype ParaStyleId = ParaStyleId String

62+newtype ParaStyleId = ParaStyleId T.Text

6363 deriving (Show, Eq, Ord, IsString, FromStyleId)

6464

6565newtype CharStyleName = CharStyleName CIString

@@ -68,25 +68,31 @@ newtype ParaStyleName = ParaStyleName CIString

6868 deriving (Show, Eq, Ord, IsString, FromStyleName)

6969

7070-- Case-insensitive comparisons

71−newtype CIString = CIString String deriving (Show, IsString, FromStyleName)

71+newtype CIString = CIString T.Text deriving (Show, IsString, FromStyleName)

7272

7373class FromStyleName a where

74− fromStyleName :: a -> String

74+ fromStyleName :: a -> T.Text

7575

7676instance FromStyleName String where

77+ fromStyleName = T.pack

78+

79+instance FromStyleName T.Text where

7780 fromStyleName = id

7881

7982class FromStyleId a where

80− fromStyleId :: a -> String

83+ fromStyleId :: a -> T.Text

8184

8285instance FromStyleId String where

86+ fromStyleId = T.pack

87+

88+instance FromStyleId T.Text where

8389 fromStyleId = id

8490

8591instance Eq CIString where

86− (==) = (==) `on` map toLower . coerce

92+ (==) = (==) `on` T.toCaseFold . coerce

8793

8894instance Ord CIString where

89− compare = compare `on` map toLower . coerce

95+ compare = compare `on` T.toCaseFold . coerce

9096

9197data VertAlign = BaseLn | SupScrpt | SubScrpt

9298 deriving Show

@@ -108,7 +114,7 @@ data RunStyle = RunStyle { isBold :: Maybe Bool

108114 deriving Show

109115

110116data ParStyle = ParStyle { headingLev :: Maybe (ParaStyleName, Int)

111− , numInfo :: Maybe (String, String)

117+ , numInfo :: Maybe (T.Text, T.Text)

112118 , psParentStyle :: Maybe ParStyle

113119 , pStyleName :: ParaStyleName

114120 , pStyleId :: ParaStyleId

@@ -146,7 +152,7 @@ isBasedOnStyle ns element parentStyle

146152 , Just styleType <- findAttrByName ns "w" "type" element

147153 , styleType == cStyleType parentStyle

148154 , Just basedOnVal <- findChildByName ns "w" "basedOn" element >>=

149− findAttrByName ns "w" "val"

155+ findAttrTextByName ns "w" "val"

150156 , Just ps <- parentStyle = basedOnVal == fromStyleId (getStyleId ps)

151157 | isElem ns "w" "style" element

152158 , Just styleType <- findAttrByName ns "w" "type" element

@@ -234,7 +240,7 @@ checkOnOff _ _ _ = Nothing

234240elemToCharStyle :: NameSpaces

235241 -> Element -> Maybe CharStyle -> Maybe CharStyle

236242elemToCharStyle ns element parentStyle

237− = CharStyle <$> (CharStyleId <$> findAttrByName ns "w" "styleId" element)

243+ = CharStyle <$> (CharStyleId <$> findAttrTextByName ns "w" "styleId" element)

238244 <*> getElementStyleName ns element

239245 <*> (Just $ elemToRunStyle ns element parentStyle)

240246

@@ -267,32 +273,32 @@ elemToRunStyle _ _ _ = defaultRunStyle

267273getHeaderLevel :: NameSpaces -> Element -> Maybe (ParaStyleName, Int)

268274getHeaderLevel ns element

269275 | Just styleName <- getElementStyleName ns element

270− , Just n <- stringToInteger =<<

271− (stripPrefix "heading " . map toLower $

276+ , Just n <- stringToInteger . T.unpack =<<

277+ (T.stripPrefix "heading " . T.toLower $

272278 fromStyleName styleName)

273279 , n > 0 = Just (styleName, fromInteger n)

274280getHeaderLevel _ _ = Nothing

275281

276−getElementStyleName :: Coercible String a => NameSpaces -> Element -> Maybe a

282+getElementStyleName :: Coercible T.Text a => NameSpaces -> Element -> Maybe a

277283getElementStyleName ns el = coerce <$>

278− ((findChildByName ns "w" "name" el >>= findAttrByName ns "w" "val")

279− <|> findAttrByName ns "w" "styleId" el)

284+ ((findChildByName ns "w" "name" el >>= findAttrTextByName ns "w" "val")

285+ <|> findAttrTextByName ns "w" "styleId" el)

280286

281−getNumInfo :: NameSpaces -> Element -> Maybe (String, String)

287+getNumInfo :: NameSpaces -> Element -> Maybe (T.Text, T.Text)

282288getNumInfo ns element = do

283289 let numPr = findChildByName ns "w" "pPr" element >>=

284290 findChildByName ns "w" "numPr"

285291 lvl = fromMaybe "0" (numPr >>=

286292 findChildByName ns "w" "ilvl" >>=

287− findAttrByName ns "w" "val")

293+ findAttrTextByName ns "w" "val")

288294 numId <- numPr >>=

289295 findChildByName ns "w" "numId" >>=

290− findAttrByName ns "w" "val"

296+ findAttrTextByName ns "w" "val"

291297 return (numId, lvl)

292298

293299elemToParStyleData :: NameSpaces -> Element -> Maybe ParStyle -> Maybe ParStyle

294300elemToParStyleData ns element parentStyle

295− | Just styleId <- findAttrByName ns "w" "styleId" element

301+ | Just styleId <- findAttrTextByName ns "w" "styleId" element

296302 , Just styleName <- getElementStyleName ns element

297303 = Just $ ParStyle

298304 {

src/Text/Pandoc/Readers/Docx/Util.hs+9

@@ -19,11 +19,14 @@ module Text.Pandoc.Readers.Docx.Util (

1919 , elemToNameSpaces

2020 , findChildByName

2121 , findChildrenByName

22+ , findAttrText

2223 , findAttrByName

24+ , findAttrTextByName

2325 ) where

2426

2527import Prelude

2628import Data.Maybe (mapMaybe)

29+import qualified Data.Text as T

2730import Text.XML.Light

2831

2932type NameSpaces = [(String, String)]

@@ -55,7 +58,13 @@ findChildrenByName ns pref name el =

5558 let ns' = ns ++ elemToNameSpaces el

5659 in findChildren (elemName ns' pref name) el

5760

61+findAttrText :: QName -> Element -> Maybe T.Text

62+findAttrText x = fmap T.pack . findAttr x

63+

5864findAttrByName :: NameSpaces -> String -> String -> Element -> Maybe String

5965findAttrByName ns pref name el =

6066 let ns' = ns ++ elemToNameSpaces el

6167 in findAttr (elemName ns' pref name) el

68+

69+findAttrTextByName :: NameSpaces -> String -> String -> Element -> Maybe T.Text

70+findAttrTextByName a b c = fmap T.pack . findAttrByName a b c

src/Text/Pandoc/Readers/DokuWiki.hs+94−93

@@ -2,6 +2,7 @@

22{-# LANGUAGE NoImplicitPrelude #-}

33{-# LANGUAGE OverloadedStrings #-}

44{-# LANGUAGE TupleSections #-}

5+{-# LANGUAGE ViewPatterns #-}

56{- |

67 Module : Text.Pandoc.Readers.DokuWiki

@@ -20,8 +21,7 @@ import Control.Monad

2021import Control.Monad.Except (throwError)

2122import Data.Char (isAlphaNum, isDigit)

2223import qualified Data.Foldable as F

23−import Data.List (intercalate, transpose, isPrefixOf, isSuffixOf)

24−import Data.List.Split (splitOn)

24+import Data.List (transpose)

2525import Data.Maybe (fromMaybe, catMaybes)

2626import Data.Text (Text)

2727import qualified Data.Text as T

@@ -31,7 +31,7 @@ import Text.Pandoc.Definition

3131import Text.Pandoc.Error (PandocError (PandocParsecError))

3232import Text.Pandoc.Options

3333import Text.Pandoc.Parsing hiding (enclosed, nested)

34−import Text.Pandoc.Shared (crFilter, trim, underlineSpan)

34+import Text.Pandoc.Shared (crFilter, trim, underlineSpan, tshow)

3535

3636-- | Read DokuWiki from an input string and return a Pandoc document.

3737readDokuWiki :: PandocMonad m

@@ -42,7 +42,7 @@ readDokuWiki opts s = do

4242 let input = crFilter s

4343 res <- runParserT parseDokuWiki def {stateOptions = opts } "source" input

4444 case res of

45− Left e -> throwError $ PandocParsecError (T.unpack input) e

45+ Left e -> throwError $ PandocParsecError input e

4646 Right d -> return d

4747

4848type DWParser = ParserT Text ParserState

@@ -71,9 +71,9 @@ parseDokuWiki =

7171 B.doc . mconcat <$> many block <* spaces <* eof

7272

7373-- | Parse <code> and <file> attributes

74−codeLanguage :: PandocMonad m => DWParser m (String, [String], [(String, String)])

74+codeLanguage :: PandocMonad m => DWParser m (Text, [Text], [(Text, Text)])

7575codeLanguage = try $ do

76− rawLang <- option "-" (spaceChar *> manyTill anyChar (lookAhead (spaceChar <|> char '>')))

76+ rawLang <- option "-" (spaceChar *> manyTillChar anyChar (lookAhead (spaceChar <|> char '>')))

7777 let attr = case rawLang of

7878 "-" -> []

7979 l -> [l]

@@ -81,16 +81,16 @@ codeLanguage = try $ do

8181

8282-- | Generic parser for <code> and <file> tags

8383codeTag :: PandocMonad m

84− => ((String, [String], [(String, String)]) -> String -> a)

85− -> String

84+ => ((Text, [Text], [(Text, Text)]) -> Text -> a)

85+ -> Text

8686 -> DWParser m a

8787codeTag f tag = try $ f

8888 <$ char '<'

89− <* string tag

89+ <* textStr tag

9090 <*> codeLanguage

9191 <* manyTill anyChar (char '>')

9292 <* optional (manyTill spaceChar eol)

93− <*> manyTill anyChar (try $ string "</" <* string tag <* char '>')

93+ <*> manyTillChar anyChar (try $ string "</" <* textStr tag <* char '>')

9494

9595-- * Inline parsers

9696

@@ -167,19 +167,19 @@ underlined :: PandocMonad m => DWParser m B.Inlines

167167underlined = try $ underlineSpan <$> enclosed (string "__") nestedInlines

168168

169169nowiki :: PandocMonad m => DWParser m B.Inlines

170−nowiki = try $ B.text <$ string "<nowiki>" <*> manyTill anyChar (try $ string "</nowiki>")

170+nowiki = try $ B.text <$ string "<nowiki>" <*> manyTillChar anyChar (try $ string "</nowiki>")

171171

172172percent :: PandocMonad m => DWParser m B.Inlines

173−percent = try $ B.text <$> enclosed (string "%%") nestedString

173+percent = try $ B.text <$> enclosed (string "%%") nestedText

174174

175−nestedString :: (Show a, PandocMonad m)

176− => DWParser m a -> DWParser m String

177−nestedString end = innerSpace <|> count 1 nonspaceChar

175+nestedText :: (Show a, PandocMonad m)

176+ => DWParser m a -> DWParser m Text

177+nestedText end = innerSpace <|> countChar 1 nonspaceChar

178178 where

179− innerSpace = try $ many1 spaceChar <* notFollowedBy end

179+ innerSpace = try $ many1Char spaceChar <* notFollowedBy end

180180

181181monospaced :: PandocMonad m => DWParser m B.Inlines

182−monospaced = try $ B.code <$> enclosed (string "''") nestedString

182+monospaced = try $ B.code <$> enclosed (string "''") nestedText

183183

184184subscript :: PandocMonad m => DWParser m B.Inlines

185185subscript = try $ B.subscript <$> between (string "") (try $ string "") nestedInlines

@@ -201,12 +201,12 @@ inlineFile :: PandocMonad m => DWParser m B.Inlines

201201inlineFile = codeTag B.codeWith "file"

202202

203203inlineHtml :: PandocMonad m => DWParser m B.Inlines

204−inlineHtml = try $ B.rawInline "html" <$ string "<html>" <*> manyTill anyChar (try $ string "</html>")

204+inlineHtml = try $ B.rawInline "html" <$ string "<html>" <*> manyTillChar anyChar (try $ string "</html>")

205205

206206inlinePhp :: PandocMonad m => DWParser m B.Inlines

207−inlinePhp = try $ B.codeWith ("", ["php"], []) <$ string "<php>" <*> manyTill anyChar (try $ string "</php>")

207+inlinePhp = try $ B.codeWith ("", ["php"], []) <$ string "<php>" <*> manyTillChar anyChar (try $ string "</php>")

208208

209−makeLink :: (String, String) -> B.Inlines

209+makeLink :: (Text, Text) -> B.Inlines

210210makeLink (text, url) = B.link url "" $ B.str text

211211

212212autoEmail :: PandocMonad m => DWParser m B.Inlines

@@ -220,7 +220,7 @@ autoLink = try $ do

220220 state <- getState

221221 guard $ stateAllowLinks state

222222 (text, url) <- uri

223− guard $ checkLink (last url)

223+ guard $ checkLink (T.last url)

224224 return $ makeLink (text, url)

225225 where

226226 checkLink c

@@ -234,10 +234,10 @@ nocache :: PandocMonad m => DWParser m B.Inlines

234234nocache = try $ mempty <$ string "~~NOCACHE~~"

235235

236236str :: PandocMonad m => DWParser m B.Inlines

237−str = B.str <$> (many1 alphaNum <|> count 1 characterReference)

237+str = B.str <$> (many1Char alphaNum <|> countChar 1 characterReference)

238238

239239symbol :: PandocMonad m => DWParser m B.Inlines

240−symbol = B.str <$> count 1 nonspaceChar

240+symbol = B.str <$> countChar 1 nonspaceChar

241241

242242link :: PandocMonad m => DWParser m B.Inlines

243243link = try $ do

@@ -248,77 +248,78 @@ link = try $ do

248248 setState $ st{ stateAllowLinks = True }

249249 return l

250250

251−isExternalLink :: String -> Bool

252−isExternalLink s =

253− case dropWhile (\c -> isAlphaNum c || (c `elem` ['-', '.', '+'])) s of

254− (':':'/':'/':_) -> True

255− _ -> False

256−

257−isAbsolutePath :: String -> Bool

258−isAbsolutePath ('.':_) = False

259−isAbsolutePath s = ':' `elem` s

260−

261−normalizeDots :: String -> String

262−normalizeDots path@('.':_) =

263− case dropWhile (== '.') path of

264− ':':_ -> path

265− _ -> takeWhile (== '.') path ++ ':':dropWhile (== '.') path

266−normalizeDots path = path

251+isExternalLink :: Text -> Bool

252+isExternalLink s = "://" `T.isPrefixOf` sSuff

253+ where

254+ sSuff = T.dropWhile (\c -> isAlphaNum c || (c `elem` ['-', '.', '+'])) s

255+

256+isAbsolutePath :: Text -> Bool

257+isAbsolutePath (T.uncons -> Just ('.', _)) = False

258+isAbsolutePath s = T.any (== ':') s

259+

260+normalizeDots :: Text -> Text

261+normalizeDots path

262+ | not (T.null pref) = case T.uncons suff of

263+ Just (':', _) -> path

264+ _ -> pref <> ":" <> suff

265+ | otherwise = path

266+ where

267+ (pref, suff) = T.span (== '.') path

267268

268−normalizeInternalPath :: String -> String

269+normalizeInternalPath :: Text -> Text

269270normalizeInternalPath path =

270271 if isAbsolutePath path

271272 then ensureAbsolute normalizedPath

272273 else normalizedPath

273274 where

274− normalizedPath = intercalate "/" $ dropWhile (== ".") $ splitOn ":" $ normalizeDots path

275− ensureAbsolute s@('/':_) = s

276− ensureAbsolute s = '/':s

275+ normalizedPath = T.intercalate "/" $ dropWhile (== ".") $ T.splitOn ":" $ normalizeDots path

276+ ensureAbsolute s@(T.uncons -> Just ('/', _)) = s

277+ ensureAbsolute s = "/" <> s

277278

278−normalizePath :: String -> String

279+normalizePath :: Text -> Text

279280normalizePath path =

280281 if isExternalLink path

281282 then path

282283 else normalizeInternalPath path

283284

284−urlToText :: String -> String

285+urlToText :: Text -> Text

285286urlToText url =

286287 if isExternalLink url

287288 then url

288− else reverse $ takeWhile (/= ':') $ reverse url

289+ else T.takeWhileEnd (/= ':') url

289290

290291-- Parse link or image

291292parseLink :: PandocMonad m

292− => (String -> Maybe B.Inlines -> B.Inlines)

293− -> String

294− -> String

293+ => (Text -> Maybe B.Inlines -> B.Inlines)

294+ -> Text

295+ -> Text

295296 -> DWParser m B.Inlines

296297parseLink f l r = f

297− <$ string l

298− <*> many1Till anyChar (lookAhead (void (char '|') <|> try (void $ string r)))

299− <*> optionMaybe (B.trimInlines . mconcat <$> (char '|' *> manyTill inline (try $ lookAhead $ string r)))

300− <* string r

298+ <$ textStr l

299+ <*> many1TillChar anyChar (lookAhead (void (char '|') <|> try (void $ textStr r)))

300+ <*> optionMaybe (B.trimInlines . mconcat <$> (char '|' *> manyTill inline (try $ lookAhead $ textStr r)))

301+ <* textStr r

301302

302303-- | Split Interwiki link into left and right part

303304-- | Return Nothing if it is not Interwiki link

304−splitInterwiki :: String -> Maybe (String, String)

305+splitInterwiki :: Text -> Maybe (Text, Text)

305306splitInterwiki path =

306− case span (\c -> isAlphaNum c || c == '.') path of

307− (l, '>':r) -> Just (l, r)

307+ case T.span (\c -> isAlphaNum c || c == '.') path of

308+ (l, T.uncons -> Just ('>', r)) -> Just (l, r)

308309 _ -> Nothing

309310

310−interwikiToUrl :: String -> String -> String

311−interwikiToUrl "callto" page = "callto://" ++ page

312−interwikiToUrl "doku" page = "https://www.dokuwiki.org/" ++ page

313−interwikiToUrl "phpfn" page = "https://secure.php.net/" ++ page

314−interwikiToUrl "tel" page = "tel:" ++ page

315−interwikiToUrl "wp" page = "https://en.wikipedia.org/wiki/" ++ page

316−interwikiToUrl "wpde" page = "https://de.wikipedia.org/wiki/" ++ page

317−interwikiToUrl "wpes" page = "https://es.wikipedia.org/wiki/" ++ page

318−interwikiToUrl "wpfr" page = "https://fr.wikipedia.org/wiki/" ++ page

319−interwikiToUrl "wpjp" page = "https://jp.wikipedia.org/wiki/" ++ page

320−interwikiToUrl "wppl" page = "https://pl.wikipedia.org/wiki/" ++ page

321−interwikiToUrl _ page = "https://www.google.com/search?q=" ++ page ++ "&btnI=lucky"

311+interwikiToUrl :: Text -> Text -> Text

312+interwikiToUrl "callto" page = "callto://" <> page

313+interwikiToUrl "doku" page = "https://www.dokuwiki.org/" <> page

314+interwikiToUrl "phpfn" page = "https://secure.php.net/" <> page

315+interwikiToUrl "tel" page = "tel:" <> page

316+interwikiToUrl "wp" page = "https://en.wikipedia.org/wiki/" <> page

317+interwikiToUrl "wpde" page = "https://de.wikipedia.org/wiki/" <> page

318+interwikiToUrl "wpes" page = "https://es.wikipedia.org/wiki/" <> page

319+interwikiToUrl "wpfr" page = "https://fr.wikipedia.org/wiki/" <> page

320+interwikiToUrl "wpjp" page = "https://jp.wikipedia.org/wiki/" <> page

321+interwikiToUrl "wppl" page = "https://pl.wikipedia.org/wiki/" <> page

322+interwikiToUrl _ page = "https://www.google.com/search?q=" <> page <> "&btnI=lucky"

322323

323324linkText :: PandocMonad m => DWParser m B.Inlines

324325linkText = parseLink fromRaw "[[" "]]"

@@ -338,23 +339,23 @@ linkText = parseLink fromRaw "[[" "]]"

338339 Just (_, r) -> r

339340

340341-- Matches strings like "100x100" (width x height) and "50" (width)

341−isWidthHeightParameter :: String -> Bool

342+isWidthHeightParameter :: Text -> Bool

342343isWidthHeightParameter s =

343− case s of

344− (x:xs) ->

345− isDigit x && case dropWhile isDigit xs of

346− ('x':ys@(_:_)) -> all isDigit ys

347− "" -> True

344+ case T.uncons s of

345+ Just (x, xs) ->

346+ isDigit x && case T.uncons $ T.dropWhile isDigit xs of

347+ Just ('x', ys) | not (T.null ys) -> T.all isDigit ys

348+ Nothing -> True

348349 _ -> False

349350 _ -> False

350351

351−parseWidthHeight :: String -> (Maybe String, Maybe String)

352+parseWidthHeight :: Text -> (Maybe Text, Maybe Text)

352353parseWidthHeight s = (width, height)

353354 where

354− width = Just $ takeWhile isDigit s

355+ width = Just $ T.takeWhile isDigit s

355356 height =

356− case dropWhile isDigit s of

357− ('x':xs) -> Just xs

357+ case T.uncons $ T.dropWhile isDigit s of

358+ Just ('x', xs) -> Just xs

358359 _ -> Nothing

359360

360361image :: PandocMonad m => DWParser m B.Inlines

@@ -365,17 +366,17 @@ image = try $ parseLink fromRaw "{{" "}}"

365366 then B.link normalizedPath "" (fromMaybe defaultDescription description)

366367 else B.imageWith ("", classes, attributes) normalizedPath "" (fromMaybe defaultDescription description)

367368 where

368− (path', parameters) = span (/= '?') $ trim path

369+ (path', parameters) = T.span (/= '?') $ trim path

369370 normalizedPath = normalizePath path'

370− leftPadding = " " `isPrefixOf` path

371− rightPadding = " " `isSuffixOf` path

371+ leftPadding = " " `T.isPrefixOf` path

372+ rightPadding = " " `T.isSuffixOf` path

372373 classes =

373374 case (leftPadding, rightPadding) of

374375 (False, False) -> []

375376 (False, True) -> ["align-left"]

376377 (True, False) -> ["align-right"]

377378 (True, True) -> ["align-center"]

378− parameterList = splitOn "&" $ drop 1 parameters

379+ parameterList = T.splitOn "&" $ T.drop 1 parameters

379380 linkOnly = "linkonly" `elem` parameterList

380381 (width, height) = maybe (Nothing, Nothing) parseWidthHeight (F.find isWidthHeightParameter parameterList)

381382 attributes = catMaybes [fmap ("width",) width, fmap ("height",) height]

@@ -389,7 +390,7 @@ block = do

389390 <|> blockElements

390391 <|> para

391392 skipMany blankline

392− trace (take 60 $ show $ B.toList res)

393+ trace (T.take 60 $ tshow $ B.toList res)

393394 return res

394395

395396blockElements :: PandocMonad m => DWParser m B.Blocks

@@ -417,30 +418,30 @@ header = try $ do

417418 attr <- registerHeader nullAttr contents

418419 return $ B.headerWith attr (7 - lev) contents

419420

420−list :: PandocMonad m => String -> DWParser m B.Blocks

421+list :: PandocMonad m => Text -> DWParser m B.Blocks

421422list prefix = bulletList prefix <|> orderedList prefix

422423

423−bulletList :: PandocMonad m => String -> DWParser m B.Blocks

424+bulletList :: PandocMonad m => Text -> DWParser m B.Blocks

424425bulletList prefix = try $ B.bulletList <$> parseList prefix '*'

425426

426−orderedList :: PandocMonad m => String -> DWParser m B.Blocks

427+orderedList :: PandocMonad m => Text -> DWParser m B.Blocks

427428orderedList prefix = try $ B.orderedList <$> parseList prefix '-'

428429

429430parseList :: PandocMonad m

430− => String

431+ => Text

431432 -> Char

432433 -> DWParser m [B.Blocks]

433434parseList prefix marker =

434435 many1 ((<>) <$> item <*> fmap mconcat (many continuation))

435436 where

436− continuation = try $ list (" " ++ prefix)

437− item = try $ string prefix *> char marker *> char ' ' *> itemContents

437+ continuation = try $ list (" " <> prefix)

438+ item = try $ textStr prefix *> char marker *> char ' ' *> itemContents

438439 itemContents = B.plain . mconcat <$> many1Till inline' eol

439440

440441indentedCode :: PandocMonad m => DWParser m B.Blocks

441−indentedCode = try $ B.codeBlock . unlines <$> many1 indentedLine

442+indentedCode = try $ B.codeBlock . T.unlines <$> many1 indentedLine

442443 where

443− indentedLine = try $ string " " *> manyTill anyChar eol

444+ indentedLine = try $ string " " *> manyTillChar anyChar eol

444445

445446quote :: PandocMonad m => DWParser m B.Blocks

446447quote = try $ nestedQuote 0

@@ -456,13 +457,13 @@ blockHtml :: PandocMonad m => DWParser m B.Blocks

456457blockHtml = try $ B.rawBlock "html"

457458 <$ string "<HTML>"

458459 <* optional (manyTill spaceChar eol)

459− <*> manyTill anyChar (try $ string "</HTML>")

460+ <*> manyTillChar anyChar (try $ string "</HTML>")

460461

461462blockPhp :: PandocMonad m => DWParser m B.Blocks

462463blockPhp = try $ B.codeBlockWith ("", ["php"], [])

463464 <$ string "<PHP>"

464465 <* optional (manyTill spaceChar eol)

465− <*> manyTill anyChar (try $ string "</PHP>")

466+ <*> manyTillChar anyChar (try $ string "</PHP>")

466467

467468table :: PandocMonad m => DWParser m B.Blocks

468469table = do

src/Text/Pandoc/Readers/EPUB.hs+28−25

@@ -1,6 +1,7 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE TupleSections #-}

33{-# LANGUAGE ViewPatterns #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers.EPUB

@@ -24,7 +25,8 @@ import Control.DeepSeq (NFData, deepseq)

2425import Control.Monad (guard, liftM, liftM2, mplus)

2526import Control.Monad.Except (throwError)

2627import qualified Data.ByteString.Lazy as BL (ByteString)

27−import Data.List (isInfixOf, isPrefixOf)

28+import Data.List (isInfixOf)

29+import qualified Data.Text as T

2830import qualified Data.Map as M (Map, elems, fromList, lookup)

2931import Data.Maybe (fromMaybe, mapMaybe)

3032import qualified Data.Text.Lazy as TL

@@ -67,9 +69,9 @@ archiveToEPUB os archive = do

6769 -- No need to collapse here as the image path is from the manifest file

6870 let coverDoc = fromMaybe mempty (imageToPandoc <$> cover)

6971 spine <- parseSpine items content

70− let escapedSpine = map (escapeURI . takeFileName . fst) spine

72+ let escapedSpine = map (escapeURI . T.pack . takeFileName . fst) spine

7173 Pandoc _ bs <-

72− foldM' (\a b -> ((a <>) . walk (prependHash escapedSpine))

74+ foldM' (\a b -> ((a <>) . walk (prependHash $ escapedSpine))

7375 `liftM` parseSpineElem root b) mempty spine

7476 let ast = coverDoc <> Pandoc meta bs

7577 fetchImages (M.elems items) root archive ast

@@ -79,7 +81,7 @@ archiveToEPUB os archive = do

7981 parseSpineElem :: PandocMonad m => FilePath -> (FilePath, MimeType) -> m Pandoc

8082 parseSpineElem (normalise -> r) (normalise -> path, mime) = do

8183 doc <- mimeToReader mime r path

82− let docSpan = B.doc $ B.para $ B.spanWith (takeFileName path, [], []) mempty

84+ let docSpan = B.doc $ B.para $ B.spanWith (T.pack $ takeFileName path, [], []) mempty

8385 return $ docSpan <> doc

8486 mimeToReader :: PandocMonad m => MimeType -> FilePath -> FilePath -> m Pandoc

8587 mimeToReader "application/xhtml+xml" (unEscapeString -> root)

@@ -108,18 +110,19 @@ fetchImages mimes root arc (query iq -> links) =

108110 <$> findEntryByPath abslink arc

109111

110112iq :: Inline -> [FilePath]

111−iq (Image _ _ (url, _)) = [url]

113+iq (Image _ _ (url, _)) = [T.unpack url]

112114iq _ = []

113115

114116-- Remove relative paths

115117renameImages :: FilePath -> Inline -> Inline

116118renameImages root img@(Image attr a (url, b))

117− | "data:" `isPrefixOf` url = img

118− | otherwise = Image attr a (collapseFilePath (root </> url), b)

119+ | "data:" `T.isPrefixOf` url = img

120+ | otherwise = Image attr a ( T.pack $ collapseFilePath (root </> T.unpack url)

121+ , b)

119122renameImages _ x = x

120123

121124imageToPandoc :: FilePath -> Pandoc

122−imageToPandoc s = B.doc . B.para $ B.image s "" mempty

125+imageToPandoc s = B.doc . B.para $ B.image (T.pack s) "" mempty

123126

124127imageMimes :: [MimeType]

125128imageMimes = ["image/gif", "image/jpeg", "image/png"]

@@ -144,7 +147,7 @@ parseManifest content coverId = do

144147 uid <- findAttrE (emptyName "id") e

145148 href <- findAttrE (emptyName "href") e

146149 mime <- findAttrE (emptyName "media-type") e

147− return (uid, (href, mime))

150+ return (uid, (href, T.pack mime))

148151

149152parseSpine :: PandocMonad m => Items -> Element -> m [(FilePath, MimeType)]

150153parseSpine is e = do

@@ -172,11 +175,11 @@ parseMeta content = do

172175-- http://www.idpf.org/epub/30/spec/epub30-publications.html#sec-metadata-elem

173176parseMetaItem :: Element -> Meta -> Meta

174177parseMetaItem e@(stripNamespace . elName -> field) meta =

175− addMetaField (renameMeta field) (B.str $ strContent e) meta

178+ addMetaField (renameMeta field) (B.str $ T.pack $ strContent e) meta

176179

177−renameMeta :: String -> String

180+renameMeta :: String -> T.Text

178181renameMeta "creator" = "author"

179−renameMeta s = s

182+renameMeta s = T.pack s

180183

181184getManifest :: PandocMonad m => Archive -> m (String, Element)

182185getManifest archive = do

@@ -197,26 +200,26 @@ getManifest archive = do

197200fixInternalReferences :: FilePath -> Pandoc -> Pandoc

198201fixInternalReferences pathToFile =

199202 walk (renameImages root)

200− . walk (fixBlockIRs filename)

203+ . walk (fixBlockIRs filename)

201204 . walk (fixInlineIRs filename)

202205 where

203− (root, escapeURI -> filename) = splitFileName pathToFile

206+ (root, T.unpack . escapeURI . T.pack -> filename) = splitFileName pathToFile

204207

205208fixInlineIRs :: String -> Inline -> Inline

206209fixInlineIRs s (Span as v) =

207210 Span (fixAttrs s as) v

208211fixInlineIRs s (Code as code) =

209212 Code (fixAttrs s as) code

210−fixInlineIRs s (Link as is ('#':url, tit)) =

213+fixInlineIRs s (Link as is (T.uncons -> Just ('#', url), tit)) =

211214 Link (fixAttrs s as) is (addHash s url, tit)

212215fixInlineIRs s (Link as is t) =

213216 Link (fixAttrs s as) is t

214217fixInlineIRs _ v = v

215218

216−prependHash :: [String] -> Inline -> Inline

219+prependHash :: [T.Text] -> Inline -> Inline

217220prependHash ps l@(Link attr is (url, tit))

218− | or [s `isPrefixOf` url | s <- ps] =

219− Link attr is ('#':url, tit)

221+ | or [s `T.isPrefixOf` url | s <- ps] =

222+ Link attr is ("#" <> url, tit)

220223 | otherwise = l

221224prependHash _ i = i

222225

@@ -230,17 +233,17 @@ fixBlockIRs s (CodeBlock as code) =

230233fixBlockIRs _ b = b

231234

232235fixAttrs :: FilePath -> B.Attr -> B.Attr

233−fixAttrs s (ident, cs, kvs) = (addHash s ident, filter (not . null) cs, removeEPUBAttrs kvs)

236+fixAttrs s (ident, cs, kvs) = (addHash s ident, filter (not . T.null) cs, removeEPUBAttrs kvs)

234237

235−addHash :: String -> String -> String

238+addHash :: String -> T.Text -> T.Text

236239addHash _ "" = ""

237−addHash s ident = takeFileName s ++ "#" ++ ident

240+addHash s ident = T.pack (takeFileName s) <> "#" <> ident

238241

239−removeEPUBAttrs :: [(String, String)] -> [(String, String)]

242+removeEPUBAttrs :: [(T.Text, T.Text)] -> [(T.Text, T.Text)]

240243removeEPUBAttrs kvs = filter (not . isEPUBAttr) kvs

241244

242−isEPUBAttr :: (String, String) -> Bool

243−isEPUBAttr (k, _) = "epub:" `isPrefixOf` k

245+isEPUBAttr :: (T.Text, a) -> Bool

246+isEPUBAttr (k, _) = "epub:" `T.isPrefixOf` k

244247

245248-- Library

246249

@@ -291,4 +294,4 @@ findElementE :: PandocMonad m => QName -> Element -> m Element

291294findElementE e x = mkE ("Unable to find element: " ++ show e) $ findElement e x

292295

293296mkE :: PandocMonad m => String -> Maybe a -> m a

294−mkE s = maybe (throwError . PandocParseError $ s) return

297+mkE s = maybe (throwError . PandocParseError $ T.pack $ s) return

src/Text/Pandoc/Readers/FB2.hs+63−60

@@ -1,5 +1,6 @@

11{-# LANGUAGE FlexibleContexts #-}

22{-# LANGUAGE TupleSections #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Readers.FB2

@@ -27,12 +28,11 @@ import Control.Monad.Except (throwError)

2728import Control.Monad.State.Strict

2829import Data.ByteString.Lazy.Char8 ( pack )

2930import Data.ByteString.Base64.Lazy

30−import Data.Char (isSpace, toUpper)

3131import Data.Functor

32−import Data.List (dropWhileEnd, intersperse)

33−import Data.List.Split (splitOn)

32+import Data.List (intersperse)

3433import qualified Data.Map as M

3534import Data.Text (Text)

35+import qualified Data.Text as T

3636import Data.Default

3737import Data.Maybe

3838import Text.HTML.TagSoup.Entity (lookupEntity)

@@ -48,8 +48,8 @@ type FB2 m = StateT FB2State m

4848

4949data FB2State = FB2State{ fb2SectionLevel :: Int

5050 , fb2Meta :: Meta

51− , fb2Authors :: [String]

52− , fb2Notes :: M.Map String Blocks

51+ , fb2Authors :: [Text]

52+ , fb2Notes :: M.Map Text Blocks

5353 } deriving Show

5454

5555instance Default FB2State where

@@ -76,19 +76,20 @@ readFB2 _ inp =

7676

7777-- * Utility functions

7878

79−trim :: String -> String

80−trim = dropWhileEnd isSpace . dropWhile isSpace

79+trim :: Text -> Text

80+trim = T.strip

8181

82−removeHash :: String -> String

83−removeHash ('#':xs) = xs

84−removeHash xs = xs

82+removeHash :: Text -> Text

83+removeHash t = case T.uncons t of

84+ Just ('#', xs) -> xs

85+ _ -> t

8586

86−convertEntity :: String -> String

87−convertEntity e = fromMaybe (map toUpper e) (lookupEntity e)

87+convertEntity :: String -> Text

88+convertEntity e = maybe (T.toUpper $ T.pack e) T.pack $ lookupEntity e

8889

8990parseInline :: PandocMonad m => Content -> FB2 m Inlines

9091parseInline (Elem e) =

91− case qName $ elName e of

92+ case T.pack $ qName $ elName e of

9293 "strong" -> strong <$> parseStyleType e

9394 "emphasis" -> emph <$> parseStyleType e

9495 "style" -> parseNamedStyle e

@@ -96,12 +97,12 @@ parseInline (Elem e) =

9697 "strikethrough" -> strikeout <$> parseStyleType e

9798 "sub" -> subscript <$> parseStyleType e

9899 "sup" -> superscript <$> parseStyleType e

99− "code" -> pure $ code $ strContent e

100+ "code" -> pure $ code $ T.pack $ strContent e

100101 "image" -> parseInlineImageElement e

101102 name -> do

102103 report $ IgnoredElement name

103104 pure mempty

104−parseInline (Text x) = pure $ text $ cdData x

105+parseInline (Text x) = pure $ text $ T.pack $ cdData x

105106parseInline (CRef r) = pure $ str $ convertEntity r

106107

107108parseSubtitle :: PandocMonad m => Element -> FB2 m Blocks

@@ -111,7 +112,7 @@ parseSubtitle e = headerWith ("", ["unnumbered"], []) <$> gets fb2SectionLevel <

111112

112113parseRootElement :: PandocMonad m => Element -> FB2 m Blocks

113114parseRootElement e =

114− case qName $ elName e of

115+ case T.pack $ qName $ elName e of

115116 "FictionBook" -> do

116117 -- Parse notes before parsing the rest of the content.

117118 case filterChild isNotesBody e of

@@ -144,7 +145,7 @@ parseNote e =

144145 Just sectionId -> do

145146 content <- mconcat <$> mapM parseSectionChild (dropTitle $ elChildren e)

146147 oldNotes <- gets fb2Notes

147− modify $ \s -> s { fb2Notes = M.insert ("#" ++ sectionId) content oldNotes }

148+ modify $ \s -> s { fb2Notes = M.insert ("#" <> T.pack sectionId) content oldNotes }

148149 pure ()

149150 where

150151 isTitle x = qName (elName x) == "title"

@@ -156,7 +157,7 @@ parseNote e =

156157-- | Parse a child of @\<FictionBook>@ element.

157158parseFictionBookChild :: PandocMonad m => Element -> FB2 m Blocks

158159parseFictionBookChild e =

159− case qName $ elName e of

160+ case T.pack $ qName $ elName e of

160161 "stylesheet" -> pure mempty -- stylesheet is ignored

161162 "description" -> mempty <$ mapM_ parseDescriptionChild (elChildren e)

162163 "body" -> if isNotesBody e

@@ -168,7 +169,7 @@ parseFictionBookChild e =

168169-- | Parse a child of @\<description>@ element.

169170parseDescriptionChild :: PandocMonad m => Element -> FB2 m ()

170171parseDescriptionChild e =

171− case qName $ elName e of

172+ case T.pack $ qName $ elName e of

172173 "title-info" -> mapM_ parseTitleInfoChild (elChildren e)

173174 "src-title-info" -> pure () -- ignore

174175 "document-info" -> pure ()

@@ -176,13 +177,13 @@ parseDescriptionChild e =

176177 "custom-info" -> pure ()

177178 "output" -> pure ()

178179 name -> do

179− report $ IgnoredElement $ name ++ " in description"

180+ report $ IgnoredElement $ name <> " in description"

180181 pure mempty

181182

182183-- | Parse a child of @\<body>@ element.

183184parseBodyChild :: PandocMonad m => Element -> FB2 m Blocks

184185parseBodyChild e =

185− case qName $ elName e of

186+ case T.pack $ qName $ elName e of

186187 "image" -> parseImageElement e

187188 "title" -> header <$> gets fb2SectionLevel <*> parseTitleType (elContent e)

188189 "epigraph" -> parseEpigraph e

@@ -196,25 +197,25 @@ parseBinaryElement e =

196197 (Nothing, _) -> report $ IgnoredElement "binary without id attribute"

197198 (Just _, Nothing) ->

198199 report $ IgnoredElement "binary without content-type attribute"

199− (Just filename, contentType) -> insertMedia filename contentType (decodeLenient (pack (strContent e)))

200+ (Just filename, contentType) -> insertMedia filename (T.pack <$> contentType) (decodeLenient (pack (strContent e)))

200201

201202-- * Type parsers

202203

203204-- | Parse @authorType@

204−parseAuthor :: PandocMonad m => Element -> FB2 m String

205−parseAuthor e = unwords . catMaybes <$> mapM parseAuthorChild (elChildren e)

205+parseAuthor :: PandocMonad m => Element -> FB2 m Text

206+parseAuthor e = T.unwords . catMaybes <$> mapM parseAuthorChild (elChildren e)

206207

207−parseAuthorChild :: PandocMonad m => Element -> FB2 m (Maybe String)

208+parseAuthorChild :: PandocMonad m => Element -> FB2 m (Maybe Text)

208209parseAuthorChild e =

209− case qName $ elName e of

210− "first-name" -> pure $ Just $ strContent e

211− "middle-name" -> pure $ Just $ strContent e

212− "last-name" -> pure $ Just $ strContent e

213− "nickname" -> pure $ Just $ strContent e

214− "home-page" -> pure $ Just $ strContent e

215− "email" -> pure $ Just $ strContent e

210+ case T.pack $ qName $ elName e of

211+ "first-name" -> pure $ Just $ T.pack $ strContent e

212+ "middle-name" -> pure $ Just $ T.pack $ strContent e

213+ "last-name" -> pure $ Just $ T.pack $ strContent e

214+ "nickname" -> pure $ Just $ T.pack $ strContent e

215+ "home-page" -> pure $ Just $ T.pack $ strContent e

216+ "email" -> pure $ Just $ T.pack $ strContent e

216217 name -> do

217− report $ IgnoredElement $ name ++ " in author"

218+ report $ IgnoredElement $ name <> " in author"

218219 pure Nothing

219220

220221-- | Parse @titleType@

@@ -236,13 +237,13 @@ parseTitleContent _ = pure Nothing

236237parseImageElement :: PandocMonad m => Element -> FB2 m Blocks

237238parseImageElement e =

238239 case href of

239− Just src -> pure $ para $ imageWith (imgId, [], []) (removeHash src) title alt

240+ Just src -> pure $ para $ imageWith (imgId, [], []) (removeHash $ T.pack src) title alt

240241 Nothing -> do

241242 report $ IgnoredElement " image without href"

242243 pure mempty

243− where alt = maybe mempty str $ findAttr (unqual "alt") e

244− title = fromMaybe "" $ findAttr (unqual "title") e

245− imgId = fromMaybe "" $ findAttr (unqual "id") e

244+ where alt = maybe mempty (str . T.pack) $ findAttr (unqual "alt") e

245+ title = maybe "" T.pack $ findAttr (unqual "title") e

246+ imgId = maybe "" T.pack $ findAttr (unqual "id") e

246247 href = findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e

247248

248249-- | Parse @pType@

@@ -256,7 +257,7 @@ parseCite e = blockQuote . mconcat <$> mapM parseCiteChild (elChildren e)

256257-- | Parse @citeType@ child

257258parseCiteChild :: PandocMonad m => Element -> FB2 m Blocks

258259parseCiteChild e =

259− case qName $ elName e of

260+ case T.pack $ qName $ elName e of

260261 "p" -> para <$> parsePType e

261262 "poem" -> parsePoem e

262263 "empty-line" -> pure horizontalRule

@@ -271,13 +272,13 @@ parsePoem e = mconcat <$> mapM parsePoemChild (elChildren e)

271272

272273parsePoemChild :: PandocMonad m => Element -> FB2 m Blocks

273274parsePoemChild e =

274− case qName $ elName e of

275+ case T.pack $ qName $ elName e of

275276 "title" -> parseTitle e

276277 "subtitle" -> parseSubtitle e

277278 "epigraph" -> parseEpigraph e

278279 "stanza" -> parseStanza e

279280 "text-author" -> para <$> parsePType e

280− "date" -> pure $ para $ text $ strContent e

281+ "date" -> pure $ para $ text $ T.pack $ strContent e

281282 name -> report (UnexpectedXmlElement name "poem") $> mempty

282283

283284parseStanza :: PandocMonad m => Element -> FB2 m Blocks

@@ -290,7 +291,7 @@ joinLineBlocks [] = []

290291

291292parseStanzaChild :: PandocMonad m => Element -> FB2 m Blocks

292293parseStanzaChild e =

293− case qName $ elName e of

294+ case T.pack $ qName $ elName e of

294295 "title" -> parseTitle e

295296 "subtitle" -> parseSubtitle e

296297 "v" -> lineBlock . (:[]) <$> parsePType e

@@ -300,11 +301,11 @@ parseStanzaChild e =

300301parseEpigraph :: PandocMonad m => Element -> FB2 m Blocks

301302parseEpigraph e =

302303 divWith (divId, ["epigraph"], []) . mconcat <$> mapM parseEpigraphChild (elChildren e)

303− where divId = fromMaybe "" $ findAttr (unqual "id") e

304+ where divId = maybe "" T.pack $ findAttr (unqual "id") e

304305

305306parseEpigraphChild :: PandocMonad m => Element -> FB2 m Blocks

306307parseEpigraphChild e =

307− case qName $ elName e of

308+ case T.pack $ qName $ elName e of

308309 "p" -> para <$> parsePType e

309310 "poem" -> parsePoem e

310311 "cite" -> parseCite e

@@ -318,7 +319,7 @@ parseAnnotation e = mconcat <$> mapM parseAnnotationChild (elChildren e)

318319

319320parseAnnotationChild :: PandocMonad m => Element -> FB2 m Blocks

320321parseAnnotationChild e =

321− case qName $ elName e of

322+ case T.pack $ qName $ elName e of

322323 "p" -> para <$> parsePType e

323324 "poem" -> parsePoem e

324325 "cite" -> parseCite e

@@ -332,14 +333,14 @@ parseSection :: PandocMonad m => Element -> FB2 m Blocks

332333parseSection e = do

333334 n <- gets fb2SectionLevel

334335 modify $ \st -> st{ fb2SectionLevel = n + 1 }

335− let sectionId = fromMaybe "" $ findAttr (unqual "id") e

336+ let sectionId = maybe "" T.pack $ findAttr (unqual "id") e

336337 bs <- divWith (sectionId, ["section"], []) . mconcat <$> mapM parseSectionChild (elChildren e)

337338 modify $ \st -> st{ fb2SectionLevel = n }

338339 pure bs

339340

340341parseSectionChild :: PandocMonad m => Element -> FB2 m Blocks

341342parseSectionChild e =

342− case qName $ elName e of

343+ case T.pack $ qName $ elName e of

343344 "title" -> parseBodyChild e

344345 "epigraph" -> parseEpigraph e

345346 "image" -> parseImageElement e

@@ -361,16 +362,16 @@ parseStyleType e = mconcat <$> mapM parseInline (elContent e)

361362parseNamedStyle :: PandocMonad m => Element -> FB2 m Inlines

362363parseNamedStyle e = do

363364 content <- mconcat <$> mapM parseNamedStyleChild (elContent e)

364− let lang = maybeToList $ ("lang",) <$> findAttr (QName "lang" Nothing (Just "xml")) e

365+ let lang = maybeToList $ ("lang",) . T.pack <$> findAttr (QName "lang" Nothing (Just "xml")) e

365366 case findAttr (unqual "name") e of

366− Just name -> pure $ spanWith ("", [name], lang) content

367+ Just name -> pure $ spanWith ("", [T.pack name], lang) content

367368 Nothing -> do

368369 report $ IgnoredElement "link without required name"

369370 pure mempty

370371

371372parseNamedStyleChild :: PandocMonad m => Content -> FB2 m Inlines

372373parseNamedStyleChild (Elem e) =

373− case qName (elName e) of

374+ case T.pack $ qName (elName e) of

374375 "strong" -> strong <$> parseStyleType e

375376 "emphasis" -> emph <$> parseStyleType e

376377 "style" -> parseNamedStyle e

@@ -378,10 +379,10 @@ parseNamedStyleChild (Elem e) =

378379 "strikethrough" -> strikeout <$> parseStyleType e

379380 "sub" -> subscript <$> parseStyleType e

380381 "sup" -> superscript <$> parseStyleType e

381− "code" -> pure $ code $ strContent e

382+ "code" -> pure $ code $ T.pack $ strContent e

382383 "image" -> parseInlineImageElement e

383384 name -> do

384− report $ IgnoredElement $ name ++ " in style"

385+ report $ IgnoredElement $ name <> " in style"

385386 pure mempty

386387parseNamedStyleChild x = parseInline x

387388

@@ -390,7 +391,7 @@ parseLinkType :: PandocMonad m => Element -> FB2 m Inlines

390391parseLinkType e = do

391392 content <- mconcat <$> mapM parseStyleLinkType (elContent e)

392393 notes <- gets fb2Notes

393− case findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e of

394+ case T.pack <$> findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e of

394395 Just href -> case findAttr (QName "type" Nothing Nothing) e of

395396 Just "note" -> case M.lookup href notes of

396397 Nothing -> pure $ link href "" content

@@ -417,19 +418,21 @@ parseTable _ = pure mempty -- TODO: tables are not supported yet

417418-- | Parse @title-infoType@

418419parseTitleInfoChild :: PandocMonad m => Element -> FB2 m ()

419420parseTitleInfoChild e =

420− case qName (elName e) of

421+ case T.pack $ qName (elName e) of

421422 "genre" -> pure ()

422423 "author" -> parseAuthor e >>= \author -> modify (\st -> st {fb2Authors = author:fb2Authors st})

423− "book-title" -> modify (setMeta "title" (text $ strContent e))

424+ "book-title" -> modify (setMeta "title" (text $ T.pack $ strContent e))

424425 "annotation" -> parseAnnotation e >>= modify . setMeta "abstract"

425− "keywords" -> modify (setMeta "keywords" (map (MetaString . trim) $ splitOn "," $ strContent e))

426− "date" -> modify (setMeta "date" (text $ strContent e))

426+ "keywords" -> modify (setMeta "keywords" (map (MetaString . trim) $ T.splitOn ","

427+ $ T.pack

428+ $ strContent e))

429+ "date" -> modify (setMeta "date" (text $ T.pack $ strContent e))

427430 "coverpage" -> parseCoverPage e

428431 "lang" -> pure ()

429432 "src-lang" -> pure ()

430433 "translator" -> pure ()

431434 "sequence" -> pure ()

432− name -> report $ IgnoredElement $ name ++ " in title-info"

435+ name -> report $ IgnoredElement $ name <> " in title-info"

433436

434437parseCoverPage :: PandocMonad m => Element -> FB2 m ()

435438parseCoverPage e =

@@ -437,7 +440,7 @@ parseCoverPage e =

437440 Just img -> case href of

438441 Just src -> modify (setMeta "cover-image" (MetaString $ removeHash src))

439442 Nothing -> pure ()

440− where href = findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) img

443+ where href = T.pack <$> findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) img

441444 Nothing -> pure ()

442445

443446-- | Parse @inlineImageType@ element

@@ -450,5 +453,5 @@ parseInlineImageElement e =

450453 Nothing -> do

451454 report $ IgnoredElement "inline image without href"

452455 pure mempty

453− where alt = maybe mempty str $ findAttr (unqual "alt") e

454− href = findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e

456+ where alt = maybe mempty (str . T.pack) $ findAttr (unqual "alt") e

457+ href = T.pack <$> findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e

src/Text/Pandoc/Readers/HTML.hs+112−119

@@ -1,5 +1,5 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE LambdaCase #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE LambdaCase #-}

33{-# LANGUAGE FlexibleContexts #-}

44{-# LANGUAGE FlexibleInstances #-}

55{-# LANGUAGE MultiParamTypeClasses #-}

@@ -35,8 +35,7 @@ import Control.Monad.Reader (ReaderT, ask, asks, lift, local, runReaderT)

3535import Data.Char (isAlphaNum, isLetter)

3636import Data.Default (Default (..), def)

3737import Data.Foldable (for_)

38−import Data.List (isPrefixOf)

39−import Data.List.Split (wordsBy, splitWhen)

38+import Data.List.Split (splitWhen)

4039import qualified Data.Map as M

4140import Data.Maybe (fromMaybe, isJust, isNothing)

4241import Data.Monoid (First (..))

@@ -62,8 +61,8 @@ import Text.Pandoc.Options (

6261 extensionEnabled)

6362import Text.Pandoc.Parsing hiding ((<|>))

6463import Text.Pandoc.Shared (addMetaField, blocksToInlines', crFilter, escapeURI,

65− extractSpaces, htmlSpanLikeElements,

66− onlySimpleTableCells, safeRead, underlineSpan)

64+ extractSpaces, htmlSpanLikeElements, elemText, splitTextBy,

65+ onlySimpleTableCells, safeRead, underlineSpan, tshow)

6766import Text.Pandoc.Walk

6867import Text.Parsec.Error

6968import Text.TeXMath (readMathML, writeTeX)

@@ -93,14 +92,14 @@ readHtml opts inp = do

9392 "source" tags

9493 case result of

9594 Right doc -> return doc

96− Left err -> throwError $ PandocParseError $ getError err

95+ Left err -> throwError $ PandocParseError $ T.pack $ getError err

9796

9897replaceNotes :: PandocMonad m => [Block] -> TagParser m [Block]

9998replaceNotes bs = do

10099 st <- getState

101100 return $ walk (replaceNotes' (noteTable st)) bs

102101

103−replaceNotes' :: [(String, Blocks)] -> Inline -> Inline

102+replaceNotes' :: [(Text, Blocks)] -> Inline -> Inline

104103replaceNotes' noteTbl (RawInline (Format "noteref") ref) =

105104 maybe (Str "") (Note . B.toList) $ lookup ref noteTbl

106105replaceNotes' _ x = x

@@ -108,9 +107,9 @@ replaceNotes' _ x = x

108107data HTMLState =

109108 HTMLState

110109 { parserState :: ParserState,

111− noteTable :: [(String, Blocks)],

110+ noteTable :: [(Text, Blocks)],

112111 baseHref :: Maybe URI,

113− identifiers :: Set.Set String,

112+ identifiers :: Set.Set Text,

114113 logMessages :: [LogMessage],

115114 macros :: M.Map Text Macro

116115 }

@@ -134,7 +133,7 @@ pHtml :: PandocMonad m => TagParser m Blocks

134133pHtml = try $ do

135134 (TagOpen "html" attr) <- lookAhead pAny

136135 for_ (lookup "lang" attr) $

137− updateState . B.setMeta "lang" . B.text . T.unpack

136+ updateState . B.setMeta "lang" . B.text

138137 pInTags "html" block

139138

140139pBody :: PandocMonad m => TagParser m Blocks

@@ -146,11 +145,11 @@ pHead = pInTags "head" $ pTitle <|> pMetaTag <|> pBaseTag <|> (mempty <$ pAny)

146145 setTitle t = mempty <$ updateState (B.setMeta "title" t)

147146 pMetaTag = do

148147 mt <- pSatisfy (matchTagOpen "meta" [])

149− let name = T.unpack $ fromAttrib "name" mt

150− if null name

148+ let name = fromAttrib "name" mt

149+ if T.null name

151150 then return mempty

152151 else do

153− let content = T.unpack $ fromAttrib "content" mt

152+ let content = fromAttrib "content" mt

154153 updateState $ \s ->

155154 let ps = parserState s in

156155 s{ parserState = ps{

@@ -187,13 +186,13 @@ block = do

187186 , pFigure

188187 , pRawHtmlBlock

189188 ]

190− trace (take 60 $ show $ B.toList res)

189+ trace (T.take 60 $ tshow $ B.toList res)

191190 return res

192191

193−namespaces :: PandocMonad m => [(String, TagParser m Inlines)]

192+namespaces :: PandocMonad m => [(Text, TagParser m Inlines)]

194193namespaces = [(mathMLNamespace, pMath True)]

195194

196−mathMLNamespace :: String

195+mathMLNamespace :: Text

197196mathMLNamespace = "http://www.w3.org/1998/Math/MathML"

198197

199198eSwitch :: (PandocMonad m, Monoid a)

@@ -233,7 +232,7 @@ eFootnote = try $ do

233232 content <- pInTags tag block

234233 addNote ident content

235234

236−addNote :: PandocMonad m => String -> Blocks -> TagParser m ()

235+addNote :: PandocMonad m => Text -> Blocks -> TagParser m ()

237236addNote uid cont = updateState (\s -> s {noteTable = (uid, cont) : noteTable s})

238237

239238eNoteref :: PandocMonad m => TagParser m Inlines

@@ -245,8 +244,8 @@ eNoteref = try $ do

245244 -> (lookup "type" as <|> lookup "epub:type" as)

246245 == Just "noteref"

247246 _ -> False)

248− ident <- case T.unpack <$> lookup "href" attr of

249− Just ('#':rest) -> return rest

247+ ident <- case lookup "href" attr >>= T.uncons of

248+ Just ('#', rest) -> return rest

250249 _ -> mzero

251250 _ <- manyTill pAny (pSatisfy (\case

252251 TagClose t -> t == tag

@@ -287,7 +286,7 @@ pListItem nonItem = do

287286 maybe id addId (lookup "id" attr) <$>

288287 pInTags "li" block <* skipMany nonItem

289288

290−parseListStyleType :: String -> ListNumberStyle

289+parseListStyleType :: Text -> ListNumberStyle

291290parseListStyleType "lower-roman" = LowerRoman

292291parseListStyleType "upper-roman" = UpperRoman

293292parseListStyleType "lower-alpha" = LowerAlpha

@@ -295,7 +294,7 @@ parseListStyleType "upper-alpha" = UpperAlpha

295294parseListStyleType "decimal" = Decimal

296295parseListStyleType _ = DefaultStyle

297296

298−parseTypeAttr :: String -> ListNumberStyle

297+parseTypeAttr :: Text -> ListNumberStyle

299298parseTypeAttr "i" = LowerRoman

300299parseTypeAttr "I" = UpperRoman

301300parseTypeAttr "a" = LowerAlpha

@@ -404,20 +403,19 @@ pDiv = try $ do

404403

405404pRawHtmlBlock :: PandocMonad m => TagParser m Blocks

406405pRawHtmlBlock = do

407− raw <- T.unpack <$>

408− (pHtmlBlock "script" <|> pHtmlBlock "style" <|> pHtmlBlock "textarea"

409− <|> pRawTag)

406+ raw <- (pHtmlBlock "script" <|> pHtmlBlock "style" <|> pHtmlBlock "textarea"

407+ <|> pRawTag)

410408 exts <- getOption readerExtensions

411− if extensionEnabled Ext_raw_html exts && not (null raw)

409+ if extensionEnabled Ext_raw_html exts && not (T.null raw)

412410 then return $ B.rawBlock "html" raw

413411 else ignore raw

414412

415−ignore :: (Monoid a, PandocMonad m) => String -> TagParser m a

413+ignore :: (Monoid a, PandocMonad m) => Text -> TagParser m a

416414ignore raw = do

417415 pos <- getPosition

418416 -- raw can be null for tags like <!DOCTYPE>; see paRawTag

419417 -- in this case we don't want a warning:

420− unless (null raw) $

418+ unless (T.null raw) $

421419 logMessage $ SkippedContent raw pos

422420 return mempty

423421

@@ -438,7 +436,7 @@ eSection = try $ do

438436

439437headerLevel :: Text -> TagParser m Int

440438headerLevel tagtype =

441− case safeRead (T.unpack (T.drop 1 tagtype)) of

439+ case safeRead (T.drop 1 tagtype) of

442440 Just level ->

443441-- try (do

444442-- guardEnabled Ext_epub_html_exts

@@ -468,7 +466,7 @@ pHeader = try $ do

468466 level <- headerLevel tagtype

469467 contents <- trimInlines . mconcat <$> manyTill inline (pCloses tagtype <|> eof)

470468 let ident = fromMaybe "" $ lookup "id" attr

471− let classes = maybe [] words $ lookup "class" attr

469+ let classes = maybe [] T.words $ lookup "class" attr

472470 let keyvals = [(k,v) | (k,v) <- attr, k /= "class", k /= "id"]

473471 attr'' <- registerHeader (ident, classes, keyvals) contents

474472 return $ if bodyTitle

@@ -529,14 +527,14 @@ pCol = try $ do

529527 optional $ pSatisfy (matchTagClose "col")

530528 skipMany pBlank

531529 let width = case lookup "width" attribs of

532− Nothing -> case lookup "style" attribs of

533− Just ('w':'i':'d':'t':'h':':':xs) | '%' `elem` xs ->

534− fromMaybe 0.0 $ safeRead (filter

535− (`notElem` (" \t\r\n%'\";" :: [Char])) xs)

536− _ -> 0.0

537− Just x | not (null x) && last x == '%' ->

538− fromMaybe 0.0 $ safeRead (init x)

539− _ -> 0.0

530+ Nothing -> case lookup "style" attribs of

531+ Just (T.stripPrefix "width:" -> Just xs) | T.any (== '%') xs ->

532+ fromMaybe 0.0 $ safeRead (T.filter

533+ (`notElem` (" \t\r\n%'\";" :: [Char])) xs)

534+ _ -> 0.0

535+ Just (T.unsnoc -> Just (xs, '%')) ->

536+ fromMaybe 0.0 $ safeRead xs

537+ _ -> 0.0

540538 if width > 0.0

541539 then return $ width / 100.0

542540 else return 0.0

@@ -562,7 +560,7 @@ pCell celltype = try $ do

562560 let extractAlign' [] = ""

563561 extractAlign' ("text-align":x:_) = x

564562 extractAlign' (_:xs) = extractAlign' xs

565− let extractAlign = extractAlign' . wordsBy (`elem` [' ','\t',';',':'])

563+ let extractAlign = extractAlign' . splitTextBy (`elemText` " \t;:")

566564 let align = case maybeFromAttrib "align" tag `mplus`

567565 (extractAlign <$> maybeFromAttrib "style" tag) of

568566 Just "left" -> AlignLeft

@@ -610,7 +608,7 @@ pFigure = try $ do

610608 let caption = fromMaybe mempty mbcap

611609 case B.toList <$> mbimg of

612610 Just [Image attr _ (url, tit)] ->

613− return $ B.para $ B.imageWith attr url ("fig:" ++ tit) caption

611+ return $ B.para $ B.imageWith attr url ("fig:" <> tit) caption

614612 _ -> mzero

615613

616614pCodeBlock :: PandocMonad m => TagParser m Blocks

@@ -618,21 +616,21 @@ pCodeBlock = try $ do

618616 TagOpen _ attr' <- pSatisfy (matchTagOpen "pre" [])

619617 let attr = toStringAttr attr'

620618 contents <- manyTill pAny (pCloses "pre" <|> eof)

621− let rawText = concatMap tagToString contents

619+ let rawText = T.concat $ map tagToText contents

622620 -- drop leading newline if any

623− let result' = case rawText of

624− '\n':xs -> xs

625− _ -> rawText

621+ let result' = case T.uncons rawText of

622+ Just ('\n', xs) -> xs

623+ _ -> rawText

626624 -- drop trailing newline if any

627− let result = case reverse result' of

628− '\n':_ -> init result'

629− _ -> result'

625+ let result = case T.unsnoc result' of

626+ Just (result'', '\n') -> result''

627+ _ -> result'

630628 return $ B.codeBlockWith (mkAttr attr) result

631629

632−tagToString :: Tag Text -> String

633−tagToString (TagText s) = T.unpack s

634−tagToString (TagOpen "br" _) = "\n"

635−tagToString _ = ""

630+tagToText :: Tag Text -> Text

631+tagToText (TagText s) = s

632+tagToText (TagOpen "br" _) = "\n"

633+tagToText _ = ""

636634

637635inline :: PandocMonad m => TagParser m Inlines

638636inline = choice

@@ -667,7 +665,7 @@ pLocation = do

667665pSat :: PandocMonad m => (Tag Text -> Bool) -> TagParser m (Tag Text)

668666pSat f = do

669667 pos <- getPosition

670− token show (const pos) (\x -> if f x then Just x else Nothing)

668+ token tshow (const pos) (\x -> if f x then Just x else Nothing)

671669

672670pSatisfy :: PandocMonad m => (Tag Text -> Bool) -> TagParser m (Tag Text)

673671pSatisfy f = try $ optional pLocation >> pSat f

@@ -688,10 +686,10 @@ pQ = choice $ map try [citedQuote, normalQuote]

688686 where citedQuote = do

689687 tag <- pSatisfy $ tagOpenLit "q" (any ((=="cite") . fst))

690688

691− url <- canonicalizeUrl $ T.unpack $ fromAttrib "cite" tag

692− let uid = fromMaybe (T.unpack $ fromAttrib "name" tag) $

689+ url <- canonicalizeUrl $ fromAttrib "cite" tag

690+ let uid = fromMaybe (fromAttrib "name" tag) $

693691 maybeFromAttrib "id" tag

694− let cls = words $ T.unpack $ fromAttrib "class" tag

692+ let cls = T.words $ fromAttrib "class" tag

695693

696694 makeQuote $ B.spanWith (uid, cls, [("cite", escapeURI url)])

697695 normalQuote = do

@@ -729,7 +727,7 @@ pSpanLike =

729727 TagOpen _ attrs <- pSatisfy $ tagOpenLit tagName (const True)

730728 let (ids, cs, kvs) = mkAttr . toStringAttr $ attrs

731729 content <- mconcat <$> manyTill inline (pCloses tagName <|> eof)

732− return $ B.spanWith (ids, T.unpack tagName : cs, kvs) content

730+ return $ B.spanWith (ids, tagName : cs, kvs) content

733731

734732pSmall :: PandocMonad m => TagParser m Inlines

735733pSmall = pInlinesInTags "small" (B.spanWith ("",["small"],[]))

@@ -753,19 +751,18 @@ pLineBreak = do

753751

754752-- Unlike fromAttrib from tagsoup, this distinguishes

755753-- between a missing attribute and an attribute with empty content.

756−maybeFromAttrib :: String -> Tag Text -> Maybe String

757−maybeFromAttrib name (TagOpen _ attrs) =

758− T.unpack <$> lookup (T.pack name) attrs

754+maybeFromAttrib :: Text -> Tag Text -> Maybe Text

755+maybeFromAttrib name (TagOpen _ attrs) = lookup name attrs

759756maybeFromAttrib _ _ = Nothing

760757

761758pLink :: PandocMonad m => TagParser m Inlines

762759pLink = try $ do

763760 tag <- pSatisfy $ tagOpenLit "a" (const True)

764− let title = T.unpack $ fromAttrib "title" tag

761+ let title = fromAttrib "title" tag

765762 -- take id from id attribute if present, otherwise name

766− let uid = fromMaybe (T.unpack $ fromAttrib "name" tag) $

763+ let uid = fromMaybe (fromAttrib "name" tag) $

767764 maybeFromAttrib "id" tag

768− let cls = words $ T.unpack $ fromAttrib "class" tag

765+ let cls = T.words $ fromAttrib "class" tag

769766 lab <- mconcat <$> manyTill inline (pCloses "a")

770767 -- check for href; if href, then a link, otherwise a span

771768 case maybeFromAttrib "href" tag of

@@ -778,34 +775,33 @@ pLink = try $ do

778775pImage :: PandocMonad m => TagParser m Inlines

779776pImage = do

780777 tag <- pSelfClosing (=="img") (isJust . lookup "src")

781− url <- canonicalizeUrl $ T.unpack $ fromAttrib "src" tag

782− let title = T.unpack $ fromAttrib "title" tag

783− let alt = T.unpack $ fromAttrib "alt" tag

784− let uid = T.unpack $ fromAttrib "id" tag

785− let cls = words $ T.unpack $ fromAttrib "class" tag

778+ url <- canonicalizeUrl $ fromAttrib "src" tag

779+ let title = fromAttrib "title" tag

780+ let alt = fromAttrib "alt" tag

781+ let uid = fromAttrib "id" tag

782+ let cls = T.words $ fromAttrib "class" tag

786783 let getAtt k = case fromAttrib k tag of

787784 "" -> []

788− v -> [(T.unpack k, T.unpack v)]

785+ v -> [(k, v)]

789786 let kvs = concatMap getAtt ["width", "height", "sizes", "srcset"]

790787 return $ B.imageWith (uid, cls, kvs) (escapeURI url) title (B.text alt)

791788

792−pCodeWithClass :: PandocMonad m => [(T.Text,String)] -> TagParser m Inlines

793−pCodeWithClass elemToClass = try $ do

789+pCodeWithClass :: PandocMonad m => [(T.Text,Text)] -> TagParser m Inlines

790+pCodeWithClass elemToClass = try $ do

794791 let tagTest = flip elem . fmap fst $ elemToClass

795792 TagOpen open attr' <- pSatisfy $ tagOpen tagTest (const True)

796793 result <- manyTill pAny (pCloses open)

797794 let (ids,cs,kvs) = mkAttr . toStringAttr $ attr'

798795 cs' = maybe cs (:cs) . lookup open $ elemToClass

799796 return . B.codeWith (ids,cs',kvs) .

800− unwords . lines . T.unpack . innerText $ result

797+ T.unwords . T.lines . innerText $ result

801798

802799pCode :: PandocMonad m => TagParser m Inlines

803800pCode = try $ do

804801 (TagOpen open attr') <- pSatisfy $ tagOpen (`elem` ["code","tt"]) (const True)

805802 let attr = toStringAttr attr'

806803 result <- manyTill pAny (pCloses open)

807− return $ B.codeWith (mkAttr attr) $ unwords $ lines $ T.unpack $

808− innerText result

804+ return $ B.codeWith (mkAttr attr) $ T.unwords $ T.lines $ innerText result

809805

810806pSpan :: PandocMonad m => TagParser m Inlines

811807pSpan = try $ do

@@ -817,7 +813,7 @@ pSpan = try $ do

817813 where styleAttr = fromMaybe "" $ lookup "style" attr

818814 fontVariant = fromMaybe "" $ pickStyleAttrProps ["font-variant"] styleAttr

819815 classes = fromMaybe [] $

820− words <$> lookup "class" attr

816+ T.words <$> lookup "class" attr

821817 let tag = if isSmallCaps then B.smallcaps else B.spanWith (mkAttr attr)

822818 return $ tag contents

823819

@@ -829,18 +825,17 @@ pRawHtmlInline = do

829825 then pSatisfy (not . isBlockTag)

830826 else pSatisfy isInlineTag

831827 exts <- getOption readerExtensions

832− let raw = T.unpack $ renderTags' [result]

828+ let raw = renderTags' [result]

833829 if extensionEnabled Ext_raw_html exts

834830 then return $ B.rawInline "html" raw

835831 else ignore raw

836832

837−mathMLToTeXMath :: String -> Either String String

833+mathMLToTeXMath :: Text -> Either Text Text

838834mathMLToTeXMath s = writeTeX <$> readMathML s

839835

840−toStringAttr :: [(Text, Text)] -> [(String, String)]

836+toStringAttr :: [(Text, Text)] -> [(Text, Text)]

841837toStringAttr = map go

842− where go (x,y) = (T.unpack (fromMaybe x $ T.stripPrefix "data-" x),

843− T.unpack y)

838+ where go (x,y) = (fromMaybe x $ T.stripPrefix "data-" x, y)

844839

845840pScriptMath :: PandocMonad m => TagParser m Inlines

846841pScriptMath = try $ do

@@ -849,8 +844,7 @@ pScriptMath = try $ do

849844 Just x | "math/tex" `T.isPrefixOf` x

850845 -> return $ "display" `T.isSuffixOf` x

851846 _ -> mzero

852− contents <- T.unpack . innerText <$>

853− manyTill pAny (pSatisfy (matchTagClose "script"))

847+ contents <- innerText <$> manyTill pAny (pSatisfy (matchTagClose "script"))

854848 return $ (if isdisplay then B.displayMath else B.math) contents

855849

856850pMath :: PandocMonad m => Bool -> TagParser m Inlines

@@ -862,11 +856,11 @@ pMath inCase = try $ do

862856 unless inCase $

863857 guard (maybe True (== mathMLNamespace) (lookup "xmlns" attr))

864858 contents <- manyTill pAny (pSatisfy (matchTagClose "math"))

865− case mathMLToTeXMath (T.unpack $ renderTags $

859+ case mathMLToTeXMath (renderTags $

866860 [open] <> contents <> [TagClose "math"]) of

867861 Left _ -> return $ B.spanWith ("",["math"],attr) $ B.text $

868− T.unpack $ innerText contents

869− Right [] -> return mempty

862+ innerText contents

863+ Right "" -> return mempty

870864 Right x -> return $ case lookup "display" attr of

871865 Just "block" -> B.displayMath x

872866 _ -> B.math x

@@ -925,7 +919,7 @@ pTagText = try $ do

925919 parsed <- lift $ lift $

926920 flip runReaderT qu $ runParserT (many pTagContents) st "text" str

927921 case parsed of

928− Left _ -> throwError $ PandocParseError $ "Could not parse `" <> T.unpack str <> "'"

922+ Left _ -> throwError $ PandocParseError $ "Could not parse `" <> str <> "'"

929923 Right result -> return $ mconcat result

930924

931925pBlank :: PandocMonad m => TagParser m ()

@@ -954,11 +948,11 @@ pRawTeX = do

954948 guardEnabled Ext_raw_tex

955949 inp <- getInput

956950 st <- getState

957− res <- lift $ runParserT (withRaw rawLaTeXInline) st "chunk" (T.unpack inp)

951+ res <- lift $ runParserT (withRaw rawLaTeXInline) st "chunk" inp

958952 case res of

959953 Left _ -> mzero

960954 Right (contents, raw) -> do

961− _ <- count (length raw) anyChar

955+ _ <- count (T.length raw) anyChar

962956 return $ B.rawInline "tex" contents

963957

964958pStr :: PandocMonad m => InlinesParser m Inlines

@@ -966,7 +960,7 @@ pStr = do

966960 result <- many1 $ satisfy $ \c ->

967961 not (isSpace c) && not (isSpecial c) && not (isBad c)

968962 updateLastStrPos

969− return $ B.str result

963+ return $ B.str $ T.pack result

970964

971965isSpecial :: Char -> Bool

972966isSpecial '"' = True

@@ -982,7 +976,7 @@ isSpecial '\8221' = True

982976isSpecial _ = False

983977

984978pSymbol :: PandocMonad m => InlinesParser m Inlines

985−pSymbol = satisfy isSpecial >>= return . B.str . (:[])

979+pSymbol = satisfy isSpecial >>= return . B.str . T.singleton

986980

987981isBad :: Char -> Bool

988982isBad c = c >= '\128' && c <= '\159' -- not allowed in HTML

@@ -1019,7 +1013,7 @@ pBad = do

10191013 '\158' -> '\382'

10201014 '\159' -> '\376'

10211015 _ -> '?'

1022− return $ B.str [c']

1016+ return $ B.str $ T.singleton c'

10231017

10241018pSpace :: PandocMonad m => InlinesParser m Inlines

10251019pSpace = many1 (satisfy isSpace) >>= \xs ->

@@ -1156,8 +1150,8 @@ _ `closes` _ = False

11561150

11571151-- | Matches a stretch of HTML in balanced tags.

11581152htmlInBalanced :: Monad m

1159− => (Tag String -> Bool)

1160− -> ParserT String st m String

1153+ => (Tag Text -> Bool)

1154+ -> ParserT Text st m Text

11611155htmlInBalanced f = try $ do

11621156 lookAhead (char '<')

11631157 inp <- getInput

@@ -1174,21 +1168,21 @@ htmlInBalanced f = try $ do

11741168 (TagClose _ : TagPosition er ec : _) -> do

11751169 let ls = er - sr

11761170 let cs = ec - sc

1177− lscontents <- unlines <$> count ls anyLine

1171+ lscontents <- T.unlines <$> count ls anyLine

11781172 cscontents <- count cs anyChar

11791173 closetag <- do

11801174 x <- many (satisfy (/='>'))

11811175 char '>'

11821176 return (x <> ">")

1183− return (lscontents <> cscontents <> closetag)

1177+ return $ lscontents <> T.pack cscontents <> T.pack closetag

11841178 _ -> mzero

11851179 _ -> mzero

11861180

1187−htmlInBalanced' :: String

1188− -> [Tag String]

1189− -> [Tag String]

1181+htmlInBalanced' :: Text

1182+ -> [Tag Text]

1183+ -> [Tag Text]

11901184htmlInBalanced' tagname ts = fromMaybe [] $ go 0 ts

1191− where go :: Int -> [Tag String] -> Maybe [Tag String]

1185+ where go :: Int -> [Tag Text] -> Maybe [Tag Text]

11921186 go n (t@(TagOpen tn' _):rest) | tn' == tagname =

11931187 (t :) <$> go (n + 1) rest

11941188 go 1 (t@(TagClose tn'):_) | tn' == tagname =

@@ -1204,8 +1198,8 @@ hasTagWarning _ = False

12041198

12051199-- | Matches a tag meeting a certain condition.

12061200htmlTag :: (HasReaderOptions st, Monad m)

1207− => (Tag String -> Bool)

1208− -> ParserT [Char] st m (Tag String, String)

1201+ => (Tag Text -> Bool)

1202+ -> ParserT Text st m (Tag Text, Text)

12091203htmlTag f = try $ do

12101204 lookAhead (char '<')

12111205 startpos <- getPosition

@@ -1213,7 +1207,7 @@ htmlTag f = try $ do

12131207 let ts = canonicalizeTags $ parseTagsOptions

12141208 parseOptions{ optTagWarning = False

12151209 , optTagPosition = True }

1216− (inp ++ " ") -- add space to ensure that

1210+ (inp <> " ") -- add space to ensure that

12171211 -- we get a TagPosition after the tag

12181212 (next, ln, col) <- case ts of

12191213 (TagPosition{} : next : TagPosition ln col : _)

@@ -1225,13 +1219,12 @@ htmlTag f = try $ do

12251219 -- so we exclude . even though it's a valid character

12261220 -- in XML element names

12271221 let isNameChar c = isAlphaNum c || c == ':' || c == '-' || c == '_'

1228− let isName s = case s of

1229− [] -> False

1230− (c:cs) -> isLetter c && all isNameChar cs

1231− let isPI s = case s of

1232− ('?':_) -> True -- processing instruction

1233− _ -> False

1234−

1222+ let isName s = case T.uncons s of

1223+ Nothing -> False

1224+ Just (c, cs) -> isLetter c && T.all isNameChar cs

1225+ let isPI s = case T.uncons s of

1226+ Just ('?', _) -> True -- processing instruction

1227+ _ -> False

12351228 let endpos = if ln == 1

12361229 then setSourceColumn startpos

12371230 (sourceColumn startpos + (col - 1))

@@ -1247,18 +1240,18 @@ htmlTag f = try $ do

12471240 -- basic sanity check, since the parser is very forgiving

12481241 -- and finds tags in stuff like x<y)

12491242 guard $ isName tagname || isPI tagname

1250− guard $ not $ null tagname

1243+ guard $ not $ T.null tagname

12511244 -- <https://example.org> should NOT be a tag either.

12521245 -- tagsoup will parse it as TagOpen "https:" [("example.org","")]

1253− guard $ last tagname /= ':'

1246+ guard $ T.last tagname /= ':'

12541247 char '<'

12551248 rendered <- manyTill anyChar endAngle

1256− return (next, "<" ++ rendered ++ ">")

1249+ return (next, T.pack $ "<" ++ rendered ++ ">")

12571250 case next of

12581251 TagComment s

1259− | "<!--" `isPrefixOf` inp -> do

1252+ | "<!--" `T.isPrefixOf` inp -> do

12601253 string "<!--"

1261− count (length s) anyChar

1254+ count (T.length s) anyChar

12621255 string "-->"

12631256 stripComments <- getOption readerStripComments

12641257 if stripComments

@@ -1272,12 +1265,12 @@ htmlTag f = try $ do

12721265 handleTag tagname

12731266 _ -> mzero

12741267

1275−mkAttr :: [(String, String)] -> Attr

1268+mkAttr :: [(Text, Text)] -> Attr

12761269mkAttr attr = (attribsId, attribsClasses, attribsKV)

12771270 where attribsId = fromMaybe "" $ lookup "id" attr

1278− attribsClasses = words (fromMaybe "" $ lookup "class" attr) <> epubTypes

1271+ attribsClasses = T.words (fromMaybe "" $ lookup "class" attr) <> epubTypes

12791272 attribsKV = filter (\(k,_) -> k /= "class" && k /= "id") attr

1280− epubTypes = words $ fromMaybe "" $ lookup "epub:type" attr

1273+ epubTypes = T.words $ fromMaybe "" $ lookup "epub:type" attr

12811274

12821275-- Strip namespace prefixes

12831276stripPrefixes :: [Tag Text] -> [Tag Text]

@@ -1304,11 +1297,11 @@ isSpace _ = False

13041297-- Utilities

13051298

13061299-- | Adjusts a url according to the document's base URL.

1307−canonicalizeUrl :: PandocMonad m => String -> TagParser m String

1300+canonicalizeUrl :: PandocMonad m => Text -> TagParser m Text

13081301canonicalizeUrl url = do

13091302 mbBaseHref <- baseHref <$> getState

1310− return $ case (parseURIReference url, mbBaseHref) of

1311− (Just rel, Just bs) -> show (rel `nonStrictRelativeTo` bs)

1303+ return $ case (parseURIReference (T.unpack url), mbBaseHref) of

1304+ (Just rel, Just bs) -> tshow (rel `nonStrictRelativeTo` bs)

13121305 _ -> url

13131306

13141307

src/Text/Pandoc/Readers/Haddock.hs+27−25

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE CPP #-}

2+{-# LANGUAGE CPP #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Readers.Haddock

@@ -17,9 +18,10 @@ module Text.Pandoc.Readers.Haddock

1718

1819import Prelude

1920import Control.Monad.Except (throwError)

20−import Data.List (intersperse, stripPrefix)

21+import Data.List (intersperse)

2122import Data.Maybe (fromMaybe)

2223import Data.Text (Text, unpack)

24+import qualified Data.Text as T

2325import Documentation.Haddock.Parser

2426import Documentation.Haddock.Types as H

2527import Text.Pandoc.Builder (Blocks, Inlines)

@@ -28,7 +30,7 @@ import Text.Pandoc.Class (PandocMonad)

2830import Text.Pandoc.Definition

2931import Text.Pandoc.Error

3032import Text.Pandoc.Options

31−import Text.Pandoc.Shared (crFilter, splitBy, trim)

33+import Text.Pandoc.Shared (crFilter, splitTextBy, trim)

3234

3335

3436-- | Parse Haddock markup and return a 'Pandoc' document.

@@ -51,7 +53,7 @@ docHToBlocks d' =

5153 case d' of

5254 DocEmpty -> mempty

5355 DocAppend (DocParagraph (DocHeader h)) (DocParagraph (DocAName ident)) ->

54− B.headerWith (ident,[],[]) (headerLevel h)

56+ B.headerWith (T.pack ident,[],[]) (headerLevel h)

5557 (docHToInlines False $ headerTitle h)

5658 DocAppend d1 d2 -> mappend (docHToBlocks d1) (docHToBlocks d2)

5759 DocString _ -> inlineFallback

@@ -73,12 +75,12 @@ docHToBlocks d' =

7375 DocDefList items -> B.definitionList (map (\(d,t) ->

7476 (docHToInlines False d,

7577 [consolidatePlains $ docHToBlocks t])) items)

76− DocCodeBlock (DocString s) -> B.codeBlockWith ("",[],[]) s

78+ DocCodeBlock (DocString s) -> B.codeBlockWith ("",[],[]) $ T.pack s

7779 DocCodeBlock d -> B.para $ docHToInlines True d

7880 DocHyperlink _ -> inlineFallback

7981 DocPic _ -> inlineFallback

8082 DocAName _ -> inlineFallback

81− DocProperty s -> B.codeBlockWith ("",["property","haskell"],[]) (trim s)

83+ DocProperty s -> B.codeBlockWith ("",["property","haskell"],[]) (trim $ T.pack s)

8284 DocExamples es -> mconcat $ map (\e ->

8385 makeExample ">>>" (exampleExpression e) (exampleResult e)) es

8486 DocTable H.Table{ tableHeaderRows = headerRows

@@ -114,58 +116,58 @@ docHToInlines isCode d' =

114116 (docHToInlines isCode d2)

115117 DocString s

116118 | isCode -> mconcat $ intersperse B.linebreak

117− $ map B.code $ splitBy (=='\n') s

118− | otherwise -> B.text s

119+ $ map B.code $ splitTextBy (=='\n') $ T.pack s

120+ | otherwise -> B.text $ T.pack s

119121 DocParagraph _ -> mempty

120122 DocIdentifier ident ->

121123 case toRegular (DocIdentifier ident) of

122− DocIdentifier s -> B.codeWith ("",["haskell","identifier"],[]) s

124+ DocIdentifier s -> B.codeWith ("",["haskell","identifier"],[]) $ T.pack s

123125 _ -> mempty

124− DocIdentifierUnchecked s -> B.codeWith ("",["haskell","identifier"],[]) s

125− DocModule s -> B.codeWith ("",["haskell","module"],[]) s

126+ DocIdentifierUnchecked s -> B.codeWith ("",["haskell","identifier"],[]) $ T.pack s

127+ DocModule s -> B.codeWith ("",["haskell","module"],[]) $ T.pack s

126128 DocWarning _ -> mempty -- TODO

127129 DocEmphasis d -> B.emph (docHToInlines isCode d)

128− DocMonospaced (DocString s) -> B.code s

130+ DocMonospaced (DocString s) -> B.code $ T.pack s

129131 DocMonospaced d -> docHToInlines True d

130132 DocBold d -> B.strong (docHToInlines isCode d)

131− DocMathInline s -> B.math s

132− DocMathDisplay s -> B.displayMath s

133+ DocMathInline s -> B.math $ T.pack s

134+ DocMathDisplay s -> B.displayMath $ T.pack s

133135 DocHeader _ -> mempty

134136 DocUnorderedList _ -> mempty

135137 DocOrderedList _ -> mempty

136138 DocDefList _ -> mempty

137139 DocCodeBlock _ -> mempty

138− DocHyperlink h -> B.link (hyperlinkUrl h) (hyperlinkUrl h)

139− (maybe (B.text $ hyperlinkUrl h) (docHToInlines isCode)

140+ DocHyperlink h -> B.link (T.pack $ hyperlinkUrl h) (T.pack $ hyperlinkUrl h)

141+ (maybe (B.text $ T.pack $ hyperlinkUrl h) (docHToInlines isCode)

140142 (hyperlinkLabel h))

141− DocPic p -> B.image (pictureUri p) (fromMaybe (pictureUri p) $ pictureTitle p)

142− (maybe mempty B.text $ pictureTitle p)

143− DocAName s -> B.spanWith (s,["anchor"],[]) mempty

143+ DocPic p -> B.image (T.pack $ pictureUri p) (T.pack $ fromMaybe (pictureUri p) $ pictureTitle p)

144+ (maybe mempty (B.text . T.pack) $ pictureTitle p)

145+ DocAName s -> B.spanWith (T.pack s,["anchor"],[]) mempty

144146 DocProperty _ -> mempty

145147 DocExamples _ -> mempty

146148 DocTable _ -> mempty

147149

148150-- | Create an 'Example', stripping superfluous characters as appropriate

149−makeExample :: String -> String -> [String] -> Blocks

151+makeExample :: T.Text -> String -> [String] -> Blocks

150152makeExample prompt expression result =

151153 B.para $ B.codeWith ("",["prompt"],[]) prompt

152154 <> B.space

153− <> B.codeWith ([], ["haskell","expr"], []) (trim expression)

155+ <> B.codeWith ("", ["haskell","expr"], []) (trim $ T.pack expression)

154156 <> B.linebreak

155157 <> mconcat (intersperse B.linebreak $ map coder result')

156158 where

157159 -- 1. drop trailing whitespace from the prompt, remember the prefix

158− prefix = takeWhile (`elem` " \t") prompt

160+ prefix = T.takeWhile (`elem` (" \t" :: String)) prompt

159161

160162 -- 2. drop, if possible, the exact same sequence of whitespace

161163 -- characters from each result line

162164 --

163165 -- 3. interpret lines that only contain the string "<BLANKLINE>" as an

164166 -- empty line

165− result' = map (substituteBlankLine . tryStripPrefix prefix) result

167+ result' = map (substituteBlankLine . tryStripPrefix prefix . T.pack) result

166168 where

167− tryStripPrefix xs ys = fromMaybe ys $ stripPrefix xs ys

169+ tryStripPrefix xs ys = fromMaybe ys $ T.stripPrefix xs ys

168170

169171 substituteBlankLine "<BLANKLINE>" = ""

170172 substituteBlankLine line = line

171− coder = B.codeWith ([], ["result"], [])

173+ coder = B.codeWith ("", ["result"], [])

src/Text/Pandoc/Readers/Ipynb.hs+39−40

@@ -4,6 +4,7 @@

44{-# LANGUAGE OverloadedStrings #-}

55{-# LANGUAGE FlexibleContexts #-}

66{-# LANGUAGE ScopedTypeVariables #-}

7+{-# LANGUAGE OverloadedStrings #-}

78{- |

89 Module : Text.Pandoc.Readers.Ipynb

@@ -19,7 +20,6 @@ module Text.Pandoc.Readers.Ipynb ( readIpynb )

1920where

2021import Prelude

2122import Data.Char (isDigit)

22−import Data.List (isPrefixOf)

2323import Data.Maybe (fromMaybe)

2424import Data.Digest.Pure.SHA (sha1, showDigest)

2525import Text.Pandoc.Options

@@ -30,6 +30,7 @@ import Text.Pandoc.Definition

3030import Data.Ipynb as Ipynb

3131import Text.Pandoc.Class

3232import Text.Pandoc.MIME (extensionFromMimeType)

33+import Text.Pandoc.Shared (tshow)

3334import Text.Pandoc.UTF8

3435import Text.Pandoc.Walk (walk)

3536import Text.Pandoc.Error

@@ -51,15 +52,15 @@ readIpynb opts t = do

5152 Left _ ->

5253 case eitherDecode src of

5354 Right (notebook3 :: Notebook NbV3) -> notebookToPandoc opts notebook3

54− Left err -> throwError $ PandocIpynbDecodingError err

55+ Left err -> throwError $ PandocIpynbDecodingError $ T.pack err

5556

5657notebookToPandoc :: PandocMonad m

5758 => ReaderOptions -> Notebook a -> m Pandoc

5859notebookToPandoc opts notebook = do

5960 let cells = notebookCells notebook

6061 let (fmt,fmtminor) = notebookFormat notebook

61− let m = M.insert "nbformat" (MetaString $ show fmt) $

62− M.insert "nbformat_minor" (MetaString $ show fmtminor) $

62+ let m = M.insert "nbformat" (MetaString $ tshow fmt) $

63+ M.insert "nbformat_minor" (MetaString $ tshow fmtminor) $

6364 jsonMetaToMeta (notebookMetadata notebook)

6465 let lang = case M.lookup "kernelspec" m of

6566 Just (MetaMap ks) ->

@@ -72,7 +73,7 @@ notebookToPandoc opts notebook = do

7273 return $ Pandoc (Meta $ M.insert "jupyter" (MetaMap m) mempty) blocks

7374

7475cellToBlocks :: PandocMonad m

75− => ReaderOptions -> String -> Cell a -> m B.Blocks

76+ => ReaderOptions -> Text -> Cell a -> m B.Blocks

7677cellToBlocks opts lang c = do

7778 let Source ts = cellSource c

7879 let source = mconcat ts

@@ -100,19 +101,18 @@ cellToBlocks opts lang c = do

100101 "text/markdown" -> "markdown"

101102 "text/x-rsrt" -> "rst"

102103 _ -> format

103− return $ B.divWith ("",["cell","raw"],kvs) $ B.rawBlock format'

104− $ T.unpack source

104+ return $ B.divWith ("",["cell","raw"],kvs) $ B.rawBlock format' source

105105 Ipynb.Code{ codeOutputs = outputs, codeExecutionCount = ec } -> do

106106 outputBlocks <- mconcat <$> mapM outputToBlock outputs

107− let kvs' = maybe kvs (\x -> ("execution_count", show x):kvs) ec

107+ let kvs' = maybe kvs (\x -> ("execution_count", tshow x):kvs) ec

108108 return $ B.divWith ("",["cell","code"],kvs') $

109− B.codeBlockWith ("",[lang],[]) (T.unpack source)

109+ B.codeBlockWith ("",[lang],[]) source

110110 <> outputBlocks

111111

112112-- Remove attachment: prefix from images...

113113fixImage :: Inline -> Inline

114114fixImage (Image attr lab (src,tit))

115− | "attachment:" `isPrefixOf` src = Image attr lab (drop 11 src, tit)

115+ | "attachment:" `T.isPrefixOf` src = Image attr lab (T.drop 11 src, tit)

116116fixImage x = x

117117

118118addAttachment :: PandocMonad m => (Text, MimeBundle) -> m ()

@@ -120,19 +120,19 @@ addAttachment (fname, mimeBundle) = do

120120 let fp = T.unpack fname

121121 case M.toList (unMimeBundle mimeBundle) of

122122 (mimeType, BinaryData bs):_ ->

123− insertMedia fp (Just $ T.unpack mimeType) (BL.fromStrict bs)

123+ insertMedia fp (Just mimeType) (BL.fromStrict bs)

124124 (mimeType, TextualData t):_ ->

125− insertMedia fp (Just $ T.unpack mimeType)

125+ insertMedia fp (Just mimeType)

126126 (BL.fromStrict $ TE.encodeUtf8 t)

127127 (mimeType, JsonData v):_ ->

128− insertMedia fp (Just $ T.unpack mimeType) (encode v)

129− [] -> report $ CouldNotFetchResource fp "no attachment"

128+ insertMedia fp (Just mimeType) (encode v)

129+ [] -> report $ CouldNotFetchResource fname "no attachment"

130130

131131outputToBlock :: PandocMonad m => Output a -> m B.Blocks

132132outputToBlock Stream{ streamName = sName,

133133 streamText = Source text } = do

134− return $ B.divWith ("",["output","stream",T.unpack sName],[])

135− $ B.codeBlock $ T.unpack . mconcat $ text

134+ return $ B.divWith ("",["output","stream",sName],[])

135+ $ B.codeBlock $ T.concat $ text

136136outputToBlock DisplayData{ displayData = data',

137137 displayMetadata = metadata' } =

138138 B.divWith ("",["output", "display_data"],[]) <$>

@@ -140,15 +140,15 @@ outputToBlock DisplayData{ displayData = data',

140140outputToBlock ExecuteResult{ executeCount = ec,

141141 executeData = data',

142142 executeMetadata = metadata' } =

143− B.divWith ("",["output", "execute_result"],[("execution_count",show ec)])

143+ B.divWith ("",["output", "execute_result"],[("execution_count",tshow ec)])

144144 <$> handleData metadata' data'

145145outputToBlock Err{ errName = ename,

146146 errValue = evalue,

147147 errTraceback = traceback } = do

148148 return $ B.divWith ("",["output","error"],

149− [("ename",T.unpack ename),

150− ("evalue",T.unpack evalue)])

151− $ B.codeBlock $ T.unpack . T.unlines $ traceback

149+ [("ename",ename),

150+ ("evalue",evalue)])

151+ $ B.codeBlock $ T.unlines $ traceback

152152

153153-- We want to display the richest output possible given

154154-- the output format.

@@ -174,54 +174,53 @@ handleData metadata (MimeBundle mb) =

174174 let metaPairs = jsonMetaToPairs meta

175175 let bl = BL.fromStrict bs

176176 -- SHA1 hash for filename

177− let mt' = T.unpack mt

178− let fname = showDigest (sha1 bl) ++

179− case extensionFromMimeType mt' of

177+ let fname = T.pack (showDigest (sha1 bl)) <>

178+ case extensionFromMimeType mt of

180179 Nothing -> ""

181− Just ext -> '.':ext

182− insertMedia fname (Just mt') bl

180+ Just ext -> "." <> ext

181+ insertMedia (T.unpack fname) (Just mt) bl

183182 return $ B.para $ B.imageWith ("",[],metaPairs) fname "" mempty

184183 | otherwise = return mempty

185184

186185 dataBlock ("text/html", TextualData t)

187− = return $ B.rawBlock "html" $ T.unpack t

186+ = return $ B.rawBlock "html" $ t

188187

189188 dataBlock ("text/latex", TextualData t)

190− = return $ B.rawBlock "latex" $ T.unpack t

189+ = return $ B.rawBlock "latex" $ t

191190

192191 dataBlock ("text/plain", TextualData t) =

193− return $ B.codeBlock $ T.unpack t

192+ return $ B.codeBlock $ t

194193

195194 dataBlock (_, JsonData v) =

196− return $ B.codeBlockWith ("",["json"],[]) $ toStringLazy $ encode v

195+ return $ B.codeBlockWith ("",["json"],[]) $ T.pack $ toStringLazy $ encode v

197196

198197 dataBlock _ = return mempty

199198

200−jsonMetaToMeta :: JSONMeta -> M.Map String MetaValue

201−jsonMetaToMeta = M.mapKeys T.unpack . M.map valueToMetaValue

199+jsonMetaToMeta :: JSONMeta -> M.Map Text MetaValue

200+jsonMetaToMeta = M.map valueToMetaValue

202201 where

203202 valueToMetaValue :: Value -> MetaValue

204203 valueToMetaValue x@(Object{}) =

205204 case fromJSON x of

206− Error s -> MetaString s

205+ Error s -> MetaString $ T.pack s

207206 Success jm' -> MetaMap $ jsonMetaToMeta jm'

208207 valueToMetaValue x@(Array{}) =

209208 case fromJSON x of

210− Error s -> MetaString s

209+ Error s -> MetaString $ T.pack s

211210 Success xs -> MetaList $ map valueToMetaValue xs

212211 valueToMetaValue (Bool b) = MetaBool b

213− valueToMetaValue (String t) = MetaString (T.unpack t)

212+ valueToMetaValue (String t) = MetaString t

214213 valueToMetaValue (Number n)

215− | Scientific.isInteger n = MetaString (show (floor n :: Integer))

216− | otherwise = MetaString (show n)

214+ | Scientific.isInteger n = MetaString (tshow (floor n :: Integer))

215+ | otherwise = MetaString (tshow n)

217216 valueToMetaValue Aeson.Null = MetaString ""

218217

219−jsonMetaToPairs :: JSONMeta -> [(String, String)]

220−jsonMetaToPairs = M.toList . M.mapKeys T.unpack . M.map

218+jsonMetaToPairs :: JSONMeta -> [(Text, Text)]

219+jsonMetaToPairs = M.toList . M.map

221220 (\case

222221 String t

223222 | not (T.all isDigit t)

224223 , t /= "true"

225224 , t /= "false"

226− -> T.unpack t

227− x -> UTF8.toStringLazy $ Aeson.encode x)

225+ -> t

226+ x -> T.pack $ UTF8.toStringLazy $ Aeson.encode x)

src/Text/Pandoc/Readers/JATS.hs+42−34

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE TupleSections #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Readers.JATS

@@ -76,13 +77,13 @@ convertEntity :: String -> String

7677convertEntity e = Data.Maybe.fromMaybe (map toUpper e) (lookupEntity e)

7778

7879-- convenience function to get an attribute value, defaulting to ""

79−attrValue :: String -> Element -> String

80+attrValue :: String -> Element -> Text

8081attrValue attr =

8182 fromMaybe "" . maybeAttrValue attr

8283

83−maybeAttrValue :: String -> Element -> Maybe String

84+maybeAttrValue :: String -> Element -> Maybe Text

8485maybeAttrValue attr elt =

85− lookupAttrBy (\x -> qName x == attr) (elAttribs elt)

86+ T.pack <$> lookupAttrBy (\x -> qName x == attr) (elAttribs elt)

8687

8788-- convenience function

8889named :: String -> Element -> Bool

@@ -90,7 +91,7 @@ named s e = qName (elName e) == s

9091

9192--

9293

93−addMeta :: PandocMonad m => ToMetaValue a => String -> a -> JATS m ()

94+addMeta :: PandocMonad m => ToMetaValue a => Text -> a -> JATS m ()

9495addMeta field val = modify (setMeta field val)

9596

9697instance HasMeta JATSState where

@@ -126,15 +127,13 @@ isBlockElement (Elem e) = qName (elName e) `S.member` blocktags

126127isBlockElement _ = False

127128

128129-- Trim leading and trailing newline characters

129−trimNl :: String -> String

130−trimNl = reverse . go . reverse . go

131− where go ('\n':xs) = xs

132− go xs = xs

130+trimNl :: Text -> Text

131+trimNl = T.dropAround (== '\n')

133132

134133-- function that is used by both graphic (in parseBlock)

135134-- and inline-graphic (in parseInline)

136135getGraphic :: PandocMonad m

137− => Maybe (Inlines, String) -> Element -> JATS m Inlines

136+ => Maybe (Inlines, Text) -> Element -> JATS m Inlines

138137getGraphic mbfigdata e = do

139138 let atVal a = attrValue a e

140139 (ident, title, caption) =

@@ -142,7 +141,7 @@ getGraphic mbfigdata e = do

142141 Just (capt, i) -> (i, "fig:" <> atVal "title", capt)

143142 Nothing -> (atVal "id", atVal "title",

144143 text (atVal "alt-text"))

145− attr = (ident, words $ atVal "role", [])

144+ attr = (ident, T.words $ atVal "role", [])

146145 imageUrl = atVal "href"

147146 return $ imageWith attr imageUrl title caption

148147

@@ -155,8 +154,8 @@ parseBlock :: PandocMonad m => Content -> JATS m Blocks

155154parseBlock (Text (CData CDataRaw _ _)) = return mempty -- DOCTYPE

156155parseBlock (Text (CData _ s _)) = if all isSpace s

157156 then return mempty

158− else return $ plain $ trimInlines $ text s

159−parseBlock (CRef x) = return $ plain $ str $ map toUpper x

157+ else return $ plain $ trimInlines $ text $ T.pack s

158+parseBlock (CRef x) = return $ plain $ str $ T.toUpper $ T.pack x

160159parseBlock (Elem e) =

161160 case qName (elName e) of

162161 "p" -> parseMixed para (elContent e)

@@ -167,7 +166,7 @@ parseBlock (Elem e) =

167166 "bullet" -> bulletList <$> listitems

168167 listType -> do

169168 let start = fromMaybe 1 $

170− (strContent <$> (filterElement (named "list-item") e

169+ (textContent <$> (filterElement (named "list-item") e

171170 >>= filterElement (named "label")))

172171 >>= safeRead

173172 orderedListWith (start, parseListStyleType listType, DefaultDelim)

@@ -204,7 +203,7 @@ parseBlock (Elem e) =

204203 "" -> []

205204 x -> [x]

206205 return $ codeBlockWith (attrValue "id" e, classes', [])

207− $ trimNl $ strContentRecursive e

206+ $ trimNl $ textContentRecursive e

208207 parseBlockquote = do

209208 attrib <- case filterChild (named "attribution") e of

210209 Nothing -> return mempty

@@ -267,9 +266,9 @@ parseBlock (Elem e) =

267266 Just "right" -> AlignRight

268267 Just "center" -> AlignCenter

269268 _ -> AlignDefault

270− let toWidth c = case findAttr (unqual "colwidth") c of

269+ let toWidth c = case findAttrText (unqual "colwidth") c of

271270 Just w -> fromMaybe 0

272− $ safeRead $ '0': filter (\x ->

271+ $ safeRead $ "0" <> T.filter (\x ->

273272 isDigit x || x == '.') w

274273 Nothing -> 0 :: Double

275274 let numrows = foldl' max 0 $ map length bodyrows

@@ -363,7 +362,7 @@ parseRefList e = do

363362 return mempty

364363

365364parseRef :: PandocMonad m

366− => Element -> JATS m (Map.Map String MetaValue)

365+ => Element -> JATS m (Map.Map Text MetaValue)

367366parseRef e = do

368367 let refId = text $ attrValue "id" e

369368 let getInlineText n = maybe (return mempty) getInlines . filterChild (named n)

@@ -396,7 +395,7 @@ parseRef e = do

396395 family <- maybe (return mempty) getInlines

397396 $ filterChild (named "surname") nm

398397 return $ toMetaValue $ Map.fromList [

399− ("given", given)

398+ ("given" :: Text, given)

400399 , ("family", family)

401400 ]

402401 personGroups <- mapM (\pg ->

@@ -406,7 +405,7 @@ parseRef e = do

406405 toMetaValue names))

407406 personGroups'

408407 return $ Map.fromList $

409− [ ("id", toMetaValue refId)

408+ [ ("id" :: Text, toMetaValue refId)

410409 , ("type", toMetaValue refType)

411410 , ("title", toMetaValue refTitle)

412411 , ("container-title", toMetaValue refContainerTitle)

@@ -415,7 +414,7 @@ parseRef e = do

415414 , ("title", toMetaValue refTitle)

416415 , ("issued", toMetaValue

417416 $ Map.fromList [

418− ("year", refYear)

417+ ("year" :: Text, refYear)

419418 ])

420419 , ("volume", toMetaValue refVolume)

421420 , ("page", toMetaValue refPages)

@@ -424,6 +423,15 @@ parseRef e = do

424423 Nothing -> return $ Map.insert "id" (toMetaValue refId) mempty

425424 -- TODO handle mixed-citation

426425

426+findAttrText :: QName -> Element -> Maybe Text

427+findAttrText x = fmap T.pack . findAttr x

428+

429+textContent :: Element -> Text

430+textContent = T.pack . strContent

431+

432+textContentRecursive :: Element -> Text

433+textContentRecursive = T.pack . strContentRecursive

434+

427435strContentRecursive :: Element -> String

428436strContentRecursive = strContent .

429437 (\e' -> e'{ elContent = map elementToStr $ elContent e' })

@@ -433,9 +441,9 @@ elementToStr (Elem e') = Text $ CData CDataText (strContentRecursive e') Nothing

433441elementToStr x = x

434442

435443parseInline :: PandocMonad m => Content -> JATS m Inlines

436−parseInline (Text (CData _ s _)) = return $ text s

444+parseInline (Text (CData _ s _)) = return $ text $ T.pack s

437445parseInline (CRef ref) =

438− return $ maybe (text $ map toUpper ref) text $ lookupEntity ref

446+ return $ maybe (text $ T.toUpper $ T.pack ref) text $ T.pack <$> lookupEntity ref

439447parseInline (Elem e) =

440448 case qName (elName e) of

441449 "italic" -> emph <$> innerInlines

@@ -464,7 +472,7 @@ parseInline (Elem e) =

464472 "xref" -> do

465473 ils <- innerInlines

466474 let rid = attrValue "rid" e

467− let rids = words rid

475+ let rids = T.words rid

468476 let refType = ("ref-type",) <$> maybeAttrValue "ref-type" e

469477 let attr = (attrValue "id" e, [], maybeToList refType)

470478 return $ if refType == Just ("ref-type","bibr")

@@ -477,13 +485,13 @@ parseInline (Elem e) =

477485 , citationNoteNum = 0

478486 , citationHash = 0}) rids)

479487 ils

480− else linkWith attr ('#' : rid) "" ils

488+ else linkWith attr ("#" <> rid) "" ils

481489 "ext-link" -> do

482490 ils <- innerInlines

483− let title = fromMaybe "" $ findAttr (QName "title" (Just "http://www.w3.org/1999/xlink") Nothing) e

491+ let title = fromMaybe "" $ findAttrText (QName "title" (Just "http://www.w3.org/1999/xlink") Nothing) e

484492 let href = case findAttr (QName "href" (Just "http://www.w3.org/1999/xlink") Nothing) e of

485− Just h -> h

486− _ -> '#' : attrValue "rid" e

493+ Just h -> T.pack h

494+ _ -> "#" <> attrValue "rid" e

487495 let ils' = if ils == mempty then str href else ils

488496 let attr = (attrValue "id" e, [], [])

489497 return $ linkWith attr href title ils'

@@ -491,23 +499,23 @@ parseInline (Elem e) =

491499 "disp-formula" -> formula displayMath

492500 "inline-formula" -> formula math

493501 "math" | qPrefix (elName e) == Just "mml" -> return . math $ mathML e

494− "tex-math" -> return . math $ strContent e

502+ "tex-math" -> return . math $ textContent e

495503

496− "email" -> return $ link ("mailto:" ++ strContent e) ""

497− $ str $ strContent e

498− "uri" -> return $ link (strContent e) "" $ str $ strContent e

504+ "email" -> return $ link ("mailto:" <> textContent e) ""

505+ $ str $ textContent e

506+ "uri" -> return $ link (textContent e) "" $ str $ textContent e

499507 "fn" -> (note . mconcat) <$>

500508 mapM parseBlock (elContent e)

501509 _ -> innerInlines

502510 where innerInlines = (trimInlines . mconcat) <$>

503511 mapM parseInline (elContent e)

504512 mathML x =

505− case readMathML . showElement $ everywhere (mkT removePrefix) x of

513+ case readMathML . T.pack . showElement $ everywhere (mkT removePrefix) x of

506514 Left _ -> mempty

507515 Right m -> writeTeX m

508516 formula constructor = do

509517 let whereToLook = fromMaybe e $ filterElement (named "alternatives") e

510− texMaths = map strContent $

518+ texMaths = map textContent $

511519 filterChildren (named "tex-math") whereToLook

512520 mathMLs = map mathML $

513521 filterChildren isMathML whereToLook

@@ -520,4 +528,4 @@ parseInline (Elem e) =

520528 let classes' = case attrValue "language" e of

521529 "" -> []

522530 l -> [l]

523− return $ codeWith (attrValue "id" e,classes',[]) $ strContentRecursive e

531+ return $ codeWith (attrValue "id" e,classes',[]) $ textContentRecursive e

src/Text/Pandoc/Readers/LaTeX.hs+168−166

@@ -1,9 +1,10 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleInstances #-}

33{-# LANGUAGE MultiParamTypeClasses #-}

44{-# LANGUAGE OverloadedStrings #-}

55{-# LANGUAGE PatternGuards #-}

66{-# LANGUAGE ScopedTypeVariables #-}

7+{-# LANGUAGE ViewPatterns #-}

78{- |

89 Module : Text.Pandoc.Readers.LaTeX

@@ -29,9 +30,9 @@ import Prelude

2930import Control.Applicative (many, optional, (<|>))

3031import Control.Monad

3132import Control.Monad.Except (throwError)

32−import Data.Char (isDigit, isLetter, toLower, toUpper, chr)

33+import Data.Char (isDigit, isLetter, toUpper, chr)

3334import Data.Default

34−import Data.List (intercalate, isPrefixOf)

35+import Data.List (intercalate)

3536import qualified Data.Map as M

3637import Data.Maybe (fromMaybe, maybeToList)

3738import qualified Data.Set as Set

@@ -44,7 +45,7 @@ import Text.Pandoc.Builder

4445import Text.Pandoc.Class (PandocMonad, PandocPure, getResourcePath, lookupEnv,

4546 readFileFromDirs, report, setResourcePath,

4647 setTranslations, translateTerm, trace, fileExists)

47−import Text.Pandoc.Error (PandocError ( PandocParseError, PandocParsecError))

48+import Text.Pandoc.Error (PandocError (PandocParseError, PandocParsecError))

4849import Text.Pandoc.Highlighting (fromListingsLanguage, languagesByExtension)

4950import Text.Pandoc.ImageSize (numUnit, showFl)

5051import Text.Pandoc.Logging

@@ -77,7 +78,7 @@ readLaTeX opts ltx = do

7778 (tokenize "source" (crFilter ltx))

7879 case parsed of

7980 Right result -> return result

80− Left e -> throwError $ PandocParsecError (T.unpack ltx) e

81+ Left e -> throwError $ PandocParsecError ltx e

8182

8283parseLaTeX :: PandocMonad m => LP m Pandoc

8384parseLaTeX = do

@@ -99,13 +100,13 @@ parseLaTeX = do

99100 walk (resolveRefs (sLabels st)) doc'

100101 return $ Pandoc meta bs'

101102

102−resolveRefs :: M.Map String [Inline] -> Inline -> Inline

103+resolveRefs :: M.Map Text [Inline] -> Inline -> Inline

103104resolveRefs labels x@(Link (ident,classes,kvs) _ _) =

104105 case (lookup "reference-type" kvs,

105106 lookup "reference" kvs) of

106107 (Just "ref", Just lab) ->

107108 case M.lookup lab labels of

108− Just txt -> Link (ident,classes,kvs) txt ('#':lab, "")

109+ Just txt -> Link (ident,classes,kvs) txt ("#" <> lab, "")

109110 Nothing -> x

110111 _ -> x

111112resolveRefs _ x = x

@@ -123,11 +124,11 @@ resolveRefs _ x = x

123124

124125

125126rawLaTeXBlock :: (PandocMonad m, HasMacros s, HasReaderOptions s)

126− => ParserT String s m String

127+ => ParserT Text s m Text

127128rawLaTeXBlock = do

128129 lookAhead (try (char '\\' >> letter))

129130 inp <- getInput

130− let toks = tokenize "source" $ T.pack inp

131+ let toks = tokenize "source" inp

131132 snd <$> (rawLaTeXParser toks False (macroDef (const mempty)) blocks

132133 <|> (rawLaTeXParser toks True

133134 (do choice (map controlSeq

@@ -151,14 +152,14 @@ beginOrEndCommand = try $ do

151152 (inlineEnvironments :: M.Map Text (LP PandocPure Inlines))

152153 then mzero

153154 else return $ rawBlock "latex"

154− (T.unpack (txt <> untokenize rawargs))

155+ (txt <> untokenize rawargs)

155156

156157rawLaTeXInline :: (PandocMonad m, HasMacros s, HasReaderOptions s)

157− => ParserT String s m String

158+ => ParserT Text s m Text

158159rawLaTeXInline = do

159160 lookAhead (try (char '\\' >> letter))

160161 inp <- getInput

161− let toks = tokenize "source" $ T.pack inp

162+ let toks = tokenize "source" inp

162163 raw <- snd <$>

163164 ( rawLaTeXParser toks True

164165 (mempty <$ (controlSeq "input" >> skipMany opt >> braced))

@@ -167,23 +168,23 @@ rawLaTeXInline = do

167168 inlines

168169 )

169170 finalbraces <- mconcat <$> many (try (string "{}")) -- see #5439

170− return $ raw <> finalbraces

171+ return $ raw <> T.pack finalbraces

171172

172−inlineCommand :: PandocMonad m => ParserT String ParserState m Inlines

173+inlineCommand :: PandocMonad m => ParserT Text ParserState m Inlines

173174inlineCommand = do

174175 lookAhead (try (char '\\' >> letter))

175176 inp <- getInput

176− let toks = tokenize "source" $ T.pack inp

177+ let toks = tokenize "source" inp

177178 fst <$> rawLaTeXParser toks True (inlineEnvironment <|> inlineCommand')

178179 inlines

179180

180181-- inline elements:

181182

182183word :: PandocMonad m => LP m Inlines

183−word = (str . T.unpack . untoken) <$> satisfyTok isWordTok

184+word = (str . untoken) <$> satisfyTok isWordTok

184185

185186regularSymbol :: PandocMonad m => LP m Inlines

186−regularSymbol = (str . T.unpack . untoken) <$> satisfyTok isRegularSymbol

187+regularSymbol = (str . untoken) <$> satisfyTok isRegularSymbol

187188 where isRegularSymbol (Tok _ Symbol t) = not $ T.any isSpecial t

188189 isRegularSymbol _ = False

189190 isSpecial c = c `Set.member` specialChars

@@ -199,14 +200,14 @@ inlineGroup = do

199200

200201doLHSverb :: PandocMonad m => LP m Inlines

201202doLHSverb =

202− (codeWith ("",["haskell"],[]) . T.unpack . untokenize)

203+ (codeWith ("",["haskell"],[]) . untokenize)

203204 <$> manyTill (satisfyTok (not . isNewlineTok)) (symbol '|')

204205

205−mkImage :: PandocMonad m => [(String, String)] -> String -> LP m Inlines

206−mkImage options src = do

206+mkImage :: PandocMonad m => [(Text, Text)] -> Text -> LP m Inlines

207+mkImage options (T.unpack -> src) = do

207208 let replaceTextwidth (k,v) =

208209 case numUnit v of

209− Just (num, "\\textwidth") -> (k, showFl (num * 100) ++ "%")

210+ Just (num, "\\textwidth") -> (k, showFl (num * 100) <> "%")

210211 _ -> (k, v)

211212 let kvs = map replaceTextwidth

212213 $ filter (\(k,_) -> k `elem` ["width", "height"]) options

@@ -223,10 +224,10 @@ mkImage options src = do

223224 then return s'

224225 else findFile s es

225226 src' <- case takeExtension src of

226− "" | not (null defaultExt) -> return $ addExtension src defaultExt

227+ "" | not (T.null defaultExt) -> return $ addExtension src $ T.unpack defaultExt

227228 | otherwise -> findFile src exts

228229 _ -> return src

229− return $ imageWith attr src' "" alt

230+ return $ imageWith attr (T.pack src') "" alt

230231

231232doxspace :: PandocMonad m => LP m Inlines

232233doxspace =

@@ -435,7 +436,7 @@ siUnitMap = M.fromList

435436 , ("zetta", str "Z")

436437 ]

437438

438−lit :: String -> LP m Inlines

439+lit :: Text -> LP m Inlines

439440lit = pure . str

440441

441442removeDoubleQuotes :: Text -> Text

@@ -471,7 +472,7 @@ quoted' :: PandocMonad m

471472 -> LP m ()

472473 -> LP m Inlines

473474quoted' f starter ender = do

474− startchs <- (T.unpack . untokenize) <$> starter

475+ startchs <- untokenize <$> starter

475476 smart <- extensionEnabled Ext_smart <$> getOption readerExtensions

476477 if smart

477478 then do

@@ -487,7 +488,7 @@ quoted' f starter ender = do

487488enquote :: PandocMonad m => Bool -> Maybe Text -> LP m Inlines

488489enquote starred mblang = do

489490 skipopts

490− let lang = (T.unpack <$> mblang) >>= babelLangToBCP47

491+ let lang = mblang >>= babelLangToBCP47

491492 let langspan = case lang of

492493 Nothing -> id

493494 Just l -> spanWith ("",[],[("lang", renderLang l)])

@@ -503,27 +504,27 @@ blockquote citations mblang = do

503504 cs <- cites NormalCitation False

504505 return $ para (cite cs mempty)

505506 else return mempty

506− let lang = (T.unpack <$> mblang) >>= babelLangToBCP47

507+ let lang = mblang >>= babelLangToBCP47

507508 let langdiv = case lang of

508509 Nothing -> id

509510 Just l -> divWith ("",[],[("lang", renderLang l)])

510511 bs <- grouped block

511512 return $ blockQuote . langdiv $ (bs <> citePar)

512513

513−doAcronym :: PandocMonad m => String -> LP m Inlines

514+doAcronym :: PandocMonad m => Text -> LP m Inlines

514515doAcronym form = do

515516 acro <- braced

516− return . mconcat $ [spanWith ("",[],[("acronym-label", toksToString acro),

517− ("acronym-form", "singular+" ++ form)])

518− $ str $ toksToString acro]

517+ return . mconcat $ [spanWith ("",[],[("acronym-label", untokenize acro),

518+ ("acronym-form", "singular+" <> form)])

519+ $ str $ untokenize acro]

519520

520−doAcronymPlural :: PandocMonad m => String -> LP m Inlines

521+doAcronymPlural :: PandocMonad m => Text -> LP m Inlines

521522doAcronymPlural form = do

522523 acro <- braced

523524 plural <- lit "s"

524− return . mconcat $ [spanWith ("",[],[("acronym-label", toksToString acro),

525− ("acronym-form", "plural+" ++ form)]) $

526− mconcat [str $ toksToString acro, plural]]

525+ return . mconcat $ [spanWith ("",[],[("acronym-label", untokenize acro),

526+ ("acronym-form", "plural+" <> form)]) $

527+ mconcat [str $ untokenize acro, plural]]

527528

528529doverb :: PandocMonad m => LP m Inlines

529530doverb = do

@@ -532,7 +533,7 @@ doverb = do

532533 Just (c, ts) | T.null ts -> return c

533534 _ -> mzero

534535 withVerbatimMode $

535− (code . T.unpack . untokenize) <$>

536+ (code . untokenize) <$>

536537 manyTill (notFollowedBy newlineTok >> verbTok marker) (symbol marker)

537538

538539verbTok :: PandocMonad m => Char -> LP m Tok

@@ -547,7 +548,7 @@ verbTok stopchar = do

547548 : totoks (incSourceColumn pos (i + 1)) (T.drop 1 t2) ++ inp

548549 return $ Tok pos toktype t1

549550

550−listingsLanguage :: [(String, String)] -> Maybe String

551+listingsLanguage :: [(Text, Text)] -> Maybe Text

551552listingsLanguage opts =

552553 case lookup "language" opts of

553554 Nothing -> Nothing

@@ -562,10 +563,10 @@ dolstinline = do

562563domintinline :: PandocMonad m => LP m Inlines

563564domintinline = do

564565 skipopts

565− cls <- toksToString <$> braced

566+ cls <- untokenize <$> braced

566567 doinlinecode [cls]

567568

568−doinlinecode :: PandocMonad m => [String] -> LP m Inlines

569+doinlinecode :: PandocMonad m => [Text] -> LP m Inlines

569570doinlinecode classes = do

570571 Tok _ Symbol t <- anySymbol

571572 marker <- case T.uncons t of

@@ -573,14 +574,14 @@ doinlinecode classes = do

573574 _ -> mzero

574575 let stopchar = if marker == '{' then '}' else marker

575576 withVerbatimMode $

576− (codeWith ("",classes,[]) . map nlToSpace . T.unpack . untokenize) <$>

577+ (codeWith ("",classes,[]) . T.map nlToSpace . untokenize) <$>

577578 manyTill (verbTok stopchar) (symbol stopchar)

578579

579580nlToSpace :: Char -> Char

580581nlToSpace '\n' = ' '

581582nlToSpace x = x

582583

583−keyval :: PandocMonad m => LP m (String, String)

584+keyval :: PandocMonad m => LP m (Text, Text)

584585keyval = try $ do

585586 Tok _ Word key <- satisfyTok isWordTok

586587 optional sp

@@ -601,35 +602,34 @@ keyval = try $ do

601602 _ -> True))))))

602603 optional (symbol ',')

603604 optional sp

604− return (T.unpack key, T.unpack $ T.strip val)

605+ return (key, T.strip val)

605606

606−keyvals :: PandocMonad m => LP m [(String, String)]

607+keyvals :: PandocMonad m => LP m [(Text, Text)]

607608keyvals = try $ symbol '[' >> manyTill keyval (symbol ']')

608609

609610accent :: PandocMonad m => Char -> Maybe Char -> LP m Inlines

610611accent combiningAccent fallBack = try $ do

611612 ils <- tok

612613 case toList ils of

613− (Str (x:xs) : ys) -> return $ fromList $

614+ (Str (T.uncons -> Just (x, xs)) : ys) -> return $ fromList $

614615 -- try to normalize to the combined character:

615− Str (T.unpack

616− (Normalize.normalize Normalize.NFC

617− (T.pack [x, combiningAccent])) ++ xs) : ys

618− [Space] -> return $ str [fromMaybe combiningAccent fallBack]

619− [] -> return $ str [fromMaybe combiningAccent fallBack]

616+ Str (Normalize.normalize Normalize.NFC

617+ (T.pack [x, combiningAccent]) <> xs) : ys

618+ [Space] -> return $ str $ T.singleton $ fromMaybe combiningAccent fallBack

619+ [] -> return $ str $ T.singleton $ fromMaybe combiningAccent fallBack

620620 _ -> return ils

621621

622−mathDisplay :: String -> Inlines

622+mathDisplay :: Text -> Inlines

623623mathDisplay = displayMath . trimMath

624624

625−mathInline :: String -> Inlines

625+mathInline :: Text -> Inlines

626626mathInline = math . trimMath

627627

628628dollarsMath :: PandocMonad m => LP m Inlines

629629dollarsMath = do

630630 symbol '$'

631631 display <- option False (True <$ symbol '$')

632− (do contents <- try $ T.unpack . untokenize <$> pDollarsMath 0

632+ (do contents <- try $ untokenize <$> pDollarsMath 0

633633 if display

634634 then (mathDisplay contents <$ symbol '$')

635635 else return $ mathInline contents)

@@ -682,10 +682,10 @@ simpleCiteArgs = try $ do

682682 }

683683 return $ addPrefix pre $ addSuffix suf $ map conv keys

684684

685−citationLabel :: PandocMonad m => LP m String

685+citationLabel :: PandocMonad m => LP m Text

686686citationLabel = do

687687 optional spaces

688− toksToString <$>

688+ untokenize <$>

689689 (many1 (satisfyTok isWordTok <|> symbolIn bibtexKeyChar)

690690 <* optional spaces

691691 <* optional (symbol ',')

@@ -729,10 +729,10 @@ cites mode multi = try $ do

729729 addMprenote _ _ = []

730730 addMpostnote = addSuffix . mpostnote

731731

732−citation :: PandocMonad m => String -> CitationMode -> Bool -> LP m Inlines

732+citation :: PandocMonad m => Text -> CitationMode -> Bool -> LP m Inlines

733733citation name mode multi = do

734734 (c,raw) <- withRaw $ cites mode multi

735− return $ cite c (rawInline "latex" $ "\\" ++ name ++ toksToString raw)

735+ return $ cite c (rawInline "latex" $ "\\" <> name <> untokenize raw)

736736

737737handleCitationPart :: Inlines -> [Citation]

738738handleCitationPart ils =

@@ -756,7 +756,7 @@ complexNatbibCitation mode = try $ do

756756 case cs of

757757 [] -> mzero

758758 (c:cits) -> return $ cite (c{ citationMode = mode }:cits)

759− (rawInline "latex" $ "\\citetext" ++ toksToString raw)

759+ (rawInline "latex" $ "\\citetext" <> untokenize raw)

760760

761761inNote :: Inlines -> Inlines

762762inNote ils =

@@ -780,10 +780,10 @@ tok :: PandocMonad m => LP m Inlines

780780tok = try $ spaces >> grouped inline <|> inlineCommand' <|> singleChar'

781781 where singleChar' = do

782782 Tok _ _ t <- singleChar

783− return (str (T.unpack t))

783+ return $ str t

784784

785785opt :: PandocMonad m => LP m Inlines

786−opt = bracketed inline <|> (str . T.unpack <$> rawopt)

786+opt = bracketed inline <|> (str <$> rawopt)

787787

788788paropt :: PandocMonad m => LP m Inlines

789789paropt = parenWrapped inline

@@ -822,26 +822,31 @@ overlayTok =

822822inBrackets :: Inlines -> Inlines

823823inBrackets x = str "[" <> x <> str "]"

824824

825−unescapeURL :: String -> String

826−unescapeURL ('\\':x:xs) | isEscapable x = x:unescapeURL xs

827− where isEscapable c = c `elem` ("#$%&~_^\\{}" :: String)

828−unescapeURL (x:xs) = x:unescapeURL xs

829−unescapeURL [] = ""

825+unescapeURL :: Text -> Text

826+unescapeURL = T.concat . go . T.splitOn "\\"

827+ where

828+ isEscapable c = c `elemText` "#$%&~_^\\{}"

829+ go (x:xs) = x : map unescapeInterior xs

830+ go [] = []

831+ unescapeInterior t

832+ | Just (c, _) <- T.uncons t

833+ , isEscapable c = t

834+ | otherwise = "\\" <> t

830835

831836mathEnvWith :: PandocMonad m

832837 => (Inlines -> a) -> Maybe Text -> Text -> LP m a

833838mathEnvWith f innerEnv name = f . mathDisplay . inner <$> mathEnv name

834839 where inner x = case innerEnv of

835840 Nothing -> x

836− Just y -> "\\begin{" ++ T.unpack y ++ "}\n" ++ x ++

837− "\\end{" ++ T.unpack y ++ "}"

841+ Just y -> "\\begin{" <> y <> "}\n" <> x <>

842+ "\\end{" <> y <> "}"

838843

839−mathEnv :: PandocMonad m => Text -> LP m String

844+mathEnv :: PandocMonad m => Text -> LP m Text

840845mathEnv name = do

841846 skipopts

842847 optional blankline

843848 res <- manyTill anyTok (end_ name)

844− return $ stripTrailingNewlines $ T.unpack $ untokenize res

849+ return $ stripTrailingNewlines $ untokenize res

845850

846851inlineEnvironment :: PandocMonad m => LP m Inlines

847852inlineEnvironment = try $ do

@@ -914,9 +919,9 @@ inlineCommands = M.union inlineLanguageCommands $ M.fromList

914919 , ("mbox", rawInlineOr "mbox" $ processHBox <$> tok)

915920 , ("hbox", rawInlineOr "hbox" $ processHBox <$> tok)

916921 , ("lettrine", optional opt >> extractSpaces (spanWith ("",["lettrine"],[])) <$> tok)

917− , ("(", mathInline . toksToString <$> manyTill anyTok (controlSeq ")"))

918− , ("[", mathDisplay . toksToString <$> manyTill anyTok (controlSeq "]"))

919− , ("ensuremath", mathInline . toksToString <$> braced)

922+ , ("(", mathInline . untokenize <$> manyTill anyTok (controlSeq ")"))

923+ , ("[", mathDisplay . untokenize <$> manyTill anyTok (controlSeq "]"))

924+ , ("ensuremath", mathInline . untokenize <$> braced)

920925 , ("texorpdfstring", const <$> tok <*> tok)

921926 , ("P", lit "¶")

922927 , ("S", lit "§")

@@ -1008,16 +1013,15 @@ inlineCommands = M.union inlineLanguageCommands $ M.fromList

10081013 , ("lstinline", dolstinline)

10091014 , ("mintinline", domintinline)

10101015 , ("Verb", doverb)

1011− , ("url", ((unescapeURL . T.unpack . untokenize) <$> bracedUrl) >>= \url ->

1016+ , ("url", ((unescapeURL . untokenize) <$> bracedUrl) >>= \url ->

10121017 pure (link url "" (str url)))

1013− , ("nolinkurl", ((unescapeURL . T.unpack . untokenize) <$> bracedUrl) >>= \url ->

1018+ , ("nolinkurl", ((unescapeURL . untokenize) <$> bracedUrl) >>= \url ->

10141019 pure (code url))

1015− , ("href", (unescapeURL . toksToString <$>

1020+ , ("href", (unescapeURL . untokenize <$>

10161021 bracedUrl <* optional sp) >>= \url ->

10171022 tok >>= \lab -> pure (link url "" lab))

10181023 , ("includegraphics", do options <- option [] keyvals

1019− src <- unescapeURL . T.unpack .

1020− removeDoubleQuotes . untokenize <$> braced

1024+ src <- unescapeURL . removeDoubleQuotes . untokenize <$> braced

10211025 mkImage options src)

10221026 , ("enquote*", enquote True Nothing)

10231027 , ("enquote", enquote False Nothing)

@@ -1172,22 +1176,21 @@ inlineCommands = M.union inlineLanguageCommands $ M.fromList

11721176ifdim :: PandocMonad m => LP m Inlines

11731177ifdim = do

11741178 contents <- manyTill anyTok (controlSeq "fi")

1175− return $ rawInline "latex" $ T.unpack $

1176− "\\ifdim" <> untokenize contents <> "\\fi"

1179+ return $ rawInline "latex" $ "\\ifdim" <> untokenize contents <> "\\fi"

11771180

11781181makeUppercase :: Inlines -> Inlines

1179−makeUppercase = fromList . walk (alterStr (map toUpper)) . toList

1182+makeUppercase = fromList . walk (alterStr T.toUpper) . toList

11801183

11811184makeLowercase :: Inlines -> Inlines

1182−makeLowercase = fromList . walk (alterStr (map toLower)) . toList

1185+makeLowercase = fromList . walk (alterStr T.toLower) . toList

11831186

1184−alterStr :: (String -> String) -> Inline -> Inline

1187+alterStr :: (Text -> Text) -> Inline -> Inline

11851188alterStr f (Str xs) = Str (f xs)

11861189alterStr _ x = x

11871190

11881191foreignlanguage :: PandocMonad m => LP m Inlines

11891192foreignlanguage = do

1190− babelLang <- T.unpack . untokenize <$> braced

1193+ babelLang <- untokenize <$> braced

11911194 case babelLangToBCP47 babelLang of

11921195 Just lang -> spanWith ("", [], [("lang", renderLang lang)]) <$> tok

11931196 _ -> tok

@@ -1196,24 +1199,24 @@ inlineLanguageCommands :: PandocMonad m => M.Map Text (LP m Inlines)

11961199inlineLanguageCommands = M.fromList $ mk <$> M.toList polyglossiaLangToBCP47

11971200 where

11981201 mk (polyglossia, bcp47Func) =

1199− ("text" <> T.pack polyglossia, inlineLanguage bcp47Func)

1202+ ("text" <> polyglossia, inlineLanguage bcp47Func)

12001203

1201−inlineLanguage :: PandocMonad m => (String -> Lang) -> LP m Inlines

1204+inlineLanguage :: PandocMonad m => (Text -> Lang) -> LP m Inlines

12021205inlineLanguage bcp47Func = do

1203− o <- option "" $ (T.unpack . T.filter (\c -> c /= '[' && c /= ']'))

1206+ o <- option "" $ T.filter (\c -> c /= '[' && c /= ']')

12041207 <$> rawopt

12051208 let lang = renderLang $ bcp47Func o

12061209 extractSpaces (spanWith ("", [], [("lang", lang)])) <$> tok

12071210

12081211hyperlink :: PandocMonad m => LP m Inlines

12091212hyperlink = try $ do

1210− src <- toksToString <$> braced

1213+ src <- untokenize <$> braced

12111214 lab <- tok

1212− return $ link ('#':src) "" lab

1215+ return $ link ("#" <> src) "" lab

12131216

12141217hypertargetBlock :: PandocMonad m => LP m Blocks

12151218hypertargetBlock = try $ do

1216− ref <- toksToString <$> braced

1219+ ref <- untokenize <$> braced

12171220 bs <- grouped block

12181221 case toList bs of

12191222 [Header 1 (ident,_,_) _] | ident == ref -> return bs

@@ -1221,7 +1224,7 @@ hypertargetBlock = try $ do

12211224

12221225hypertargetInline :: PandocMonad m => LP m Inlines

12231226hypertargetInline = try $ do

1224− ref <- toksToString <$> braced

1227+ ref <- untokenize <$> braced

12251228 ils <- grouped inline

12261229 return $ spanWith (ref, [], []) ils

12271230

@@ -1231,7 +1234,7 @@ romanNumeralUpper =

12311234

12321235romanNumeralLower :: (PandocMonad m) => LP m Inlines

12331236romanNumeralLower =

1234− str . map toLower . toRomanNumeral <$> romanNumeralArg

1237+ str . T.toLower . toRomanNumeral <$> romanNumeralArg

12351238

12361239romanNumeralArg :: (PandocMonad m) => LP m Int

12371240romanNumeralArg = spaces *> (parser <|> inBraces)

@@ -1248,18 +1251,18 @@ romanNumeralArg = spaces *> (parser <|> inBraces)

12481251 let (digits, rest) = T.span isDigit s

12491252 unless (T.null rest) $

12501253 Prelude.fail "Non-digits in argument to \\Rn or \\RN"

1251− safeRead $ T.unpack digits

1254+ safeRead digits

12521255

12531256newToggle :: (Monoid a, PandocMonad m) => [Tok] -> LP m a

12541257newToggle name = do

12551258 updateState $ \st ->

1256− st{ sToggles = M.insert (toksToString name) False (sToggles st) }

1259+ st{ sToggles = M.insert (untokenize name) False (sToggles st) }

12571260 return mempty

12581261

12591262setToggle :: (Monoid a, PandocMonad m) => Bool -> [Tok] -> LP m a

12601263setToggle on name = do

12611264 updateState $ \st ->

1262− st{ sToggles = M.adjust (const on) (toksToString name) (sToggles st) }

1265+ st{ sToggles = M.adjust (const on) (untokenize name) (sToggles st) }

12631266 return mempty

12641267

12651268ifToggle :: PandocMonad m => LP m ()

@@ -1271,7 +1274,7 @@ ifToggle = do

12711274 no <- braced

12721275 toggles <- sToggles <$> getState

12731276 inp <- getInput

1274− let name' = toksToString name

1277+ let name' = untokenize name

12751278 case M.lookup name' toggles of

12761279 Just True -> setInput (yes ++ inp)

12771280 Just False -> setInput (no ++ inp)

@@ -1294,11 +1297,11 @@ ifstrequal = do

12941297 else getInput >>= setInput . (ifnotequal ++)

12951298 return mempty

12961299

1297−coloredInline :: PandocMonad m => String -> LP m Inlines

1300+coloredInline :: PandocMonad m => Text -> LP m Inlines

12981301coloredInline stylename = do

12991302 skipopts

13001303 color <- braced

1301− spanWith ("",[],[("style",stylename ++ ": " ++ toksToString color)]) <$> tok

1304+ spanWith ("",[],[("style",stylename <> ": " <> untokenize color)]) <$> tok

13021305

13031306ttfamily :: PandocMonad m => LP m Inlines

13041307ttfamily = (code . stringify . toList) <$> tok

@@ -1313,12 +1316,12 @@ rawInlineOr name' fallback = do

13131316processHBox :: Inlines -> Inlines

13141317processHBox = walk convert

13151318 where

1316− convert Space = Str [chr 160] -- non-breakable space

1317− convert SoftBreak = Str [chr 160] -- non-breakable space

1319+ convert Space = Str $ T.singleton $ chr 160 -- non-breakable space

1320+ convert SoftBreak = Str $ T.singleton $ chr 160 -- non-breakable space

13181321 convert LineBreak = Str ""

13191322 convert x = x

13201323

1321−getRawCommand :: PandocMonad m => Text -> Text -> LP m String

1324+getRawCommand :: PandocMonad m => Text -> Text -> LP m Text

13221325getRawCommand name txt = do

13231326 (_, rawargs) <- withRaw $

13241327 case name of

@@ -1336,7 +1339,7 @@ getRawCommand name txt = do

13361339 skipopts

13371340 option "" (try dimenarg)

13381341 void $ many braced

1339− return $ T.unpack (txt <> untokenize rawargs)

1342+ return $ txt <> untokenize rawargs

13401343

13411344isFontSizeCommand :: Text -> Bool

13421345isFontSizeCommand "tiny" = True

@@ -1396,17 +1399,17 @@ treatAsInline = Set.fromList

13961399dolabel :: PandocMonad m => LP m Inlines

13971400dolabel = do

13981401 v <- braced

1399− let refstr = toksToString v

1402+ let refstr = untokenize v

14001403 return $ spanWith (refstr,[],[("label", refstr)])

1401− $ inBrackets $ str $ toksToString v

1404+ $ inBrackets $ str $ untokenize v

14021405

1403−doref :: PandocMonad m => String -> LP m Inlines

1406+doref :: PandocMonad m => Text -> LP m Inlines

14041407doref cls = do

14051408 v <- braced

1406− let refstr = toksToString v

1409+ let refstr = untokenize v

14071410 return $ linkWith ("",[],[ ("reference-type", cls)

14081411 , ("reference", refstr)])

1409− ('#':refstr)

1412+ ("#" <> refstr)

14101413 ""

14111414 (inBrackets $ str refstr)

14121415

@@ -1435,11 +1438,11 @@ inline = (mempty <$ comment)

14351438 <|> (str "\160" <$ symbol '~')

14361439 <|> dollarsMath

14371440 <|> (guardEnabled Ext_literate_haskell *> symbol '|' *> doLHSverb)

1438− <|> (str . (:[]) <$> primEscape)

1441+ <|> (str . T.singleton <$> primEscape)

14391442 <|> regularSymbol

14401443 <|> (do res <- symbolIn "#^'`\"[]&"

14411444 pos <- getPosition

1442− let s = T.unpack (untoken res)

1445+ let s = untoken res

14431446 report $ ParsingUnescaped s pos

14441447 return $ str s)

14451448

@@ -1498,7 +1501,7 @@ include name = do

14981501 -- note, we can have cc_by_4.0 for example...

14991502 _ | name == "usepackage" -> addExtension f ".sty"

15001503 | otherwise -> addExtension f ".tex"

1501− dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

1504+ dirs <- (map T.unpack . splitTextBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

15021505 mapM_ (insertIncluded dirs) (map addExt fs)

15031506 return mempty

15041507

@@ -1509,19 +1512,19 @@ insertIncluded :: PandocMonad m

15091512insertIncluded dirs f = do

15101513 pos <- getPosition

15111514 containers <- getIncludeFiles <$> getState

1512− when (f `elem` containers) $

1513− throwError $ PandocParseError $ "Include file loop at " ++ show pos

1514− updateState $ addIncludeFile f

1515+ when (T.pack f `elem` containers) $

1516+ throwError $ PandocParseError $ T.pack $ "Include file loop at " ++ show pos

1517+ updateState $ addIncludeFile $ T.pack f

15151518 mbcontents <- readFileFromDirs dirs f

15161519 contents <- case mbcontents of

15171520 Just s -> return s

15181521 Nothing -> do

1519− report $ CouldNotLoadIncludeFile f pos

1522+ report $ CouldNotLoadIncludeFile (T.pack f) pos

15201523 return ""

1521− getInput >>= setInput . (tokenize f (T.pack contents) ++)

1524+ getInput >>= setInput . (tokenize f contents ++)

15221525 updateState dropLatestIncludeFile

15231526

1524−addMeta :: PandocMonad m => ToMetaValue a => String -> a -> LP m ()

1527+addMeta :: PandocMonad m => ToMetaValue a => Text -> a -> LP m ()

15251528addMeta field val = updateState $ \st ->

15261529 st{ sMeta = addMetaField field val $ sMeta st }

15271530

@@ -1536,10 +1539,10 @@ authors = try $ do

15361539 egroup

15371540 addMeta "author" (map trimInlines auths)

15381541

1539−macroDef :: (PandocMonad m, Monoid a) => (String -> a) -> LP m a

1542+macroDef :: (PandocMonad m, Monoid a) => (Text -> a) -> LP m a

15401543macroDef constructor = do

15411544 (_, s) <- withRaw (commandDef <|> environmentDef)

1542− (constructor (T.unpack $ untokenize s) <$

1545+ (constructor (untokenize s) <$

15431546 guardDisabled Ext_latex_macros)

15441547 <|> return mempty

15451548 where commandDef = do

@@ -1632,7 +1635,7 @@ newcommand = do

16321635 case M.lookup name macros of

16331636 Just macro

16341637 | mtype == "newcommand" -> do

1635− report $ MacroAlreadyDefined (T.unpack txt) pos

1638+ report $ MacroAlreadyDefined txt pos

16361639 return (name, macro)

16371640 | mtype == "providecommand" -> return (name, macro)

16381641 _ -> return (name, Macro ExpandWhenUsed argspecs optarg contents)

@@ -1658,7 +1661,7 @@ newenvironment = do

16581661 case M.lookup name macros of

16591662 Just _

16601663 | mtype == "newenvironment" -> do

1661− report $ MacroAlreadyDefined (T.unpack name) pos

1664+ report $ MacroAlreadyDefined name pos

16621665 return Nothing

16631666 | mtype == "provideenvironment" -> do

16641667 return Nothing

@@ -1669,7 +1672,7 @@ newenvironment = do

16691672bracketedNum :: PandocMonad m => LP m Int

16701673bracketedNum = do

16711674 ds <- untokenize <$> bracketedToks

1672− case safeRead (T.unpack ds) of

1675+ case safeRead ds of

16731676 Just i -> return i

16741677 _ -> return 0

16751678

@@ -1709,7 +1712,7 @@ section (ident, classes, kvs) lvl = do

17091712 contents <- grouped inline

17101713 lab <- option ident $

17111714 try (spaces >> controlSeq "label"

1712− >> spaces >> toksToString <$> braced)

1715+ >> spaces >> untokenize <$> braced)

17131716 when (lvl == 0) $

17141717 updateState $ \st -> st{ sHasChapters = True }

17151718 unless ("unnumbered" `elem` classes) $ do

@@ -1836,9 +1839,9 @@ blockCommands = M.fromList

18361839 , ("centerline", (para . trimInlines) <$> (skipopts *> tok))

18371840 , ("caption", skipopts *> setCaption)

18381841 , ("bibliography", mempty <$ (skipopts *> braced >>=

1839− addMeta "bibliography" . splitBibs . toksToString))

1842+ addMeta "bibliography" . splitBibs . untokenize))

18401843 , ("addbibresource", mempty <$ (skipopts *> braced >>=

1841− addMeta "bibliography" . splitBibs . toksToString))

1844+ addMeta "bibliography" . splitBibs . untokenize))

18421845 , ("endinput", mempty <$ skipMany tok)

18431846 -- includes

18441847 , ("lstinputlisting", inputListing)

@@ -1941,18 +1944,18 @@ rawEnv name = do

19411944 (bs, raw) <- withRaw $ env name blocks

19421945 if parseRaw

19431946 then return $ rawBlock "latex"

1944− $ T.unpack $ beginCommand <> untokenize raw

1947+ $ beginCommand <> untokenize raw

19451948 else do

1946− report $ SkippedContent (T.unpack beginCommand) pos1

1949+ report $ SkippedContent beginCommand pos1

19471950 pos2 <- getPosition

1948− report $ SkippedContent ("\\end{" ++ T.unpack name ++ "}") pos2

1951+ report $ SkippedContent ("\\end{" <> name <> "}") pos2

19491952 return bs

19501953

19511954rawVerbEnv :: PandocMonad m => Text -> LP m Blocks

19521955rawVerbEnv name = do

19531956 pos <- getPosition

19541957 (_, raw) <- withRaw $ verbEnv name

1955− let raw' = "\\begin{" ++ T.unpack name ++ "}" ++ toksToString raw

1958+ let raw' = "\\begin{" <> name <> "}" <> untokenize raw

19561959 exts <- getOption readerExtensions

19571960 let parseRaw = extensionEnabled Ext_raw_tex exts

19581961 if parseRaw

@@ -1961,12 +1964,11 @@ rawVerbEnv name = do

19611964 report $ SkippedContent raw' pos

19621965 return mempty

19631966

1964−verbEnv :: PandocMonad m => Text -> LP m String

1967+verbEnv :: PandocMonad m => Text -> LP m Text

19651968verbEnv name = withVerbatimMode $ do

19661969 optional blankline

19671970 res <- manyTill anyTok (end_ name)

1968− return $ T.unpack

1969− $ stripTrailingNewline

1971+ return $ stripTrailingNewline

19701972 $ untokenize

19711973 $ res

19721974

@@ -2010,11 +2012,11 @@ minted = do

20102012mintedAttr :: PandocMonad m => LP m Attr

20112013mintedAttr = do

20122014 options <- option [] keyvals

2013− lang <- toksToString <$> braced

2015+ lang <- untokenize <$> braced

20142016 let kvs = [ (if k == "firstnumber"

20152017 then "startFrom"

20162018 else k, v) | (k,v) <- options ]

2017− let classes = [ lang | not (null lang) ] ++

2019+ let classes = [ lang | not (T.null lang) ] ++

20182020 [ "numberLines" |

20192021 lookup "linenos" options == Just "true" ]

20202022 return ("",classes,kvs)

@@ -2023,14 +2025,14 @@ inputMinted :: PandocMonad m => LP m Blocks

20232025inputMinted = do

20242026 pos <- getPosition

20252027 attr <- mintedAttr

2026− f <- filter (/='"') . toksToString <$> braced

2027− dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

2028− mbCode <- readFileFromDirs dirs f

2028+ f <- T.filter (/='"') . untokenize <$> braced

2029+ dirs <- (map T.unpack . splitTextBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

2030+ mbCode <- readFileFromDirs dirs (T.unpack f)

20292031 rawcode <- case mbCode of

20302032 Just s -> return s

20312033 Nothing -> do

20322034 report $ CouldNotLoadIncludeFile f pos

2033− return []

2035+ return ""

20342036 return $ B.codeBlockWith attr rawcode

20352037

20362038letterContents :: PandocMonad m => LP m Blocks

@@ -2052,10 +2054,10 @@ figure = try $ do

20522054addImageCaption :: PandocMonad m => Blocks -> LP m Blocks

20532055addImageCaption = walkM go

20542056 where go (Image attr@(_, cls, kvs) alt (src,tit))

2055− | not ("fig:" `isPrefixOf` tit) = do

2057+ | not ("fig:" `T.isPrefixOf` tit) = do

20562058 (mbcapt, mblab) <- sCaption <$> getState

20572059 let (alt', tit') = case mbcapt of

2058− Just ils -> (toList ils, "fig:" ++ tit)

2060+ Just ils -> (toList ils, "fig:" <> tit)

20592061 Nothing -> (alt, tit)

20602062 attr' = case mblab of

20612063 Just lab -> (lab, cls, kvs)

@@ -2090,23 +2092,23 @@ addImageCaption = walkM go

20902092 return $ Image attr' alt' (src, tit')

20912093 go x = return x

20922094

2093−coloredBlock :: PandocMonad m => String -> LP m Blocks

2095+coloredBlock :: PandocMonad m => Text -> LP m Blocks

20942096coloredBlock stylename = try $ do

20952097 skipopts

20962098 color <- braced

20972099 notFollowedBy (grouped inline)

2098− let constructor = divWith ("",[],[("style",stylename ++ ": " ++ toksToString color)])

2100+ let constructor = divWith ("",[],[("style",stylename <> ": " <> untokenize color)])

20992101 constructor <$> grouped block

21002102

21012103graphicsPath :: PandocMonad m => LP m Blocks

21022104graphicsPath = do

2103− ps <- map toksToString <$>

2105+ ps <- map (T.unpack . untokenize) <$>

21042106 (bgroup *> spaces *> manyTill (braced <* spaces) egroup)

2105− getResourcePath >>= setResourcePath . (++ ps)

2107+ getResourcePath >>= setResourcePath . (<> ps)

21062108 return mempty

21072109

2108−splitBibs :: String -> [Inlines]

2109−splitBibs = map (str . flip replaceExtension "bib" . trim) . splitBy (==',')

2110+splitBibs :: Text -> [Inlines]

2111+splitBibs = map (str . T.pack . flip replaceExtension "bib" . T.unpack . trim) . splitTextBy (==',')

21102112

21112113alltt :: Blocks -> Blocks

21122114alltt = walk strToCode

@@ -2115,7 +2117,7 @@ alltt = walk strToCode

21152117 strToCode SoftBreak = LineBreak

21162118 strToCode x = x

21172119

2118−parseListingsOptions :: [(String, String)] -> Attr

2120+parseListingsOptions :: [(Text, Text)] -> Attr

21192121parseListingsOptions options =

21202122 let kvs = [ (if k == "firstnumber"

21212123 then "startFrom"

@@ -2129,23 +2131,23 @@ inputListing :: PandocMonad m => LP m Blocks

21292131inputListing = do

21302132 pos <- getPosition

21312133 options <- option [] keyvals

2132− f <- filter (/='"') . toksToString <$> braced

2133− dirs <- (splitBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

2134− mbCode <- readFileFromDirs dirs f

2134+ f <- T.filter (/='"') . untokenize <$> braced

2135+ dirs <- (map T.unpack . splitTextBy (==':') . fromMaybe ".") <$> lookupEnv "TEXINPUTS"

2136+ mbCode <- readFileFromDirs dirs (T.unpack f)

21352137 codeLines <- case mbCode of

2136− Just s -> return $ lines s

2138+ Just s -> return $ T.lines s

21372139 Nothing -> do

21382140 report $ CouldNotLoadIncludeFile f pos

21392141 return []

21402142 let (ident,classes,kvs) = parseListingsOptions options

21412143 let classes' =

21422144 (case listingsLanguage options of

2143− Nothing -> (take 1 (languagesByExtension (takeExtension f)) ++)

2145+ Nothing -> (take 1 (languagesByExtension (T.pack $ takeExtension $ T.unpack f)) <>)

21442146 Just _ -> id) classes

21452147 let firstline = fromMaybe 1 $ lookup "firstline" options >>= safeRead

21462148 let lastline = fromMaybe (length codeLines) $

21472149 lookup "lastline" options >>= safeRead

2148− let codeContents = intercalate "\n" $ take (1 + lastline - firstline) $

2150+ let codeContents = T.intercalate "\n" $ take (1 + lastline - firstline) $

21492151 drop (firstline - 1) codeLines

21502152 return $ codeBlockWith (ident,classes',kvs) codeContents

21512153

@@ -2176,12 +2178,12 @@ orderedList' = try $ do

21762178 spaces

21772179 let markerSpec = do

21782180 symbol '['

2179− ts <- toksToString <$> manyTill anyTok (symbol ']')

2181+ ts <- untokenize <$> manyTill anyTok (symbol ']')

21802182 case runParser anyOrderedListMarker def "option" ts of

21812183 Right r -> return r

21822184 Left _ -> do

21832185 pos <- getPosition

2184− report $ SkippedContent ("[" ++ ts ++ "]") pos

2186+ report $ SkippedContent ("[" <> ts <> "]") pos

21852187 return (1, DefaultStyle, DefaultDelim)

21862188 (_, style, delim) <- option (1, DefaultStyle, DefaultDelim) markerSpec

21872189 spaces

@@ -2191,17 +2193,17 @@ orderedList' = try $ do

21912193 spaces

21922194 start <- option 1 $ try $ do pos <- getPosition

21932195 controlSeq "setcounter"

2194− ctr <- toksToString <$> braced

2195− guard $ "enum" `isPrefixOf` ctr

2196− guard $ all (`elem` ['i','v']) (drop 4 ctr)

2196+ ctr <- untokenize <$> braced

2197+ guard $ "enum" `T.isPrefixOf` ctr

2198+ guard $ T.all (`elem` ['i','v']) (T.drop 4 ctr)

21972199 optional sp

2198− num <- toksToString <$> braced

2200+ num <- untokenize <$> braced

21992201 case safeRead num of

22002202 Just i -> return (i + 1 :: Int)

22012203 Nothing -> do

22022204 report $ SkippedContent

2203− ("\\setcounter{" ++ ctr ++

2204− "}{" ++ num ++ "}") pos

2205+ ("\\setcounter{" <> ctr <>

2206+ "}{" <> num <> "}") pos

22052207 return 1

22062208 bs <- listenv "enumerate" (many item)

22072209 return $ orderedListWith (start, style, delim) bs

@@ -2235,7 +2237,7 @@ splitWordTok = do

22352237 inp <- getInput

22362238 case inp of

22372239 (Tok spos Word t : rest) ->

2238− setInput $ map (Tok spos Symbol . T.singleton) (T.unpack t) ++ rest

2240+ setInput $ map (Tok spos Symbol . T.singleton) (T.unpack t) <> rest

22392241 _ -> return ()

22402242

22412243parseAligns :: PandocMonad m => LP m [(Alignment, Double, ([Tok], [Tok]))]

@@ -2256,7 +2258,7 @@ parseAligns = try $ do

22562258 let alignSuffix = symbol '<' >> braced

22572259 let colWidth = try $ do

22582260 symbol '{'

2259− ds <- trim . toksToString <$> manyTill anyTok (controlSeq "linewidth")

2261+ ds <- trim . untokenize <$> manyTill anyTok (controlSeq "linewidth")

22602262 spaces

22612263 symbol '}'

22622264 case safeRead ds of

@@ -2266,7 +2268,7 @@ parseAligns = try $ do

22662268 pref <- option [] alignPrefix

22672269 spaces

22682270 al <- alignChar

2269− width <- colWidth <|> option 0.0 (do s <- toksToString <$> braced

2271+ width <- colWidth <|> option 0.0 (do s <- untokenize <$> braced

22702272 pos <- getPosition

22712273 report $ SkippedContent s pos

22722274 return 0.0)

@@ -2276,13 +2278,13 @@ parseAligns = try $ do

22762278 let starAlign = do -- '*{2}{r}' == 'rr', we just expand like a macro

22772279 symbol '*'

22782280 spaces

2279− ds <- trim . toksToString <$> braced

2281+ ds <- trim . untokenize <$> braced

22802282 spaces

22812283 spec <- braced

22822284 case safeRead ds of

22832285 Just n ->

22842286 getInput >>= setInput . (mconcat (replicate n spec) ++)

2285− Nothing -> Prelude.fail $ "Could not parse " ++ ds ++ " as number"

2287+ Nothing -> Prelude.fail $ "Could not parse " <> T.unpack ds <> " as number"

22862288 bgroup

22872289 spaces

22882290 maybeBar

@@ -2379,7 +2381,7 @@ block = do

23792381 <|> blockCommand

23802382 <|> paragraph

23812383 <|> grouped block

2382− trace (take 60 $ show $ B.toList res)

2384+ trace (T.take 60 $ tshow $ B.toList res)

23832385 return res

23842386

23852387blocks :: PandocMonad m => LP m Blocks

@@ -2387,9 +2389,9 @@ blocks = mconcat <$> many block

23872389

23882390setDefaultLanguage :: PandocMonad m => LP m Blocks

23892391setDefaultLanguage = do

2390− o <- option "" $ (T.unpack . T.filter (\c -> c /= '[' && c /= ']'))

2392+ o <- option "" $ T.filter (\c -> c /= '[' && c /= ']')

23912393 <$> rawopt

2392− polylang <- toksToString <$> braced

2394+ polylang <- untokenize <$> braced

23932395 case M.lookup polylang polyglossiaLangToBCP47 of

23942396 Nothing -> return mempty -- TODO mzero? warning?

23952397 Just langFunc -> do

src/Text/Pandoc/Readers/LaTeX/Lang.hs+9−7

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.LaTeX.Lang

@@ -18,11 +19,12 @@ module Text.Pandoc.Readers.LaTeX.Lang

1819where

1920import Prelude

2021import qualified Data.Map as M

22+import qualified Data.Text as T

2123import Text.Pandoc.BCP47 (Lang(..))

2224

23−polyglossiaLangToBCP47 :: M.Map String (String -> Lang)

25+polyglossiaLangToBCP47 :: M.Map T.Text (T.Text -> Lang)

2426polyglossiaLangToBCP47 = M.fromList

25− [ ("arabic", \o -> case filter (/=' ') o of

27+ [ ("arabic", \o -> case T.filter (/=' ') o of

2628 "locale=algeria" -> Lang "ar" "" "DZ" []

2729 "locale=mashriq" -> Lang "ar" "" "SY" []

2830 "locale=libya" -> Lang "ar" "" "LY" []

@@ -30,7 +32,7 @@ polyglossiaLangToBCP47 = M.fromList

3032 "locale=mauritania" -> Lang "ar" "" "MR" []

3133 "locale=tunisia" -> Lang "ar" "" "TN" []

3234 _ -> Lang "ar" "" "" [])

33− , ("german", \o -> case filter (/=' ') o of

35+ , ("german", \o -> case T.filter (/=' ') o of

3436 "spelling=old" -> Lang "de" "" "DE" ["1901"]

3537 "variant=austrian,spelling=old"

3638 -> Lang "de" "" "AT" ["1901"]

@@ -40,11 +42,11 @@ polyglossiaLangToBCP47 = M.fromList

4042 "variant=swiss" -> Lang "de" "" "CH" []

4143 _ -> Lang "de" "" "" [])

4244 , ("lsorbian", \_ -> Lang "dsb" "" "" [])

43− , ("greek", \o -> case filter (/=' ') o of

45+ , ("greek", \o -> case T.filter (/=' ') o of

4446 "variant=poly" -> Lang "el" "" "polyton" []

4547 "variant=ancient" -> Lang "grc" "" "" []

4648 _ -> Lang "el" "" "" [])

47− , ("english", \o -> case filter (/=' ') o of

49+ , ("english", \o -> case T.filter (/=' ') o of

4850 "variant=australian" -> Lang "en" "" "AU" []

4951 "variant=canadian" -> Lang "en" "" "CA" []

5052 "variant=british" -> Lang "en" "" "GB" []

@@ -52,7 +54,7 @@ polyglossiaLangToBCP47 = M.fromList

5254 "variant=american" -> Lang "en" "" "US" []

5355 _ -> Lang "en" "" "" [])

5456 , ("usorbian", \_ -> Lang "hsb" "" "" [])

55− , ("latin", \o -> case filter (/=' ') o of

57+ , ("latin", \o -> case T.filter (/=' ') o of

5658 "variant=classic" -> Lang "la" "" "" ["x-classic"]

5759 _ -> Lang "la" "" "" [])

5860 , ("slovenian", \_ -> Lang "sl" "" "" [])

@@ -133,7 +135,7 @@ polyglossiaLangToBCP47 = M.fromList

133135 , ("vietnamese", \_ -> Lang "vi" "" "" [])

134136 ]

135137

136−babelLangToBCP47 :: String -> Maybe Lang

138+babelLangToBCP47 :: T.Text -> Maybe Lang

137139babelLangToBCP47 s =

138140 case s of

139141 "austrian" -> Just $ Lang "de" "" "AT" ["1901"]

src/Text/Pandoc/Readers/LaTeX/Parsing.hs+16−16

@@ -97,8 +97,8 @@ import Text.Parsec.Pos

9797newtype DottedNum = DottedNum [Int]

9898 deriving (Show)

9999

100−renderDottedNum :: DottedNum -> String

101−renderDottedNum (DottedNum xs) =

100+renderDottedNum :: DottedNum -> T.Text

101+renderDottedNum (DottedNum xs) = T.pack $

102102 intercalate "." (map show xs)

103103

104104incrementDottedNum :: Int -> DottedNum -> DottedNum

@@ -111,18 +111,18 @@ data LaTeXState = LaTeXState{ sOptions :: ReaderOptions

111111 , sMeta :: Meta

112112 , sQuoteContext :: QuoteContext

113113 , sMacros :: M.Map Text Macro

114− , sContainers :: [String]

114+ , sContainers :: [Text]

115115 , sLogMessages :: [LogMessage]

116− , sIdentifiers :: Set.Set String

116+ , sIdentifiers :: Set.Set Text

117117 , sVerbatimMode :: Bool

118− , sCaption :: (Maybe Inlines, Maybe String)

118+ , sCaption :: (Maybe Inlines, Maybe Text)

119119 , sInListItem :: Bool

120120 , sInTableCell :: Bool

121121 , sLastHeaderNum :: DottedNum

122122 , sLastFigureNum :: DottedNum

123− , sLabels :: M.Map String [Inline]

123+ , sLabels :: M.Map Text [Inline]

124124 , sHasChapters :: Bool

125− , sToggles :: M.Map String Bool

125+ , sToggles :: M.Map Text Bool

126126 , sExpanded :: Bool

127127 }

128128 deriving Show

@@ -202,7 +202,7 @@ withVerbatimMode parser = do

202202

203203rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s)

204204 => [Tok] -> Bool -> LP m a -> LP m a

205− -> ParserT String s m (a, String)

205+ -> ParserT Text s m (a, Text)

206206rawLaTeXParser toks retokenize parser valParser = do

207207 pstate <- getState

208208 let lstate = def{ sOptions = extractReaderOptions pstate }

@@ -233,16 +233,16 @@ rawLaTeXParser toks retokenize parser valParser = do

233233 , not (" " `T.isSuffixOf` result)

234234 -> result <> " "

235235 _ -> result

236− return (val, T.unpack result')

236+ return (val, result')

237237

238238applyMacros :: (PandocMonad m, HasMacros s, HasReaderOptions s)

239− => String -> ParserT String s m String

239+ => Text -> ParserT Text s m Text

240240applyMacros s = (guardDisabled Ext_latex_macros >> return s) <|>

241− do let retokenize = toksToString <$> many (satisfyTok (const True))

241+ do let retokenize = untokenize <$> many (satisfyTok (const True))

242242 pstate <- getState

243243 let lstate = def{ sOptions = extractReaderOptions pstate

244244 , sMacros = extractMacros pstate }

245− res <- runParserT retokenize lstate "math" (tokenize "math" (T.pack s))

245+ res <- runParserT retokenize lstate "math" (tokenize "math" s)

246246 case res of

247247 Left e -> Prelude.fail (show e)

248248 Right s' -> return s'

@@ -307,7 +307,7 @@ totoks pos t =

307307 : totoks (incSourceColumn pos 2) rest'

308308 | c == '#' ->

309309 let (t1, t2) = T.span (\d -> d >= '0' && d <= '9') rest

310− in case safeRead (T.unpack t1) of

310+ in case safeRead t1 of

311311 Just i ->

312312 Tok pos (Arg i) ("#" <> t1)

313313 : totoks (incSourceColumn pos (1 + T.length t1)) t2

@@ -447,7 +447,7 @@ doMacros' n inp = do

447447

448448 handleMacros n' spos name ts = do

449449 when (n' > 20) -- detect macro expansion loops

450− $ throwError $ PandocMacroLoop (T.unpack name)

450+ $ throwError $ PandocMacroLoop name

451451 macros <- sMacros <$> getState

452452 case M.lookup name macros of

453453 Nothing -> mzero

@@ -588,7 +588,7 @@ primEscape = do

588588 | c >= '\64' && c <= '\127' -> return (chr (ord c - 64))

589589 | otherwise -> return (chr (ord c + 64))

590590 Nothing -> Prelude.fail "Empty content of Esc1"

591− Esc2 -> case safeRead ('0':'x':T.unpack (T.drop 2 t)) of

591+ Esc2 -> case safeRead ("0x" <> T.drop 2 t) of

592592 Just x -> return (chr x)

593593 Nothing -> Prelude.fail $ "Could not read: " ++ T.unpack t

594594 _ -> Prelude.fail "Expected an Esc1 or Esc2 token" -- should not happen

@@ -677,7 +677,7 @@ dimenarg = try $ do

677677 guard $ rest `elem` ["", "pt","pc","in","bp","cm","mm","dd","cc","sp"]

678678 return $ T.pack ['=' | ch] <> minus <> s

679679

680−ignore :: (Monoid a, PandocMonad m) => String -> ParserT s u m a

680+ignore :: (Monoid a, PandocMonad m) => Text -> ParserT s u m a

681681ignore raw = do

682682 pos <- getPosition

683683 report $ SkippedContent raw pos

src/Text/Pandoc/Readers/Man.hs+17−15

@@ -1,4 +1,6 @@

1−{-# LANGUAGE FlexibleContexts #-}

1+{-# LANGUAGE FlexibleContexts #-}

2+{-# LANGUAGE ViewPatterns #-}

3+{-# LANGUAGE OverloadedStrings #-}

24{- |

35 Module : Text.Pandoc.Readers.Man

@@ -63,7 +65,7 @@ readWithMTokens :: PandocMonad m

6365 -> [RoffToken] -- ^ input

6466 -> m (Either PandocError a)

6567readWithMTokens parser state input =

66− let leftF = PandocParsecError . intercalate "\n" $ show <$> input

68+ let leftF = PandocParsecError . T.pack . intercalate "\n" $ show <$> input

6769 in mapLeft leftF `liftM` runParserT parser state "source" input

6870

6971parseMan :: PandocMonad m => ManParser m Pandoc

@@ -141,7 +143,7 @@ parseTable = do

141143 isHrule ([cellfmt], _) = columnType cellfmt `elem` ['_','-','=']

142144 isHrule (_, [RoffTokens ss]) =

143145 case Foldable.toList ss of

144− [TextLine [RoffStr [c]]] -> c `elem` ['_','-','=']

146+ [TextLine [RoffStr (T.unpack -> [c])]] -> c `elem` ['_','-','=']

145147 _ -> False

146148 isHrule _ = False

147149

@@ -191,7 +193,7 @@ memptyLine = msatisfy isEmptyLine where

191193 isEmptyLine EmptyLine = True

192194 isEmptyLine _ = False

193195

194−mmacro :: PandocMonad m => String -> ManParser m RoffToken

196+mmacro :: PandocMonad m => T.Text -> ManParser m RoffToken

195197mmacro mk = msatisfy isControlLine where

196198 isControlLine (ControlLine mk' _ _) | mk == mk' = True

197199 | otherwise = False

@@ -284,7 +286,7 @@ parseInline = try $ do

284286 _ -> mzero

285287

286288handleInlineMacro :: PandocMonad m

287− => String -> [Arg] -> SourcePos -> ManParser m Inlines

289+ => T.Text -> [Arg] -> SourcePos -> ManParser m Inlines

288290handleInlineMacro mname args _pos = do

289291 case mname of

290292 "UR" -> parseLink args

@@ -339,7 +341,7 @@ bareIP = msatisfy isBareIP where

339341 isBareIP (ControlLine "IP" [] _) = True

340342 isBareIP _ = False

341343

342−endmacro :: PandocMonad m => String -> ManParser m ()

344+endmacro :: PandocMonad m => T.Text -> ManParser m ()

343345endmacro name = void (mmacro name)

344346 <|> lookAhead (void newBlockMacro)

345347 <|> lookAhead eof

@@ -356,7 +358,7 @@ parseCodeBlock = try $ do

356358 toks <- (mmacro "nf" *> manyTill codeline (endmacro "fi"))

357359 <|> (mmacro "EX" *> manyTill codeline (endmacro "EE"))

358360 optional (mmacro "in")

359− return $ codeBlock (intercalate "\n" $ catMaybes toks)

361+ return $ codeBlock (T.intercalate "\n" $ catMaybes toks)

360362

361363 where

362364

@@ -366,7 +368,7 @@ parseCodeBlock = try $ do

366368 ControlLine "PP" _ _ -> return $ Just "" -- .PP sometimes used for blank line

367369 ControlLine mname args pos -> do

368370 (Just . query getText <$> handleInlineMacro mname args pos) <|>

369− do report $ SkippedContent ('.':mname) pos

371+ do report $ SkippedContent ("." <> mname) pos

370372 return Nothing

371373 Tbl _ _ pos -> do

372374 report $ SkippedContent "TABLE" pos

@@ -375,12 +377,12 @@ parseCodeBlock = try $ do

375377 TextLine ss

376378 | not (null ss)

377379 , all isFontToken ss -> return Nothing

378− | otherwise -> return $ Just $ linePartsToString ss

380+ | otherwise -> return $ Just $ linePartsToText ss

379381

380382 isFontToken Font{} = True

381383 isFontToken _ = False

382384

383− getText :: Inline -> String

385+ getText :: Inline -> T.Text

384386 getText (Str s) = s

385387 getText Space = " "

386388 getText (Code _ s) = s

@@ -416,8 +418,8 @@ listItem mbListType = try $ do

416418 (ControlLine _ args _) <- mmacro "IP"

417419 case args of

418420 (arg1 : _) -> do

419− let cs = linePartsToString arg1

420− let cs' = if not ('.' `elem` cs || ')' `elem` cs) then cs ++ "." else cs

421+ let cs = linePartsToText arg1

422+ let cs' = if not (T.any (== '.') cs || T.any (== ')') cs) then cs <> "." else cs

421423 let lt = case Parsec.runParser anyOrderedListMarker defaultParserState

422424 "list marker" cs' of

423425 Right (start, listtype, listdelim)

@@ -467,7 +469,7 @@ parseLink args = do

467469 ControlLine _ endargs _ <- mmacro "UE"

468470 let url = case args of

469471 [] -> ""

470− (x:_) -> linePartsToString x

472+ (x:_) -> linePartsToText x

471473 return $ link url "" contents <>

472474 case endargs of

473475 [] -> mempty

@@ -479,7 +481,7 @@ parseEmailLink args = do

479481 ControlLine _ endargs _ <- mmacro "ME"

480482 let url = case args of

481483 [] -> ""

482− (x:_) -> "mailto:" ++ linePartsToString x

484+ (x:_) -> "mailto:" <> linePartsToText x

483485 return $ link url "" contents <>

484486 case endargs of

485487 [] -> mempty

@@ -490,6 +492,6 @@ skipUnknownMacro = do

490492 tok <- mmacroAny

491493 case tok of

492494 ControlLine mkind _ pos -> do

493− report $ SkippedContent ('.':mkind) pos

495+ report $ SkippedContent ("." <> mkind) pos

494496 return mempty

495497 _ -> Prelude.fail "the impossible happened"

src/Text/Pandoc/Readers/Markdown.hs+215−211

@@ -1,7 +1,9 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE RelaxedPolyRec #-}

33{-# LANGUAGE ScopedTypeVariables #-}

4−{-# LANGUAGE TupleSections #-}

4+{-# LANGUAGE TupleSections #-}

5+{-# LANGUAGE OverloadedStrings #-}

6+{-# LANGUAGE ViewPatterns #-}

57{- |

68 Module : Text.Pandoc.Readers.Markdown

@@ -19,14 +21,15 @@ import Prelude

1921import Control.Monad

2022import Control.Monad.Except (throwError)

2123import qualified Data.ByteString.Lazy as BS

22−import Data.Char (isAlphaNum, isPunctuation, isSpace, toLower)

23−import Data.List (intercalate, sortBy, transpose, elemIndex)

24+import Data.Char (isAlphaNum, isPunctuation, isSpace)

25+import Data.List (sortBy, transpose, elemIndex)

2426import qualified Data.Map as M

2527import Data.Maybe

2628import Data.Ord (comparing)

2729import qualified Data.Set as Set

2830import Data.Text (Text)

2931import qualified Data.Text as T

32+import qualified Data.Text.Lazy as TL

3033import qualified Data.YAML as YAML

3134import qualified Data.YAML.Event as YE

3235import System.FilePath (addExtension, takeExtension)

@@ -47,7 +50,7 @@ import Text.Pandoc.Shared

4750import qualified Text.Pandoc.UTF8 as UTF8

4851import Text.Pandoc.XML (fromEntities)

4952

50−type MarkdownParser m = ParserT [Char] ParserState m

53+type MarkdownParser m = ParserT Text ParserState m

5154

5255-- | Read markdown from an input string and return a Pandoc document.

5356readMarkdown :: PandocMonad m

@@ -56,7 +59,7 @@ readMarkdown :: PandocMonad m

5659 -> m Pandoc

5760readMarkdown opts s = do

5861 parsed <- readWithM parseMarkdown def{ stateOptions = opts }

59− (T.unpack (crFilter s) ++ "\n\n")

62+ (crFilter s <> "\n\n")

6063 case parsed of

6164 Right result -> return result

6265 Left e -> throwError e

@@ -77,7 +80,7 @@ isHruleChar '-' = True

7780isHruleChar '_' = True

7881isHruleChar _ = False

7982

80−setextHChars :: String

83+setextHChars :: [Char]

8184setextHChars = "=-"

8285

8386isBlank :: Char -> Bool

@@ -96,30 +99,30 @@ inList = do

9699 ctx <- stateParserContext <$> getState

97100 guard (ctx == ListItemState)

98101

99−spnl :: PandocMonad m => ParserT [Char] st m ()

102+spnl :: PandocMonad m => ParserT Text st m ()

100103spnl = try $ do

101104 skipSpaces

102105 optional newline

103106 skipSpaces

104107 notFollowedBy (char '\n')

105108

106−spnl' :: PandocMonad m => ParserT [Char] st m String

109+spnl' :: PandocMonad m => ParserT Text st m Text

107110spnl' = try $ do

108111 xs <- many spaceChar

109112 ys <- option "" $ try $ (:) <$> newline

110113 <*> (many spaceChar <* notFollowedBy (char '\n'))

111− return (xs ++ ys)

114+ return $ T.pack $ xs ++ ys

112115

113−indentSpaces :: PandocMonad m => MarkdownParser m String

116+indentSpaces :: PandocMonad m => MarkdownParser m Text

114117indentSpaces = try $ do

115118 tabStop <- getOption readerTabStop

116− count tabStop (char ' ') <|>

117− string "\t" <?> "indentation"

119+ countChar tabStop (char ' ') <|>

120+ textStr "\t" <?> "indentation"

118121

119−nonindentSpaces :: PandocMonad m => MarkdownParser m String

122+nonindentSpaces :: PandocMonad m => MarkdownParser m Text

120123nonindentSpaces = do

121124 n <- skipNonindentSpaces

122− return $ replicate n ' '

125+ return $ T.replicate n " "

123126

124127-- returns number of spaces parsed

125128skipNonindentSpaces :: PandocMonad m => MarkdownParser m Int

@@ -139,8 +142,9 @@ inlinesInBalancedBrackets :: PandocMonad m => MarkdownParser m (F Inlines)

139142inlinesInBalancedBrackets =

140143 try $ char '[' >> withRaw (go 1) >>=

141144 parseFromString inlines . stripBracket . snd

142− where stripBracket [] = []

143− stripBracket xs = if last xs == ']' then init xs else xs

145+ where stripBracket t = case T.unsnoc t of

146+ Just (t', ']') -> t'

147+ _ -> t

144148 go :: PandocMonad m => Int -> MarkdownParser m ()

145149 go 0 = return ()

146150 go openBrackets =

@@ -160,7 +164,7 @@ inlinesInBalancedBrackets =

160164-- document structure

161165--

162166

163−rawTitleBlockLine :: PandocMonad m => MarkdownParser m String

167+rawTitleBlockLine :: PandocMonad m => MarkdownParser m Text

164168rawTitleBlockLine = do

165169 char '%'

166170 skipSpaces

@@ -169,7 +173,7 @@ rawTitleBlockLine = do

169173 notFollowedBy blankline

170174 skipSpaces

171175 anyLine

172− return $ trim $ unlines (first:rest)

176+ return $ trim $ T.unlines (first:rest)

173177

174178titleLine :: PandocMonad m => MarkdownParser m (F Inlines)

175179titleLine = try $ do

@@ -222,9 +226,9 @@ yamlMetaBlock = try $ do

222226 notFollowedBy blankline -- if --- is followed by a blank it's an HRULE

223227 rawYamlLines <- manyTill anyLine stopLine

224228 -- by including --- and ..., we allow yaml blocks with just comments:

225− let rawYaml = unlines ("---" : (rawYamlLines ++ ["..."]))

229+ let rawYaml = T.unlines ("---" : (rawYamlLines ++ ["..."]))

226230 optional blanklines

227− newMetaF <- yamlBsToMeta $ UTF8.fromStringLazy rawYaml

231+ newMetaF <- yamlBsToMeta $ UTF8.fromTextLazy $ TL.fromStrict rawYaml

228232 -- Since `<>` is left-biased, existing values are not touched:

229233 updateState $ \st -> st{ stateMeta' = (stateMeta' st) <> newMetaF }

230234 return mempty

@@ -255,7 +259,7 @@ yamlBsToMeta bstr = do

255259 return . return $ mempty

256260 Left (_pos, err') -> do

257261 logMessage $ CouldNotParseYamlMetadata

258− err' pos

262+ (T.pack err') pos

259263 return . return $ mempty

260264

261265nodeToKey :: PandocMonad m => YAML.Node YE.Pos -> m Text

@@ -270,11 +274,11 @@ toMetaValue x =

270274 -- Note: a standard quoted or unquoted YAML value will

271275 -- not end in a newline, but a "block" set off with

272276 -- `|` or `>` will.

273− if (T.pack "\n") `T.isSuffixOf` x

274− then parseFromString' (asBlocks <$> parseBlocks) (xstring <> "\n")

277+ if "\n" `T.isSuffixOf` x

278+ then parseFromString' (asBlocks <$> parseBlocks) (x <> "\n")

275279 else parseFromString'

276280 ((asInlines <$> try pInlines) <|> (asBlocks <$> parseBlocks))

277− xstring

281+ x

278282 where pInlines = trimInlinesF . mconcat <$> manyTill inline eof

279283 asBlocks p = do

280284 p' <- p

@@ -282,7 +286,6 @@ toMetaValue x =

282286 asInlines p = do

283287 p' <- p

284288 return $ MetaInlines (B.toList p')

285− xstring = T.unpack x

286289

287290checkBoolean :: Text -> Maybe Bool

288291checkBoolean t =

@@ -298,8 +301,8 @@ yamlToMetaValue (YAML.Scalar _ x) =

298301 case x of

299302 YAML.SStr t -> toMetaValue t

300303 YAML.SBool b -> return $ return $ MetaBool b

301− YAML.SFloat d -> return $ return $ MetaString (show d)

302− YAML.SInt i -> return $ return $ MetaString (show i)

304+ YAML.SFloat d -> return $ return $ MetaString $ tshow d

305+ YAML.SInt i -> return $ return $ MetaString $ tshow i

303306 YAML.SUnknown _ t ->

304307 case checkBoolean t of

305308 Just b -> return $ return $ MetaBool b

@@ -315,7 +318,7 @@ yamlToMetaValue _ = return $ return $ MetaString ""

315318

316319yamlMap :: PandocMonad m

317320 => M.Map (YAML.Node YE.Pos) (YAML.Node YE.Pos)

318− -> MarkdownParser m (F (M.Map String MetaValue))

321+ -> MarkdownParser m (F (M.Map Text MetaValue))

319322yamlMap o = do

320323 kvs <- forM (M.toList o) $ \(key, v) -> do

321324 k <- nodeToKey key

@@ -323,12 +326,12 @@ yamlMap o = do

323326 let kvs' = filter (not . ignorable . fst) kvs

324327 (fmap M.fromList . sequence) <$> mapM toMeta kvs'

325328 where

326− ignorable t = (T.pack "_") `T.isSuffixOf` t

329+ ignorable t = "_" `T.isSuffixOf` t

327330 toMeta (k, v) = do

328331 fv <- yamlToMetaValue v

329332 return $ do

330333 v' <- fv

331− return (T.unpack k, v')

334+ return (k, v')

332335

333336stopLine :: PandocMonad m => MarkdownParser m ()

334337stopLine = try $ (string "---" <|> string "...") >> blankline >> return ()

@@ -343,14 +346,14 @@ mmdTitleBlock = try $ do

343346 updateState $ \st -> st{ stateMeta' = stateMeta' st <>

344347 return (Meta $ M.fromList kvPairs) }

345348

346−kvPair :: PandocMonad m => Bool -> MarkdownParser m (String, MetaValue)

349+kvPair :: PandocMonad m => Bool -> MarkdownParser m (Text, MetaValue)

347350kvPair allowEmpty = try $ do

348− key <- many1Till (alphaNum <|> oneOf "_- ") (char ':')

349− val <- trim <$> manyTill anyChar

351+ key <- many1TillChar (alphaNum <|> oneOf "_- ") (char ':')

352+ val <- trim <$> manyTillChar anyChar

350353 (try $ newline >> lookAhead (blankline <|> nonspaceChar))

351− guard $ allowEmpty || not (null val)

352− let key' = concat $ words $ map toLower key

353− let val' = MetaBlocks $ B.toList $ B.plain $B.text val

354+ guard $ allowEmpty || not (T.null val)

355+ let key' = T.concat $ T.words $ T.toLower key

356+ let val' = MetaBlocks $ B.toList $ B.plain $ B.text val

354357 return (key',val')

355358

356359parseMarkdown :: PandocMonad m => MarkdownParser m Pandoc

@@ -380,13 +383,13 @@ referenceKey = try $ do

380383 (_,raw) <- reference

381384 char ':'

382385 skipSpaces >> optional newline >> skipSpaces >> notFollowedBy (char '[')

383− let sourceURL = fmap unwords $ many $ try $ do

386+ let sourceURL = fmap T.unwords $ many $ try $ do

384387 skipMany spaceChar

385388 notFollowedBy' referenceTitle

386389 notFollowedBy' $ guardEnabled Ext_link_attributes >> attributes

387390 notFollowedBy' (() <$ reference)

388− many1 $ notFollowedBy space >> litChar

389− let betweenAngles = try $ char '<' >> manyTill litChar (char '>')

391+ many1Char $ notFollowedBy space >> litChar

392+ let betweenAngles = try $ char '<' >> manyTillChar litChar (char '>')

390393 src <- try betweenAngles <|> sourceURL

391394 tit <- option "" referenceTitle

392395 attr <- option nullAttr $ try $

@@ -411,20 +414,20 @@ referenceKey = try $ do

411414 updateState $ \s -> s { stateKeys = M.insert key (target, attr') oldkeys }

412415 return $ return mempty

413416

414−referenceTitle :: PandocMonad m => MarkdownParser m String

417+referenceTitle :: PandocMonad m => MarkdownParser m Text

415418referenceTitle = try $ do

416419 skipSpaces >> optional newline >> skipSpaces

417420 quotedTitle '"' <|> quotedTitle '\'' <|> charsInBalanced '(' ')' litChar

418421

419422-- A link title in quotes

420−quotedTitle :: PandocMonad m => Char -> MarkdownParser m String

423+quotedTitle :: PandocMonad m => Char -> MarkdownParser m Text

421424quotedTitle c = try $ do

422425 char c

423426 notFollowedBy spaces

424427 let pEnder = try $ char c >> notFollowedBy (satisfy isAlphaNum)

425− let regChunk = many1 (noneOf ['\\','\n','&',c]) <|> count 1 litChar

426− let nestedChunk = (\x -> [c] ++ x ++ [c]) <$> quotedTitle c

427− unwords . words . concat <$> manyTill (nestedChunk <|> regChunk) pEnder

428+ let regChunk = many1Char (noneOf ['\\','\n','&',c]) <|> countChar 1 litChar

429+ let nestedChunk = (\x -> T.singleton c <> x <> T.singleton c) <$> quotedTitle c

430+ T.unwords . T.words . T.concat <$> manyTill (nestedChunk <|> regChunk) pEnder

428431

429432-- | PHP Markdown Extra style abbreviation key. Currently

430433-- we just skip them, since Pandoc doesn't have an element for

@@ -440,21 +443,21 @@ abbrevKey = do

440443 blanklines

441444 return $ return mempty

442445

443−noteMarker :: PandocMonad m => MarkdownParser m String

444−noteMarker = string "[^" >> many1Till (satisfy $ not . isBlank) (char ']')

446+noteMarker :: PandocMonad m => MarkdownParser m Text

447+noteMarker = string "[^" >> many1TillChar (satisfy $ not . isBlank) (char ']')

445448

446−rawLine :: PandocMonad m => MarkdownParser m String

449+rawLine :: PandocMonad m => MarkdownParser m Text

447450rawLine = try $ do

448451 notFollowedBy blankline

449452 notFollowedBy' $ try $ skipNonindentSpaces >> noteMarker

450453 optional indentSpaces

451454 anyLine

452455

453−rawLines :: PandocMonad m => MarkdownParser m String

456+rawLines :: PandocMonad m => MarkdownParser m Text

454457rawLines = do

455458 first <- anyLine

456459 rest <- many rawLine

457− return $ unlines (first:rest)

460+ return $ T.unlines (first:rest)

458461

459462noteBlock :: PandocMonad m => MarkdownParser m (F Blocks)

460463noteBlock = try $ do

@@ -466,7 +469,7 @@ noteBlock = try $ do

466469 optional indentSpaces

467470 first <- rawLines

468471 rest <- many $ try $ blanklines >> indentSpaces >> rawLines

469− let raw = unlines (first:rest) ++ "\n"

472+ let raw = T.unlines (first:rest) <> "\n"

470473 optional blanklines

471474 parsed <- parseFromString' parseBlocks raw

472475 oldnotes <- stateNotes' <$> getState

@@ -510,7 +513,7 @@ block = do

510513 , para

511514 , plain

512515 ] <?> "block"

513− trace (take 60 $ show $ B.toList $ runF res defaultParserState)

516+ trace (T.take 60 $ tshow $ B.toList $ runF res defaultParserState)

514517 return res

515518

516519--

@@ -570,7 +573,7 @@ mmdHeaderIdentifier :: PandocMonad m => MarkdownParser m Attr

570573mmdHeaderIdentifier = do

571574 (_, raw) <- reference

572575 let raw' = trim $ stripFirstAndLast raw

573− let ident = concat $ words $ map toLower raw'

576+ let ident = T.concat $ T.words $ T.toLower raw'

574577 let attr = (ident, [], [])

575578 guardDisabled Ext_implicit_header_references

576579 <|> registerImplicitHeader raw' attr

@@ -600,20 +603,20 @@ setextHeader = try $ do

600603 <|> registerImplicitHeader raw attr'

601604 return $ B.headerWith attr' level <$> text

602605

603−registerImplicitHeader :: PandocMonad m => String -> Attr -> MarkdownParser m ()

606+registerImplicitHeader :: PandocMonad m => Text -> Attr -> MarkdownParser m ()

604607registerImplicitHeader raw attr@(ident, _, _)

605− | null raw = return ()

608+ | T.null raw = return ()

606609 | otherwise = do

607− let key = toKey $ "[" ++ raw ++ "]"

610+ let key = toKey $ "[" <> raw <> "]"

608611 updateState $ \s ->

609− s { stateHeaderKeys = M.insert key (('#':ident,""), attr)

612+ s { stateHeaderKeys = M.insert key (("#" <> ident,""), attr)

610613 (stateHeaderKeys s) }

611614

612615--

613616-- hrule block

614617--

615618

616−hrule :: PandocMonad m => ParserT [Char] st m (F Blocks)

619+hrule :: PandocMonad m => ParserT Text st m (F Blocks)

617620hrule = try $ do

618621 skipSpaces

619622 start <- satisfy isHruleChar

@@ -627,13 +630,13 @@ hrule = try $ do

627630-- code blocks

628631--

629632

630−indentedLine :: PandocMonad m => MarkdownParser m String

633+indentedLine :: PandocMonad m => MarkdownParser m Text

631634indentedLine = indentSpaces >> anyLineNewline

632635

633636blockDelimiter :: PandocMonad m

634637 => (Char -> Bool)

635638 -> Maybe Int

636− -> ParserT [Char] ParserState m Int

639+ -> ParserT Text ParserState m Int

637640blockDelimiter f len = try $ do

638641 skipNonindentSpaces

639642 c <- lookAhead (satisfy f)

@@ -652,11 +655,11 @@ attributes = try $ do

652655attribute :: PandocMonad m => MarkdownParser m (Attr -> Attr)

653656attribute = identifierAttr <|> classAttr <|> keyValAttr <|> specialAttr

654657

655−identifier :: PandocMonad m => MarkdownParser m String

658+identifier :: PandocMonad m => MarkdownParser m Text

656659identifier = do

657660 first <- letter

658661 rest <- many $ alphaNum <|> oneOf "-_:."

659− return (first:rest)

662+ return $ T.pack (first:rest)

660663

661664identifierAttr :: PandocMonad m => MarkdownParser m (Attr -> Attr)

662665identifierAttr = try $ do

@@ -674,15 +677,15 @@ keyValAttr :: PandocMonad m => MarkdownParser m (Attr -> Attr)

674677keyValAttr = try $ do

675678 key <- identifier

676679 char '='

677− val <- enclosed (char '"') (char '"') litChar

678− <|> enclosed (char '\'') (char '\'') litChar

680+ val <- T.pack <$> enclosed (char '"') (char '"') litChar

681+ <|> T.pack <$> enclosed (char '\'') (char '\'') litChar

679682 <|> ("" <$ try (string "\"\""))

680683 <|> ("" <$ try (string "''"))

681− <|> many (escapedChar' <|> noneOf " \t\n\r}")

684+ <|> manyChar (escapedChar' <|> noneOf " \t\n\r}")

682685 return $ \(id',cs,kvs) ->

683686 case key of

684687 "id" -> (val,cs,kvs)

685− "class" -> (id',cs ++ words val,kvs)

688+ "class" -> (id',cs ++ T.words val,kvs)

686689 _ -> (id',cs,kvs ++ [(key,val)])

687690

688691specialAttr :: PandocMonad m => MarkdownParser m (Attr -> Attr)

@@ -690,12 +693,12 @@ specialAttr = do

690693 char '-'

691694 return $ \(id',cs,kvs) -> (id',cs ++ ["unnumbered"],kvs)

692695

693−rawAttribute :: PandocMonad m => MarkdownParser m String

696+rawAttribute :: PandocMonad m => MarkdownParser m Text

694697rawAttribute = do

695698 char '{'

696699 skipMany spaceChar

697700 char '='

698− format <- many1 $ satisfy (\c -> isAlphaNum c || c `elem` "-_")

701+ format <- many1Char $ satisfy (\c -> isAlphaNum c || c `elem` ['-', '_'])

699702 skipMany spaceChar

700703 char '}'

701704 return format

@@ -703,7 +706,7 @@ rawAttribute = do

703706codeBlockFenced :: PandocMonad m => MarkdownParser m (F Blocks)

704707codeBlockFenced = try $ do

705708 indentchars <- nonindentSpaces

706− let indentLevel = length indentchars

709+ let indentLevel = T.length indentchars

707710 c <- try (guardEnabled Ext_fenced_code_blocks >> lookAhead (char '~'))

708711 <|> (guardEnabled Ext_backtick_code_blocks >> lookAhead (char '`'))

709712 size <- blockDelimiter (== c) Nothing

@@ -713,9 +716,9 @@ codeBlockFenced = try $ do

713716 <|>

714717 (Right <$> option ("",[],[])

715718 (try (guardEnabled Ext_fenced_code_attributes >> attributes)

716− <|> ((\x -> ("",[toLanguageId x],[])) <$> many1 nonspaceChar)))

719+ <|> ((\x -> ("",[toLanguageId x],[])) <$> many1Char nonspaceChar)))

717720 blankline

718− contents <- intercalate "\n" <$>

721+ contents <- T.intercalate "\n" <$>

719722 manyTill (gobbleAtMostSpaces indentLevel >> anyLine)

720723 (try $ do

721724 blockDelimiter (== c) (Just size)

@@ -726,8 +729,8 @@ codeBlockFenced = try $ do

726729 Right attr -> B.codeBlockWith attr contents

727730

728731-- correctly handle github language identifiers

729−toLanguageId :: String -> String

730−toLanguageId = map toLower . go

732+toLanguageId :: Text -> Text

733+toLanguageId = T.toLower . go

731734 where go "c++" = "cpp"

732735 go "objective-c" = "objectivec"

733736 go x = x

@@ -737,11 +740,11 @@ codeBlockIndented = do

737740 contents <- many1 (indentedLine <|>

738741 try (do b <- blanklines

739742 l <- indentedLine

740− return $ b ++ l))

743+ return $ b <> l))

741744 optional blanklines

742745 classes <- getOption readerIndentedCodeClasses

743746 return $ return $ B.codeBlockWith ("", classes, []) $

744− stripTrailingNewlines $ concat contents

747+ stripTrailingNewlines $ T.concat contents

745748

746749lhsCodeBlock :: PandocMonad m => MarkdownParser m (F Blocks)

747750lhsCodeBlock = do

@@ -751,33 +754,33 @@ lhsCodeBlock = do

751754 <|> (return . B.codeBlockWith ("",["haskell"],[]) <$>

752755 lhsCodeBlockInverseBird)

753756

754−lhsCodeBlockLaTeX :: PandocMonad m => MarkdownParser m String

757+lhsCodeBlockLaTeX :: PandocMonad m => MarkdownParser m Text

755758lhsCodeBlockLaTeX = try $ do

756759 string "\\begin{code}"

757760 manyTill spaceChar newline

758− contents <- many1Till anyChar (try $ string "\\end{code}")

761+ contents <- many1TillChar anyChar (try $ string "\\end{code}")

759762 blanklines

760763 return $ stripTrailingNewlines contents

761764

762−lhsCodeBlockBird :: PandocMonad m => MarkdownParser m String

765+lhsCodeBlockBird :: PandocMonad m => MarkdownParser m Text

763766lhsCodeBlockBird = lhsCodeBlockBirdWith '>'

764767

765−lhsCodeBlockInverseBird :: PandocMonad m => MarkdownParser m String

768+lhsCodeBlockInverseBird :: PandocMonad m => MarkdownParser m Text

766769lhsCodeBlockInverseBird = lhsCodeBlockBirdWith '<'

767770

768−lhsCodeBlockBirdWith :: PandocMonad m => Char -> MarkdownParser m String

771+lhsCodeBlockBirdWith :: PandocMonad m => Char -> MarkdownParser m Text

769772lhsCodeBlockBirdWith c = try $ do

770773 pos <- getPosition

771774 when (sourceColumn pos /= 1) $ Prelude.fail "Not in first column"

772775 lns <- many1 $ birdTrackLine c

773776 -- if (as is normal) there is always a space after >, drop it

774− let lns' = if all (\ln -> null ln || take 1 ln == " ") lns

775− then map (drop 1) lns

777+ let lns' = if all (\ln -> T.null ln || T.take 1 ln == " ") lns

778+ then map (T.drop 1) lns

776779 else lns

777780 blanklines

778− return $ intercalate "\n" lns'

781+ return $ T.intercalate "\n" lns'

779782

780−birdTrackLine :: PandocMonad m => Char -> ParserT [Char] st m String

783+birdTrackLine :: PandocMonad m => Char -> ParserT Text st m Text

781784birdTrackLine c = try $ do

782785 char c

783786 -- allow html tags on left margin:

@@ -791,12 +794,12 @@ birdTrackLine c = try $ do

791794emailBlockQuoteStart :: PandocMonad m => MarkdownParser m Char

792795emailBlockQuoteStart = try $ skipNonindentSpaces >> char '>' <* optional (char ' ')

793796

794−emailBlockQuote :: PandocMonad m => MarkdownParser m [String]

797+emailBlockQuote :: PandocMonad m => MarkdownParser m [Text]

795798emailBlockQuote = try $ do

796799 emailBlockQuoteStart

797− let emailLine = many $ nonEndline <|> try

798− (endline >> notFollowedBy emailBlockQuoteStart >>

799− return '\n')

800+ let emailLine = manyChar $ nonEndline <|> try

801+ (endline >> notFollowedBy emailBlockQuoteStart >>

802+ return '\n')

800803 let emailSep = try (newline >> emailBlockQuoteStart)

801804 first <- emailLine

802805 rest <- many $ try $ emailSep >> emailLine

@@ -809,7 +812,7 @@ blockQuote :: PandocMonad m => MarkdownParser m (F Blocks)

809812blockQuote = do

810813 raw <- emailBlockQuote

811814 -- parse the extracted block, which may contain various block elements:

812− contents <- parseFromString' parseBlocks $ intercalate "\n" raw ++ "\n\n"

815+ contents <- parseFromString' parseBlocks $ T.intercalate "\n" raw <> "\n\n"

813816 return $ B.blockQuote <$> contents

814817

815818--

@@ -833,7 +836,7 @@ orderedListStart mbstydelim = try $ do

833836 skipNonindentSpaces

834837 notFollowedBy $ string "p." >> spaceChar >> digit -- page number

835838 (do guardDisabled Ext_fancy_lists

836− start <- many1 digit >>= safeRead

839+ start <- many1Char digit >>= safeRead

837840 char '.'

838841 gobbleSpaces 1 <|> () <$ lookAhead newline

839842 optional $ try (gobbleAtMostSpaces 3 >> notFollowedBy spaceChar)

@@ -857,7 +860,7 @@ orderedListStart mbstydelim = try $ do

857860listStart :: PandocMonad m => MarkdownParser m ()

858861listStart = bulletListStart <|> Control.Monad.void (orderedListStart Nothing)

859862

860−listLine :: PandocMonad m => Int -> MarkdownParser m String

863+listLine :: PandocMonad m => Int -> MarkdownParser m Text

861864listLine continuationIndent = try $ do

862865 notFollowedBy' (do gobbleSpaces continuationIndent

863866 skipMany spaceChar

@@ -867,19 +870,19 @@ listLine continuationIndent = try $ do

867870 optional (() <$ gobbleSpaces continuationIndent)

868871 listLineCommon

869872

870−listLineCommon :: PandocMonad m => MarkdownParser m String

871−listLineCommon = concat <$> manyTill

872− ( many1 (satisfy $ \c -> c `notElem` ['\n', '<', '`'])

873+listLineCommon :: PandocMonad m => MarkdownParser m Text

874+listLineCommon = T.concat <$> manyTill

875+ ( many1Char (satisfy $ \c -> c `notElem` ['\n', '<', '`'])

873876 <|> fmap snd (withRaw code)

874877 <|> fmap snd (htmlTag isCommentTag)

875− <|> count 1 anyChar

878+ <|> countChar 1 anyChar

876879 ) newline

877880

878881-- parse raw text for one list item, excluding start marker and continuations

879882rawListItem :: PandocMonad m

880883 => Bool -- four space rule

881884 -> MarkdownParser m a

882− -> MarkdownParser m (String, Int)

885+ -> MarkdownParser m (Text, Int)

883886rawListItem fourSpaceRule start = try $ do

884887 pos1 <- getPosition

885888 start

@@ -892,14 +895,14 @@ rawListItem fourSpaceRule start = try $ do

892895 notFollowedBy (() <$ codeBlockFenced)

893896 notFollowedBy blankline

894897 listLine continuationIndent)

895− blanks <- many blankline

896− let result = unlines (first:rest) ++ blanks

898+ blanks <- manyChar blankline

899+ let result = T.unlines (first:rest) <> blanks

897900 return (result, continuationIndent)

898901

899902-- continuation of a list item - indented and separated by blankline

900903-- or (in compact lists) endline.

901904-- note: nested lists are parsed as continuations

902−listContinuation :: PandocMonad m => Int -> MarkdownParser m String

905+listContinuation :: PandocMonad m => Int -> MarkdownParser m Text

903906listContinuation continuationIndent = try $ do

904907 x <- try $ do

905908 notFollowedBy blankline

@@ -913,12 +916,12 @@ listContinuation continuationIndent = try $ do

913916 notFollowedByDivCloser

914917 gobbleSpaces continuationIndent <|> notFollowedBy' listStart

915918 anyLineNewline

916− blanks <- many blankline

917− return $ concat (x:xs) ++ blanks

919+ blanks <- manyChar blankline

920+ return $ T.concat (x:xs) <> blanks

918921

919922-- Variant of blanklines that doesn't require blank lines

920923-- before a fence or eof.

921−blanklines' :: PandocMonad m => MarkdownParser m [Char]

924+blanklines' :: PandocMonad m => MarkdownParser m Text

922925blanklines' = blanklines <|> try checkDivCloser

923926 where checkDivCloser = do

924927 guardEnabled Ext_fenced_divs

@@ -954,7 +957,7 @@ listItem fourSpaceRule start = try $ do

954957 (first, continuationIndent) <- rawListItem fourSpaceRule start

955958 continuations <- many (listContinuation continuationIndent)

956959 -- parse the extracted block, which may contain various block elements:

957− let raw = concat (first:continuations)

960+ let raw = T.concat (first:continuations)

958961 contents <- parseFromString' parseBlocks raw

959962 updateState (\st -> st {stateParserContext = oldContext})

960963 exts <- getOption readerExtensions

@@ -990,7 +993,7 @@ defListMarker = do

990993 sps <- nonindentSpaces

991994 char ':' <|> char '~'

992995 tabStop <- getOption readerTabStop

993− let remaining = tabStop - (length sps + 1)

996+ let remaining = tabStop - (T.length sps + 1)

994997 if remaining > 0

995998 then try (count remaining (char ' ')) <|> string "\t" <|> many1 spaceChar

996999 else mzero

@@ -1001,11 +1004,11 @@ definitionListItem compact = try $ do

10011004 rawLine' <- anyLine

10021005 raw <- many1 $ defRawBlock compact

10031006 term <- parseFromString' (trimInlinesF <$> inlines) rawLine'

1004− contents <- mapM (parseFromString' parseBlocks . (++"\n")) raw

1007+ contents <- mapM (parseFromString' parseBlocks . (<> "\n")) raw

10051008 optional blanklines

10061009 return $ liftM2 (,) term (sequence contents)

10071010

1008−defRawBlock :: PandocMonad m => Bool -> MarkdownParser m String

1011+defRawBlock :: PandocMonad m => Bool -> MarkdownParser m Text

10091012defRawBlock compact = try $ do

10101013 hasBlank <- option False $ blankline >> return True

10111014 defListMarker

@@ -1020,13 +1023,13 @@ defRawBlock compact = try $ do

10201023 <|> notFollowedBy defListMarker

10211024 anyLine )

10221025 rawlines <- many dline

1023− cont <- fmap concat $ many $ try $ do

1026+ cont <- fmap T.concat $ many $ try $ do

10241027 trailing <- option "" blanklines

10251028 ln <- indentSpaces >> notFollowedBy blankline >> anyLine

10261029 lns <- many dline

1027− return $ trailing ++ unlines (ln:lns)

1028− return $ trimr (firstline ++ unlines rawlines ++ cont) ++

1029− if hasBlank || not (null cont) then "\n\n" else ""

1030+ return $ trailing <> T.unlines (ln:lns)

1031+ return $ trimr (firstline <> T.unlines rawlines <> cont) <>

1032+ if hasBlank || not (T.null cont) then "\n\n" else ""

10301033

10311034definitionList :: PandocMonad m => MarkdownParser m (F Blocks)

10321035definitionList = try $ do

@@ -1063,7 +1066,7 @@ para = try $ do

10631066 | not (null alt) ->

10641067 -- the fig: at beginning of title indicates a figure

10651068 return $ B.singleton

1066− $ Image attr alt (src,'f':'i':'g':':':tit)

1069+ $ Image attr alt (src, "fig:" <> tit)

10671070 _ -> return x'

10681071 | otherwise = x

10691072 result <- implicitFigures . trimInlinesF <$> inlines1

@@ -1082,7 +1085,7 @@ para = try $ do

10821085 inHtmlBlock <- stateInHtmlBlock <$> getState

10831086 case inHtmlBlock of

10841087 Just "div" -> () <$

1085− lookAhead (htmlTag (~== TagClose "div"))

1088+ lookAhead (htmlTag (~== TagClose ("div" :: Text)))

10861089 _ -> mzero

10871090 <|> do guardEnabled Ext_fenced_divs

10881091 divLevel <- stateFencedDivLevel <$> getState

@@ -1098,7 +1101,7 @@ plain = fmap B.plain . trimInlinesF <$> inlines1

10981101-- raw html

10991102--

11001103

1101−htmlElement :: PandocMonad m => MarkdownParser m String

1104+htmlElement :: PandocMonad m => MarkdownParser m Text

11021105htmlElement = rawVerbatimBlock

11031106 <|> strictHtmlBlock

11041107 <|> fmap snd (htmlTag isBlockTag)

@@ -1132,14 +1135,14 @@ htmlBlock' = try $ do

11321135 first <- htmlElement

11331136 skipMany spaceChar

11341137 optional blanklines

1135− return $ if null first

1138+ return $ if T.null first

11361139 then mempty

11371140 else return $ B.rawBlock "html" first

11381141

1139−strictHtmlBlock :: PandocMonad m => MarkdownParser m String

1142+strictHtmlBlock :: PandocMonad m => MarkdownParser m Text

11401143strictHtmlBlock = htmlInBalanced (not . isInlineTag)

11411144

1142−rawVerbatimBlock :: PandocMonad m => MarkdownParser m String

1145+rawVerbatimBlock :: PandocMonad m => MarkdownParser m Text

11431146rawVerbatimBlock = htmlInBalanced isVerbTag

11441147 where isVerbTag (TagOpen "pre" _) = True

11451148 isVerbTag (TagOpen "style" _) = True

@@ -1150,13 +1153,13 @@ rawVerbatimBlock = htmlInBalanced isVerbTag

11501153rawTeXBlock :: PandocMonad m => MarkdownParser m (F Blocks)

11511154rawTeXBlock = do

11521155 guardEnabled Ext_raw_tex

1153− result <- (B.rawBlock "tex" . trim . concat <$>

1154− many1 ((++) <$> rawConTeXtEnvironment <*> spnl'))

1155− <|> (B.rawBlock "tex" . trim . concat <$>

1156− many1 ((++) <$> rawLaTeXBlock <*> spnl'))

1156+ result <- (B.rawBlock "tex" . trim . T.concat <$>

1157+ many1 ((<>) <$> rawConTeXtEnvironment <*> spnl'))

1158+ <|> (B.rawBlock "tex" . trim . T.concat <$>

1159+ many1 ((<>) <$> rawLaTeXBlock <*> spnl'))

11571160 return $ case B.toList result of

11581161 [RawBlock _ cs]

1159− | all (`elem` [' ','\t','\n']) cs -> return mempty

1162+ | T.all (`elem` [' ','\t','\n']) cs -> return mempty

11601163 -- don't create a raw block for suppressed macro defs

11611164 _ -> return result

11621165

@@ -1186,7 +1189,7 @@ rawHtmlBlocks = do

11861189 return result

11871190

11881191-- remove markdown="1" attribute

1189−stripMarkdownAttribute :: String -> String

1192+stripMarkdownAttribute :: Text -> Text

11901193stripMarkdownAttribute s = renderTags' $ map filterAttrib $ parseTags s

11911194 where filterAttrib (TagOpen t as) = TagOpen t

11921195 [(k,v) | (k,v) <- as, k /= "markdown"]

@@ -1211,7 +1214,7 @@ lineBlock = try $ do

12111214-- and the length including trailing space.

12121215dashedLine :: PandocMonad m

12131216 => Char

1214− -> ParserT [Char] st m (Int, Int)

1217+ -> ParserT Text st m (Int, Int)

12151218dashedLine ch = do

12161219 dashes <- many1 (char ch)

12171220 sp <- many spaceChar

@@ -1232,9 +1235,9 @@ simpleTableHeader headless = try $ do

12321235 dashes <- many1 (dashedLine '-')

12331236 newline

12341237 let (lengths, lines') = unzip dashes

1235− let indices = scanl (+) (length initSp) lines'

1238+ let indices = scanl (+) (T.length initSp) lines'

12361239 -- If no header, calculate alignment on basis of first row of text

1237− rawHeads <- fmap (tail . splitStringByIndices (init indices)) $

1240+ rawHeads <- fmap (tail . splitTextByIndices (init indices)) $

12381241 if headless

12391242 then lookAhead anyLine

12401243 else return rawContent

@@ -1250,15 +1253,15 @@ simpleTableHeader headless = try $ do

12501253-- Returns an alignment type for a table, based on a list of strings

12511254-- (the rows of the column header) and a number (the length of the

12521255-- dashed line under the rows.

1253−alignType :: [String]

1256+alignType :: [Text]

12541257 -> Int

12551258 -> Alignment

12561259alignType [] _ = AlignDefault

12571260alignType strLst len =

1258− let nonempties = filter (not . null) $ map trimr strLst

1261+ let nonempties = filter (not . T.null) $ map trimr strLst

12591262 (leftSpace, rightSpace) =

1260− case sortBy (comparing length) nonempties of

1261− (x:_) -> (head x `elem` " \t", length x < len)

1263+ case sortBy (comparing T.length) nonempties of

1264+ (x:_) -> (T.head x `elem` [' ', 't'], T.length x < len)

12621265 [] -> (False, False)

12631266 in case (leftSpace, rightSpace) of

12641267 (True, False) -> AlignRight

@@ -1267,7 +1270,7 @@ alignType strLst len =

12671270 (False, False) -> AlignDefault

12681271

12691272-- Parse a table footer - dashed lines followed by blank line.

1270−tableFooter :: PandocMonad m => MarkdownParser m String

1273+tableFooter :: PandocMonad m => MarkdownParser m Text

12711274tableFooter = try $ skipNonindentSpaces >> many1 (dashedLine '-') >> blanklines'

12721275

12731276-- Parse a table separator - dashed line.

@@ -1277,12 +1280,12 @@ tableSep = try $ skipNonindentSpaces >> many1 (dashedLine '-') >> char '\n'

12771280-- Parse a raw line and split it into chunks by indices.

12781281rawTableLine :: PandocMonad m

12791282 => [Int]

1280− -> MarkdownParser m [String]

1283+ -> MarkdownParser m [Text]

12811284rawTableLine indices = do

12821285 notFollowedBy' (blanklines' <|> tableFooter)

1283− line <- many1Till anyChar newline

1286+ line <- many1TillChar anyChar newline

12841287 return $ map trim $ tail $

1285− splitStringByIndices (init indices) line

1288+ splitTextByIndices (init indices) line

12861289

12871290-- Parse a table line and return a list of lists of blocks (columns).

12881291tableLine :: PandocMonad m

@@ -1297,7 +1300,7 @@ multilineRow :: PandocMonad m

12971300 -> MarkdownParser m (F [Blocks])

12981301multilineRow indices = do

12991302 colLines <- many1 (rawTableLine indices)

1300− let cols = map unlines $ transpose colLines

1303+ let cols = map T.unlines $ transpose colLines

13011304 fmap sequence $ mapM (parseFromString' (mconcat <$> many plain)) cols

13021305

13031306-- Parses a table caption: inlines beginning with 'Table:'

@@ -1344,7 +1347,7 @@ multilineTableHeader headless = try $ do

13441347 dashes <- many1 (dashedLine '-')

13451348 newline

13461349 let (lengths, lines') = unzip dashes

1347− let indices = scanl (+) (length initSp) lines'

1350+ let indices = scanl (+) (T.length initSp) lines'

13481351 -- compensate for the fact that intercolumn spaces are

13491352 -- not included in the last index:

13501353 let indices' = case reverse indices of

@@ -1352,14 +1355,14 @@ multilineTableHeader headless = try $ do

13521355 (x:xs) -> reverse (x+1:xs)

13531356 rawHeadsList <- if headless

13541357 then fmap (map (:[]) . tail .

1355− splitStringByIndices (init indices')) $ lookAhead anyLine

1358+ splitTextByIndices (init indices')) $ lookAhead anyLine

13561359 else return $ transpose $ map

1357− (tail . splitStringByIndices (init indices'))

1360+ (tail . splitTextByIndices (init indices'))

13581361 rawContent

13591362 let aligns = zipWith alignType rawHeadsList lengths

13601363 let rawHeads = if headless

13611364 then replicate (length dashes) ""

1362− else map (unlines . map trim) rawHeadsList

1365+ else map (T.unlines . map trim) rawHeadsList

13631366 heads <- fmap sequence $

13641367 mapM ((parseFromString' (mconcat <$> many plain)).trim) rawHeads

13651368 return (heads, aligns, indices')

@@ -1393,7 +1396,7 @@ pipeTable = try $ do

13931396 lines' <- many pipeTableRow

13941397 let lines'' = map (take (length aligns) <$>) lines'

13951398 let maxlength = maximum $

1396− map (\x -> length . stringify $ runF x def) (heads' : lines'')

1399+ map (\x -> T.length . stringify $ runF x def) (heads' : lines'')

13971400 numColumns <- getOption readerColumns

13981401 let widths = if maxlength > numColumns

13991402 then map (\len ->

@@ -1430,7 +1433,7 @@ pipeTableCell =

14301433 return $ B.plain <$> result)

14311434 <|> return mempty

14321435

1433−pipeTableHeaderPart :: PandocMonad m => ParserT [Char] st m (Alignment, Int)

1436+pipeTableHeaderPart :: PandocMonad m => ParserT Text st m (Alignment, Int)

14341437pipeTableHeaderPart = try $ do

14351438 skipMany spaceChar

14361439 left <- optionMaybe (char ':')

@@ -1446,12 +1449,12 @@ pipeTableHeaderPart = try $ do

14461449 (Just _,Just _) -> AlignCenter, len)

14471450

14481451-- Succeed only if current line contains a pipe.

1449−scanForPipe :: PandocMonad m => ParserT [Char] st m ()

1452+scanForPipe :: PandocMonad m => ParserT Text st m ()

14501453scanForPipe = do

14511454 inp <- getInput

1452− case break (\c -> c == '\n' || c == '|') inp of

1453− (_,'|':_) -> return ()

1454− _ -> mzero

1455+ case T.break (\c -> c == '\n' || c == '|') inp of

1456+ (_, T.uncons -> Just ('|', _)) -> return ()

1457+ _ -> mzero

14551458

14561459-- | Parse a table using 'headerParser', 'rowParser',

14571460-- 'lineParser', and 'footerParser'. Variant of the version in

@@ -1561,7 +1564,7 @@ escapedChar = do

15611564 result <- escapedChar'

15621565 case result of

15631566 ' ' -> return $ return $ B.str "\160" -- "\ " is a nonbreaking space

1564− _ -> return $ return $ B.str [result]

1567+ _ -> return $ return $ B.str $ T.singleton result

15651568

15661569ltSign :: PandocMonad m => MarkdownParser m (F Inlines)

15671570ltSign = do

@@ -1574,12 +1577,12 @@ exampleRef :: PandocMonad m => MarkdownParser m (F Inlines)

15741577exampleRef = try $ do

15751578 guardEnabled Ext_example_lists

15761579 char '@'

1577− lab <- many1 (alphaNum <|> oneOf "-_")

1580+ lab <- many1Char (alphaNum <|> oneOf "-_")

15781581 return $ do

15791582 st <- askF

15801583 return $ case M.lookup lab (stateExamples st) of

1581− Just n -> B.str (show n)

1582− Nothing -> B.str ('@':lab)

1584+ Just n -> B.str $ tshow n

1585+ Nothing -> B.str $ "@" <> lab

15831586

15841587symbol :: PandocMonad m => MarkdownParser m (F Inlines)

15851588symbol = do

@@ -1587,16 +1590,16 @@ symbol = do

15871590 <|> try (do lookAhead $ char '\\'

15881591 notFollowedBy' (() <$ rawTeXBlock)

15891592 char '\\')

1590− return $ return $ B.str [result]

1593+ return $ return $ B.str $ T.singleton result

15911594

15921595-- parses inline code, between n `s and n `s

15931596code :: PandocMonad m => MarkdownParser m (F Inlines)

15941597code = try $ do

15951598 starts <- many1 (char '`')

15961599 skipSpaces

1597− result <- (trim . concat) <$>

1600+ result <- (trim . T.concat) <$>

15981601 manyTill (notFollowedBy (inList >> listStart) >>

1599− (many1 (noneOf "`\n") <|> many1 (char '`') <|>

1602+ (many1Char (noneOf "`\n") <|> many1Char (char '`') <|>

16001603 (char '\n' >> notFollowedBy' blankline >> return " ")))

16011604 (try (skipSpaces >> count (length starts) (char '`') >>

16021605 notFollowedBy (char '`')))

@@ -1627,10 +1630,10 @@ enclosure c = do

16271630 guardDisabled Ext_intraword_underscores

16281631 <|> guard (c == '*')

16291632 <|> (guard =<< notAfterString)

1630− cs <- many1 (char c)

1633+ cs <- many1Char (char c)

16311634 (return (B.str cs) <>) <$> whitespace

16321635 <|>

1633− case length cs of

1636+ case T.length cs of

16341637 3 -> three c

16351638 2 -> two c mempty

16361639 1 -> one c mempty

@@ -1653,7 +1656,7 @@ three c = do

16531656 (ender c 3 >> updateLastStrPos >> return ((B.strong . B.emph) <$> contents))

16541657 <|> (ender c 2 >> updateLastStrPos >> one c (B.strong <$> contents))

16551658 <|> (ender c 1 >> updateLastStrPos >> two c (B.emph <$> contents))

1656− <|> return (return (B.str [c,c,c]) <> contents)

1659+ <|> return (return (B.str $ T.pack [c,c,c]) <> contents)

16571660

16581661-- Parse inlines til you hit two c's, and emit strong.

16591662-- If you never do hit two cs, emit ** plus inlines parsed.

@@ -1662,7 +1665,7 @@ two c prefix' = do

16621665 contents <- mconcat <$> many (try $ notFollowedBy (ender c 2) >> inline)

16631666 (ender c 2 >> updateLastStrPos >>

16641667 return (B.strong <$> (prefix' <> contents)))

1665− <|> return (return (B.str [c,c]) <> (prefix' <> contents))

1668+ <|> return (return (B.str $ T.pack [c,c]) <> (prefix' <> contents))

16661669

16671670-- Parse inlines til you hit a c, and emit emph.

16681671-- If you never hit a c, emit * plus inlines parsed.

@@ -1673,7 +1676,7 @@ one c prefix' = do

16731676 notFollowedBy (ender c 1) >>

16741677 two c mempty) )

16751678 (ender c 1 >> updateLastStrPos >> return (B.emph <$> (prefix' <> contents)))

1676− <|> return (return (B.str [c]) <> (prefix' <> contents))

1679+ <|> return (return (B.str $ T.singleton c) <> (prefix' <> contents))

16771680

16781681strongOrEmph :: PandocMonad m => MarkdownParser m (F Inlines)

16791682strongOrEmph = enclosure '*' <|> enclosure '_'

@@ -1717,16 +1720,16 @@ whitespace = spaceChar >> return <$> (lb <|> regsp) <?> "whitespace"

17171720 where lb = spaceChar >> skipMany spaceChar >> option B.space (endline >> return B.linebreak)

17181721 regsp = skipMany spaceChar >> return B.space

17191722

1720−nonEndline :: PandocMonad m => ParserT [Char] st m Char

1723+nonEndline :: PandocMonad m => ParserT Text st m Char

17211724nonEndline = satisfy (/='\n')

17221725

17231726str :: PandocMonad m => MarkdownParser m (F Inlines)

17241727str = do

1725− result <- many1 (alphaNum <|> try (char '.' <* notFollowedBy (char '.')))

1728+ result <- many1Char (alphaNum <|> try (char '.' <* notFollowedBy (char '.')))

17261729 updateLastStrPos

17271730 (do guardEnabled Ext_smart

17281731 abbrevs <- getOption readerAbbreviations

1729− if not (null result) && last result == '.' && result `Set.member` abbrevs

1732+ if not (T.null result) && T.last result == '.' && result `Set.member` abbrevs

17301733 then try (do ils <- whitespace

17311734 -- ?? lookAhead alphaNum

17321735 -- replace space after with nonbreaking space

@@ -1766,36 +1769,36 @@ endline = try $ do

17661769--

17671770

17681771-- a reference label for a link

1769−reference :: PandocMonad m => MarkdownParser m (F Inlines, String)

1772+reference :: PandocMonad m => MarkdownParser m (F Inlines, Text)

17701773reference = do

17711774 guardDisabled Ext_footnotes <|> notFollowedBy' (string "[^")

17721775 guardDisabled Ext_citations <|> notFollowedBy' (string "[@")

17731776 withRaw $ trimInlinesF <$> inlinesInBalancedBrackets

17741777

1775−parenthesizedChars :: PandocMonad m => MarkdownParser m [Char]

1778+parenthesizedChars :: PandocMonad m => MarkdownParser m Text

17761779parenthesizedChars = do

17771780 result <- charsInBalanced '(' ')' litChar

1778− return $ '(' : result ++ ")"

1781+ return $ "(" <> result <> ")"

17791782

17801783-- source for a link, with optional title

1781−source :: PandocMonad m => MarkdownParser m (String, String)

1784+source :: PandocMonad m => MarkdownParser m (Text, Text)

17821785source = do

17831786 char '('

17841787 skipSpaces

17851788 let urlChunk =

17861789 try parenthesizedChars

1787− <|> (notFollowedBy (oneOf " )") >> count 1 litChar)

1788− <|> try (many1 spaceChar <* notFollowedBy (oneOf "\"')"))

1789− let sourceURL = (unwords . words . concat) <$> many urlChunk

1790+ <|> (notFollowedBy (oneOf " )") >> countChar 1 litChar)

1791+ <|> try (many1Char spaceChar <* notFollowedBy (oneOf "\"')"))

1792+ let sourceURL = (T.unwords . T.words . T.concat) <$> many urlChunk

17901793 let betweenAngles = try $

1791− char '<' >> manyTill litChar (char '>')

1794+ char '<' >> manyTillChar litChar (char '>')

17921795 src <- try betweenAngles <|> sourceURL

17931796 tit <- option "" $ try $ spnl >> linkTitle

17941797 skipSpaces

17951798 char ')'

17961799 return (escapeURI $ trimr src, tit)

17971800

1798−linkTitle :: PandocMonad m => MarkdownParser m String

1801+linkTitle :: PandocMonad m => MarkdownParser m Text

17991802linkTitle = quotedTitle '"' <|> quotedTitle '\''

18001803

18011804link :: PandocMonad m => MarkdownParser m (F Inlines)

@@ -1823,13 +1826,13 @@ isSmallCaps :: Attr -> Bool

18231826isSmallCaps ("",["smallcaps"],[]) = True

18241827isSmallCaps ("",[],kvs) =

18251828 case lookup "style" kvs of

1826− Just s -> map toLower (filter (`notElem` " \t;") s) ==

1829+ Just s -> T.toLower (T.filter (`notElem` [' ', '\t', ';']) s) ==

18271830 "font-variant:small-caps"

18281831 Nothing -> False

18291832isSmallCaps _ = False

18301833

18311834regLink :: PandocMonad m

1832− => (Attr -> String -> String -> Inlines -> Inlines)

1835+ => (Attr -> Text -> Text -> Inlines -> Inlines)

18331836 -> F Inlines

18341837 -> MarkdownParser m (F Inlines)

18351838regLink constructor lab = try $ do

@@ -1840,8 +1843,8 @@ regLink constructor lab = try $ do

18401843

18411844-- a link like [this][ref] or [this][] or [this]

18421845referenceLink :: PandocMonad m

1843− => (Attr -> String -> String -> Inlines -> Inlines)

1844− -> (F Inlines, String)

1846+ => (Attr -> Text -> Text -> Inlines -> Inlines)

1847+ -> (F Inlines, Text)

18451848 -> MarkdownParser m (F Inlines)

18461849referenceLink constructor (lab, raw) = do

18471850 sp <- (True <$ lookAhead (char ' ')) <|> return False

@@ -1863,7 +1866,7 @@ referenceLink constructor (lab, raw) = do

18631866 parsedRaw' <- parsedRaw

18641867 fallback' <- fallback

18651868 return $ B.str "[" <> fallback' <> B.str "]" <>

1866− (if sp && not (null raw) then B.space else mempty) <>

1869+ (if sp && not (T.null raw) then B.space else mempty) <>

18671870 parsedRaw'

18681871 return $ do

18691872 keys <- asksF stateKeys

@@ -1878,19 +1881,19 @@ referenceLink constructor (lab, raw) = do

18781881 else makeFallback

18791882 Just ((src,tit), attr) -> constructor attr src tit <$> lab

18801883

1881−dropBrackets :: String -> String

1882−dropBrackets = reverse . dropRB . reverse . dropLB

1883− where dropRB (']':xs) = xs

1884− dropRB xs = xs

1885− dropLB ('[':xs) = xs

1886− dropLB xs = xs

1884+dropBrackets :: Text -> Text

1885+dropBrackets = dropRB . dropLB

1886+ where dropRB (T.unsnoc -> Just (xs,']')) = xs

1887+ dropRB xs = xs

1888+ dropLB (T.uncons -> Just ('[',xs)) = xs

1889+ dropLB xs = xs

18871890

18881891bareURL :: PandocMonad m => MarkdownParser m (F Inlines)

18891892bareURL = try $ do

18901893 guardEnabled Ext_autolink_bare_uris

18911894 getState >>= guard . stateAllowLinks

18921895 (cls, (orig, src)) <- (("uri",) <$> uri) <|> (("email",) <$> emailAddress)

1893− notFollowedBy $ try $ spaces >> htmlTag (~== TagClose "a")

1896+ notFollowedBy $ try $ spaces >> htmlTag (~== TagClose ("a" :: Text))

18941897 return $ return $ B.linkWith ("",[cls],[]) src "" (B.str orig)

18951898

18961899autoLink :: PandocMonad m => MarkdownParser m (F Inlines)

@@ -1902,19 +1905,20 @@ autoLink = try $ do

19021905 -- is finished, because the uri parser tries to avoid parsing

19031906 -- final punctuation. for example: in `<http://hi---there>`,

19041907 -- the URI parser will stop before the dashes.

1905− extra <- fromEntities <$> manyTill nonspaceChar (char '>')

1908+ extra <- fromEntities <$> manyTillChar nonspaceChar (char '>')

19061909 attr <- option ("", [cls], []) $ try $

19071910 guardEnabled Ext_link_attributes >> attributes

1908− return $ return $ B.linkWith attr (src ++ escapeURI extra) ""

1909− (B.str $ orig ++ extra)

1911+ return $ return $ B.linkWith attr (src <> escapeURI extra) ""

1912+ (B.str $ orig <> extra)

19101913

19111914image :: PandocMonad m => MarkdownParser m (F Inlines)

19121915image = try $ do

19131916 char '!'

19141917 (lab,raw) <- reference

19151918 defaultExt <- getOption readerDefaultImageExtension

1916− let constructor attr' src = case takeExtension src of

1917− "" -> B.imageWith attr' (addExtension src defaultExt)

1919+ let constructor attr' src = case takeExtension (T.unpack src) of

1920+ "" -> B.imageWith attr' (T.pack $ addExtension (T.unpack src)

1921+ $ T.unpack defaultExt)

19181922 _ -> B.imageWith attr' src

19191923 regLink constructor lab <|> referenceLink constructor (lab,raw)

19201924

@@ -1926,7 +1930,7 @@ note = try $ do

19261930 return $ do

19271931 notes <- asksF stateNotes'

19281932 case M.lookup ref notes of

1929− Nothing -> return $ B.str $ "[^" ++ ref ++ "]"

1933+ Nothing -> return $ B.str $ "[^" <> ref <> "]"

19301934 Just (_pos, contents) -> do

19311935 st <- askF

19321936 -- process the note in a context that doesn't resolve

@@ -1949,29 +1953,29 @@ rawLaTeXInline' = try $ do

19491953 s <- rawLaTeXInline

19501954 return $ return $ B.rawInline "tex" s -- "tex" because it might be context

19511955

1952−rawConTeXtEnvironment :: PandocMonad m => ParserT [Char] st m String

1956+rawConTeXtEnvironment :: PandocMonad m => ParserT Text st m Text

19531957rawConTeXtEnvironment = try $ do

19541958 string "\\start"

19551959 completion <- inBrackets (letter <|> digit <|> spaceChar)

1956− <|> many1 letter

1957− contents <- manyTill (rawConTeXtEnvironment <|> count 1 anyChar)

1958− (try $ string "\\stop" >> string completion)

1959− return $ "\\start" ++ completion ++ concat contents ++ "\\stop" ++ completion

1960+ <|> many1Char letter

1961+ contents <- manyTill (rawConTeXtEnvironment <|> countChar 1 anyChar)

1962+ (try $ string "\\stop" >> textStr completion)

1963+ return $ "\\start" <> completion <> T.concat contents <> "\\stop" <> completion

19601964

1961−inBrackets :: PandocMonad m => ParserT [Char] st m Char -> ParserT [Char] st m String

1965+inBrackets :: PandocMonad m => ParserT Text st m Char -> ParserT Text st m Text

19621966inBrackets parser = do

19631967 char '['

1964− contents <- many parser

1968+ contents <- manyChar parser

19651969 char ']'

1966− return $ "[" ++ contents ++ "]"

1970+ return $ "[" <> contents <> "]"

19671971

19681972spanHtml :: PandocMonad m => MarkdownParser m (F Inlines)

19691973spanHtml = try $ do

19701974 guardEnabled Ext_native_spans

1971− (TagOpen _ attrs, _) <- htmlTag (~== TagOpen "span" [])

1972− contents <- mconcat <$> manyTill inline (htmlTag (~== TagClose "span"))

1975+ (TagOpen _ attrs, _) <- htmlTag (~== TagOpen ("span" :: Text) [])

1976+ contents <- mconcat <$> manyTill inline (htmlTag (~== TagClose ("span" :: Text)))

19731977 let ident = fromMaybe "" $ lookup "id" attrs

1974− let classes = maybe [] words $ lookup "class" attrs

1978+ let classes = maybe [] T.words $ lookup "class" attrs

19751979 let keyvals = [(k,v) | (k,v) <- attrs, k /= "id" && k /= "class"]

19761980 return $ if isSmallCaps (ident, classes, keyvals)

19771981 then B.smallcaps <$> contents

@@ -1980,20 +1984,20 @@ spanHtml = try $ do

19801984divHtml :: PandocMonad m => MarkdownParser m (F Blocks)

19811985divHtml = try $ do

19821986 guardEnabled Ext_native_divs

1983− (TagOpen _ attrs, rawtag) <- htmlTag (~== TagOpen "div" [])

1987+ (TagOpen _ attrs, rawtag) <- htmlTag (~== TagOpen ("div" :: Text) [])

19841988 -- we set stateInHtmlBlock so that closing tags that can be either block or

19851989 -- inline will not be parsed as inline tags

19861990 oldInHtmlBlock <- stateInHtmlBlock <$> getState

19871991 updateState $ \st -> st{ stateInHtmlBlock = Just "div" }

19881992 bls <- option "" (blankline >> option "" blanklines)

19891993 contents <- mconcat <$>

1990− many (notFollowedBy' (htmlTag (~== TagClose "div")) >> block)

1991− closed <- option False (True <$ htmlTag (~== TagClose "div"))

1994+ many (notFollowedBy' (htmlTag (~== TagClose ("div" :: Text))) >> block)

1995+ closed <- option False (True <$ htmlTag (~== TagClose ("div" :: Text)))

19921996 if closed

19931997 then do

19941998 updateState $ \st -> st{ stateInHtmlBlock = oldInHtmlBlock }

19951999 let ident = fromMaybe "" $ lookup "id" attrs

1996− let classes = maybe [] words $ lookup "class" attrs

2000+ let classes = maybe [] T.words $ lookup "class" attrs

19972001 let keyvals = [(k,v) | (k,v) <- attrs, k /= "id" && k /= "class"]

19982002 return $ B.divWith (ident, classes, keyvals) <$> contents

19992003 else -- avoid backtracing

@@ -2005,7 +2009,7 @@ divFenced = try $ do

20052009 string ":::"

20062010 skipMany (char ':')

20072011 skipMany spaceChar

2008− attribs <- attributes <|> ((\x -> ("",[x],[])) <$> many1 nonspaceChar)

2012+ attribs <- attributes <|> ((\x -> ("",[x],[])) <$> many1Char nonspaceChar)

20092013 skipMany spaceChar

20102014 skipMany (char ':')

20112015 blankline

@@ -2047,7 +2051,7 @@ emoji :: PandocMonad m => MarkdownParser m (F Inlines)

20472051emoji = try $ do

20482052 guardEnabled Ext_emoji

20492053 char ':'

2050− emojikey <- many1 (oneOf emojiChars)

2054+ emojikey <- many1Char (oneOf emojiChars)

20512055 char ':'

20522056 case emojiToInline emojikey of

20532057 Just i -> return (return $ B.singleton i)

@@ -2077,14 +2081,14 @@ textualCite = try $ do

20772081 mbrest <- option Nothing $ try $ spnl >> Just <$> withRaw normalCite

20782082 case mbrest of

20792083 Just (rest, raw) ->

2080− return $ (flip B.cite (B.text $ '@':key ++ " " ++ raw) . (first:))

2084+ return $ (flip B.cite (B.text $ "@" <> key <> " " <> raw) . (first:))

20812085 <$> rest

20822086 Nothing ->

20832087 (do

20842088 (cs, raw) <- withRaw $ bareloc first

2085− let (spaces',raw') = span isSpace raw

2086− spc | null spaces' = mempty

2087− | otherwise = B.space

2089+ let (spaces',raw') = T.span isSpace raw

2090+ spc | T.null spaces' = mempty

2091+ | otherwise = B.space

20882092 lab <- parseFromString' inlines $ dropBrackets raw'

20892093 fallback <- referenceLink B.linkWith (lab,raw')

20902094 return $ do

@@ -2092,12 +2096,12 @@ textualCite = try $ do

20922096 cs' <- cs

20932097 return $

20942098 case B.toList fallback' of

2095− Link{}:_ -> B.cite [first] (B.str $ '@':key) <> spc <> fallback'

2096− _ -> B.cite cs' (B.text $ '@':key ++ " " ++ raw))

2099+ Link{}:_ -> B.cite [first] (B.str $ "@" <> key) <> spc <> fallback'

2100+ _ -> B.cite cs' (B.text $ "@" <> key <> " " <> raw))

20972101 <|> return (do st <- askF

20982102 return $ case M.lookup key (stateExamples st) of

2099− Just n -> B.str (show n)

2100− _ -> B.cite [first] $ B.str $ '@':key)

2103+ Just n -> B.str $ tshow n

2104+ _ -> B.cite [first] $ B.str $ "@" <> key)

21012105

21022106bareloc :: PandocMonad m => Citation -> MarkdownParser m (F [Citation])

21032107bareloc c = try $ do

src/Text/Pandoc/Readers/MediaWiki.hs+99−99

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE RelaxedPolyRec #-}

2+{-# LANGUAGE OverloadedStrings #-}

3+{-# LANGUAGE RelaxedPolyRec #-}

34-- RelaxedPolyRec needed for inlinesBetween on GHC < 7

45{- |

56 Module : Text.Pandoc.Readers.MediaWiki

@@ -24,11 +25,12 @@ import Control.Monad

2425import Control.Monad.Except (throwError)

2526import Data.Char (isDigit, isSpace)

2627import qualified Data.Foldable as F

27−import Data.List (intercalate, intersperse, isPrefixOf)

28+import Data.List (intersperse)

2829import Data.Maybe (fromMaybe, maybeToList)

2930import Data.Sequence (ViewL (..), viewl, (<|))

3031import qualified Data.Set as Set

31−import Data.Text (Text, unpack)

32+import Data.Text (Text)

33+import qualified Data.Text as T

3234import Text.HTML.TagSoup

3335import Text.Pandoc.Builder (Blocks, Inlines, trimInlines)

3436import qualified Text.Pandoc.Builder as B

@@ -39,7 +41,7 @@ import Text.Pandoc.Options

3941import Text.Pandoc.Parsing hiding (nested)

4042import Text.Pandoc.Readers.HTML (htmlTag, isBlockTag, isCommentTag)

4143import Text.Pandoc.Shared (crFilter, safeRead, stringify, stripTrailingNewlines,

42− trim)

44+ trim, splitTextBy, tshow)

4345import Text.Pandoc.Walk (walk)

4446import Text.Pandoc.XML (fromEntities)

4547

@@ -57,7 +59,7 @@ readMediaWiki opts s = do

5759 , mwLogMessages = []

5860 , mwInTT = False

5961 }

60− (unpack (crFilter s) ++ "\n")

62+ (crFilter s <> "\n")

6163 case parsed of

6264 Right result -> return result

6365 Left e -> throwError e

@@ -66,12 +68,12 @@ data MWState = MWState { mwOptions :: ReaderOptions

6668 , mwMaxNestingLevel :: Int

6769 , mwNextLinkNumber :: Int

6870 , mwCategoryLinks :: [Inlines]

69− , mwIdentifierList :: Set.Set String

71+ , mwIdentifierList :: Set.Set Text

7072 , mwLogMessages :: [LogMessage]

7173 , mwInTT :: Bool

7274 }

7375

74−type MWParser m = ParserT [Char] MWState m

76+type MWParser m = ParserT Text MWState m

7577

7678instance HasReaderOptions MWState where

7779 extractReaderOptions = mwOptions

@@ -105,58 +107,58 @@ specialChars = "'[]<=&*{}|\":\\"

105107spaceChars :: [Char]

106108spaceChars = " \n\t"

107109

108−sym :: PandocMonad m => String -> MWParser m ()

109−sym s = () <$ try (string s)

110+sym :: PandocMonad m => Text -> MWParser m ()

111+sym s = () <$ try (string $ T.unpack s)

110112

111−newBlockTags :: [String]

113+newBlockTags :: [Text]

112114newBlockTags = ["haskell","syntaxhighlight","source","gallery","references"]

113115

114−isBlockTag' :: Tag String -> Bool

116+isBlockTag' :: Tag Text -> Bool

115117isBlockTag' tag@(TagOpen t _) = (isBlockTag tag || t `elem` newBlockTags) &&

116118 t `notElem` eitherBlockOrInline

117119isBlockTag' tag@(TagClose t) = (isBlockTag tag || t `elem` newBlockTags) &&

118120 t `notElem` eitherBlockOrInline

119121isBlockTag' tag = isBlockTag tag

120122

121−isInlineTag' :: Tag String -> Bool

123+isInlineTag' :: Tag Text -> Bool

122124isInlineTag' (TagComment _) = True

123125isInlineTag' t = not (isBlockTag' t)

124126

125−eitherBlockOrInline :: [String]

127+eitherBlockOrInline :: [Text]

126128eitherBlockOrInline = ["applet", "button", "del", "iframe", "ins",

127129 "map", "area", "object"]

128130

129131htmlComment :: PandocMonad m => MWParser m ()

130132htmlComment = () <$ htmlTag isCommentTag

131133

132−inlinesInTags :: PandocMonad m => String -> MWParser m Inlines

134+inlinesInTags :: PandocMonad m => Text -> MWParser m Inlines

133135inlinesInTags tag = try $ do

134136 (_,raw) <- htmlTag (~== TagOpen tag [])

135− if '/' `elem` raw -- self-closing tag

137+ if T.any (== '/') raw -- self-closing tag

136138 then return mempty

137139 else trimInlines . mconcat <$>

138140 manyTill inline (htmlTag (~== TagClose tag))

139141

140−blocksInTags :: PandocMonad m => String -> MWParser m Blocks

142+blocksInTags :: PandocMonad m => Text -> MWParser m Blocks

141143blocksInTags tag = try $ do

142144 (_,raw) <- htmlTag (~== TagOpen tag [])

143145 let closer = if tag == "li"

144− then htmlTag (~== TagClose "li")

146+ then htmlTag (~== TagClose ("li" :: Text))

145147 <|> lookAhead (

146− htmlTag (~== TagOpen "li" [])

147− <|> htmlTag (~== TagClose "ol")

148− <|> htmlTag (~== TagClose "ul"))

148+ htmlTag (~== TagOpen ("li" :: Text) [])

149+ <|> htmlTag (~== TagClose ("ol" :: Text))

150+ <|> htmlTag (~== TagClose ("ul" :: Text)))

149151 else htmlTag (~== TagClose tag)

150− if '/' `elem` raw -- self-closing tag

152+ if T.any (== '/') raw -- self-closing tag

151153 then return mempty

152154 else mconcat <$> manyTill block closer

153155

154−charsInTags :: PandocMonad m => String -> MWParser m [Char]

155−charsInTags tag = try $ do

156+textInTags :: PandocMonad m => Text -> MWParser m Text

157+textInTags tag = try $ do

156158 (_,raw) <- htmlTag (~== TagOpen tag [])

157− if '/' `elem` raw -- self-closing tag

159+ if T.any (== '/') raw -- self-closing tag

158160 then return ""

159− else manyTill anyChar (htmlTag (~== TagClose tag))

161+ else T.pack <$> manyTill anyChar (htmlTag (~== TagClose tag))

160162

161163--

162164-- main parser

@@ -192,7 +194,7 @@ block = do

192194 <|> blockTag

193195 <|> (B.rawBlock "mediawiki" <$> template)

194196 <|> para

195− trace (take 60 $ show $ B.toList res)

197+ trace (T.take 60 $ tshow $ B.toList res)

196198 return res

197199

198200para :: PandocMonad m => MWParser m Blocks

@@ -234,16 +236,16 @@ table = do

234236 else (replicate cols mempty, hdr:rows')

235237 return $ B.table caption cellspecs headers rows

236238

237−parseAttrs :: PandocMonad m => MWParser m [(String,String)]

239+parseAttrs :: PandocMonad m => MWParser m [(Text,Text)]

238240parseAttrs = many1 parseAttr

239241

240−parseAttr :: PandocMonad m => MWParser m (String, String)

242+parseAttr :: PandocMonad m => MWParser m (Text, Text)

241243parseAttr = try $ do

242244 skipMany spaceChar

243− k <- many1 letter

245+ k <- many1Char letter

244246 char '='

245− v <- (char '"' >> many1Till (satisfy (/='\n')) (char '"'))

246− <|> many1 (satisfy $ \c -> not (isSpace c) && c /= '|')

247+ v <- (char '"' >> many1TillChar (satisfy (/='\n')) (char '"'))

248+ <|> many1Char (satisfy $ \c -> not (isSpace c) && c /= '|')

247249 return (k,v)

248250

249251tableStart :: PandocMonad m => MWParser m ()

@@ -293,8 +295,8 @@ tableCell = try $ do

293295 notFollowedBy (char '|')

294296 skipMany spaceChar

295297 pos' <- getPosition

296− ls <- concat <$> many (notFollowedBy (cellsep <|> rowsep <|> tableEnd) *>

297− ((snd <$> withRaw table) <|> count 1 anyChar))

298+ ls <- T.concat <$> many (notFollowedBy (cellsep <|> rowsep <|> tableEnd) *>

299+ ((snd <$> withRaw table) <|> countChar 1 anyChar))

298300 bs <- parseFromString (do setPosition pos'

299301 mconcat <$> many block) ls

300302 let align = case lookup "align" attrs of

@@ -307,48 +309,49 @@ tableCell = try $ do

307309 Nothing -> 0.0

308310 return ((align, width), bs)

309311

310−parseWidth :: String -> Maybe Double

312+parseWidth :: Text -> Maybe Double

311313parseWidth s =

312− case reverse s of

313− ('%':ds) | all isDigit ds -> safeRead ('0':'.':reverse ds)

314− _ -> Nothing

314+ case T.unsnoc s of

315+ Just (ds, '%') | T.all isDigit ds -> safeRead $ "0." <> ds

316+ _ -> Nothing

315317

316−template :: PandocMonad m => MWParser m String

318+template :: PandocMonad m => MWParser m Text

317319template = try $ do

318320 string "{{"

319321 notFollowedBy (char '{')

320322 lookAhead $ letter <|> digit <|> char ':'

321− let chunk = template <|> variable <|> many1 (noneOf "{}") <|> count 1 anyChar

323+ let chunk = template <|> variable <|> many1Char (noneOf "{}") <|> countChar 1 anyChar

322324 contents <- manyTill chunk (try $ string "}}")

323− return $ "{{" ++ concat contents ++ "}}"

325+ return $ "{{" <> T.concat contents <> "}}"

324326

325327blockTag :: PandocMonad m => MWParser m Blocks

326328blockTag = do

327329 (tag, _) <- lookAhead $ htmlTag isBlockTag'

328330 case tag of

329331 TagOpen "blockquote" _ -> B.blockQuote <$> blocksInTags "blockquote"

330− TagOpen "pre" _ -> B.codeBlock . trimCode <$> charsInTags "pre"

332+ TagOpen "pre" _ -> B.codeBlock . trimCode <$> textInTags "pre"

331333 TagOpen "syntaxhighlight" attrs -> syntaxhighlight "syntaxhighlight" attrs

332334 TagOpen "source" attrs -> syntaxhighlight "source" attrs

333335 TagOpen "haskell" _ -> B.codeBlockWith ("",["haskell"],[]) . trimCode <$>

334− charsInTags "haskell"

336+ textInTags "haskell"

335337 TagOpen "gallery" _ -> blocksInTags "gallery"

336338 TagOpen "p" _ -> mempty <$ htmlTag (~== tag)

337339 TagClose "p" -> mempty <$ htmlTag (~== tag)

338340 _ -> B.rawBlock "html" . snd <$> htmlTag (~== tag)

339341

340−trimCode :: String -> String

341−trimCode ('\n':xs) = stripTrailingNewlines xs

342−trimCode xs = stripTrailingNewlines xs

342+trimCode :: Text -> Text

343+trimCode t = case T.uncons t of

344+ Just ('\n', xs) -> stripTrailingNewlines xs

345+ _ -> stripTrailingNewlines t

343346

344−syntaxhighlight :: PandocMonad m => String -> [Attribute String] -> MWParser m Blocks

347+syntaxhighlight :: PandocMonad m => Text -> [Attribute Text] -> MWParser m Blocks

345348syntaxhighlight tag attrs = try $ do

346349 let mblang = lookup "lang" attrs

347350 let mbstart = lookup "start" attrs

348351 let mbline = lookup "line" attrs

349352 let classes = maybeToList mblang ++ maybe [] (const ["numberLines"]) mbline

350353 let kvs = maybe [] (\x -> [("startFrom",x)]) mbstart

351− contents <- charsInTags tag

354+ contents <- textInTags tag

352355 return $ B.codeBlockWith ("",classes,kvs) $ trimCode contents

353356

354357hrule :: PandocMonad m => MWParser m Blocks

@@ -362,17 +365,17 @@ preformatted = try $ do

362365 guardColumnOne

363366 char ' '

364367 let endline' = B.linebreak <$ try (newline <* char ' ')

365− let whitespace' = B.str <$> many1 ('\160' <$ spaceChar)

368+ let whitespace' = B.str <$> many1Char ('\160' <$ spaceChar)

366369 let spToNbsp ' ' = '\160'

367370 spToNbsp x = x

368371 let nowiki' = mconcat . intersperse B.linebreak . map B.str .

369− lines . fromEntities . map spToNbsp <$> try

370− (htmlTag (~== TagOpen "nowiki" []) *>

371− manyTill anyChar (htmlTag (~== TagClose "nowiki")))

372+ T.lines . fromEntities . T.map spToNbsp <$> try

373+ (htmlTag (~== TagOpen ("nowiki" :: Text) []) *>

374+ manyTillChar anyChar (htmlTag (~== TagClose ("nowiki" :: Text))))

372375 let inline' = whitespace' <|> endline' <|> nowiki'

373376 <|> try (notFollowedBy newline *> inline)

374377 contents <- mconcat <$> many1 inline'

375− let spacesStr (Str xs) = all isSpace xs

378+ let spacesStr (Str xs) = T.all isSpace xs

376379 spacesStr _ = False

377380 if F.all spacesStr contents

378381 then return mempty

@@ -385,7 +388,7 @@ encode = B.fromList . normalizeCode . B.toList . walk strToCode

385388 strToCode x = x

386389 normalizeCode [] = []

387390 normalizeCode (Code a1 x : Code a2 y : zs) | a1 == a2 =

388− normalizeCode $ Code a1 (x ++ y) : zs

391+ normalizeCode $ Code a1 (x <> y) : zs

389392 normalizeCode (x:xs) = x : normalizeCode xs

390393

391394header :: PandocMonad m => MWParser m Blocks

@@ -400,22 +403,22 @@ header = try $ do

400403-- See #4731:

401404modifyIdentifier :: Attr -> Attr

402405modifyIdentifier (ident,cl,kv) = (ident',cl,kv)

403− where ident' = map (\c -> if c == '-' then '_' else c) ident

406+ where ident' = T.map (\c -> if c == '-' then '_' else c) ident

404407

405408bulletList :: PandocMonad m => MWParser m Blocks

406409bulletList = B.bulletList <$>

407410 ( many1 (listItem '*')

408− <|> (htmlTag (~== TagOpen "ul" []) *> spaces *> many (listItem '*' <|> li) <*

409− optional (htmlTag (~== TagClose "ul"))) )

411+ <|> (htmlTag (~== TagOpen ("ul" :: Text) []) *> spaces *> many (listItem '*' <|> li) <*

412+ optional (htmlTag (~== TagClose ("ul" :: Text)))) )

410413

411414orderedList :: PandocMonad m => MWParser m Blocks

412415orderedList =

413416 (B.orderedList <$> many1 (listItem '#'))

414417 <|> try

415− (do (tag,_) <- htmlTag (~== TagOpen "ol" [])

418+ (do (tag,_) <- htmlTag (~== TagOpen ("ol" :: Text) [])

416419 spaces

417420 items <- many (listItem '#' <|> li)

418− optional (htmlTag (~== TagClose "ol"))

421+ optional (htmlTag (~== TagClose ("ol" :: Text)))

419422 let start = fromMaybe 1 $ safeRead $ fromAttrib "start" tag

420423 return $ B.orderedListWith (start, DefaultStyle, DefaultDelim) items)

421424

@@ -452,7 +455,7 @@ anyListStart :: PandocMonad m => MWParser m Char

452455anyListStart = guardColumnOne >> oneOf "*#:;"

453456

454457li :: PandocMonad m => MWParser m Blocks

455−li = lookAhead (htmlTag (~== TagOpen "li" [])) *>

458+li = lookAhead (htmlTag (~== TagOpen ("li" :: Text) [])) *>

456459 (firstParaToPlain <$> blocksInTags "li") <* spaces

457460

458461listItem :: PandocMonad m => Char -> MWParser m Blocks

@@ -464,13 +467,13 @@ listItem c = try $ do

464467 else do

465468 skipMany spaceChar

466469 pos' <- getPosition

467− first <- concat <$> manyTill listChunk newline

470+ first <- T.concat <$> manyTill listChunk newline

468471 rest <- many

469472 (try $ string extras *> lookAhead listStartChar *>

470− (concat <$> manyTill listChunk newline))

473+ (T.concat <$> manyTill listChunk newline))

471474 contents <- parseFromString (do setPosition pos'

472475 many1 $ listItem' c)

473− (unlines (first : rest))

476+ (T.unlines (first : rest))

474477 case c of

475478 '*' -> return $ B.bulletList contents

476479 '#' -> return $ B.orderedList contents

@@ -484,20 +487,20 @@ listItem c = try $ do

484487-- }}

485488-- * next list item

486489-- which seems to be valid mediawiki.

487−listChunk :: PandocMonad m => MWParser m String

488−listChunk = template <|> count 1 anyChar

490+listChunk :: PandocMonad m => MWParser m Text

491+listChunk = template <|> countChar 1 anyChar

489492

490493listItem' :: PandocMonad m => Char -> MWParser m Blocks

491494listItem' c = try $ do

492495 listStart c

493496 skipMany spaceChar

494497 pos' <- getPosition

495− first <- concat <$> manyTill listChunk newline

498+ first <- T.concat <$> manyTill listChunk newline

496499 rest <- many (try $ char c *> lookAhead listStartChar *>

497− (concat <$> manyTill listChunk newline))

500+ (T.concat <$> manyTill listChunk newline))

498501 parseFromString (do setPosition pos'

499502 firstParaToPlain . mconcat <$> many1 block)

500− $ unlines $ first : rest

503+ $ T.unlines $ first : rest

501504

502505firstParaToPlain :: Blocks -> Blocks

503506firstParaToPlain contents =

@@ -528,23 +531,23 @@ inline = whitespace

528531 <|> special

529532

530533str :: PandocMonad m => MWParser m Inlines

531−str = B.str <$> many1 (noneOf $ specialChars ++ spaceChars)

534+str = B.str <$> many1Char (noneOf $ specialChars ++ spaceChars)

532535

533536math :: PandocMonad m => MWParser m Inlines

534−math = (B.displayMath . trim <$> try (many1 (char ':') >> charsInTags "math"))

535− <|> (B.math . trim <$> charsInTags "math")

536− <|> (B.displayMath . trim <$> try (dmStart *> manyTill anyChar dmEnd))

537− <|> (B.math . trim <$> try (mStart *> manyTill (satisfy (/='\n')) mEnd))

537+math = (B.displayMath . trim <$> try (many1 (char ':') >> textInTags "math"))

538+ <|> (B.math . trim <$> textInTags "math")

539+ <|> (B.displayMath . trim <$> try (dmStart *> manyTillChar anyChar dmEnd))

540+ <|> (B.math . trim <$> try (mStart *> manyTillChar (satisfy (/='\n')) mEnd))

538541 where dmStart = string "\\["

539542 dmEnd = try (string "\\]")

540543 mStart = string "\\("

541544 mEnd = try (string "\\)")

542545

543−variable :: PandocMonad m => MWParser m String

546+variable :: PandocMonad m => MWParser m Text

544547variable = try $ do

545548 string "{{{"

546− contents <- manyTill anyChar (try $ string "}}}")

547− return $ "{{{" ++ contents ++ "}}}"

549+ contents <- manyTillChar anyChar (try $ string "}}}")

550+ return $ "{{{" <> contents <> "}}}"

548551

549552inlineTag :: PandocMonad m => MWParser m Inlines

550553inlineTag = do

@@ -553,11 +556,11 @@ inlineTag = do

553556 TagOpen "ref" _ -> B.note . B.plain <$> inlinesInTags "ref"

554557 TagOpen "nowiki" _ -> try $ do

555558 (_,raw) <- htmlTag (~== tag)

556− if '/' `elem` raw

559+ if T.any (== '/') raw

557560 then return mempty

558561 else B.text . fromEntities <$>

559− manyTill anyChar (htmlTag (~== TagClose "nowiki"))

560− TagOpen "br" _ -> B.linebreak <$ (htmlTag (~== TagOpen "br" []) -- will get /> too

562+ manyTillChar anyChar (htmlTag (~== TagClose ("nowiki" :: Text)))

563+ TagOpen "br" _ -> B.linebreak <$ (htmlTag (~== TagOpen ("br" :: Text) []) -- will get /> too

561564 *> optional blankline)

562565 TagOpen "strike" _ -> B.strikeout <$> inlinesInTags "strike"

563566 TagOpen "del" _ -> B.strikeout <$> inlinesInTags "del"

@@ -570,12 +573,12 @@ inlineTag = do

570573 result <- encode <$> inlinesInTags "tt"

571574 updateState $ \st -> st{ mwInTT = inTT }

572575 return result

573− TagOpen "hask" _ -> B.codeWith ("",["haskell"],[]) <$> charsInTags "hask"

576+ TagOpen "hask" _ -> B.codeWith ("",["haskell"],[]) <$> textInTags "hask"

574577 _ -> B.rawInline "html" . snd <$> htmlTag (~== tag)

575578

576579special :: PandocMonad m => MWParser m Inlines

577−special = B.str <$> count 1 (notFollowedBy' (htmlTag isBlockTag') *>

578− oneOf specialChars)

580+special = B.str <$> countChar 1 (notFollowedBy' (htmlTag isBlockTag') *>

581+ oneOf specialChars)

579582

580583inlineHtml :: PandocMonad m => MWParser m Inlines

581584inlineHtml = B.rawInline "html" . snd <$> htmlTag isInlineTag'

@@ -594,7 +597,7 @@ endline = () <$ try (newline <*

594597 notFollowedBy anyListStart)

595598

596599imageIdentifiers :: PandocMonad m => [MWParser m ()]

597−imageIdentifiers = [sym (identifier ++ ":") | identifier <- identifiers]

600+imageIdentifiers = [sym (identifier <> ":") | identifier <- identifiers]

598601 where identifiers = ["File", "Image", "Archivo", "Datei", "Fichier",

599602 "Bild"]

600603

@@ -602,9 +605,9 @@ image :: PandocMonad m => MWParser m Inlines

602605image = try $ do

603606 sym "[["

604607 choice imageIdentifiers

605− fname <- addUnderscores <$> many1 (noneOf "|]")

608+ fname <- addUnderscores <$> many1Char (noneOf "|]")

606609 _ <- many imageOption

607− dims <- try (char '|' *> sepBy (many digit) (char 'x') <* string "px")

610+ dims <- try (char '|' *> sepBy (manyChar digit) (char 'x') <* string "px")

608611 <|> return []

609612 _ <- many imageOption

610613 let kvs = case dims of

@@ -614,9 +617,9 @@ image = try $ do

614617 let attr = ("", [], kvs)

615618 caption <- (B.str fname <$ sym "]]")

616619 <|> try (char '|' *> (mconcat <$> manyTill inline (sym "]]")))

617− return $ B.imageWith attr fname ("fig:" ++ stringify caption) caption

620+ return $ B.imageWith attr fname ("fig:" <> stringify caption) caption

618621

619−imageOption :: PandocMonad m => MWParser m String

622+imageOption :: PandocMonad m => MWParser m Text

620623imageOption = try $ char '|' *> opt

621624 where

622625 opt = try (oneOfStrings [ "border", "thumbnail", "frameless"

@@ -624,30 +627,27 @@ imageOption = try $ char '|' *> opt

624627 , "center", "none", "baseline", "sub"

625628 , "super", "top", "text-top", "middle"

626629 , "bottom", "text-bottom" ])

627− <|> try (string "frame")

630+ <|> try (textStr "frame")

628631 <|> try (oneOfStrings ["link=","alt=","page=","class="] <* many (noneOf "|]"))

629632

630−collapseUnderscores :: String -> String

631−collapseUnderscores [] = []

632−collapseUnderscores ('_':'_':xs) = collapseUnderscores ('_':xs)

633−collapseUnderscores (x:xs) = x : collapseUnderscores xs

634−

635−addUnderscores :: String -> String

636−addUnderscores = collapseUnderscores . intercalate "_" . words

633+addUnderscores :: Text -> Text

634+addUnderscores = T.intercalate "_" . splitTextBy sep

635+ where

636+ sep c = isSpace c || c == '_'

637637

638638internalLink :: PandocMonad m => MWParser m Inlines

639639internalLink = try $ do

640640 sym "[["

641− pagename <- unwords . words <$> many (noneOf "|]")

641+ pagename <- T.unwords . T.words <$> manyChar (noneOf "|]")

642642 label <- option (B.text pagename) $ char '|' *>

643643 ( (mconcat <$> many1 (notFollowedBy (char ']') *> inline))

644644 -- the "pipe trick"

645645 -- [[Help:Contents|] -> "Contents"

646− <|> return (B.text $ drop 1 $ dropWhile (/=':') pagename) )

646+ <|> return (B.text $ T.drop 1 $ T.dropWhile (/=':') pagename) )

647647 sym "]]"

648− linktrail <- B.text <$> many letter

648+ linktrail <- B.text <$> manyChar letter

649649 let link = B.link (addUnderscores pagename) "wikilink" (label <> linktrail)

650− if "Category:" `isPrefixOf` pagename

650+ if "Category:" `T.isPrefixOf` pagename

651651 then do

652652 updateState $ \st -> st{ mwCategoryLinks = link : mwCategoryLinks st }

653653 return mempty

@@ -662,7 +662,7 @@ externalLink = try $ do

662662 <|> do char ']'

663663 num <- mwNextLinkNumber <$> getState

664664 updateState $ \st -> st{ mwNextLinkNumber = num + 1 }

665− return $ B.str $ show num

665+ return $ B.str $ tshow num

666666 return $ B.link src "" lab

667667

668668url :: PandocMonad m => MWParser m Inlines

src/Text/Pandoc/Readers/Muse.hs+75−69

@@ -1,6 +1,7 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE FlexibleContexts #-}

3−{-# LANGUAGE TupleSections #-}

2+{-# LANGUAGE FlexibleContexts #-}

3+{-# LANGUAGE TupleSections #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers.Muse

@@ -24,12 +25,12 @@ import Control.Monad.Reader

2425import Control.Monad.Except (throwError)

2526import Data.Bifunctor

2627import Data.Default

27−import Data.List (intercalate, transpose, uncons)

28−import Data.List.Split (splitOn)

28+import Data.List (transpose, uncons)

2929import qualified Data.Map as M

3030import qualified Data.Set as Set

3131import Data.Maybe (fromMaybe, isNothing, maybeToList)

32−import Data.Text (Text, unpack)

32+import Data.Text (Text)

33+import qualified Data.Text as T

3334import Text.Pandoc.Builder (Blocks, Inlines)

3435import qualified Text.Pandoc.Builder as B

3536import Text.Pandoc.Class (PandocMonad (..))

@@ -38,7 +39,7 @@ import Text.Pandoc.Error (PandocError (PandocParsecError))

3839import Text.Pandoc.Logging

3940import Text.Pandoc.Options

4041import Text.Pandoc.Parsing hiding (F)

41−import Text.Pandoc.Shared (crFilter, trimr, underlineSpan)

42+import Text.Pandoc.Shared (crFilter, trimr, underlineSpan, tshow)

4243

4344-- | Read Muse from an input string and return a Pandoc document.

4445readMuse :: PandocMonad m

@@ -49,18 +50,18 @@ readMuse opts s = do

4950 let input = crFilter s

5051 res <- flip runReaderT def $ runParserT parseMuse def{ museOptions = opts } "source" input

5152 case res of

52− Left e -> throwError $ PandocParsecError (unpack input) e

53+ Left e -> throwError $ PandocParsecError input e

5354 Right d -> return d

5455

5556type F = Future MuseState

5657

5758data MuseState = MuseState { museMeta :: F Meta -- ^ Document metadata

5859 , museOptions :: ReaderOptions

59− , museIdentifierList :: Set.Set String

60+ , museIdentifierList :: Set.Set Text

6061 , museLastSpacePos :: Maybe SourcePos -- ^ Position after last space or newline parsed

6162 , museLastStrPos :: Maybe SourcePos -- ^ Position after last str parsed

6263 , museLogMessages :: [LogMessage]

63− , museNotes :: M.Map String (SourcePos, F Blocks)

64+ , museNotes :: M.Map Text (SourcePos, F Blocks)

6465 }

6566

6667instance Default MuseState where

@@ -116,22 +117,27 @@ parseMuse = do

116117-- * Utility functions

117118

118119-- | Trim up to one newline from the beginning of the string.

119−lchop :: String -> String

120−lchop ('\n':xs) = xs

121−lchop s = s

120+lchop :: Text -> Text

121+lchop s = case T.uncons s of

122+ Just ('\n', xs) -> xs

123+ _ -> s

122124

123125-- | Trim up to one newline from the end of the string.

124−rchop :: String -> String

125−rchop = reverse . lchop . reverse

126+rchop :: Text -> Text

127+rchop s = case T.unsnoc s of

128+ Just (xs, '\n') -> xs

129+ _ -> s

126130

127−unindent :: String -> String

128−unindent = rchop . intercalate "\n" . dropSpacePrefix . splitOn "\n" . lchop

131+unindent :: Text -> Text

132+unindent = rchop . T.intercalate "\n" . dropSpacePrefix . T.splitOn "\n" . lchop

129133

130−dropSpacePrefix :: [String] -> [String]

131−dropSpacePrefix lns = drop maxIndent <$> lns

134+dropSpacePrefix :: [Text] -> [Text]

135+dropSpacePrefix lns = T.drop maxIndent <$> lns

132136 where isSpaceChar c = c == ' ' || c == '\t'

133− maxIndent = length $ takeWhile (isSpaceChar . head) $ takeWhile same $ transpose lns

134− same = and . (zipWith (==) <*> drop 1)

137+ maxIndent = length $ takeWhile (isSpaceChar . T.head) $ takeWhile same $ T.transpose lns

138+ same t = case T.uncons t of

139+ Just (c, cs) -> T.all (== c) cs

140+ Nothing -> True

135141

136142atStart :: PandocMonad m => MuseParser m ()

137143atStart = do

@@ -160,29 +166,29 @@ getIndent = subtract 1 . sourceColumn <$ many spaceChar <*> getPosition

160166

161167-- ** HTML parsers

162168

163−openTag :: PandocMonad m => String -> MuseParser m [(String, String)]

169+openTag :: PandocMonad m => Text -> MuseParser m [(Text, Text)]

164170openTag tag = try $

165− char '<' *> string tag *> manyTill attr (char '>')

171+ char '<' *> textStr tag *> manyTill attr (char '>')

166172 where

167173 attr = try $ (,)

168174 <$ many1 spaceChar

169− <*> many1 (noneOf "=\n")

175+ <*> many1Char (noneOf "=\n")

170176 <* string "=\""

171− <*> manyTill (noneOf "\"") (char '"')

177+ <*> manyTillChar (noneOf "\"") (char '"')

172178

173−closeTag :: PandocMonad m => String -> MuseParser m ()

174−closeTag tag = try $ string "</" *> string tag *> void (char '>')

179+closeTag :: PandocMonad m => Text -> MuseParser m ()

180+closeTag tag = try $ string "</" *> textStr tag *> void (char '>')

175181

176182-- | Convert HTML attributes to Pandoc 'Attr'

177−htmlAttrToPandoc :: [(String, String)] -> Attr

183+htmlAttrToPandoc :: [(Text, Text)] -> Attr

178184htmlAttrToPandoc attrs = (ident, classes, keyvals)

179185 where

180186 ident = fromMaybe "" $ lookup "id" attrs

181− classes = maybe [] words $ lookup "class" attrs

187+ classes = maybe [] T.words $ lookup "class" attrs

182188 keyvals = [(k,v) | (k,v) <- attrs, k /= "id", k /= "class"]

183189

184190parseHtmlContent :: PandocMonad m

185− => String -- ^ Tag name

191+ => Text -- ^ Tag name

186192 -> MuseParser m (Attr, F Blocks)

187193parseHtmlContent tag = try $ getIndent >>= \indent -> (,)

188194 <$> fmap htmlAttrToPandoc (openTag tag)

@@ -193,16 +199,16 @@ parseHtmlContent tag = try $ getIndent >>= \indent -> (,)

193199-- ** Directive parsers

194200

195201-- While not documented, Emacs Muse allows "-" in directive name

196−parseDirectiveKey :: PandocMonad m => MuseParser m String

197−parseDirectiveKey = char '#' *> many (letter <|> char '-')

202+parseDirectiveKey :: PandocMonad m => MuseParser m Text

203+parseDirectiveKey = char '#' *> manyChar (letter <|> char '-')

198204

199−parseEmacsDirective :: PandocMonad m => MuseParser m (String, F Inlines)

205+parseEmacsDirective :: PandocMonad m => MuseParser m (Text, F Inlines)

200206parseEmacsDirective = (,)

201207 <$> parseDirectiveKey

202208 <* spaceChar

203209 <*> (trimInlinesF . mconcat <$> manyTill inline' eol)

204210

205−parseAmuseDirective :: PandocMonad m => MuseParser m (String, F Inlines)

211+parseAmuseDirective :: PandocMonad m => MuseParser m (Text, F Inlines)

206212parseAmuseDirective = (,)

207213 <$> parseDirectiveKey

208214 <* many1 spaceChar

@@ -289,7 +295,7 @@ listItemContentsUntil col pre end = p

289295parseBlock :: PandocMonad m => MuseParser m (F Blocks)

290296parseBlock = do

291297 res <- blockElements <|> para

292− trace (take 60 $ show $ B.toList $ runF res def)

298+ trace (T.take 60 $ tshow $ B.toList $ runF res def)

293299 return res

294300 where para = fst <$> paraUntil (try (eof <|> void (lookAhead blockElements)))

295301

@@ -337,7 +343,7 @@ pagebreak = try $ pure (B.divWith ("", [], [("style", "page-break-before: always

337343 <* string "* * * * *"

338344 <* manyTill spaceChar eol

339345

340−headingStart :: PandocMonad m => MuseParser m (String, Int)

346+headingStart :: PandocMonad m => MuseParser m (Text, Int)

341347headingStart = try $ (,)

342348 <$> option "" (try (parseAnchor <* manyTill spaceChar eol))

343349 <* firstColumn

@@ -371,14 +377,14 @@ example :: PandocMonad m => MuseParser m (F Blocks)

371377example = try $ pure . B.codeBlock

372378 <$ string "{{{"

373379 <* many spaceChar

374− <*> (unindent <$> manyTill anyChar (string "}}}"))

380+ <*> (unindent <$> manyTillChar anyChar (string "}}}"))

375381

376382-- | Parse an @\<example>@ tag.

377383exampleTag :: PandocMonad m => MuseParser m (F Blocks)

378384exampleTag = try $ fmap pure $ B.codeBlockWith

379385 <$ many spaceChar

380386 <*> (htmlAttrToPandoc <$> openTag "example")

381− <*> (unindent <$> manyTill anyChar (closeTag "example"))

387+ <*> (unindent <$> manyTillChar anyChar (closeTag "example"))

382388 <* manyTill spaceChar eol

383389

384390-- | Parse a @\<literal>@ tag as a raw block.

@@ -388,7 +394,7 @@ literalTag = try $ fmap pure $ B.rawBlock

388394 <$ many spaceChar

389395 <*> (fromMaybe "html" . lookup "style" <$> openTag "literal") -- FIXME: Emacs Muse inserts <literal> without style into all output formats, but we assume HTML

390396 <* manyTill spaceChar eol

391− <*> (unindent <$> manyTill anyChar (closeTag "literal"))

397+ <*> (unindent <$> manyTillChar anyChar (closeTag "literal"))

392398 <* manyTill spaceChar eol

393399

394400-- | Parse @\<center>@ tag.

@@ -428,7 +434,7 @@ playTag = do

428434

429435verseLine :: PandocMonad m => MuseParser m (F Inlines)

430436verseLine = (<>)

431− <$> fmap pure (option mempty (B.str <$> many1 ('\160' <$ char ' ')))

437+ <$> fmap pure (option mempty (B.str <$> many1Char ('\160' <$ char ' ')))

432438 <*> fmap (trimInlinesF . mconcat) (manyTill inline' eol)

433439

434440-- | Parse @\<verse>@ tag.

@@ -466,17 +472,17 @@ paraUntil end = do

466472noteMarker' :: PandocMonad m

467473 => Char

468474 -> Char

469− -> MuseParser m String

470−noteMarker' l r = try $ (\x y -> l:x:y ++ [r])

475+ -> MuseParser m Text

476+noteMarker' l r = try $ (\x y -> T.pack $ l:x:y ++ [r])

471477 <$ char l

472478 <*> oneOf "123456789"

473479 <*> manyTill digit (char r)

474480

475−noteMarker :: PandocMonad m => MuseParser m String

481+noteMarker :: PandocMonad m => MuseParser m Text

476482noteMarker = noteMarker' '[' ']' <|> noteMarker' '{' '}'

477483

478484addNote :: PandocMonad m

479− => String

485+ => Text

480486 -> SourcePos

481487 -> F Blocks

482488 -> MuseParser m ()

@@ -674,15 +680,15 @@ museGridTableRow :: PandocMonad m

674680 -> MuseParser m (F [Blocks])

675681museGridTableRow indent indices = try $ do

676682 lns <- many1 $ try (indentWith indent *> museGridTableRawLine indices)

677− let cols = map (unlines . map trimr) $ transpose lns

683+ let cols = map (T.unlines . map trimr) $ transpose lns

678684 indentWith indent *> museGridTableHeader

679685 sequence <$> mapM (parseFromString' parseBlocks) cols

680686

681687museGridTableRawLine :: PandocMonad m

682688 => [Int]

683− -> MuseParser m [String]

689+ -> MuseParser m [Text]

684690museGridTableRawLine indices =

685− char '|' *> forM indices (\n -> count n anyChar <* char '|') <* manyTill spaceChar eol

691+ char '|' *> forM indices (\n -> countChar n anyChar <* char '|') <* manyTill spaceChar eol

686692

687693museGridTable :: PandocMonad m => MuseParser m (F Blocks)

688694museGridTable = try $ do

@@ -767,12 +773,12 @@ inline = endline <|> inline'

767773endline :: PandocMonad m => MuseParser m (F Inlines)

768774endline = try $ pure B.softbreak <$ newline <* notFollowedBy blankline <* updateLastSpacePos

769775

770−parseAnchor :: PandocMonad m => MuseParser m String

771−parseAnchor = try $ (:)

776+parseAnchor :: PandocMonad m => MuseParser m Text

777+parseAnchor = try $ T.cons

772778 <$ firstColumn

773779 <* char '#'

774780 <*> letter

775− <*> many (letter <|> digit <|> char '-')

781+ <*> manyChar (letter <|> digit <|> char '-')

776782

777783anchor :: PandocMonad m => MuseParser m (F Inlines)

778784anchor = try $ do

@@ -813,7 +819,7 @@ emphasisBetween p = try $ trimInlinesF . mconcat

813819

814820-- | Parse an inline tag, such as @\@ and @\@.

815821inlineTag :: PandocMonad m

816− => String -- ^ Tag name

822+ => Text -- ^ Tag name

817823 -> MuseParser m (F Inlines)

818824inlineTag tag = try $ mconcat

819825 <$ openTag tag

@@ -862,12 +868,12 @@ strikeoutTag = fmap B.strikeout <$> inlineTag "del"

862868verbatimTag :: PandocMonad m => MuseParser m (F Inlines)

863869verbatimTag = return . B.text

864870 <$ openTag "verbatim"

865− <*> manyTill anyChar (closeTag "verbatim")

871+ <*> manyTillChar anyChar (closeTag "verbatim")

866872

867873-- | Parse @\<class>@ tag.

868874classTag :: PandocMonad m => MuseParser m (F Inlines)

869875classTag = do

870− classes <- maybe [] words . lookup "name" <$> openTag "class"

876+ classes <- maybe [] T.words . lookup "name" <$> openTag "class"

871877 fmap (B.spanWith ("", classes, [])) . mconcat <$> manyTill inline (closeTag "class")

872878

873879-- | Parse @\<\<\<RTL>>>@ text.

@@ -886,43 +892,43 @@ nbsp = try $ pure (B.str "\160") <$ string "~~"

886892

887893-- | Parse code markup, indicated by @\'=\'@ characters.

888894code :: PandocMonad m => MuseParser m (F Inlines)

889−code = try $ fmap pure $ B.code . uncurry (++)

895+code = try $ fmap pure $ B.code . uncurry (<>)

890896 <$ atStart

891897 <* char '='

892898 <* notFollowedBy (spaceChar <|> newline)

893− <*> manyUntil (noneOf "\n\r" <|> (newline <* notFollowedBy newline)) (try $ fmap pure $ noneOf " \t\n\r=" <* char '=')

899+ <*> manyUntilChar (noneOf "\n\r" <|> (newline <* notFollowedBy newline)) (try $ fmap T.singleton $ noneOf " \t\n\r=" <* char '=')

894900 <* notFollowedBy alphaNum

895901

896902-- | Parse @\<code>@ tag.

897903codeTag :: PandocMonad m => MuseParser m (F Inlines)

898904codeTag = fmap pure $ B.codeWith

899905 <$> (htmlAttrToPandoc <$> openTag "code")

900− <*> manyTill anyChar (closeTag "code")

906+ <*> manyTillChar anyChar (closeTag "code")

901907

902908-- | Parse @\<math>@ tag.

903909-- @\<math>@ tag is an Emacs Muse extension enabled by @(require 'muse-latex2png)@

904910mathTag :: PandocMonad m => MuseParser m (F Inlines)

905911mathTag = return . B.math

906912 <$ openTag "math"

907− <*> manyTill anyChar (closeTag "math")

913+ <*> manyTillChar anyChar (closeTag "math")

908914

909915-- | Parse inline @\<literal>@ tag as a raw inline.

910916inlineLiteralTag :: PandocMonad m => MuseParser m (F Inlines)

911917inlineLiteralTag = try $ fmap pure $ B.rawInline

912918 <$> (fromMaybe "html" . lookup "style" <$> openTag "literal") -- FIXME: Emacs Muse inserts <literal> without style into all output formats, but we assume HTML

913− <*> manyTill anyChar (closeTag "literal")

919+ <*> manyTillChar anyChar (closeTag "literal")

914920

915921str :: PandocMonad m => MuseParser m (F Inlines)

916−str = return . B.str <$> many1 alphaNum <* updateLastStrPos

922+str = return . B.str <$> many1Char alphaNum <* updateLastStrPos

917923

918924-- | Consume asterisks that were not used as emphasis opening.

919925-- This prevents series of asterisks from being split into

920926-- literal asterisk and emphasis opening.

921927asterisks :: PandocMonad m => MuseParser m (F Inlines)

922−asterisks = pure . B.str <$> many1 (char '*')

928+asterisks = pure . B.str <$> many1Char (char '*')

923929

924930symbol :: PandocMonad m => MuseParser m (F Inlines)

925−symbol = pure . B.str . pure <$> nonspaceChar

931+symbol = pure . B.str . T.singleton <$> nonspaceChar

926932

927933-- | Parse a link or image.

928934linkOrImage :: PandocMonad m => MuseParser m (F Inlines)

@@ -934,12 +940,12 @@ linkContent = trimInlinesF . mconcat

934940 <*> manyTill inline (char ']')

935941

936942-- | Parse a link starting with (possibly null) prefix

937−link :: PandocMonad m => String -> MuseParser m (F Inlines)

943+link :: PandocMonad m => Text -> MuseParser m (F Inlines)

938944link prefix = try $ do

939945 inLink <- asks museInLink

940946 guard $ not inLink

941− string $ "[[" ++ prefix

942− url <- manyTill anyChar $ char ']'

947+ textStr $ "[[" <> prefix

948+ url <- manyTillChar anyChar $ char ']'

943949 content <- option (pure $ B.str url) (local (\s -> s { museInLink = True }) linkContent)

944950 char ']'

945951 return $ B.link url "" <$> content

@@ -947,27 +953,27 @@ link prefix = try $ do

947953image :: PandocMonad m => MuseParser m (F Inlines)

948954image = try $ do

949955 string "[["

950− (url, (ext, width, align)) <- manyUntil (noneOf "]") (imageExtensionAndOptions <* char ']')

956+ (url, (ext, width, align)) <- manyUntilChar (noneOf "]") (imageExtensionAndOptions <* char ']')

951957 content <- option mempty linkContent

952958 char ']'

953959 let widthAttr = case align of

954− Just 'f' -> [("width", fromMaybe "100" width ++ "%"), ("height", "75%")]

955− _ -> maybeToList (("width",) . (++ "%") <$> width)

960+ Just 'f' -> [("width", fromMaybe "100" width <> "%"), ("height", "75%")]

961+ _ -> maybeToList (("width",) . (<> "%") <$> width)

956962 let alignClass = case align of

957963 Just 'r' -> ["align-right"]

958964 Just 'l' -> ["align-left"]

959965 Just 'f' -> []

960966 _ -> []

961− return $ B.imageWith ("", alignClass, widthAttr) (url ++ ext) mempty <$> content

967+ return $ B.imageWith ("", alignClass, widthAttr) (url <> ext) mempty <$> content

962968 where -- Taken from muse-image-regexp defined in Emacs Muse file lisp/muse-regexps.el

963969 imageExtensions = [".eps", ".gif", ".jpg", ".jpeg", ".pbm", ".png", ".tiff", ".xbm", ".xpm"]

964− imageExtension = choice (try . string <$> imageExtensions)

970+ imageExtension = choice (try . textStr <$> imageExtensions)

965971 imageExtensionAndOptions = do

966972 ext <- imageExtension

967973 (width, align) <- option (Nothing, Nothing) imageAttrs

968974 return (ext, width, align)

969975 imageAttrs = (,)

970976 <$ many1 spaceChar

971− <*> optionMaybe (many1 digit)

977+ <*> optionMaybe (many1Char digit)

972978 <* many spaceChar

973979 <*> optionMaybe (oneOf "rlf")

src/Text/Pandoc/Readers/Native.hs+7−6

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Native

@@ -19,7 +20,7 @@ import Text.Pandoc.Options (ReaderOptions)

1920import Text.Pandoc.Shared (safeRead)

2021

2122import Control.Monad.Except (throwError)

22−import Data.Text (Text, unpack)

23+import Data.Text (Text)

2324import Text.Pandoc.Class

2425import Text.Pandoc.Error

2526

@@ -38,18 +39,18 @@ readNative :: PandocMonad m

3839 -> Text -- ^ String to parse (assuming @'\n'@ line endings)

3940 -> m Pandoc

4041readNative _ s =

41− case maybe (Pandoc nullMeta <$> readBlocks s) Right (safeRead (unpack s)) of

42+ case maybe (Pandoc nullMeta <$> readBlocks s) Right (safeRead s) of

4243 Right doc -> return doc

4344 Left _ -> throwError $ PandocParseError "couldn't read native"

4445

4546readBlocks :: Text -> Either PandocError [Block]

46−readBlocks s = maybe ((:[]) <$> readBlock s) Right (safeRead (unpack s))

47+readBlocks s = maybe ((:[]) <$> readBlock s) Right (safeRead s)

4748

4849readBlock :: Text -> Either PandocError Block

49−readBlock s = maybe (Plain <$> readInlines s) Right (safeRead (unpack s))

50+readBlock s = maybe (Plain <$> readInlines s) Right (safeRead s)

5051

5152readInlines :: Text -> Either PandocError [Inline]

52−readInlines s = maybe ((:[]) <$> readInline s) Right (safeRead (unpack s))

53+readInlines s = maybe ((:[]) <$> readInline s) Right (safeRead s)

5354

5455readInline :: Text -> Either PandocError Inline

55−readInline s = maybe (Left . PandocParseError $ "Could not read: " ++ unpack s) Right (safeRead (unpack s))

56+readInline s = maybe (Left . PandocParseError $ "Could not read: " <> s) Right (safeRead s)

src/Text/Pandoc/Readers/OPML.hs+17−12

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.OPML

@@ -18,7 +19,8 @@ import Data.Char (toUpper)

1819import Data.Default

1920import Data.Generics

2021import Data.Maybe (fromMaybe)

21−import Data.Text (Text, pack, unpack)

22+import Data.Text (Text)

23+import qualified Data.Text as T

2224import Text.HTML.TagSoup.Entity (lookupEntity)

2325import Text.Pandoc.Builder

2426import Text.Pandoc.Class (PandocMonad)

@@ -50,7 +52,7 @@ readOPML :: PandocMonad m => ReaderOptions -> Text -> m Pandoc

5052readOPML opts inp = do

5153 (bs, st') <- runStateT

5254 (mapM parseBlock $ normalizeTree $

53− parseXML (unpack (crFilter inp))) def{ opmlOptions = opts }

55+ parseXML (T.unpack (crFilter inp))) def{ opmlOptions = opts }

5456 return $

5557 setTitle (opmlDocTitle st') $

5658 setAuthors (opmlDocAuthors st') $

@@ -76,23 +78,26 @@ convertEntity :: String -> String

7678convertEntity e = Data.Maybe.fromMaybe (map toUpper e) (lookupEntity e)

7779

7880-- convenience function to get an attribute value, defaulting to ""

79−attrValue :: String -> Element -> String

81+attrValue :: String -> Element -> Text

8082attrValue attr elt =

81− fromMaybe "" (lookupAttrBy (\x -> qName x == attr) (elAttribs elt))

83+ maybe "" T.pack (lookupAttrBy (\x -> qName x == attr) (elAttribs elt))

84+

85+textContent :: Element -> Text

86+textContent = T.pack . strContent

8287

8388-- exceptT :: PandocMonad m => Either PandocError a -> OPML m a

8489-- exceptT = either throwError return

8590

86−asHtml :: PandocMonad m => String -> OPML m Inlines

91+asHtml :: PandocMonad m => Text -> OPML m Inlines

8792asHtml s = do

8893 opts <- gets opmlOptions

89− Pandoc _ bs <- readHtml def{ readerExtensions = readerExtensions opts } (pack s)

94+ Pandoc _ bs <- readHtml def{ readerExtensions = readerExtensions opts } s

9095 return $ blocksToInlines' bs

9196

92−asMarkdown :: PandocMonad m => String -> OPML m Blocks

97+asMarkdown :: PandocMonad m => Text -> OPML m Blocks

9398asMarkdown s = do

9499 opts <- gets opmlOptions

95− Pandoc _ bs <- readMarkdown def{ readerExtensions = readerExtensions opts } (pack s)

100+ Pandoc _ bs <- readMarkdown def{ readerExtensions = readerExtensions opts } s

96101 return $ fromList bs

97102

98103getBlocks :: PandocMonad m => Element -> OPML m Blocks

@@ -102,11 +107,11 @@ parseBlock :: PandocMonad m => Content -> OPML m Blocks

102107parseBlock (Elem e) =

103108 case qName (elName e) of

104109 "ownerName" -> mempty <$ modify (\st ->

105− st{opmlDocAuthors = [text $ strContent e]})

110+ st{opmlDocAuthors = [text $ textContent e]})

106111 "dateModified" -> mempty <$ modify (\st ->

107− st{opmlDocDate = text $ strContent e})

112+ st{opmlDocDate = text $ textContent e})

108113 "title" -> mempty <$ modify (\st ->

109− st{opmlDocTitle = text $ strContent e})

114+ st{opmlDocTitle = text $ textContent e})

110115 "outline" -> gets opmlSectionLevel >>= sect . (+1)

111116 "?xml" -> return mempty

112117 _ -> getBlocks e

@@ -115,7 +120,7 @@ parseBlock (Elem e) =

115120 modify $ \st -> st{ opmlSectionLevel = n }

116121 bs <- getBlocks e

117122 modify $ \st -> st{ opmlSectionLevel = n - 1 }

118− let headerText' = case map toUpper (attrValue "type" e) of

123+ let headerText' = case T.toUpper (attrValue "type" e) of

119124 "LINK" -> link

120125 (attrValue "url" e) "" headerText

121126 _ -> headerText

src/Text/Pandoc/Readers/Odt.hs+2−1

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE PatternGuards #-}

2+{-# LANGUAGE PatternGuards #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Reader.Odt

src/Text/Pandoc/Readers/Odt/ContentReader.hs+31−30

@@ -1,11 +1,12 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE Arrows #-}

3−{-# LANGUAGE DeriveFoldable #-}

2+{-# LANGUAGE Arrows #-}

3+{-# LANGUAGE DeriveFoldable #-}

44{-# LANGUAGE GeneralizedNewtypeDeriving #-}

5−{-# LANGUAGE PatternGuards #-}

6−{-# LANGUAGE RecordWildCards #-}

7−{-# LANGUAGE TupleSections #-}

8−{-# LANGUAGE ViewPatterns #-}

5+{-# LANGUAGE PatternGuards #-}

6+{-# LANGUAGE RecordWildCards #-}

7+{-# LANGUAGE TupleSections #-}

8+{-# LANGUAGE ViewPatterns #-}

9+{-# LANGUAGE OverloadedStrings #-}

910{- |

1011 Module : Text.Pandoc.Readers.Odt.ContentReader

@@ -29,8 +30,9 @@ import Control.Arrow

2930

3031import qualified Data.ByteString.Lazy as B

3132import Data.Foldable (fold)

32−import Data.List (find, intercalate, stripPrefix)

33+import Data.List (find, stripPrefix)

3334import qualified Data.Map as M

35+import qualified Data.Text as T

3436import Data.Maybe

3537import Data.Semigroup (First(..), Option(..))

3638

@@ -59,7 +61,7 @@ import qualified Data.Set as Set

5961-- State

6062--------------------------------------------------------------------------------

6163

62−type Anchor = String

64+type Anchor = T.Text

6365type Media = [(FilePath, B.ByteString)]

6466

6567data ReaderState

@@ -204,21 +206,21 @@ updateMediaWithResource = keepingTheValue (

204206 )

205207 >>^ fst

206208

207−lookupResource :: OdtReaderSafe String (FilePath, B.ByteString)

209+lookupResource :: OdtReaderSafe FilePath (FilePath, B.ByteString)

208210lookupResource = proc target -> do

209211 state <- getExtraState -< ()

210212 case lookup target (getMediaEnv state) of

211213 Just bs -> returnV (target, bs) -<< ()

212214 Nothing -> returnV ("", B.empty) -< ()

213215

214−type AnchorPrefix = String

216+type AnchorPrefix = T.Text

215217

216218-- | An adaptation of 'uniqueIdent' from "Text.Pandoc.Shared" that generates a

217219-- unique identifier but without assuming that the id should be for a header.

218220-- Second argument is a list of already used identifiers.

219221uniqueIdentFrom :: AnchorPrefix -> [Anchor] -> Anchor

220222uniqueIdentFrom baseIdent usedIdents =

221− let numIdent n = baseIdent ++ "-" ++ show n

223+ let numIdent n = baseIdent <> "-" <> T.pack (show n)

222224 in if baseIdent `elem` usedIdents

223225 then case find (\x -> numIdent x `notElem` usedIdents) ([1..60000] :: [Int]) of

224226 Just x -> numIdent x

@@ -305,7 +307,7 @@ withNewStyle a = proc x -> do

305307 isCodeStyle _ = False

306308

307309 inlineCode :: Inlines -> Inlines

308− inlineCode = code . intercalate "" . map stringify . toList

310+ inlineCode = code . T.concat . map stringify . toList

309311

310312type PropertyTriple = (ReaderState, TextProperties, Maybe StyleFamily)

311313type InlineModifier = Inlines -> Inlines

@@ -535,7 +537,6 @@ matchChildContent :: (Monoid result)

535537 -> OdtReaderSafe _x result

536538matchChildContent ls fallback = returnV mempty >>> matchContent ls fallback

537539

538−

539540--------------------------------------------

540541-- Matchers

541542--------------------------------------------

@@ -556,8 +557,8 @@ read_plain_text = fst ^&&& read_plain_text' >>% recover

556557 )

557558 >>?% mappend

558559 --

559− extractText :: XML.Content -> Fallible String

560− extractText (XML.Text cData) = succeedWith (XML.cdData cData)

560+ extractText :: XML.Content -> Fallible T.Text

561+ extractText (XML.Text cData) = succeedWith (T.pack $ XML.cdData cData)

561562 extractText _ = failEmpty

562563

563564read_text_seq :: InlineMatcher

@@ -675,8 +676,8 @@ read_list_item = matchingElement NsText "list-item"

675676read_link :: InlineMatcher

676677read_link = matchingElement NsText "a"

677678 $ liftA3 link

678− ( findAttrWithDefault NsXLink "href" "" )

679− ( findAttrWithDefault NsOffice "title" "" )

679+ ( findAttrTextWithDefault NsXLink "href" "" )

680+ ( findAttrTextWithDefault NsOffice "title" "" )

680681 ( matchChildContent [ read_span

681682 , read_note

682683 , read_citation

@@ -709,12 +710,12 @@ read_citation :: InlineMatcher

709710read_citation = matchingElement NsText "bibliography-mark"

710711 $ liftA2 cite

711712 ( liftA2 makeCitation

712− ( findAttrWithDefault NsText "identifier" "" )

713+ ( findAttrTextWithDefault NsText "identifier" "" )

713714 ( readAttrWithDefault NsText "number" 0 )

714715 )

715716 ( matchChildContent [] read_plain_text )

716717 where

717− makeCitation :: String -> Int -> [Citation]

718+ makeCitation :: T.Text -> Int -> [Citation]

718719 makeCitation citeId num = [Citation citeId [] [] NormalCitation num 0]

719720

720721

@@ -779,17 +780,17 @@ read_frame_img =

779780 let exts = extensionsFromList [Ext_auto_identifiers]

780781 resource <- lookupResource -< src'

781782 _ <- updateMediaWithResource -< resource

782− w <- findAttr' NsSVG "width" -< ()

783− h <- findAttr' NsSVG "height" -< ()

783+ w <- findAttrText' NsSVG "width" -< ()

784+ h <- findAttrText' NsSVG "height" -< ()

784785 titleNodes <- matchChildContent' [ read_frame_title ] -< ()

785786 alt <- matchChildContent [] read_plain_text -< ()

786787 arr (firstMatch . uncurry4 imageWith) -<

787− (image_attributes w h, src', inlineListToIdentifier exts (toList titleNodes), alt)

788+ (image_attributes w h, T.pack src', inlineListToIdentifier exts (toList titleNodes), alt)

788789

789790read_frame_title :: InlineMatcher

790791read_frame_title = matchingElement NsSVG "title" (matchChildContent [] read_plain_text)

791792

792−image_attributes :: Maybe String -> Maybe String -> Attr

793+image_attributes :: Maybe T.Text -> Maybe T.Text -> Attr

793794image_attributes x y =

794795 ( "", [], (dim "width" x) ++ (dim "height" y))

795796 where

@@ -806,7 +807,7 @@ read_frame_mathml =

806807 src' -> do

807808 let path = fromMaybe src' (stripPrefix "./" src') ++ "/content.xml"

808809 (_, mathml) <- lookupResource -< path

809− case readMathML (UTF8.toString $ B.toStrict mathml) of

810+ case readMathML (UTF8.toText $ B.toStrict mathml) of

810811 Left _ -> returnV mempty -< ()

811812 Right exps -> arr (firstMatch . displayMath . writeTeX) -< exps

812813

@@ -817,9 +818,9 @@ read_frame_text_box = proc box -> do

817818

818819read_img_with_caption :: [Block] -> FirstMatch Inlines

819820read_img_with_caption (Para [Image attr alt (src,title)] : _) =

820− firstMatch $ singleton (Image attr alt (src, 'f':'i':'g':':':title)) -- no text, default caption

821+ firstMatch $ singleton (Image attr alt (src, "fig:" <> title)) -- no text, default caption

821822read_img_with_caption (Para (Image attr _ (src,title) : txt) : _) =

822− firstMatch $ singleton (Image attr txt (src, 'f':'i':'g':':':title) ) -- override caption with the text that follows

823+ firstMatch $ singleton (Image attr txt (src, "fig:" <> title) ) -- override caption with the text that follows

823824read_img_with_caption ( Para (_ : xs) : ys) =

824825 read_img_with_caption (Para xs : ys)

825826read_img_with_caption _ =

@@ -829,12 +830,12 @@ read_img_with_caption _ =

829830-- Internal links

830831----------------------

831832

832−_ANCHOR_PREFIX_ :: String

833+_ANCHOR_PREFIX_ :: T.Text

833834_ANCHOR_PREFIX_ = "anchor"

834835

835836--

836837readAnchorAttr :: OdtReader _x Anchor

837−readAnchorAttr = findAttr NsText "name"

838+readAnchorAttr = findAttrText NsText "name"

838839

839840-- | Beware: may fail

840841findAnchorName :: OdtReader AnchorPrefix Anchor

@@ -875,7 +876,7 @@ read_reference_start = matchingElement NsText "reference-mark-start"

875876

876877-- | Beware: may fail

877878findAnchorRef :: OdtReader _x Anchor

878−findAnchorRef = ( findAttr NsText "ref-name"

879+findAnchorRef = ( findAttrText NsText "ref-name"

879880 >>?^ (_ANCHOR_PREFIX_,)

880881 ) >>?! getPrettyAnchor

881882

@@ -890,7 +891,7 @@ maybeInAnchorRef = proc inlines -> do

890891 Left _ -> returnA -< inlines

891892 where

892893 toAnchorRef :: Anchor -> Inlines -> Inlines

893− toAnchorRef anchor = link ('#':anchor) "" -- no title

894+ toAnchorRef anchor = link ("#" <> anchor) "" -- no title

894895

895896--

896897read_bookmark_ref :: InlineMatcher

src/Text/Pandoc/Readers/Odt/Generic/XMLConverter.hs+33

@@ -38,8 +38,11 @@ module Text.Pandoc.Readers.Odt.Generic.XMLConverter

3838, lookupAttr'

3939, lookupDefaultingAttr

4040, findAttr'

41+, findAttrText'

4142, findAttr

43+, findAttrText

4244, findAttrWithDefault

45+, findAttrTextWithDefault

4346, readAttr

4447, readAttr'

4548, readAttrWithDefault

@@ -59,6 +62,7 @@ import Control.Arrow

5962

6063import Data.Either ( rights )

6164import qualified Data.Map as M

65+import qualified Data.Text as T

6266import Data.Default

6367import Data.Maybe

6468

@@ -79,6 +83,7 @@ import Text.Pandoc.Readers.Odt.Generic.Fallible

7983type ElementName = String

8084type AttributeName = String

8185type AttributeValue = String

86+type TextAttributeValue = T.Text

8287

8388--

8489type NameSpacePrefix = String

@@ -466,6 +471,16 @@ findAttr' nsID attrName = qualifyName nsID attrName

466471 &&& getCurrentElement

467472 >>% XML.findAttr

468473

474+-- | Return value as a (Maybe Text)

475+findAttrText' :: (NameSpaceID nsID)

476+ => nsID -> AttributeName

477+ -> XMLConverter nsID extraState x (Maybe TextAttributeValue)

478+findAttrText' nsID attrName

479+ = qualifyName nsID attrName

480+ &&& getCurrentElement

481+ >>% XML.findAttr

482+ >>^ fmap T.pack

483+

469484-- | Return value as string or fail

470485findAttr :: (NameSpaceID nsID)

471486 => nsID -> AttributeName

@@ -473,6 +488,15 @@ findAttr :: (NameSpaceID nsID)

473488findAttr nsID attrName = findAttr' nsID attrName

474489 >>> maybeToChoice

475490

491+-- | Return value as text or fail

492+findAttrText :: (NameSpaceID nsID)

493+ => nsID -> AttributeName

494+ -> FallibleXMLConverter nsID extraState x TextAttributeValue

495+findAttrText nsID attrName

496+ = findAttr' nsID attrName

497+ >>^ fmap T.pack

498+ >>> maybeToChoice

499+

476500-- | Return value as string or return provided default value

477501findAttrWithDefault :: (NameSpaceID nsID)

478502 => nsID -> AttributeName

@@ -482,6 +506,15 @@ findAttrWithDefault nsID attrName deflt

482506 = findAttr' nsID attrName

483507 >>^ fromMaybe deflt

484508

509+-- | Return value as string or return provided default value

510+findAttrTextWithDefault :: (NameSpaceID nsID)

511+ => nsID -> AttributeName

512+ -> TextAttributeValue

513+ -> XMLConverter nsID extraState x TextAttributeValue

514+findAttrTextWithDefault nsID attrName deflt

515+ = findAttr' nsID attrName

516+ >>^ maybe deflt T.pack

517+

485518-- | Read and return value or fail

486519readAttr :: (NameSpaceID nsID, Read attrValue)

487520 => nsID -> AttributeName

src/Text/Pandoc/Readers/Odt/StyleReader.hs+5−5

@@ -548,11 +548,11 @@ readListLevelStyle :: ListLevelType -> StyleReader _x (Int, ListLevelStyle)

548548readListLevelStyle levelType = readAttr NsText "level"

549549 >>?! keepingTheValue

550550 ( liftA5 toListLevelStyle

551− ( returnV levelType )

552− ( findAttr' NsStyle "num-prefix" )

553− ( findAttr' NsStyle "num-suffix" )

554− ( getAttr NsStyle "num-format" )

555− ( findAttr' NsText "start-value" )

551+ ( returnV levelType )

552+ ( findAttr' NsStyle "num-prefix" )

553+ ( findAttr' NsStyle "num-suffix" )

554+ ( getAttr NsStyle "num-format" )

555+ ( findAttrText' NsText "start-value" )

556556 )

557557 where

558558 toListLevelStyle _ p s LinfNone b = ListLevelStyle LltBullet p s LinfNone (startValue b)

src/Text/Pandoc/Readers/Org.hs+2−2

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Org

@@ -26,7 +27,6 @@ import Control.Monad.Except (throwError)

2627import Control.Monad.Reader (runReaderT)

2728

2829import Data.Text (Text)

29−import qualified Data.Text as T

3030

3131-- | Parse org-mode string and return a Pandoc document.

3232readOrg :: PandocMonad m

@@ -36,7 +36,7 @@ readOrg :: PandocMonad m

3636readOrg opts s = do

3737 parsed <- flip runReaderT def $

3838 readWithM parseOrg (optionsToParserState opts)

39− (T.unpack (crFilter s) ++ "\n\n")

39+ (crFilter s <> "\n\n")

4040 case parsed of

4141 Right result -> return result

4242 Left _ -> throwError $ PandocParseError "problem parsing org"

src/Text/Pandoc/Readers/Org/BlockStarts.hs+14−11

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Org.BlockStarts

@@ -25,6 +26,8 @@ module Text.Pandoc.Readers.Org.BlockStarts

2526

2627import Prelude

2728import Control.Monad (void)

29+import Data.Text (Text)

30+import qualified Data.Text as T

2831import Text.Pandoc.Readers.Org.Parsing

2932

3033-- | Horizontal Line (five -- dashes or more)

@@ -49,15 +52,15 @@ gridTableStart :: Monad m => OrgParser m ()

4952gridTableStart = try $ skipSpaces <* char '+' <* char '-'

5053

5154

52−latexEnvStart :: Monad m => OrgParser m String

55+latexEnvStart :: Monad m => OrgParser m Text

5356latexEnvStart = try $

5457 skipSpaces *> string "\\begin{"

5558 *> latexEnvName

5659 <* string "}"

5760 <* blankline

5861 where

59− latexEnvName :: Monad m => OrgParser m String

60− latexEnvName = try $ mappend <$> many1 alphaNum <*> option "" (string "*")

62+ latexEnvName :: Monad m => OrgParser m Text

63+ latexEnvName = try $ mappend <$> many1Char alphaNum <*> option "" (textStr "*")

6164

6265bulletListStart :: Monad m => OrgParser m Int

6366bulletListStart = try $ do

@@ -68,7 +71,7 @@ bulletListStart = try $ do

6871 return (ind + 1)

6972

7073genericListStart :: Monad m

71− => OrgParser m String

74+ => OrgParser m Text

7275 -> OrgParser m Int

7376genericListStart listMarker = try $ do

7477 ind <- length <$> many spaceChar

@@ -82,11 +85,11 @@ eol = void (char '\n')

8285orderedListStart :: Monad m => OrgParser m Int

8386orderedListStart = genericListStart orderedListMarker

8487 -- Ordered list markers allowed in org-mode

85− where orderedListMarker = mappend <$> many1 digit <*> (pure <$> oneOf ".)")

88+ where orderedListMarker = T.snoc <$> many1Char digit <*> oneOf ".)"

8689

87−drawerStart :: Monad m => OrgParser m String

90+drawerStart :: Monad m => OrgParser m Text

8891drawerStart = try $ skipSpaces *> drawerName <* skipSpaces <* newline

89− where drawerName = char ':' *> manyTill nonspaceChar (char ':')

92+ where drawerName = char ':' *> manyTillChar nonspaceChar (char ':')

9093

9194metaLineStart :: Monad m => OrgParser m ()

9295metaLineStart = try $ skipSpaces <* string "#+"

@@ -99,12 +102,12 @@ commentLineStart = try $

99102exampleLineStart :: Monad m => OrgParser m ()

100103exampleLineStart = () <$ try (skipSpaces *> string ": ")

101104

102−noteMarker :: Monad m => OrgParser m String

105+noteMarker :: Monad m => OrgParser m Text

103106noteMarker = try $ do

104107 char '['

105− choice [ many1Till digit (char ']')

106− , (++) <$> string "fn:"

107− <*> many1Till (noneOf "\n\r\t ") (char ']')

108+ choice [ many1TillChar digit (char ']')

109+ , (<>) <$> textStr "fn:"

110+ <*> many1TillChar (noneOf "\n\r\t ") (char ']')

108111 ]

109112

110113-- | Succeeds if the parser is at the end of a block.

src/Text/Pandoc/Readers/Org/Blocks.hs+111−108

@@ -1,6 +1,7 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE FlexibleContexts #-}

3−{-# LANGUAGE RecordWildCards #-}

2+{-# LANGUAGE FlexibleContexts #-}

3+{-# LANGUAGE RecordWildCards #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers.Org.Blocks

@@ -23,7 +24,7 @@ import Text.Pandoc.Readers.Org.Inlines

2324import Text.Pandoc.Readers.Org.Meta (metaExport, metaKey, metaLine)

2425import Text.Pandoc.Readers.Org.ParserState

2526import Text.Pandoc.Readers.Org.Parsing

26−import Text.Pandoc.Readers.Org.Shared (cleanLinkString, isImageFilename,

27+import Text.Pandoc.Readers.Org.Shared (cleanLinkText, isImageFilename,

2728 originalLang, translateLang, exportsCode)

2829

2930import Text.Pandoc.Builder (Blocks, Inlines)

@@ -33,11 +34,13 @@ import Text.Pandoc.Options

3334import Text.Pandoc.Shared (compactify, compactifyDL, safeRead)

3435

3536import Control.Monad (foldM, guard, mzero, void)

36−import Data.Char (isSpace, toLower, toUpper)

37+import Data.Char (isSpace)

3738import Data.Default (Default)

38−import Data.List (foldl', isPrefixOf)

39+import Data.List (foldl')

3940import Data.Maybe (fromMaybe, isJust, isNothing)

41+import Data.Text (Text)

4042

43+import qualified Data.Text as T

4144import qualified Text.Pandoc.Builder as B

4245import qualified Text.Pandoc.Walk as Walk

4346

@@ -90,10 +93,10 @@ horizontalRule = return B.horizontalRule <$ try hline

9093

9194-- | Attributes that may be added to figures (like a name or caption).

9295data BlockAttributes = BlockAttributes

93− { blockAttrName :: Maybe String

94− , blockAttrLabel :: Maybe String

96+ { blockAttrName :: Maybe Text

97+ , blockAttrLabel :: Maybe Text

9598 , blockAttrCaption :: Maybe (F Inlines)

96− , blockAttrKeyValues :: [(String, String)]

99+ , blockAttrKeyValues :: [(Text, Text)]

97100 }

98101

99102-- | Convert BlockAttributes into pandoc Attr

@@ -103,14 +106,14 @@ attrFromBlockAttributes BlockAttributes{..} =

103106 ident = fromMaybe mempty $ lookup "id" blockAttrKeyValues

104107 classes = case lookup "class" blockAttrKeyValues of

105108 Nothing -> []

106− Just clsStr -> words clsStr

109+ Just clsStr -> T.words clsStr

107110 kv = filter ((`notElem` ["id", "class"]) . fst) blockAttrKeyValues

108111 in (ident, classes, kv)

109112

110−stringyMetaAttribute :: Monad m => OrgParser m (String, String)

113+stringyMetaAttribute :: Monad m => OrgParser m (Text, Text)

111114stringyMetaAttribute = try $ do

112115 metaLineStart

113− attrName <- map toUpper <$> many1Till nonspaceChar (char ':')

116+ attrName <- T.toUpper <$> many1TillChar nonspaceChar (char ':')

114117 skipSpaces

115118 attrValue <- anyLine <|> ("" <$ newline)

116119 return (attrName, attrValue)

@@ -129,8 +132,8 @@ blockAttributes = try $ do

129132 let label = lookup "LABEL" kv

130133 caption' <- case caption of

131134 Nothing -> return Nothing

132− Just s -> Just <$> parseFromString inlines (s ++ "\n")

133− kvAttrs' <- parseFromString keyValues . (++ "\n") $ fromMaybe mempty kvAttrs

135+ Just s -> Just <$> parseFromString inlines (s <> "\n")

136+ kvAttrs' <- parseFromString keyValues . (<> "\n") $ fromMaybe mempty kvAttrs

134137 return BlockAttributes

135138 { blockAttrName = name

136139 , blockAttrLabel = label

@@ -138,31 +141,31 @@ blockAttributes = try $ do

138141 , blockAttrKeyValues = kvAttrs'

139142 }

140143 where

141− isBlockAttr :: String -> Bool

144+ isBlockAttr :: Text -> Bool

142145 isBlockAttr = flip elem

143146 [ "NAME", "LABEL", "CAPTION"

144147 , "ATTR_HTML", "ATTR_LATEX"

145148 , "RESULTS"

146149 ]

147150

148− appendValues :: String -> Maybe String -> (String, String) -> Maybe String

151+ appendValues :: Text -> Maybe Text -> (Text, Text) -> Maybe Text

149152 appendValues attrName accValue (key, value) =

150153 if key /= attrName

151154 then accValue

152155 else case accValue of

153− Just acc -> Just $ acc ++ ' ':value

156+ Just acc -> Just $ acc <> " " <> value

154157 Nothing -> Just value

155158

156159-- | Parse key-value pairs for HTML attributes

157−keyValues :: Monad m => OrgParser m [(String, String)]

160+keyValues :: Monad m => OrgParser m [(Text, Text)]

158161keyValues = try $

159162 manyTill ((,) <$> key <*> value) newline

160163 where

161− key :: Monad m => OrgParser m String

162− key = try $ skipSpaces *> char ':' *> many1 nonspaceChar

164+ key :: Monad m => OrgParser m Text

165+ key = try $ skipSpaces *> char ':' *> many1Char nonspaceChar

163166

164− value :: Monad m => OrgParser m String

165− value = skipSpaces *> manyTill anyChar endOfValue

167+ value :: Monad m => OrgParser m Text

168+ value = skipSpaces *> manyTillChar anyChar endOfValue

166169

167170 endOfValue :: Monad m => OrgParser m ()

168171 endOfValue =

@@ -180,7 +183,7 @@ orgBlock = try $ do

180183 blockAttrs <- blockAttributes

181184 blkType <- blockHeaderStart

182185 ($ blkType) $

183− case map toLower blkType of

186+ case T.toLower blkType of

184187 "export" -> exportBlock

185188 "comment" -> rawBlockLines (const mempty)

186189 "html" -> rawBlockLines (return . B.rawBlock (lowercase blkType))

@@ -194,13 +197,13 @@ orgBlock = try $ do

194197 let (ident, classes, kv) = attrFromBlockAttributes blockAttrs

195198 in fmap $ B.divWith (ident, classes ++ [blkType], kv)

196199 where

197− blockHeaderStart :: Monad m => OrgParser m String

200+ blockHeaderStart :: Monad m => OrgParser m Text

198201 blockHeaderStart = try $ skipSpaces *> stringAnyCase "#+begin_" *> orgArgWord

199202

200− lowercase :: String -> String

201− lowercase = map toLower

203+ lowercase :: Text -> Text

204+ lowercase = T.toLower

202205

203−exampleBlock :: PandocMonad m => BlockAttributes -> String -> OrgParser m (F Blocks)

206+exampleBlock :: PandocMonad m => BlockAttributes -> Text -> OrgParser m (F Blocks)

204207exampleBlock blockAttrs _label = do

205208 skipSpaces

206209 (classes, kv) <- switchesAsAttributes

@@ -210,54 +213,54 @@ exampleBlock blockAttrs _label = do

210213 let codeBlck = B.codeBlockWith (id', "example":classes, kv) content

211214 return . return $ codeBlck

212215

213−rawBlockLines :: Monad m => (String -> F Blocks) -> String -> OrgParser m (F Blocks)

216+rawBlockLines :: Monad m => (Text -> F Blocks) -> Text -> OrgParser m (F Blocks)

214217rawBlockLines f blockType = ignHeaders *> (f <$> rawBlockContent blockType)

215218

216−parseBlockLines :: PandocMonad m => (F Blocks -> F Blocks) -> String -> OrgParser m (F Blocks)

219+parseBlockLines :: PandocMonad m => (F Blocks -> F Blocks) -> Text -> OrgParser m (F Blocks)

217220parseBlockLines f blockType = ignHeaders *> (f <$> parsedBlockContent)

218221 where

219222 parsedBlockContent :: PandocMonad m => OrgParser m (F Blocks)

220223 parsedBlockContent = try $ do

221224 raw <- rawBlockContent blockType

222− parseFromString blocks (raw ++ "\n")

225+ parseFromString blocks (raw <> "\n")

223226

224227-- | Read the raw string content of a block

225−rawBlockContent :: Monad m => String -> OrgParser m String

228+rawBlockContent :: Monad m => Text -> OrgParser m Text

226229rawBlockContent blockType = try $ do

227230 blkLines <- manyTill rawLine blockEnder

228231 tabLen <- getOption readerTabStop

229232 trimP <- orgStateTrimLeadBlkIndent <$> getState

230− let stripIndent strs = if trimP then map (drop (shortestIndent strs)) strs else strs

231− (unlines

233+ let stripIndent strs = if trimP then map (T.drop (shortestIndent strs)) strs else strs

234+ (T.unlines

232235 . stripIndent

233236 . map (tabsToSpaces tabLen . commaEscaped)

234237 $ blkLines)

235238 <$ updateState (\s -> s { orgStateTrimLeadBlkIndent = True })

236239 where

237− rawLine :: Monad m => OrgParser m String

240+ rawLine :: Monad m => OrgParser m Text

238241 rawLine = try $ ("" <$ blankline) <|> anyLine

239242

240243 blockEnder :: Monad m => OrgParser m ()

241244 blockEnder = try $ skipSpaces <* stringAnyCase ("#+end_" <> blockType)

242245

243− shortestIndent :: [String] -> Int

244− shortestIndent = foldr (min . length . takeWhile isSpace) maxBound

245− . filter (not . null)

246−

247− tabsToSpaces :: Int -> String -> String

248− tabsToSpaces _ [] = []

249− tabsToSpaces tabLen cs'@(c:cs) =

250− case c of

251− ' ' -> ' ':tabsToSpaces tabLen cs

252− '\t' -> replicate tabLen ' ' ++ tabsToSpaces tabLen cs

253− _ -> cs'

254−

255− commaEscaped :: String -> String

256− commaEscaped (',':cs@('*':_)) = cs

257− commaEscaped (',':cs@('#':'+':_)) = cs

258− commaEscaped (' ':cs) = ' ':commaEscaped cs

259− commaEscaped ('\t':cs) = '\t':commaEscaped cs

260− commaEscaped cs = cs

246+ shortestIndent :: [Text] -> Int

247+ shortestIndent = foldr (min . T.length . T.takeWhile isSpace) maxBound

248+ . filter (not . T.null)

249+

250+ tabsToSpaces :: Int -> Text -> Text

251+ tabsToSpaces tabStop t =

252+ let (ind, suff) = T.span (\c -> c == ' ' || c == '\t') t

253+ tabNum = T.length $ T.filter (== '\n') ind

254+ spaceNum = T.length ind - tabNum

255+ in T.replicate (spaceNum + tabStop * tabNum) " " <> suff

256+

257+ commaEscaped t =

258+ let (ind, suff) = T.span (\c -> c == ' ' || c == '\t') t

259+ in case T.uncons suff of

260+ Just (',', cs)

261+ | "*" <- T.take 1 cs -> ind <> cs

262+ | "#+" <- T.take 2 cs -> ind <> cs

263+ _ -> t

261264

262265-- | Read but ignore all remaining block headers.

263266ignHeaders :: Monad m => OrgParser m ()

@@ -265,34 +268,34 @@ ignHeaders = (() <$ newline) <|> (() <$ anyLine)

265268

266269-- | Read a block containing code intended for export in specific backends

267270-- only.

268−exportBlock :: Monad m => String -> OrgParser m (F Blocks)

271+exportBlock :: Monad m => Text -> OrgParser m (F Blocks)

269272exportBlock blockType = try $ do

270273 exportType <- skipSpaces *> orgArgWord <* ignHeaders

271274 contents <- rawBlockContent blockType

272− returnF (B.rawBlock (map toLower exportType) contents)

275+ returnF (B.rawBlock (T.toLower exportType) contents)

273276

274−verseBlock :: PandocMonad m => String -> OrgParser m (F Blocks)

277+verseBlock :: PandocMonad m => Text -> OrgParser m (F Blocks)

275278verseBlock blockType = try $ do

276279 ignHeaders

277280 content <- rawBlockContent blockType

278281 fmap B.lineBlock . sequence

279− <$> mapM parseVerseLine (lines content)

282+ <$> mapM parseVerseLine (T.lines content)

280283 where

281284 -- replace initial spaces with nonbreaking spaces to preserve

282285 -- indentation, parse the rest as normal inline

283− parseVerseLine :: PandocMonad m => String -> OrgParser m (F Inlines)

286+ parseVerseLine :: PandocMonad m => Text -> OrgParser m (F Inlines)

284287 parseVerseLine cs = do

285− let (initialSpaces, indentedLine) = span isSpace cs

286− let nbspIndent = if null initialSpaces

288+ let (initialSpaces, indentedLine) = T.span isSpace cs

289+ let nbspIndent = if T.null initialSpaces

287290 then mempty

288− else B.str $ map (const '\160') initialSpaces

289− line <- parseFromString inlines (indentedLine ++ "\n")

291+ else B.str $ T.map (const '\160') initialSpaces

292+ line <- parseFromString inlines (indentedLine <> "\n")

290293 return (trimInlinesF $ pure nbspIndent <> line)

291294

292295-- | Read a code block and the associated results block if present. Which of

293296-- boths blocks is included in the output is determined using the "exports"

294297-- argument in the block header.

295−codeBlock :: PandocMonad m => BlockAttributes -> String -> OrgParser m (F Blocks)

298+codeBlock :: PandocMonad m => BlockAttributes -> Text -> OrgParser m (F Blocks)

296299codeBlock blockAttrs blockType = do

297300 skipSpaces

298301 (classes, kv) <- codeHeaderArgs <|> (mempty <$ ignHeaders)

@@ -314,7 +317,7 @@ codeBlock blockAttrs blockType = do

314317 labelledBlock :: F Inlines -> F Blocks

315318 labelledBlock = fmap (B.plain . B.spanWith ("", ["label"], []))

316319

317− exportsResults :: [(String, String)] -> Bool

320+ exportsResults :: [(Text, Text)] -> Bool

318321 exportsResults = maybe False (`elem` ["results", "both"]) . lookup "exports"

319322

320323-- | Parse the result of an evaluated babel code block.

@@ -329,7 +332,7 @@ babelResultsBlock = try $ do

329332 resultsMarker = try . void $ stringAnyCase "#+RESULTS:" *> blankline

330333

331334-- | Parse code block arguments

332−codeHeaderArgs :: Monad m => OrgParser m ([String], [(String, String)])

335+codeHeaderArgs :: Monad m => OrgParser m ([Text], [(Text, Text)])

333336codeHeaderArgs = try $ do

334337 language <- skipSpaces *> orgArgWord

335338 (switchClasses, switchKv) <- switchesAsAttributes

@@ -338,14 +341,14 @@ codeHeaderArgs = try $ do

338341 , originalLang language <> switchKv <> parameters

339342 )

340343

341−switchesAsAttributes :: Monad m => OrgParser m ([String], [(String, String)])

344+switchesAsAttributes :: Monad m => OrgParser m ([Text], [(Text, Text)])

342345switchesAsAttributes = try $ do

343346 switches <- skipSpaces *> try (switch `sepBy` many1 spaceChar)

344347 return $ foldr addToAttr ([], []) switches

345348 where

346− addToAttr :: (Char, Maybe String, SwitchPolarity)

347− -> ([String], [(String, String)])

348− -> ([String], [(String, String)])

349+ addToAttr :: (Char, Maybe Text, SwitchPolarity)

350+ -> ([Text], [(Text, Text)])

351+ -> ([Text], [(Text, Text)])

349352 addToAttr ('n', lineNum, pol) (cls, kv) =

350353 let kv' = case lineNum of

351354 Just num -> ("startFrom", num):kv

@@ -365,15 +368,15 @@ switchPolarity :: Monad m => OrgParser m SwitchPolarity

365368switchPolarity = (SwitchMinus <$ char '-') <|> (SwitchPlus <$ char '+')

366369

367370-- | Parses a source block switch option.

368−switch :: Monad m => OrgParser m (Char, Maybe String, SwitchPolarity)

371+switch :: Monad m => OrgParser m (Char, Maybe Text, SwitchPolarity)

369372switch = try $ lineNumberSwitch <|> labelSwitch

370373 <|> whitespaceSwitch <|> simpleSwitch

371374 where

372375 simpleSwitch = (\pol c -> (c, Nothing, pol)) <$> switchPolarity <*> letter

373376 labelSwitch = genericSwitch 'l' $

374− char '"' *> many1Till nonspaceChar (char '"')

377+ char '"' *> many1TillChar nonspaceChar (char '"')

375378

376−whitespaceSwitch :: Monad m => OrgParser m (Char, Maybe String, SwitchPolarity)

379+whitespaceSwitch :: Monad m => OrgParser m (Char, Maybe Text, SwitchPolarity)

377380whitespaceSwitch = do

378381 string "-i"

379382 updateState $ \s -> s { orgStateTrimLeadBlkIndent = False }

@@ -382,8 +385,8 @@ whitespaceSwitch = do

382385-- | Generic source block switch-option parser.

383386genericSwitch :: Monad m

384387 => Char

385− -> OrgParser m String

386− -> OrgParser m (Char, Maybe String, SwitchPolarity)

388+ -> OrgParser m Text

389+ -> OrgParser m (Char, Maybe Text, SwitchPolarity)

387390genericSwitch c p = try $ do

388391 polarity <- switchPolarity <* char c <* skipSpaces

389392 arg <- optionMaybe p

@@ -391,17 +394,17 @@ genericSwitch c p = try $ do

391394

392395-- | Reads a line number switch option. The line number switch can be used with

393396-- example and source blocks.

394−lineNumberSwitch :: Monad m => OrgParser m (Char, Maybe String, SwitchPolarity)

395−lineNumberSwitch = genericSwitch 'n' (many digit)

397+lineNumberSwitch :: Monad m => OrgParser m (Char, Maybe Text, SwitchPolarity)

398+lineNumberSwitch = genericSwitch 'n' (manyChar digit)

396399

397−blockOption :: Monad m => OrgParser m (String, String)

400+blockOption :: Monad m => OrgParser m (Text, Text)

398401blockOption = try $ do

399402 argKey <- orgArgKey

400403 paramValue <- option "yes" orgParamValue

401404 return (argKey, paramValue)

402405

403−orgParamValue :: Monad m => OrgParser m String

404−orgParamValue = try $

406+orgParamValue :: Monad m => OrgParser m Text

407+orgParamValue = try $ fmap T.pack $

405408 skipSpaces

406409 *> notFollowedBy orgArgKey

407410 *> noneOf "\n\r" `many1Till` endOfValue

@@ -420,7 +423,7 @@ orgParamValue = try $

420423-- export setting.

421424genericDrawer :: PandocMonad m => OrgParser m (F Blocks)

422425genericDrawer = try $ do

423− name <- map toUpper <$> drawerStart

426+ name <- T.toUpper <$> drawerStart

424427 content <- manyTill drawerLine (try drawerEnd)

425428 state <- getState

426429 -- Include drawer if it is explicitly included in or not explicitly excluded

@@ -432,16 +435,16 @@ genericDrawer = try $ do

432435 Right names | name `notElem` names -> return mempty

433436 _ -> drawerDiv name <$> parseLines content

434437 where

435− parseLines :: PandocMonad m => [String] -> OrgParser m (F Blocks)

436− parseLines = parseFromString blocks . (++ "\n") . unlines

438+ parseLines :: PandocMonad m => [Text] -> OrgParser m (F Blocks)

439+ parseLines = parseFromString blocks . (<> "\n") . T.unlines

437440

438− drawerDiv :: String -> F Blocks -> F Blocks

441+ drawerDiv :: Text -> F Blocks -> F Blocks

439442 drawerDiv drawerName = fmap $ B.divWith (mempty, [drawerName, "drawer"], mempty)

440443

441−drawerLine :: Monad m => OrgParser m String

444+drawerLine :: Monad m => OrgParser m Text

442445drawerLine = anyLine

443446

444−drawerEnd :: Monad m => OrgParser m String

447+drawerEnd :: Monad m => OrgParser m Text

445448drawerEnd = try $

446449 skipSpaces *> stringAnyCase ":END:" <* skipSpaces <* newline

447450

@@ -456,17 +459,17 @@ figure :: PandocMonad m => OrgParser m (F Blocks)

456459figure = try $ do

457460 figAttrs <- blockAttributes

458461 src <- skipSpaces *> selfTarget <* skipSpaces <* endOfParagraph

459− case cleanLinkString src of

462+ case cleanLinkText src of

460463 Nothing -> mzero

461464 Just imgSrc -> do

462465 guard (isImageFilename imgSrc)

463466 let isFigure = isJust $ blockAttrCaption figAttrs

464467 return $ imageBlock isFigure figAttrs imgSrc

465468 where

466− selfTarget :: PandocMonad m => OrgParser m String

469+ selfTarget :: PandocMonad m => OrgParser m Text

467470 selfTarget = try $ char '[' *> linkTarget <* char ']'

468471

469− imageBlock :: Bool -> BlockAttributes -> String -> F Blocks

472+ imageBlock :: Bool -> BlockAttributes -> Text -> F Blocks

470473 imageBlock isFigure figAttrs imgSrc =

471474 let

472475 figName = fromMaybe mempty $ blockAttrName figAttrs

@@ -478,11 +481,11 @@ figure = try $ do

478481 in

479482 B.para . B.imageWith attr imgSrc figTitle <$> figCaption

480483

481− withFigPrefix :: String -> String

484+ withFigPrefix :: Text -> Text

482485 withFigPrefix cs =

483− if "fig:" `isPrefixOf` cs

486+ if "fig:" `T.isPrefixOf` cs

484487 then cs

485− else "fig:" ++ cs

488+ else "fig:" <> cs

486489

487490-- | Succeeds if looking at the end of the current paragraph

488491endOfParagraph :: Monad m => OrgParser m ()

@@ -495,12 +498,12 @@ endOfParagraph = try $ skipSpaces *> newline *> endOfBlock

495498

496499-- | Example code marked up by a leading colon.

497500example :: Monad m => OrgParser m (F Blocks)

498−example = try $ returnF . exampleCode =<< unlines <$> many1 exampleLine

501+example = try $ returnF . exampleCode =<< T.unlines <$> many1 exampleLine

499502 where

500− exampleLine :: Monad m => OrgParser m String

503+ exampleLine :: Monad m => OrgParser m Text

501504 exampleLine = try $ exampleLineStart *> anyLine

502505

503−exampleCode :: String -> Blocks

506+exampleCode :: Text -> Blocks

504507exampleCode = B.codeBlockWith ("", ["example"], [])

505508

506509

@@ -516,7 +519,7 @@ include :: PandocMonad m => OrgParser m (F Blocks)

516519include = try $ do

517520 metaLineStart <* stringAnyCase "include:" <* skipSpaces

518521 filename <- includeTarget

519− includeArgs <- many (try $ skipSpaces *> many1 alphaNum)

522+ includeArgs <- many (try $ skipSpaces *> many1Char alphaNum)

520523 params <- keyValues

521524 blocksParser <- case includeArgs of

522525 ("example" : _) -> return $ pure . B.codeBlock <$> parseRaw

@@ -535,10 +538,10 @@ include = try $ do

535538 char '"'

536539 manyTill (noneOf "\n\r\t") (char '"')

537540

538− parseRaw :: PandocMonad m => OrgParser m String

539− parseRaw = many anyChar

541+ parseRaw :: PandocMonad m => OrgParser m Text

542+ parseRaw = manyChar anyChar

540543

541− blockFilter :: [(String, String)] -> [Block] -> [Block]

544+ blockFilter :: [(Text, Text)] -> [Block] -> [Block]

542545 blockFilter params blks =

543546 let minlvl = lookup "minlevel" params

544547 in case (minlvl >>= safeRead :: Maybe Int) of

@@ -660,7 +663,7 @@ columnPropertyCell = emptyCell <|> propCell <?> "alignment info"

660663 <$> (skipSpaces

661664 *> char '<'

662665 *> optionMaybe tableAlignFromChar)

663− <*> (optionMaybe (many1 digit >>= safeRead)

666+ <*> (optionMaybe (many1Char digit >>= safeRead)

664667 <* char '>'

665668 <* emptyCell)

666669

@@ -739,10 +742,10 @@ latexFragment = try $ do

739742 , "\\end{", e, "}\n"

740743 ]

741744

742−latexEnd :: Monad m => String -> OrgParser m ()

745+latexEnd :: Monad m => Text -> OrgParser m ()

743746latexEnd envName = try $

744747 () <$ skipSpaces

745− <* string ("\\end{" ++ envName ++ "}")

748+ <* textStr ("\\end{" <> envName <> "}")

746749 <* blankline

747750

748751

@@ -813,12 +816,12 @@ definitionListItem :: PandocMonad m

813816 -> OrgParser m (F (Inlines, [Blocks]))

814817definitionListItem parseIndentedMarker = try $ do

815818 markerLength <- parseIndentedMarker

816− term <- manyTill (noneOf "\n\r") (try definitionMarker)

819+ term <- manyTillChar (noneOf "\n\r") (try definitionMarker)

817820 line1 <- anyLineNewline

818821 blank <- option "" ("\n" <$ blankline)

819− cont <- concat <$> many (listContinuation markerLength)

822+ cont <- T.concat <$> many (listContinuation markerLength)

820823 term' <- parseFromString inlines term

821− contents' <- parseFromString blocks $ line1 ++ blank ++ cont

824+ contents' <- parseFromString blocks $ line1 <> blank <> cont

822825 return $ (,) <$> term' <*> fmap (:[]) contents'

823826 where

824827 definitionMarker =

@@ -832,16 +835,16 @@ listItem parseIndentedMarker = try . withContext ListItemState $ do

832835 markerLength <- try parseIndentedMarker

833836 firstLine <- anyLineNewline

834837 blank <- option "" ("\n" <$ blankline)

835− rest <- concat <$> many (listContinuation markerLength)

836− parseFromString blocks $ firstLine ++ blank ++ rest

838+ rest <- T.concat <$> many (listContinuation markerLength)

839+ parseFromString blocks $ firstLine <> blank <> rest

837840

838841-- continuation of a list item - indented and separated by blankline or endline.

839842-- Note: nested lists are parsed as continuations.

840−listContinuation :: PandocMonad m => Int -> OrgParser m String

843+listContinuation :: PandocMonad m => Int -> OrgParser m Text

841844listContinuation markerLength = try $ do

842845 notFollowedBy' blankline

843− mappend <$> (concat <$> many1 (listContinuation' markerLength))

844− <*> many blankline

846+ mappend <$> (T.concat <$> many1 (listContinuation' markerLength))

847+ <*> manyChar blankline

845848 where

846849 listContinuation' indentation =

847850 blockLines indentation <|> listLine indentation

@@ -853,6 +856,6 @@ listContinuation markerLength = try $ do

853856 >> blockAttributes

854857 >>= (\blockAttrs ->

855858 case attrFromBlockAttributes blockAttrs of

856− ("", [], []) -> count 1 anyChar

859+ ("", [], []) -> countChar 1 anyChar

857860 _ -> indentWith indentation))

858861 >> (snd <$> withRaw orgBlock)

src/Text/Pandoc/Readers/Org/DocumentTree.hs+23−21

@@ -1,5 +1,6 @@

1−{-# LANGUAGE FlexibleContexts #-}

1+{-# LANGUAGE FlexibleContexts #-}

22{-# LANGUAGE NoImplicitPrelude #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Readers.Org.DocumentTree

@@ -17,9 +18,9 @@ module Text.Pandoc.Readers.Org.DocumentTree

1718import Prelude

1819import Control.Arrow ((***))

1920import Control.Monad (guard, void)

20−import Data.Char (toLower, toUpper)

2121import Data.List (intersperse)

2222import Data.Maybe (mapMaybe)

23+import Data.Text (Text)

2324import Text.Pandoc.Builder (Blocks, Inlines)

2425import Text.Pandoc.Class (PandocMonad)

2526import Text.Pandoc.Definition

@@ -28,6 +29,7 @@ import Text.Pandoc.Readers.Org.ParserState

2829import Text.Pandoc.Readers.Org.Parsing

2930

3031import qualified Data.Set as Set

32+import qualified Data.Text as T

3133import qualified Text.Pandoc.Builder as B

3234

3335--

@@ -59,28 +61,28 @@ documentTree blocks inline = do

5961 }

6062

6163-- | Create a tag containing the given string.

62−toTag :: String -> Tag

64+toTag :: Text -> Tag

6365toTag = Tag

6466

6567-- | The key (also called name or type) of a property.

66−newtype PropertyKey = PropertyKey { fromKey :: String }

68+newtype PropertyKey = PropertyKey { fromKey :: Text }

6769 deriving (Show, Eq, Ord)

6870

6971-- | Create a property key containing the given string. Org mode keys are

7072-- case insensitive and are hence converted to lower case.

71−toPropertyKey :: String -> PropertyKey

72−toPropertyKey = PropertyKey . map toLower

73+toPropertyKey :: Text -> PropertyKey

74+toPropertyKey = PropertyKey . T.toLower

7375

7476-- | The value assigned to a property.

75−newtype PropertyValue = PropertyValue { fromValue :: String }

77+newtype PropertyValue = PropertyValue { fromValue :: Text }

7678

7779-- | Create a property value containing the given string.

78−toPropertyValue :: String -> PropertyValue

80+toPropertyValue :: Text -> PropertyValue

7981toPropertyValue = PropertyValue

8082

8183-- | Check whether the property value is non-nil (i.e. truish).

8284isNonNil :: PropertyValue -> Bool

83−isNonNil p = map toLower (fromValue p) `notElem` ["()", "{}", "nil"]

85+isNonNil p = T.toLower (fromValue p) `notElem` ["()", "{}", "nil"]

8486

8587-- | Key/value pairs from a PROPERTIES drawer

8688type Properties = [(PropertyKey, PropertyValue)]

@@ -273,7 +275,7 @@ headlineToHeader hdln = do

273275todoKeyword :: Monad m => OrgParser m TodoMarker

274276todoKeyword = try $ do

275277 taskStates <- activeTodoMarkers <$> getState

276− let kwParser tdm = try (tdm <$ string (todoMarkerName tdm)

278+ let kwParser tdm = try (tdm <$ textStr (todoMarkerName tdm)

277279 <* spaceChar

278280 <* updateLastPreCharPos)

279281 choice (map kwParser taskStates)

@@ -281,26 +283,26 @@ todoKeyword = try $ do

281283todoKeywordToInlines :: TodoMarker -> Inlines

282284todoKeywordToInlines tdm =

283285 let todoText = todoMarkerName tdm

284− todoState = map toLower . show $ todoMarkerState tdm

286+ todoState = T.toLower . T.pack . show $ todoMarkerState tdm

285287 classes = [todoState, todoText]

286288 in B.spanWith (mempty, classes, mempty) (B.str todoText)

287289

288290propertiesToAttr :: Properties -> Attr

289291propertiesToAttr properties =

290292 let

291− toStringPair = fromKey *** fromValue

293+ toTextPair = fromKey *** fromValue

292294 customIdKey = toPropertyKey "custom_id"

293295 classKey = toPropertyKey "class"

294296 unnumberedKey = toPropertyKey "unnumbered"

295297 specialProperties = [customIdKey, classKey, unnumberedKey]

296298 id' = maybe mempty fromValue . lookup customIdKey $ properties

297299 cls = maybe mempty fromValue . lookup classKey $ properties

298− kvs' = map toStringPair . filter ((`notElem` specialProperties) . fst)

300+ kvs' = map toTextPair . filter ((`notElem` specialProperties) . fst)

299301 $ properties

300302 isUnnumbered =

301303 maybe False isNonNil . lookup unnumberedKey $ properties

302304 in

303− (id', words cls ++ ["unnumbered" | isUnnumbered], kvs')

305+ (id', T.words cls ++ ["unnumbered" | isUnnumbered], kvs')

304306

305307tagsToInlines :: [Tag] -> Inlines

306308tagsToInlines [] = mempty

@@ -336,15 +338,15 @@ planningToBlock planning = do

336338 <> B.emph (B.str time)

337339

338340-- | An Org timestamp, including repetition marks. TODO: improve

339−type Timestamp = String

341+type Timestamp = Text

340342

341343timestamp :: Monad m => OrgParser m Timestamp

342344timestamp = try $ do

343345 openChar <- oneOf "<["

344346 let isActive = openChar == '<'

345347 let closeChar = if isActive then '>' else ']'

346− content <- many1Till anyChar (char closeChar)

347− return (openChar : content ++ [closeChar])

348+ content <- many1TillChar anyChar (char closeChar)

349+ return $ T.cons openChar $ content `T.snoc` closeChar

348350

349351-- | Planning information for a subtree/headline.

350352data PlanningInfo = PlanningInfo

@@ -374,7 +376,7 @@ planningInfo = try $ do

374376propertiesDrawer :: Monad m => OrgParser m Properties

375377propertiesDrawer = try $ do

376378 drawerType <- drawerStart

377− guard $ map toUpper drawerType == "PROPERTIES"

379+ guard $ T.toUpper drawerType == "PROPERTIES"

378380 manyTill property (try endOfDrawer)

379381 where

380382 property :: Monad m => OrgParser m (PropertyKey, PropertyValue)

@@ -382,12 +384,12 @@ propertiesDrawer = try $ do

382384

383385 key :: Monad m => OrgParser m PropertyKey

384386 key = fmap toPropertyKey . try $

385− skipSpaces *> char ':' *> many1Till nonspaceChar (char ':')

387+ skipSpaces *> char ':' *> many1TillChar nonspaceChar (char ':')

386388

387389 value :: Monad m => OrgParser m PropertyValue

388390 value = fmap toPropertyValue . try $

389− skipSpaces *> manyTill anyChar (try $ skipSpaces *> newline)

391+ skipSpaces *> manyTillChar anyChar (try $ skipSpaces *> newline)

390392

391− endOfDrawer :: Monad m => OrgParser m String

393+ endOfDrawer :: Monad m => OrgParser m Text

392394 endOfDrawer = try $

393395 skipSpaces *> stringAnyCase ":END:" <* skipSpaces <* newline

src/Text/Pandoc/Readers/Org/ExportSettings.hs+21−19

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Org.ExportSettings

@@ -21,6 +22,7 @@ import Text.Pandoc.Readers.Org.Parsing

2122import Control.Monad (mzero, void)

2223import Data.Char (toLower)

2324import Data.Maybe (listToMaybe)

25+import Data.Text (Text)

2426

2527-- | Read and handle space separated org-mode export settings.

2628exportSettings :: PandocMonad m => OrgParser m ()

@@ -70,11 +72,11 @@ exportSetting = choice

7072

7173genericExportSetting :: Monad m

7274 => OrgParser m a

73− -> String

75+ -> Text

7476 -> ExportSettingSetter a

7577 -> OrgParser m ()

7678genericExportSetting optionParser settingIdentifier setter = try $ do

77− _ <- string settingIdentifier *> char ':'

79+ _ <- textStr settingIdentifier *> char ':'

7880 value <- optionParser

7981 updateState $ modifyExportSettings value

8082 where

@@ -82,11 +84,11 @@ genericExportSetting optionParser settingIdentifier setter = try $ do

8284 st { orgStateExportSettings = setter val . orgStateExportSettings $ st }

8385

8486-- | A boolean option, either nil (False) or non-nil (True).

85−booleanSetting :: Monad m => String -> ExportSettingSetter Bool -> OrgParser m ()

87+booleanSetting :: Monad m => Text -> ExportSettingSetter Bool -> OrgParser m ()

8688booleanSetting = genericExportSetting elispBoolean

8789

8890-- | An integer-valued option.

89−integerSetting :: Monad m => String -> ExportSettingSetter Int -> OrgParser m ()

91+integerSetting :: Monad m => Text -> ExportSettingSetter Int -> OrgParser m ()

9092integerSetting = genericExportSetting parseInt

9193 where

9294 parseInt = try $

@@ -95,7 +97,7 @@ integerSetting = genericExportSetting parseInt

9597-- | Either the string "headline" or an elisp boolean and treated as an

9698-- @ArchivedTreesOption@.

9799archivedTreeSetting :: Monad m

98− => String

100+ => Text

99101 -> ExportSettingSetter ArchivedTreesOption

100102 -> OrgParser m ()

101103archivedTreeSetting =

@@ -115,42 +117,42 @@ archivedTreeSetting =

115117

116118-- | A list or a complement list (i.e. a list starting with `not`).

117119complementableListSetting :: Monad m

118− => String

119− -> ExportSettingSetter (Either [String] [String])

120+ => Text

121+ -> ExportSettingSetter (Either [Text] [Text])

120122 -> OrgParser m ()

121123complementableListSetting = genericExportSetting $ choice

122− [ Left <$> complementStringList

124+ [ Left <$> complementTextList

123125 , Right <$> stringList

124126 , (\b -> if b then Left [] else Right []) <$> elispBoolean

125127 ]

126128 where

127129 -- Read a plain list of strings.

128− stringList :: Monad m => OrgParser m [String]

130+ stringList :: Monad m => OrgParser m [Text]

129131 stringList = try $

130132 char '('

131− *> sepBy elispString spaces

133+ *> sepBy elispText spaces

132134 <* char ')'

133135

134136 -- Read an emacs lisp list specifying a complement set.

135− complementStringList :: Monad m => OrgParser m [String]

136− complementStringList = try $

137+ complementTextList :: Monad m => OrgParser m [Text]

138+ complementTextList = try $

137139 string "(not "

138− *> sepBy elispString spaces

140+ *> sepBy elispText spaces

139141 <* char ')'

140142

141− elispString :: Monad m => OrgParser m String

142− elispString = try $

143+ elispText :: Monad m => OrgParser m Text

144+ elispText = try $

143145 char '"'

144− *> manyTill alphaNum (char '"')

146+ *> manyTillChar alphaNum (char '"')

145147

146148-- | Read but ignore the export setting.

147−ignoredSetting :: Monad m => String -> OrgParser m ()

148−ignoredSetting s = try (() <$ string s <* char ':' <* many1 nonspaceChar)

149+ignoredSetting :: Monad m => Text -> OrgParser m ()

150+ignoredSetting s = try (() <$ textStr s <* char ':' <* many1 nonspaceChar)

149151

150152-- | Read any setting string, but ignore it and emit a warning.

151153ignoreAndWarn :: PandocMonad m => OrgParser m ()

152154ignoreAndWarn = try $ do

153− opt <- many1 nonspaceChar

155+ opt <- many1Char nonspaceChar

154156 report (UnknownOrgExportOption opt)

155157 return ()

156158

src/Text/Pandoc/Readers/Org/Inlines.hs+76−78

@@ -20,7 +20,7 @@ import Prelude

2020import Text.Pandoc.Readers.Org.BlockStarts (endOfBlock, noteMarker)

2121import Text.Pandoc.Readers.Org.ParserState

2222import Text.Pandoc.Readers.Org.Parsing

23−import Text.Pandoc.Readers.Org.Shared (cleanLinkString, isImageFilename,

23+import Text.Pandoc.Readers.Org.Shared (cleanLinkText, isImageFilename,

2424 originalLang, translateLang, exportsCode)

2525

2626import Text.Pandoc.Builder (Inlines)

@@ -38,12 +38,14 @@ import Control.Monad.Trans (lift)

3838import Data.Char (isAlphaNum, isSpace)

3939import Data.List (intersperse)

4040import qualified Data.Map as M

41+import Data.Text (Text)

42+import qualified Data.Text as T

4143import Data.Maybe (fromMaybe)

4244

4345--

4446-- Functions acting on the parser state

4547--

46−recordAnchorId :: PandocMonad m => String -> OrgParser m ()

48+recordAnchorId :: PandocMonad m => Text -> OrgParser m ()

4749recordAnchorId i = updateState $ \s ->

4850 s{ orgStateAnchorIds = i : orgStateAnchorIds s }

4951

@@ -127,7 +129,7 @@ linebreak :: PandocMonad m => OrgParser m (F Inlines)

127129linebreak = try $ pure B.linebreak <$ string "\\\\" <* skipSpaces <* newline

128130

129131str :: PandocMonad m => OrgParser m (F Inlines)

130−str = return . B.str <$> many1 (noneOf $ specialChars ++ "\n\r ")

132+str = return . B.str <$> many1Char (noneOf $ specialChars ++ "\n\r ")

131133 <* updateLastStrPos

132134

133135-- | An endline character that can be treated as a space, not a structural

@@ -321,7 +323,7 @@ linkLikeOrgRefCite = try $ do

321323

322324-- | Read a citation key. The characters allowed in citation keys are taken

323325-- from the `org-ref-cite-re` variable in `org-ref.el`.

324−orgRefCiteKey :: PandocMonad m => OrgParser m String

326+orgRefCiteKey :: PandocMonad m => OrgParser m Text

325327orgRefCiteKey =

326328 let citeKeySpecialChars = "-_:\\./," :: String

327329 isCiteKeySpecialChar c = c `elem` citeKeySpecialChars

@@ -329,7 +331,7 @@ orgRefCiteKey =

329331 endOfCitation = try $ do

330332 many $ satisfy isCiteKeySpecialChar

331333 satisfy $ not . isCiteKeyChar

332− in try $ satisfy isCiteKeyChar `many1Till` lookAhead endOfCitation

334+ in try $ satisfy isCiteKeyChar `many1TillChar` lookAhead endOfCitation

333335

334336

335337-- | Supported citation types. Only a small subset of org-ref types is

@@ -384,11 +386,11 @@ footnote = try $ inlineNote <|> referencedNote

384386inlineNote :: PandocMonad m => OrgParser m (F Inlines)

385387inlineNote = try $ do

386388 string "[fn:"

387− ref <- many alphaNum

389+ ref <- manyChar alphaNum

388390 char ':'

389391 note <- fmap B.para . trimInlinesF . mconcat <$> many1Till inline (char ']')

390− unless (null ref) $

391− addToNotesTable ("fn:" ++ ref, note)

392+ unless (T.null ref) $

393+ addToNotesTable ("fn:" <> ref, note)

392394 return $ B.note <$> note

393395

394396referencedNote :: PandocMonad m => OrgParser m (F Inlines)

@@ -397,7 +399,7 @@ referencedNote = try $ do

397399 return $ do

398400 notes <- asksF orgStateNotes'

399401 case lookup ref notes of

400− Nothing -> return . B.str $ "[" ++ ref ++ "]"

402+ Nothing -> return . B.str $ "[" <> ref <> "]"

401403 Just contents -> do

402404 st <- askF

403405 let contents' = runF contents st{ orgStateNotes' = [] }

@@ -420,7 +422,7 @@ explicitOrImageLink = try $ do

420422 return $ do

421423 src <- srcF

422424 title <- titleF

423− case cleanLinkString descr of

425+ case cleanLinkText descr of

424426 Just imgSrc | isImageFilename imgSrc ->

425427 return . B.link src "" $ B.image imgSrc mempty mempty

426428 _ ->

@@ -429,10 +431,10 @@ explicitOrImageLink = try $ do

429431selflinkOrImage :: PandocMonad m => OrgParser m (F Inlines)

430432selflinkOrImage = try $ do

431433 target <- char '[' *> linkTarget <* char ']'

432− case cleanLinkString target of

433− Nothing -> case target of

434− '#':_ -> returnF $ B.link target "" (B.str target)

435− _ -> return $ internalLink target (B.str target)

434+ case cleanLinkText target of

435+ Nothing -> case T.uncons target of

436+ Just ('#', _) -> returnF $ B.link target "" (B.str target)

437+ _ -> return $ internalLink target (B.str target)

436438 Just nonDocTgt -> if isImageFilename nonDocTgt

437439 then returnF $ B.image nonDocTgt "" ""

438440 else returnF $ B.link nonDocTgt "" (B.str target)

@@ -449,35 +451,35 @@ angleLink = try $ do

449451 char '>'

450452 return link

451453

452−linkTarget :: PandocMonad m => OrgParser m String

453−linkTarget = enclosedByPair1 '[' ']' (noneOf "\n\r[]")

454+linkTarget :: PandocMonad m => OrgParser m Text

455+linkTarget = T.pack <$> enclosedByPair1 '[' ']' (noneOf "\n\r[]")

454456

455−possiblyEmptyLinkTarget :: PandocMonad m => OrgParser m String

457+possiblyEmptyLinkTarget :: PandocMonad m => OrgParser m Text

456458possiblyEmptyLinkTarget = try linkTarget <|> ("" <$ string "[]")

457459

458−applyCustomLinkFormat :: String -> OrgParser m (F String)

460+applyCustomLinkFormat :: Text -> OrgParser m (F Text)

459461applyCustomLinkFormat link = do

460− let (linkType, rest) = break (== ':') link

462+ let (linkType, rest) = T.break (== ':') link

461463 return $ do

462464 formatter <- M.lookup linkType <$> asksF orgStateLinkFormatters

463− return $ maybe link ($ drop 1 rest) formatter

465+ return $ maybe link ($ T.drop 1 rest) formatter

464466

465467-- | Take a link and return a function which produces new inlines when given

466468-- description inlines.

467−linkToInlinesF :: String -> Inlines -> F Inlines

469+linkToInlinesF :: Text -> Inlines -> F Inlines

468470linkToInlinesF linkStr =

469− case linkStr of

470− "" -> pure . B.link mempty "" -- wiki link (empty by convention)

471− ('#':_) -> pure . B.link linkStr "" -- document-local fraction

472− _ -> case cleanLinkString linkStr of

473− Just extTgt -> return . B.link extTgt ""

474− Nothing -> internalLink linkStr -- other internal link

475−

476−internalLink :: String -> Inlines -> F Inlines

471+ case T.uncons linkStr of

472+ Nothing -> pure . B.link mempty "" -- wiki link (empty by convention)

473+ Just ('#', _) -> pure . B.link linkStr "" -- document-local fraction

474+ _ -> case cleanLinkText linkStr of

475+ Just extTgt -> return . B.link extTgt ""

476+ Nothing -> internalLink linkStr -- other internal link

477+

478+internalLink :: Text -> Inlines -> F Inlines

477479internalLink link title = do

478480 anchorB <- (link `elem`) <$> asksF orgStateAnchorIds

479481 if anchorB

480− then return $ B.link ('#':link) "" title

482+ then return $ B.link ("#" <> link) "" title

481483 else return $ B.emph title

482484

483485-- | Parse an anchor like @<<anchor-id>>@ and return an empty span with

@@ -493,15 +495,15 @@ anchor = try $ do

493495 returnF $ B.spanWith (solidify anchorId, [], []) mempty

494496 where

495497 parseAnchor = string "<<"

496− *> many1 (noneOf "\t\n\r<>\"' ")

498+ *> many1Char (noneOf "\t\n\r<>\"' ")

497499 <* string ">>"

498500 <* skipSpaces

499501

500502-- | Replace every char but [a-zA-Z0-9_.-:] with a hyphen '-'. This mirrors

501503-- the org function @org-export-solidify-link-text@.

502504

503−solidify :: String -> String

504−solidify = map replaceSpecialChar

505+solidify :: Text -> Text

506+solidify = T.map replaceSpecialChar

505507 where replaceSpecialChar c

506508 | isAlphaNum c = c

507509 | c `elem` ("_.-:" :: String) = c

@@ -511,25 +513,25 @@ solidify = map replaceSpecialChar

511513inlineCodeBlock :: PandocMonad m => OrgParser m (F Inlines)

512514inlineCodeBlock = try $ do

513515 string "src_"

514− lang <- many1 orgArgWordChar

516+ lang <- many1Char orgArgWordChar

515517 opts <- option [] $ enclosedByPair '[' ']' inlineBlockOption

516− inlineCode <- enclosedByPair1 '{' '}' (noneOf "\n\r")

518+ inlineCode <- T.pack <$> enclosedByPair1 '{' '}' (noneOf "\n\r")

517519 let attrClasses = [translateLang lang]

518520 let attrKeyVal = originalLang lang <> opts

519521 let codeInlineBlck = B.codeWith ("", attrClasses, attrKeyVal) inlineCode

520522 returnF $ if exportsCode opts then codeInlineBlck else mempty

521523 where

522− inlineBlockOption :: PandocMonad m => OrgParser m (String, String)

524+ inlineBlockOption :: PandocMonad m => OrgParser m (Text, Text)

523525 inlineBlockOption = try $ do

524526 argKey <- orgArgKey

525527 paramValue <- option "yes" orgInlineParamValue

526528 return (argKey, paramValue)

527529

528− orgInlineParamValue :: PandocMonad m => OrgParser m String

530+ orgInlineParamValue :: PandocMonad m => OrgParser m Text

529531 orgInlineParamValue = try $

530532 skipSpaces

531533 *> notFollowedBy (char ':')

532− *> many1 (noneOf "\t\n\r ]")

534+ *> many1Char (noneOf "\t\n\r ]")

533535 <* skipSpaces

534536

535537

@@ -584,7 +586,7 @@ superscript = fmap B.superscript <$> try (char '^' *> subOrSuperExpr)

584586

585587math :: PandocMonad m => OrgParser m (F Inlines)

586588math = return . B.math <$> choice [ math1CharBetween '$'

587− , mathStringBetween '$'

589+ , mathTextBetween '$'

588590 , rawMathBetween "\$" "\$"

589591 ]

590592

@@ -604,7 +606,7 @@ updatePositions c = do

604606 return c

605607

606608symbol :: PandocMonad m => OrgParser m (F Inlines)

607−symbol = return . B.str . (: "") <$> (oneOf specialChars >>= updatePositions)

609+symbol = return . B.str . T.singleton <$> (oneOf specialChars >>= updatePositions)

608610

609611emphasisBetween :: PandocMonad m

610612 => Char

@@ -619,7 +621,7 @@ emphasisBetween c = try $ do

619621

620622verbatimBetween :: PandocMonad m

621623 => Char

622− -> OrgParser m String

624+ -> OrgParser m Text

623625verbatimBetween c = try $

624626 emphasisStart c *>

625627 many1TillNOrLessNewlines 1 verbatimChar (emphasisEnd c)

@@ -627,33 +629,33 @@ verbatimBetween c = try $

627629 verbatimChar = noneOf "\n\r" >>= updatePositions

628630

629631-- | Parses a raw string delimited by @c@ using Org's math rules

630−mathStringBetween :: PandocMonad m

632+mathTextBetween :: PandocMonad m

631633 => Char

632− -> OrgParser m String

633−mathStringBetween c = try $ do

634+ -> OrgParser m Text

635+mathTextBetween c = try $ do

634636 mathStart c

635637 body <- many1TillNOrLessNewlines mathAllowedNewlines

636638 (noneOf (c:"\n\r"))

637639 (lookAhead $ mathEnd c)

638640 final <- mathEnd c

639− return $ body ++ [final]

641+ return $ T.snoc body final

640642

641643-- | Parse a single character between @c@ using math rules

642644math1CharBetween :: PandocMonad m

643645 => Char

644− -> OrgParser m String

646+ -> OrgParser m Text

645647math1CharBetween c = try $ do

646648 char c

647649 res <- noneOf $ c:mathForbiddenBorderChars

648650 char c

649651 eof <|> () <$ lookAhead (oneOf mathPostChars)

650− return [res]

652+ return $ T.singleton res

651653

652654rawMathBetween :: PandocMonad m

653− => String

654− -> String

655− -> OrgParser m String

656−rawMathBetween s e = try $ string s *> manyTill anyChar (try $ string e)

655+ => Text

656+ -> Text

657+ -> OrgParser m Text

658+rawMathBetween s e = try $ textStr s *> manyTillChar anyChar (try $ textStr e)

657659

658660-- | Parses the start (opening character) of emphasis

659661emphasisStart :: PandocMonad m => Char -> OrgParser m Char

@@ -702,10 +704,10 @@ enclosedInlines start end = try $

702704

703705enclosedRaw :: (PandocMonad m, Show b) => OrgParser m a

704706 -> OrgParser m b

705− -> OrgParser m String

707+ -> OrgParser m Text

706708enclosedRaw start end = try $

707709 start *> (onSingleLine <|> spanningTwoLines)

708− where onSingleLine = try $ many1Till (noneOf "\n\r") end

710+ where onSingleLine = try $ many1TillChar (noneOf "\n\r") end

709711 spanningTwoLines = try $

710712 anyLine >>= \f -> mappend (f <> " ") <$> onSingleLine

711713

@@ -714,7 +716,7 @@ enclosedRaw start end = try $

714716many1TillNOrLessNewlines :: PandocMonad m => Int

715717 -> OrgParser m Char

716718 -> OrgParser m a

717− -> OrgParser m String

719+ -> OrgParser m Text

718720many1TillNOrLessNewlines n p end = try $

719721 nMoreLines (Just n) mempty >>= oneOrMore

720722 where

@@ -726,7 +728,7 @@ many1TillNOrLessNewlines n p end = try $

726728 rest m cs = (\x -> (minus1 <$> m, cs ++ x ++ "\n")) <$> try (manyTill p newline)

727729 finalLine = try $ manyTill p end

728730 minus1 k = k - 1

729− oneOrMore cs = cs <$ guard (not $ null cs)

731+ oneOrMore cs = T.pack cs <$ guard (not $ null cs)

730732

731733-- Org allows customization of the way it reads emphasis. We use the defaults

732734-- here (see, e.g., the Emacs Lisp variable `org-emphasis-regexp-components`

@@ -773,17 +775,17 @@ subOrSuperExpr :: PandocMonad m => OrgParser m (F Inlines)

773775subOrSuperExpr = try $

774776 choice [ charsInBalanced '{' '}' (noneOf "\n\r")

775777 , enclosing ('(', ')') <$> charsInBalanced '(' ')' (noneOf "\n\r")

776− , simpleSubOrSuperString

778+ , simpleSubOrSuperText

777779 ] >>= parseFromString (mconcat <$> many inline)

778− where enclosing (left, right) s = left : s ++ [right]

780+ where enclosing (left, right) s = T.cons left $ T.snoc s right

779781

780−simpleSubOrSuperString :: PandocMonad m => OrgParser m String

781−simpleSubOrSuperString = try $ do

782+simpleSubOrSuperText :: PandocMonad m => OrgParser m Text

783+simpleSubOrSuperText = try $ do

782784 state <- getState

783785 guard . exportSubSuperscripts . orgStateExportSettings $ state

784− choice [ string "*"

785− , mappend <$> option [] ((:[]) <$> oneOf "+-")

786− <*> many1 alphaNum

786+ choice [ textStr "*"

787+ , mappend <$> option "" (T.singleton <$> oneOf "+-")

788+ <*> many1Char alphaNum

787789 ]

788790

789791inlineLaTeX :: PandocMonad m => OrgParser m (F Inlines)

@@ -793,28 +795,28 @@ inlineLaTeX = try $ do

793795 maybe mzero returnF $

794796 parseAsMathMLSym cmd `mplus` parseAsMath cmd `mplus` ils

795797 where

796− parseAsMath :: String -> Maybe Inlines

798+ parseAsMath :: Text -> Maybe Inlines

797799 parseAsMath cs = B.fromList <$> texMathToPandoc cs

798800

799− parseAsInlineLaTeX :: PandocMonad m => String -> m (Maybe Inlines)

801+ parseAsInlineLaTeX :: PandocMonad m => Text -> m (Maybe Inlines)

800802 parseAsInlineLaTeX cs = maybeRight <$> runParserT inlineCommand state "" cs

801803

802− parseAsMathMLSym :: String -> Maybe Inlines

804+ parseAsMathMLSym :: Text -> Maybe Inlines

803805 parseAsMathMLSym cs = B.str <$> MathMLEntityMap.getUnicode (clean cs)

804806 -- drop initial backslash and any trailing "{}"

805− where clean = dropWhileEnd (`elem` ("{}" :: String)) . drop 1

807+ where clean = T.dropWhileEnd (`elem` ("{}" :: String)) . T.drop 1

806808

807809 state :: ParserState

808810 state = def{ stateOptions = def{ readerExtensions =

809811 enableExtension Ext_raw_tex (readerExtensions def) } }

810812

811− texMathToPandoc :: String -> Maybe [Inline]

813+ texMathToPandoc :: Text -> Maybe [Inline]

812814 texMathToPandoc cs = maybeRight (readTeX cs) >>= writePandoc DisplayInline

813815

814816maybeRight :: Either a b -> Maybe b

815817maybeRight = either (const Nothing) Just

816818

817−inlineLaTeXCommand :: PandocMonad m => OrgParser m String

819+inlineLaTeXCommand :: PandocMonad m => OrgParser m Text

818820inlineLaTeXCommand = try $ do

819821 rest <- getInput

820822 st <- getState

@@ -823,21 +825,17 @@ inlineLaTeXCommand = try $ do

823825 Right cs -> do

824826 -- drop any trailing whitespace, those are not be part of the command as

825827 -- far as org mode is concerned.

826− let cmdNoSpc = dropWhileEnd isSpace cs

827− let len = length cmdNoSpc

828+ let cmdNoSpc = T.dropWhileEnd isSpace cs

829+ let len = T.length cmdNoSpc

828830 count len anyChar

829831 return cmdNoSpc

830832 _ -> mzero

831833

832−-- Taken from Data.OldList.

833−dropWhileEnd :: (a -> Bool) -> [a] -> [a]

834−dropWhileEnd p = foldr (\x xs -> if p x && null xs then [] else x : xs) []

835−

836834exportSnippet :: PandocMonad m => OrgParser m (F Inlines)

837835exportSnippet = try $ do

838836 string "@@"

839− format <- many1Till (alphaNum <|> char '-') (char ':')

840− snippet <- manyTill anyChar (try $ string "@@")

837+ format <- many1TillChar (alphaNum <|> char '-') (char ':')

838+ snippet <- manyTillChar anyChar (try $ string "@@")

841839 returnF $ B.rawInline format snippet

842840

843841macro :: PandocMonad m => OrgParser m (F Inlines)

@@ -845,7 +843,7 @@ macro = try $ do

845843 recursionDepth <- orgStateMacroDepth <$> getState

846844 guard $ recursionDepth < 15

847845 string "{{{"

848− name <- many alphaNum

846+ name <- manyChar alphaNum

849847 args <- ([] <$ string "}}}")

850848 <|> char '(' *> argument `sepBy` char ',' <* eoa

851849 expander <- lookupMacro name <$> getState

@@ -857,7 +855,7 @@ macro = try $ do

857855 updateState $ \s -> s { orgStateMacroDepth = recursionDepth }

858856 return res

859857 where

860− argument = many $ notFollowedBy eoa *> noneOf ","

858+ argument = manyChar $ notFollowedBy eoa *> noneOf ","

861859 eoa = string ")}}}"

862860

863861smart :: PandocMonad m => OrgParser m (F Inlines)

src/Text/Pandoc/Readers/Org/Meta.hs+37−35

@@ -1,6 +1,7 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE FlexibleContexts #-}

3−{-# LANGUAGE TupleSections #-}

2+{-# LANGUAGE FlexibleContexts #-}

3+{-# LANGUAGE TupleSections #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers.Org.Meta

@@ -30,11 +31,12 @@ import Text.Pandoc.Definition

3031import Text.Pandoc.Shared (safeRead)

3132

3233import Control.Monad (mzero, void, when)

33−import Data.Char (toLower)

3434import Data.List (intersperse)

3535import Data.Maybe (fromMaybe)

3636import qualified Data.Map as M

3737import qualified Data.Set as Set

38+import Data.Text (Text)

39+import qualified Data.Text as T

3840import Network.HTTP (urlEncode)

3941

4042-- | Returns the current meta, respecting export options.

@@ -47,7 +49,7 @@ metaExport = do

4749 . (if exportWithEmail settings then id else removeMeta "email")

4850 <$> orgStateMeta st

4951

50−removeMeta :: String -> Meta -> Meta

52+removeMeta :: Text -> Meta -> Meta

5153removeMeta key meta' =

5254 let metaMap = unMeta meta'

5355 in Meta $ M.delete key metaMap

@@ -60,18 +62,18 @@ metaLine = mempty <$ metaLineStart <* (optionLine <|> declarationLine)

6062

6163declarationLine :: PandocMonad m => OrgParser m ()

6264declarationLine = try $ do

63− key <- map toLower <$> metaKey

65+ key <- T.toLower <$> metaKey

6466 (key', value) <- metaValue key

6567 let addMetaValue st =

6668 st { orgStateMeta = B.setMeta key' <$> value <*> orgStateMeta st }

6769 when (key' /= "results") $ updateState addMetaValue

6870

69−metaKey :: Monad m => OrgParser m String

70−metaKey = map toLower <$> many1 (noneOf ": \n\r")

71− <* char ':'

72− <* skipSpaces

71+metaKey :: Monad m => OrgParser m Text

72+metaKey = T.toLower <$> many1Char (noneOf ": \n\r")

73+ <* char ':'

74+ <* skipSpaces

7375

74−metaValue :: PandocMonad m => String -> OrgParser m (String, F MetaValue)

76+metaValue :: PandocMonad m => Text -> OrgParser m (Text, F MetaValue)

7577metaValue key =

7678 let inclKey = "header-includes"

7779 in case key of

@@ -88,7 +90,7 @@ metaValue key =

8890 -- Org-mode expects class options to contain the surrounding brackets,

8991 -- pandoc does not.

9092 "latex_class_options" -> ("classoption",) <$>

91− metaModifiedString (filter (`notElem` "[]"))

93+ metaModifiedString (T.filter (`notElem` ("[]" :: String)))

9294 "html_head" -> (inclKey,) <$>

9395 accumulatingList inclKey (metaExportSnippet "html")

9496 _ -> (key,) <$> metaString

@@ -98,25 +100,25 @@ metaInlines = fmap (MetaInlines . B.toList) <$> inlinesTillNewline

98100

99101metaInlinesCommaSeparated :: PandocMonad m => OrgParser m (F MetaValue)

100102metaInlinesCommaSeparated = do

101− itemStrs <- many1 (noneOf ",\n") `sepBy1` char ','

103+ itemStrs <- many1Char (noneOf ",\n") `sepBy1` char ','

102104 newline

103− items <- mapM (parseFromString inlinesTillNewline . (++ "\n")) itemStrs

105+ items <- mapM (parseFromString inlinesTillNewline . (<> "\n")) itemStrs

104106 let toMetaInlines = MetaInlines . B.toList

105107 return $ MetaList . map toMetaInlines <$> sequence items

106108

107109metaString :: Monad m => OrgParser m (F MetaValue)

108110metaString = metaModifiedString id

109111

110−metaModifiedString :: Monad m => (String -> String) -> OrgParser m (F MetaValue)

112+metaModifiedString :: Monad m => (Text -> Text) -> OrgParser m (F MetaValue)

111113metaModifiedString f = return . MetaString . f <$> anyLine

112114

113115-- | Read an format specific meta definition

114−metaExportSnippet :: Monad m => String -> OrgParser m (F MetaValue)

116+metaExportSnippet :: Monad m => Text -> OrgParser m (F MetaValue)

115117metaExportSnippet format =

116118 return . MetaInlines . B.toList . B.rawInline format <$> anyLine

117119

118120-- | Accumulate the result of the @parser@ in a list under @key@.

119−accumulatingList :: Monad m => String

121+accumulatingList :: Monad m => Text

120122 -> OrgParser m (F MetaValue)

121123 -> OrgParser m (F MetaValue)

122124accumulatingList key p = do

@@ -147,33 +149,33 @@ optionLine = try $ do

147149 "pandoc-emphasis-post" -> emphChars >>= updateState . setEmphasisPostChar

148150 _ -> mzero

149151

150−addLinkFormat :: Monad m => String

151− -> (String -> String)

152+addLinkFormat :: Monad m => Text

153+ -> (Text -> Text)

152154 -> OrgParser m ()

153155addLinkFormat key formatter = updateState $ \s ->

154156 let fs = orgStateLinkFormatters s

155157 in s{ orgStateLinkFormatters = M.insert key formatter fs }

156158

157−parseLinkFormat :: Monad m => OrgParser m (String, String -> String)

159+parseLinkFormat :: Monad m => OrgParser m (Text, Text -> Text)

158160parseLinkFormat = try $ do

159− linkType <- (:) <$> letter <*> many (alphaNum <|> oneOf "-_") <* skipSpaces

161+ linkType <- T.cons <$> letter <*> manyChar (alphaNum <|> oneOf "-_") <* skipSpaces

160162 linkSubst <- parseFormat

161163 return (linkType, linkSubst)

162164

163165-- | An ad-hoc, single-argument-only implementation of a printf-style format

164166-- parser.

165−parseFormat :: Monad m => OrgParser m (String -> String)

167+parseFormat :: Monad m => OrgParser m (Text -> Text)

166168parseFormat = try $ replacePlain <|> replaceUrl <|> justAppend

167169 where

168170 -- inefficient, but who cares

169− replacePlain = try $ (\x -> concat . flip intersperse x)

171+ replacePlain = try $ (\x -> T.concat . flip intersperse x)

170172 <$> sequence [tillSpecifier 's', rest]

171− replaceUrl = try $ (\x -> concat . flip intersperse x . urlEncode)

173+ replaceUrl = try $ (\x -> T.concat . flip intersperse x . T.pack . urlEncode . T.unpack)

172174 <$> sequence [tillSpecifier 'h', rest]

173− justAppend = try $ (++) <$> rest

175+ justAppend = try $ (<>) <$> rest

174176

175− rest = manyTill anyChar (eof <|> () <$ oneOf "\n\r")

176− tillSpecifier c = manyTill (noneOf "\n\r") (try $ string ('%':c:""))

177+ rest = manyTillChar anyChar (eof <|> () <$ oneOf "\n\r")

178+ tillSpecifier c = manyTillChar (noneOf "\n\r") (try $ string ('%':c:""))

177179

178180tagList :: Monad m => OrgParser m [Tag]

179181tagList = do

@@ -231,41 +233,41 @@ todoSequence = try $ do

231233 (x:xs) -> return $ keywordsToSequence (reverse xs) [x]

232234

233235 where

234− todoKeywords :: Monad m => OrgParser m [String]

236+ todoKeywords :: Monad m => OrgParser m [Text]

235237 todoKeywords = try $

236− let keyword = many1 nonspaceChar <* skipSpaces

238+ let keyword = many1Char nonspaceChar <* skipSpaces

237239 endOfKeywords = todoDoneSep <|> void newline

238240 in manyTill keyword (lookAhead endOfKeywords)

239241

240242 todoDoneSep :: Monad m => OrgParser m ()

241243 todoDoneSep = void . try $ skipSpaces *> char '|' <* skipSpaces1

242244

243− keywordsToSequence :: [String] -> [String] -> TodoSequence

245+ keywordsToSequence :: [Text] -> [Text] -> TodoSequence

244246 keywordsToSequence todo done =

245247 let todoMarkers = map (TodoMarker Todo) todo

246248 doneMarkers = map (TodoMarker Done) done

247249 in todoMarkers ++ doneMarkers

248250

249−macroDefinition :: Monad m => OrgParser m (String, [String] -> String)

251+macroDefinition :: Monad m => OrgParser m (Text, [Text] -> Text)

250252macroDefinition = try $ do

251− macroName <- many1 nonspaceChar <* skipSpaces

253+ macroName <- many1Char nonspaceChar <* skipSpaces

252254 firstPart <- expansionPart

253255 (elemOrder, parts) <- unzip <$> many ((,) <$> placeholder <*> expansionPart)

254256 let expander = mconcat . alternate (firstPart:parts) . reorder elemOrder

255257 return (macroName, expander)

256258 where

257259 placeholder :: Monad m => OrgParser m Int

258− placeholder = try . fmap (fromMaybe 1 . safeRead) $ char '$' *> many1 digit

260+ placeholder = try . fmap (fromMaybe 1 . safeRead) $ char '$' *> many1Char digit

259261

260− expansionPart :: Monad m => OrgParser m String

261− expansionPart = try $ many (notFollowedBy placeholder *> noneOf "\n\r")

262+ expansionPart :: Monad m => OrgParser m Text

263+ expansionPart = try $ manyChar (notFollowedBy placeholder *> noneOf "\n\r")

262264

263265 alternate :: [a] -> [a] -> [a]

264266 alternate [] ys = ys

265267 alternate xs [] = xs

266268 alternate (x:xs) (y:ys) = x : y : alternate xs ys

267269

268− reorder :: [Int] -> [String] -> [String]

270+ reorder :: [Int] -> [Text] -> [Text]

269271 reorder perm xs =

270272 let element n = take 1 $ drop (n - 1) xs

271273 in concatMap element perm

src/Text/Pandoc/Readers/Org/ParserState.hs+14−13

@@ -1,6 +1,7 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleInstances #-}

33{-# LANGUAGE MultiParamTypeClasses #-}

4+{-# LANGUAGE OverloadedStrings #-}

45{- |

56 Module : Text.Pandoc.Readers.Org.ParserState

@@ -63,16 +64,16 @@ import Text.Pandoc.Readers.LaTeX.Types (Macro)

6364type F = Future OrgParserState

6465

6566-- | An inline note / footnote containing the note key and its (inline) value.

66−type OrgNoteRecord = (String, F Blocks)

67+type OrgNoteRecord = (Text, F Blocks)

6768-- | Table of footnotes

6869type OrgNoteTable = [OrgNoteRecord]

6970-- | Map of functions for link transformations. The map key is refers to the

7071-- link-type, the corresponding function transforms the given link string.

71−type OrgLinkFormatters = M.Map String (String -> String)

72+type OrgLinkFormatters = M.Map Text (Text -> Text)

7273-- | Macro expander function

73−type MacroExpander = [String] -> String

74+type MacroExpander = [Text] -> Text

7475-- | Tag

75−newtype Tag = Tag { fromTag :: String }

76+newtype Tag = Tag { fromTag :: Text }

7677 deriving (Show, Eq, Ord)

7778

7879-- | The states in which a todo item can be

@@ -82,7 +83,7 @@ data TodoState = Todo | Done

8283-- | A ToDo keyword like @TODO@ or @DONE@.

8384data TodoMarker = TodoMarker

8485 { todoMarkerState :: TodoState

85− , todoMarkerName :: String

86+ , todoMarkerName :: Text

8687 }

8788 deriving (Show, Eq)

8889

@@ -91,7 +92,7 @@ type TodoSequence = [TodoMarker]

9192

9293-- | Org-mode parser state

9394data OrgParserState = OrgParserState

94− { orgStateAnchorIds :: [String]

95+ { orgStateAnchorIds :: [Text]

9596 , orgStateEmphasisCharStack :: [Char]

9697 , orgStateEmphasisPreChars :: [Char] -- ^ Chars allowed to occur before

9798 -- emphasis; spaces and newlines are

@@ -102,13 +103,13 @@ data OrgParserState = OrgParserState

102103 , orgStateExcludeTags :: Set.Set Tag

103104 , orgStateExcludeTagsChanged :: Bool

104105 , orgStateExportSettings :: ExportSettings

105− , orgStateIdentifiers :: Set.Set String

106− , orgStateIncludeFiles :: [String]

106+ , orgStateIdentifiers :: Set.Set Text

107+ , orgStateIncludeFiles :: [Text]

107108 , orgStateLastForbiddenCharPos :: Maybe SourcePos

108109 , orgStateLastPreCharPos :: Maybe SourcePos

109110 , orgStateLastStrPos :: Maybe SourcePos

110111 , orgStateLinkFormatters :: OrgLinkFormatters

111− , orgStateMacros :: M.Map String MacroExpander

112+ , orgStateMacros :: M.Map Text MacroExpander

112113 , orgStateMacroDepth :: Int

113114 , orgStateMeta :: F Meta

114115 , orgStateNotes' :: OrgNoteTable

@@ -212,10 +213,10 @@ activeTodoSequences st =

212213activeTodoMarkers :: OrgParserState -> TodoSequence

213214activeTodoMarkers = concat . activeTodoSequences

214215

215−lookupMacro :: String -> OrgParserState -> Maybe MacroExpander

216+lookupMacro :: Text -> OrgParserState -> Maybe MacroExpander

216217lookupMacro macroName = M.lookup macroName . orgStateMacros

217218

218−registerMacro :: (String, MacroExpander) -> OrgParserState -> OrgParserState

219+registerMacro :: (Text, MacroExpander) -> OrgParserState -> OrgParserState

219220registerMacro (name, expander) st =

220221 let curMacros = orgStateMacros st

221222 in st{ orgStateMacros = M.insert name expander curMacros }

@@ -236,7 +237,7 @@ data ArchivedTreesOption =

236237-- These settings can be changed via OPTIONS statements.

237238data ExportSettings = ExportSettings

238239 { exportArchivedTrees :: ArchivedTreesOption -- ^ How to treat archived trees

239− , exportDrawers :: Either [String] [String]

240+ , exportDrawers :: Either [Text] [Text]

240241 -- ^ Specify drawer names which should be exported. @Left@ names are

241242 -- explicitly excluded from the resulting output while @Right@ means that

242243 -- only the listed drawer names should be included.

src/Text/Pandoc/Readers/Org/Parsing.hs+17−10

@@ -32,7 +32,13 @@ module Text.Pandoc.Readers.Org.Parsing

3232 , orgTagWordChar

3333 -- * Re-exports from Text.Pandoc.Parser

3434 , ParserContext (..)

35+ , textStr

36+ , countChar

37+ , manyChar

38+ , many1Char

39+ , manyTillChar

3540 , many1Till

41+ , many1TillChar

3642 , notFollowedBy'

3743 , spaceChar

3844 , nonspaceChar

@@ -98,6 +104,7 @@ module Text.Pandoc.Readers.Org.Parsing

98104 ) where

99105

100106import Prelude

107+import Data.Text (Text)

101108import Text.Pandoc.Readers.Org.ParserState

102109

103110import Text.Pandoc.Parsing hiding (F, anyLine, blanklines, newline,

@@ -108,14 +115,14 @@ import Control.Monad (guard)

108115import Control.Monad.Reader (ReaderT)

109116

110117-- | The parser used to read org files.

111−type OrgParser m = ParserT [Char] OrgParserState (ReaderT OrgParserLocal m)

118+type OrgParser m = ParserT Text OrgParserState (ReaderT OrgParserLocal m)

112119

113120--

114121-- Adaptions and specializations of parsing utilities

115122--

116123

117124-- | Parse any line of text

118−anyLine :: Monad m => OrgParser m String

125+anyLine :: Monad m => OrgParser m Text

119126anyLine =

120127 P.anyLine

121128 <* updateLastPreCharPos

@@ -123,7 +130,7 @@ anyLine =

123130

124131-- | Like @'Text.Pandoc.Parsing'@, but resets the position of the last character

125132-- allowed before emphasised text.

126−parseFromString :: Monad m => OrgParser m a -> String -> OrgParser m a

133+parseFromString :: Monad m => OrgParser m a -> Text -> OrgParser m a

127134parseFromString parser str' = do

128135 updateState $ \s -> s{ orgStateLastPreCharPos = Nothing }

129136 result <- P.parseFromString parser str'

@@ -142,7 +149,7 @@ newline =

142149 <* updateLastForbiddenCharPos

143150

144151-- | Like @Text.Parsec.Char.blanklines@, but causes additional state changes.

145−blanklines :: Monad m => OrgParser m [Char]

152+blanklines :: Monad m => OrgParser m Text

146153blanklines =

147154 P.blanklines

148155 <* updateLastPreCharPos

@@ -192,21 +199,21 @@ updateLastPreCharPos = getPosition >>= \p ->

192199--

193200

194201-- | Read the key of a plist style key-value list.

195−orgArgKey :: Monad m => OrgParser m String

202+orgArgKey :: Monad m => OrgParser m Text

196203orgArgKey = try $

197204 skipSpaces *> char ':'

198− *> many1 orgArgWordChar

205+ *> many1Char orgArgWordChar

199206

200207-- | Read the value of a plist style key-value list.

201−orgArgWord :: Monad m => OrgParser m String

202−orgArgWord = many1 orgArgWordChar

208+orgArgWord :: Monad m => OrgParser m Text

209+orgArgWord = many1Char orgArgWordChar

203210

204211-- | Chars treated as part of a word in plists.

205212orgArgWordChar :: Monad m => OrgParser m Char

206213orgArgWordChar = alphaNum <|> oneOf "-_"

207214

208−orgTagWord :: Monad m => OrgParser m String

209−orgTagWord = many1 orgTagWordChar

215+orgTagWord :: Monad m => OrgParser m Text

216+orgTagWord = many1Char orgTagWordChar

210217

211218orgTagWordChar :: Monad m => OrgParser m Char

212219orgTagWordChar = alphaNum <|> oneOf "@%#_"

src/Text/Pandoc/Readers/Org/Shared.hs+25−25

@@ -10,7 +10,7 @@

1010Utility functions used in other Pandoc Org modules.

1111-}

1212module Text.Pandoc.Readers.Org.Shared

13− ( cleanLinkString

13+ ( cleanLinkText

1414 , isImageFilename

1515 , originalLang

1616 , translateLang

@@ -19,44 +19,44 @@ module Text.Pandoc.Readers.Org.Shared

1919

2020import Prelude

2121import Data.Char (isAlphaNum)

22−import Data.List (isPrefixOf)

22+import Data.Text (Text)

23+import qualified Data.Text as T

2324import System.FilePath (isValid, takeExtension)

24−

25+import Text.Pandoc.Shared (elemText)

2526

2627-- | Check whether the given string looks like the path to of URL of an image.

27−isImageFilename :: String -> Bool

28−isImageFilename fp = hasImageExtension && (isValid fp || isKnownProtocolUri)

28+isImageFilename :: Text -> Bool

29+isImageFilename fp = hasImageExtension && (isValid (T.unpack fp) || isKnownProtocolUri)

2930 where

30− hasImageExtension = takeExtension fp `elem` imageExtensions

31− isKnownProtocolUri = any (\x -> (x ++ "://") `isPrefixOf` fp) protocols

31+ hasImageExtension = takeExtension (T.unpack fp) `elem` imageExtensions

32+ isKnownProtocolUri = any (\x -> (x <> "://") `T.isPrefixOf` fp) protocols

3233

3334 imageExtensions = [ ".jpeg", ".jpg", ".png", ".gif", ".svg" ]

3435 protocols = [ "file", "http", "https" ]

3536

3637-- | Cleanup and canonicalize a string describing a link. Return @Nothing@ if

3738-- the string does not appear to be a link.

38−cleanLinkString :: String -> Maybe String

39−cleanLinkString s =

40− case s of

41− '/':_ -> Just $ "file://" ++ s -- absolute path

42− '.':'/':_ -> Just s -- relative path

43− '.':'.':'/':_ -> Just s -- relative path

44− -- Relative path or URL (file schema)

45− 'f':'i':'l':'e':':':s' -> Just $ if "//" `isPrefixOf` s' then s else s'

46− _ -> if isUrl s then Just s else Nothing

47− where

48− isUrl :: String -> Bool

49− isUrl cs =

50− let (scheme, path) = break (== ':') cs

51− in all (\c -> isAlphaNum c || c `elem` (".-"::String)) scheme

52− && not (null path)

39+cleanLinkText :: Text -> Maybe Text

40+cleanLinkText s

41+ | Just _ <- T.stripPrefix "/" s = Just $ "file://" <> s -- absolute path

42+ | Just _ <- T.stripPrefix "./" s = Just s -- relative path

43+ | Just _ <- T.stripPrefix "../" s = Just s -- relative path

44+ -- Relative path or URL (file schema)

45+ | Just s' <- T.stripPrefix "file:" s = Just $ if "//" `T.isPrefixOf` s' then s else s'

46+ | otherwise = if isUrl s then Just s else Nothing

47+ where

48+ isUrl :: Text -> Bool

49+ isUrl cs =

50+ let (scheme, path) = T.break (== ':') cs

51+ in T.all (\c -> isAlphaNum c || c `elemText` ".-") scheme

52+ && not (T.null path)

5353

5454-- | Creates an key-value pair marking the original language name specified for

5555-- a piece of source code.

5656

5757-- | Creates an key-value attributes marking the original language name

5858-- specified for a piece of source code.

59−originalLang :: String -> [(String, String)]

59+originalLang :: Text -> [(Text, Text)]

6060originalLang lang =

6161 let transLang = translateLang lang

6262 in if transLang == lang

@@ -66,7 +66,7 @@ originalLang lang =

6666-- | Translate from Org-mode's programming language identifiers to those used

6767-- by Pandoc. This is useful to allow for proper syntax highlighting in

6868-- Pandoc output.

69−translateLang :: String -> String

69+translateLang :: Text -> Text

7070translateLang cs =

7171 case cs of

7272 "C" -> "c"

@@ -79,5 +79,5 @@ translateLang cs =

7979 "sqlite" -> "sql"

8080 _ -> cs

8181

82−exportsCode :: [(String, String)] -> Bool

82+exportsCode :: [(Text, Text)] -> Bool

8383exportsCode = maybe True (`elem` ["code", "both"]) . lookup "exports"

src/Text/Pandoc/Readers/RST.hs+244−242

@@ -1,7 +1,8 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleContexts #-}

33{-# LANGUAGE OverloadedStrings #-}

44{-# LANGUAGE ScopedTypeVariables #-}

5+{-# LANGUAGE ViewPatterns #-}

56{- |

67 Module : Text.Pandoc.Readers.RST

@@ -19,9 +20,8 @@ import Control.Arrow (second)

1920import Control.Monad (forM_, guard, liftM, mplus, mzero, when)

2021import Control.Monad.Except (throwError)

2122import Control.Monad.Identity (Identity (..))

22−import Data.Char (isHexDigit, isSpace, toLower, toUpper, isAlphaNum)

23−import Data.List (deleteFirstsBy, elemIndex, intercalate, isInfixOf, isSuffixOf,

24− nub, sort, transpose)

23+import Data.Char (isHexDigit, isSpace, toUpper, isAlphaNum)

24+import Data.List (deleteFirstsBy, elemIndex, nub, sort, transpose)

2525import qualified Data.Map as M

2626import Data.Maybe (fromMaybe)

2727import Data.Sequence (ViewR (..), viewr)

@@ -47,16 +47,16 @@ import Text.Printf (printf)

4747-- | Parse reStructuredText string and return Pandoc document.

4848readRST :: PandocMonad m

4949 => ReaderOptions -- ^ Reader options

50− -> Text -- ^ String to parse (assuming @'\n'@ line endings)

50+ -> Text -- ^ Text to parse (assuming @'\n'@ line endings)

5151 -> m Pandoc

5252readRST opts s = do

5353 parsed <- readWithM parseRST def{ stateOptions = opts }

54− (T.unpack (crFilter s) ++ "\n\n")

54+ (crFilter s <> "\n\n")

5555 case parsed of

5656 Right result -> return result

5757 Left e -> throwError e

5858

59−type RSTParser m = ParserT [Char] ParserState m

59+type RSTParser m = ParserT Text ParserState m

6060

6161--

6262-- Constants and data structure definitions

@@ -113,7 +113,7 @@ titleTransform (bs, meta) =

113113

114114metaFromDefList :: [([Inline], [[Block]])] -> Meta -> Meta

115115metaFromDefList ds meta = adjustAuthors $ foldr f meta ds

116− where f (k,v) = setMeta (map toLower $ stringify k) (mconcat $ map fromList v)

116+ where f (k,v) = setMeta (T.toLower $ stringify k) (mconcat $ map fromList v)

117117 adjustAuthors (Meta metamap) = Meta $ M.adjust splitAuthors "author"

118118 $ M.adjust toPlain "date"

119119 $ M.adjust toPlain "title"

@@ -136,13 +136,13 @@ metaFromDefList ds meta = adjustAuthors $ foldr f meta ds

136136 isSp LineBreak = True

137137 isSp _ = False

138138 splitOnSemi = splitBy (==Str ";")

139− factorSemi (Str []) = []

140− factorSemi (Str s) = case break (==';') s of

141− (xs,[]) -> [Str xs]

142− (xs,';':ys) -> Str xs : Str ";" :

143− factorSemi (Str ys)

144− (xs,ys) -> Str xs :

145− factorSemi (Str ys)

139+ factorSemi (Str "") = []

140+ factorSemi (Str s) = case T.break (==';') s of

141+ (xs,"") -> [Str xs]

142+ (xs,T.uncons -> Just (';',ys)) -> Str xs : Str ";" :

143+ factorSemi (Str ys)

144+ (xs,ys) -> Str xs :

145+ factorSemi (Str ys)

146146 factorSemi x = [x]

147147

148148parseRST :: PandocMonad m => RSTParser m Pandoc

@@ -151,7 +151,7 @@ parseRST = do

151151 startPos <- getPosition

152152 -- go through once just to get list of reference keys and notes

153153 -- docMinusKeys is the raw document with blanks where the keys were...

154− docMinusKeys <- concat <$>

154+ docMinusKeys <- T.concat <$>

155155 manyTill (referenceKey <|> anchorDef <|>

156156 noteBlock <|> citationBlock <|>

157157 (snd <$> withRaw comment) <|>

@@ -180,7 +180,7 @@ parseRST = do

180180 return $ Pandoc meta' (blocks' ++ refBlock)

181181

182182parseCitation :: PandocMonad m

183− => (String, String) -> RSTParser m (Inlines, [Blocks])

183+ => (Text, Text) -> RSTParser m (Inlines, [Blocks])

184184parseCitation (ref, raw) = do

185185 contents <- parseFromString' parseBlocks raw

186186 return (B.spanWith (ref, ["citation-label"], []) (B.str ref),

@@ -215,23 +215,23 @@ block = choice [ codeBlock

215215-- field list

216216--

217217

218−rawFieldListItem :: Monad m => Int -> RSTParser m (String, String)

218+rawFieldListItem :: Monad m => Int -> RSTParser m (Text, Text)

219219rawFieldListItem minIndent = try $ do

220220 indent <- length <$> many (char ' ')

221221 guard $ indent >= minIndent

222222 char ':'

223− name <- many1Till (noneOf "\n") (char ':')

223+ name <- many1TillChar (noneOf "\n") (char ':')

224224 (() <$ lookAhead newline) <|> skipMany1 spaceChar

225225 first <- anyLine

226226 rest <- option "" $ try $ do lookAhead (count indent (char ' ') >> spaceChar)

227227 indentedBlock

228− let raw = (if null first then "" else (first ++ "\n")) ++ rest ++ "\n"

228+ let raw = (if T.null first then "" else (first <> "\n")) <> rest <> "\n"

229229 return (name, raw)

230230

231231fieldListItem :: PandocMonad m => Int -> RSTParser m (Inlines, [Blocks])

232232fieldListItem minIndent = try $ do

233233 (name, raw) <- rawFieldListItem minIndent

234− term <- parseInlineFromString name

234+ term <- parseInlineFromText name

235235 contents <- parseFromString' parseBlocks raw

236236 optional blanklines

237237 return (term, [contents])

@@ -251,12 +251,12 @@ fieldList = try $ do

251251lineBlock :: PandocMonad m => RSTParser m Blocks

252252lineBlock = try $ do

253253 lines' <- lineBlockLines

254− lines'' <- mapM parseInlineFromString lines'

254+ lines'' <- mapM parseInlineFromText lines'

255255 return $ B.lineBlock lines''

256256

257−lineBlockDirective :: PandocMonad m => String -> RSTParser m Blocks

257+lineBlockDirective :: PandocMonad m => Text -> RSTParser m Blocks

258258lineBlockDirective body = do

259− lines' <- mapM parseInlineFromString $ lines $ stripTrailingNewlines body

259+ lines' <- mapM parseInlineFromText $ T.lines $ stripTrailingNewlines body

260260 return $ B.lineBlock lines'

261261

262262--

@@ -271,9 +271,9 @@ para = try $ do

271271 newline

272272 blanklines

273273 case viewr (B.unMany result) of

274− ys :> Str xs | "::" `isSuffixOf` xs -> do

274+ ys :> Str xs | "::" `T.isSuffixOf` xs -> do

275275 raw <- option mempty codeBlockBody

276− return $ B.para (B.Many ys <> B.str (take (length xs - 1) xs))

276+ return $ B.para (B.Many ys <> B.str (T.take (T.length xs - 1) xs))

277277 <> raw

278278 _ -> return (B.para result)

279279

@@ -349,7 +349,7 @@ singleHeader' = try $ do

349349-- hrule block

350350--

351351

352−hrule :: Monad m => ParserT [Char] st m Blocks

352+hrule :: Monad m => ParserT Text st m Blocks

353353hrule = try $ do

354354 chr <- oneOf underlineChars

355355 count 3 (char chr)

@@ -364,7 +364,7 @@ hrule = try $ do

364364

365365-- read a line indented by a given string

366366indentedLine :: (HasReaderOptions st, Monad m)

367− => Int -> ParserT [Char] st m [Char]

367+ => Int -> ParserT Text st m Text

368368indentedLine indents = try $ do

369369 lookAhead spaceChar

370370 gobbleAtMostSpaces indents

@@ -373,29 +373,29 @@ indentedLine indents = try $ do

373373-- one or more indented lines, possibly separated by blank lines.

374374-- any amount of indentation will work.

375375indentedBlock :: (HasReaderOptions st, Monad m)

376− => ParserT [Char] st m [Char]

376+ => ParserT Text st m Text

377377indentedBlock = try $ do

378378 indents <- length <$> lookAhead (many1 spaceChar)

379379 lns <- many1 $ try $ do b <- option "" blanklines

380380 l <- indentedLine indents

381− return (b ++ l)

381+ return (b <> l)

382382 optional blanklines

383− return $ unlines lns

383+ return $ T.unlines lns

384384

385−quotedBlock :: Monad m => ParserT [Char] st m [Char]

385+quotedBlock :: Monad m => ParserT Text st m Text

386386quotedBlock = try $ do

387387 quote <- lookAhead $ oneOf "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"

388388 lns <- many1 $ lookAhead (char quote) >> anyLine

389389 optional blanklines

390− return $ unlines lns

390+ return $ T.unlines lns

391391

392−codeBlockStart :: Monad m => ParserT [Char] st m Char

392+codeBlockStart :: Monad m => ParserT Text st m Char

393393codeBlockStart = string "::" >> blankline >> blankline

394394

395−codeBlock :: (HasReaderOptions st, Monad m) => ParserT [Char] st m Blocks

395+codeBlock :: (HasReaderOptions st, Monad m) => ParserT Text st m Blocks

396396codeBlock = try $ codeBlockStart >> codeBlockBody

397397

398−codeBlockBody :: (HasReaderOptions st, Monad m) => ParserT [Char] st m Blocks

398+codeBlockBody :: (HasReaderOptions st, Monad m) => ParserT Text st m Blocks

399399codeBlockBody = try $ B.codeBlock . stripTrailingNewlines <$>

400400 (indentedBlock <|> quotedBlock)

401401

@@ -407,24 +407,24 @@ lhsCodeBlock = try $ do

407407 lns <- latexCodeBlock <|> birdCodeBlock

408408 blanklines

409409 return $ B.codeBlockWith ("", ["haskell","literate"], [])

410− $ intercalate "\n" lns

410+ $ T.intercalate "\n" lns

411411

412−latexCodeBlock :: Monad m => ParserT [Char] st m [[Char]]

412+latexCodeBlock :: Monad m => ParserT Text st m [Text]

413413latexCodeBlock = try $ do

414414 try (latexBlockLine "\\begin{code}")

415415 many1Till anyLine (try $ latexBlockLine "\\end{code}")

416416 where

417417 latexBlockLine s = skipMany spaceChar >> string s >> blankline

418418

419−birdCodeBlock :: Monad m => ParserT [Char] st m [[Char]]

419+birdCodeBlock :: Monad m => ParserT Text st m [Text]

420420birdCodeBlock = filterSpace <$> many1 birdTrackLine

421421 where filterSpace lns =

422422 -- if (as is normal) there is always a space after >, drop it

423− if all (\ln -> null ln || take 1 ln == " ") lns

424− then map (drop 1) lns

423+ if all (\ln -> T.null ln || T.take 1 ln == " ") lns

424+ then map (T.drop 1) lns

425425 else lns

426426

427−birdTrackLine :: Monad m => ParserT [Char] st m [Char]

427+birdTrackLine :: Monad m => ParserT Text st m Text

428428birdTrackLine = char '>' >> anyLine

429429

430430--

@@ -435,7 +435,7 @@ blockQuote :: PandocMonad m => RSTParser m Blocks

435435blockQuote = do

436436 raw <- indentedBlock

437437 -- parse the extracted block, which may contain various block elements:

438− contents <- parseFromString' parseBlocks $ raw ++ "\n\n"

438+ contents <- parseFromString' parseBlocks $ raw <> "\n\n"

439439 return $ B.blockQuote contents

440440

441441{-

@@ -445,12 +445,12 @@ encoding

445445-}

446446

447447includeDirective :: PandocMonad m

448− => String -> [(String, String)] -> String

448+ => Text -> [(Text, Text)] -> Text

449449 -> RSTParser m Blocks

450450includeDirective top fields body = do

451451 let f = trim top

452− guard $ not (null f)

453− guard $ null (trim body)

452+ guard $ not (T.null f)

453+ guard $ T.null (trim body)

454454 -- options

455455 let (startLine :: Maybe Int) = lookup "start-line" fields >>= safeRead

456456 let (endLine :: Maybe Int) = lookup "end-line" fields >>= safeRead

@@ -458,11 +458,11 @@ includeDirective top fields body = do

458458 oldInput <- getInput

459459 containers <- stateContainers <$> getState

460460 when (f `elem` containers) $

461− throwError $ PandocParseError $ "Include file loop at " ++ show oldPos

461+ throwError $ PandocParseError $ "Include file loop at " <> tshow oldPos

462462 updateState $ \s -> s{ stateContainers = f : stateContainers s }

463− mbContents <- readFileFromDirs ["."] f

463+ mbContents <- readFileFromDirs ["."] $ T.unpack f

464464 contentLines <- case mbContents of

465− Just s -> return $ lines s

465+ Just s -> return $ T.lines s

466466 Nothing -> do

467467 logMessage $ CouldNotLoadIncludeFile f oldPos

468468 return []

@@ -478,23 +478,23 @@ includeDirective top fields body = do

478478 let contentLines' = drop (startLine' - 1)

479479 $ take (endLine' - 1) contentLines

480480 let contentLines'' = (case trim <$> lookup "end-before" fields of

481− Just patt -> takeWhile (not . (patt `isInfixOf`))

481+ Just patt -> takeWhile (not . (patt `T.isInfixOf`))

482482 Nothing -> id) .

483483 (case trim <$> lookup "start-after" fields of

484484 Just patt -> drop 1 .

485− dropWhile (not . (patt `isInfixOf`))

485+ dropWhile (not . (patt `T.isInfixOf`))

486486 Nothing -> id) $ contentLines'

487− let contents' = unlines contentLines'' ++ "\n"

487+ let contents' = T.unlines contentLines'' <> "\n"

488488 case lookup "code" fields of

489489 Just lang -> do

490490 let numberLines = lookup "number-lines" fields

491− let classes = maybe [] words (lookup "class" fields)

491+ let classes = maybe [] T.words (lookup "class" fields)

492492 let ident = maybe "" trimr $ lookup "name" fields

493493 codeblock ident classes numberLines (trimr lang) contents' False

494494 Nothing -> case lookup "literal" fields of

495495 Just _ -> return $ B.rawBlock "rst" contents'

496496 Nothing -> do

497− setPosition $ newPos f 1 1

497+ setPosition $ newPos (T.unpack f) 1 1

498498 setInput contents'

499499 bs <- optional blanklines >>

500500 (mconcat <$> many block)

@@ -519,14 +519,14 @@ definitionListItem = try $ do

519519 term <- trimInlines . mconcat <$> many1Till inline endline

520520 raw <- indentedBlock

521521 -- parse the extracted block, which may contain various block elements:

522− contents <- parseFromString' parseBlocks $ raw ++ "\n"

522+ contents <- parseFromString' parseBlocks $ raw <> "\n"

523523 return (term, [contents])

524524

525525definitionList :: PandocMonad m => RSTParser m Blocks

526526definitionList = B.definitionList <$> many1 definitionListItem

527527

528528-- parses bullet list start and returns its length (inc. following whitespace)

529−bulletListStart :: Monad m => ParserT [Char] st m Int

529+bulletListStart :: Monad m => ParserT Text st m Int

530530bulletListStart = try $ do

531531 notFollowedBy' hrule -- because hrules start out just like lists

532532 marker <- oneOf bulletListMarkers

@@ -543,7 +543,7 @@ orderedListStart style delim = try $ do

543543 return $ markerLen + length white

544544

545545-- parse a line of a list item

546−listLine :: Monad m => Int -> RSTParser m [Char]

546+listLine :: Monad m => Int -> RSTParser m Text

547547listLine markerLength = try $ do

548548 notFollowedBy blankline

549549 indentWith markerLength

@@ -551,21 +551,21 @@ listLine markerLength = try $ do

551551

552552-- parse raw text for one list item, excluding start marker and continuations

553553rawListItem :: Monad m => RSTParser m Int

554− -> RSTParser m (Int, [Char])

554+ -> RSTParser m (Int, Text)

555555rawListItem start = try $ do

556556 markerLength <- start

557557 firstLine <- anyLineNewline

558558 restLines <- many (listLine markerLength)

559− return (markerLength, firstLine ++ concat restLines)

559+ return (markerLength, firstLine <> T.concat restLines)

560560

561561-- continuation of a list item - indented and separated by blankline or

562562-- (in compact lists) endline.

563563-- Note: nested lists are parsed as continuations.

564−listContinuation :: Monad m => Int -> RSTParser m [Char]

564+listContinuation :: Monad m => Int -> RSTParser m Text

565565listContinuation markerLength = try $ do

566− blanks <- many1 blankline

566+ blanks <- many1Char blankline

567567 result <- many1 (listLine markerLength)

568− return $ blanks ++ concat result

568+ return $ blanks <> T.concat result

569569

570570listItem :: PandocMonad m

571571 => RSTParser m Int

@@ -581,7 +581,7 @@ listItem start = try $ do

581581 let oldContext = stateParserContext state

582582 setState $ state {stateParserContext = ListItemState}

583583 -- parse the extracted block, which may itself contain block elements

584− parsed <- parseFromString' parseBlocks $ concat (first:rest) ++ "\n"

584+ parsed <- parseFromString' parseBlocks $ T.concat (first:rest) <> "\n"

585585 updateState (\st -> st {stateParserContext = oldContext})

586586 return $ case B.toList parsed of

587587 [Para xs] ->

@@ -617,9 +617,9 @@ comment = try $ do

617617 optional indentedBlock

618618 return mempty

619619

620−directiveLabel :: Monad m => RSTParser m String

621−directiveLabel = map toLower

622− <$> many1Till (letter <|> char '-') (try $ string "::")

620+directiveLabel :: Monad m => RSTParser m Text

621+directiveLabel = T.toLower

622+ <$> many1TillChar (letter <|> char '-') (try $ string "::")

623623

624624directive :: PandocMonad m => RSTParser m Blocks

625625directive = try $ do

@@ -631,7 +631,7 @@ directive' = do

631631 skipMany1 spaceChar

632632 label <- directiveLabel

633633 skipMany spaceChar

634− top <- many $ satisfy (/='\n')

634+ top <- manyChar $ satisfy (/='\n')

635635 <|> try (char '\n' <*

636636 notFollowedBy' (rawFieldListItem 1) <*

637637 many1 (char ' ') <*

@@ -644,35 +644,33 @@ directive' = do

644644 else many $ rawFieldListItem fieldIndent

645645 body <- option "" $ try $ blanklines >> indentedBlock

646646 optional blanklines

647− let body' = body ++ "\n\n"

647+ let body' = body <> "\n\n"

648648 name = trim $ fromMaybe "" (lookup "name" fields)

649− classes = words $ maybe "" trim (lookup "class" fields)

649+ classes = T.words $ maybe "" trim (lookup "class" fields)

650650 keyvals = [(k, trim v) | (k, v) <- fields, k /= "name", k /= "class"]

651651 imgAttr cl = (name, classes ++ alignClasses, widthAttr ++ heightAttr)

652652 where

653− alignClasses = words $ maybe "" trim (lookup cl fields) ++

654− maybe "" (\x -> "align-" ++ trim x)

653+ alignClasses = T.words $ maybe "" trim (lookup cl fields) <>

654+ maybe "" (\x -> "align-" <> trim x)

655655 (lookup "align" fields)

656656 scale = case trim <$> lookup "scale" fields of

657− Just v -> case reverse v of

658− '%':vv ->

659− case safeRead (reverse vv) of

660− Just (percent :: Double)

661− -> percent / 100.0

662− Nothing -> 1.0

663− _ ->

664− case safeRead v of

665− Just (s :: Double) -> s

666− Nothing -> 1.0

667− Nothing -> 1.0

657+ Just v -> case T.unsnoc v of

658+ Just (vv, '%') -> case safeRead vv of

659+ Just (percent :: Double)

660+ -> percent / 100.0

661+ Nothing -> 1.0

662+ _ -> case safeRead v of

663+ Just (s :: Double) -> s

664+ Nothing -> 1.0

665+ Nothing -> 1.0

668666 widthAttr = maybe [] (\x -> [("width",

669− show $ scaleDimension scale x)])

667+ tshow $ scaleDimension scale x)])

670668 $ lookup "width" fields >>=

671− (lengthToDim . filter (not . isSpace))

669+ (lengthToDim . T.filter (not . isSpace))

672670 heightAttr = maybe [] (\x -> [("height",

673− show $ scaleDimension scale x)])

671+ tshow $ scaleDimension scale x)])

674672 $ lookup "height" fields >>=

675− (lengthToDim . filter (not . isSpace))

673+ (lengthToDim . T.filter (not . isSpace))

676674 case label of

677675 "include" -> includeDirective top fields body'

678676 "table" -> tableDirective top fields body'

@@ -682,36 +680,37 @@ directive' = do

682680 "raw" -> return $ B.rawBlock (trim top) (stripTrailingNewlines body)

683681 "role" -> addNewRole top $ map (second trim) fields

684682 "container" -> B.divWith

685− (name, "container" : words top ++ classes, []) <$>

683+ (name, "container" : T.words top ++ classes, []) <$>

686684 parseFromString' parseBlocks body'

687685 "replace" -> B.para <$> -- consumed by substKey

688− parseInlineFromString (trim top)

686+ parseInlineFromText (trim top)

689687 "unicode" -> B.para <$> -- consumed by substKey

690− parseInlineFromString (trim $ unicodeTransform top)

688+ parseInlineFromText (trim $ unicodeTransform top)

691689 "compound" -> parseFromString' parseBlocks body'

692690 "pull-quote" -> B.blockQuote <$> parseFromString' parseBlocks body'

693691 "epigraph" -> B.blockQuote <$> parseFromString' parseBlocks body'

694692 "highlights" -> B.blockQuote <$> parseFromString' parseBlocks body'

695− "rubric" -> B.para . B.strong <$> parseInlineFromString top

693+ "rubric" -> B.para . B.strong <$> parseInlineFromText top

696694 _ | label `elem` ["attention","caution","danger","error","hint",

697695 "important","note","tip","warning","admonition"] ->

698− do bod <- parseFromString' parseBlocks $ top ++ "\n\n" ++ body'

696+ do bod <- parseFromString' parseBlocks $ top <> "\n\n" <> body'

699697 let lab = case label of

700− "admonition" -> mempty

701− (l:ls) -> B.divWith ("",["title"],[])

702− (B.para (B.str (toUpper l : ls)))

703− [] -> mempty

698+ "admonition" -> mempty

699+ (T.uncons -> Just (l, ls))

700+ -> B.divWith ("",["title"],[])

701+ (B.para (B.str $ T.cons (toUpper l) ls))

702+ _ -> mempty

704703 return $ B.divWith (name,label:classes,keyvals) (lab <> bod)

705704 "sidebar" ->

706705 do let subtit = maybe "" trim $ lookup "subtitle" fields

707− tit <- B.para . B.strong <$> parseInlineFromString

708− (trim top ++ if null subtit

706+ tit <- B.para . B.strong <$> parseInlineFromText

707+ (trim top <> if T.null subtit

709708 then ""

710− else (": " ++ subtit))

709+ else (": " <> subtit))

711710 bod <- parseFromString' parseBlocks body'

712711 return $ B.divWith (name,"sidebar":classes,keyvals) $ tit <> bod

713712 "topic" ->

714− do tit <- B.para . B.strong <$> parseInlineFromString top

713+ do tit <- B.para . B.strong <$> parseInlineFromText top

715714 bod <- parseFromString' parseBlocks body'

716715 return $ B.divWith (name,"topic":classes,keyvals) $ tit <> bod

717716 "default-role" -> mempty <$ updateState (\s ->

@@ -726,7 +725,7 @@ directive' = do

726725 let attribs = (name, ["aafig"], map (second trimr) fields)

727726 return $ B.codeBlockWith attribs $ stripTrailingNewlines body

728727 "math" -> return $ B.para $ mconcat $ map B.displayMath

729− $ toChunks $ top ++ "\n\n" ++ body

728+ $ toChunks $ top <> "\n\n" <> body

730729 "figure" -> do

731730 (caption, legend) <- parseFromString' extractCaption body'

732731 let src = escapeURI $ trim top

@@ -742,7 +741,7 @@ directive' = do

742741 $ B.imageWith attr src "" alt

743742 Nothing -> B.imageWith attr src "" alt

744743 "class" -> do

745− let attrs = (name, words (trim top), map (second trimr) fields)

744+ let attrs = (name, T.words (trim top), map (second trimr) fields)

746745 -- directive content or the first immediately following element

747746 children <- case body of

748747 "" -> block

@@ -750,12 +749,12 @@ directive' = do

750749 return $ B.divWith attrs children

751750 other -> do

752751 pos <- getPosition

753− logMessage $ SkippedContent (".. " ++ other) pos

754− bod <- parseFromString' parseBlocks $ top ++ "\n\n" ++ body'

752+ logMessage $ SkippedContent (".. " <> other) pos

753+ bod <- parseFromString' parseBlocks $ top <> "\n\n" <> body'

755754 return $ B.divWith (name, other:classes, keyvals) bod

756755

757756tableDirective :: PandocMonad m

758− => String -> [(String, String)] -> String -> RSTParser m Blocks

757+ => Text -> [(Text, Text)] -> Text -> RSTParser m Blocks

759758tableDirective top fields body = do

760759 bs <- parseFromString' parseBlocks body

761760 case B.toList bs of

@@ -770,7 +769,7 @@ tableDirective top fields body = do

770769 Just "grid" -> widths'

771770 Just specs -> normWidths

772771 $ map (fromMaybe (0 :: Double) . safeRead)

773− $ splitBy (`elem` (" ," :: String)) specs

772+ $ splitTextBy (`elem` (" ," :: String)) specs

774773 Nothing -> widths'

775774 -- align is not applicable since we can't represent whole table align

776775 return $ B.singleton $ Table (B.toList title)

@@ -783,7 +782,7 @@ tableDirective top fields body = do

783782-- since Pandoc doesn't support a table with multiple header rows.

784783-- We don't need to parse :align: as it represents the whole table align.

785784listTableDirective :: PandocMonad m

786− => String -> [(String, String)] -> String

785+ => Text -> [(Text, Text)] -> Text

787786 -> RSTParser m Blocks

788787listTableDirective top fields body = do

789788 bs <- parseFromString' parseBlocks body

@@ -799,7 +798,7 @@ listTableDirective top fields body = do

799798 widths = case trim <$> lookup "widths" fields of

800799 Just "auto" -> replicate numOfCols 0

801800 Just specs -> normWidths $ map (fromMaybe (0 :: Double) . safeRead) $

802− splitBy (`elem` (" ," :: String)) specs

801+ splitTextBy (`elem` (" ," :: String)) specs

803802 _ -> replicate numOfCols 0

804803 return $ B.table title

805804 (zip (replicate numOfCols AlignDefault) widths)

@@ -812,7 +811,7 @@ listTableDirective top fields body = do

812811 normWidths ws = map (/ max 1 (sum ws)) ws

813812

814813csvTableDirective :: PandocMonad m

815− => String -> [(String, String)] -> String

814+ => Text -> [(Text, Text)] -> Text

816815 -> RSTParser m Blocks

817816csvTableDirective top fields rawcsv = do

818817 let explicitHeader = trim <$> lookup "header" fields

@@ -820,14 +819,17 @@ csvTableDirective top fields rawcsv = do

820819 csvDelim = case trim <$> lookup "delim" fields of

821820 Just "tab" -> '\t'

822821 Just "space" -> ' '

823− Just [c] -> c

822+ Just (T.unpack -> [c])

823+ -> c

824824 _ -> ','

825825 , csvQuote = case trim <$> lookup "quote" fields of

826− Just [c] -> c

827− _ -> '"'

826+ Just (T.unpack -> [c])

827+ -> c

828+ _ -> '"'

828829 , csvEscape = case trim <$> lookup "escape" fields of

829− Just [c] -> Just c

830− _ -> Nothing

830+ Just (T.unpack -> [c])

831+ -> Just c

832+ _ -> Nothing

831833 , csvKeepSpace = case trim <$> lookup "keepspace" fields of

832834 Just "true" -> True

833835 _ -> False

@@ -840,16 +842,16 @@ csvTableDirective top fields rawcsv = do

840842 lookup "file" fields `mplus` lookup "url" fields of

841843 Just u -> do

842844 (bs, _) <- fetchItem u

843− return $ UTF8.toString bs

845+ return $ UTF8.toText bs

844846 Nothing -> return rawcsv

845− let res = parseCSV opts (T.pack $ case explicitHeader of

846− Just h -> h ++ "\n" ++ rawcsv'

847− Nothing -> rawcsv')

847+ let res = parseCSV opts (case explicitHeader of

848+ Just h -> h <> "\n" <> rawcsv'

849+ Nothing -> rawcsv')

848850 case res of

849851 Left e ->

850852 throwError $ PandocParsecError "csv table" e

851853 Right rawrows -> do

852− let parseCell = parseFromString' (plain <|> return mempty) . T.unpack

854+ let parseCell = parseFromString' (plain <|> return mempty)

853855 let parseRow = mapM parseCell

854856 rows <- mapM parseRow rawrows

855857 let (headerRow,bodyRows,numOfCols) =

@@ -865,7 +867,7 @@ csvTableDirective top fields rawcsv = do

865867 Just "auto" -> replicate numOfCols 0

866868 Just specs -> normWidths

867869 $ map (fromMaybe (0 :: Double) . safeRead)

868− $ splitBy (`elem` (" ," :: String)) specs

870+ $ splitTextBy (`elem` (" ," :: String)) specs

869871 _ -> replicate numOfCols 0

870872 return $ B.table title

871873 (zip (replicate numOfCols AlignDefault) widths)

@@ -876,10 +878,10 @@ csvTableDirective top fields rawcsv = do

876878-- - Only supports :format: fields with a single format for :raw: roles,

877879-- change Text.Pandoc.Definition.Format to fix

878880addNewRole :: PandocMonad m

879− => String -> [(String, String)] -> RSTParser m Blocks

880−addNewRole roleString fields = do

881+ => Text -> [(Text, Text)] -> RSTParser m Blocks

882+addNewRole roleText fields = do

881883 pos <- getPosition

882− (role, parentRole) <- parseFromString' inheritedRole roleString

884+ (role, parentRole) <- parseFromString' inheritedRole roleText

883885 customRoles <- stateRstCustomRoles <$> getState

884886 let getBaseRole (r, f, a) roles =

885887 case M.lookup r roles of

@@ -888,7 +890,7 @@ addNewRole roleString fields = do

888890 (baseRole, baseFmt, baseAttr) =

889891 getBaseRole (parentRole, Nothing, nullAttr) customRoles

890892 fmt = if parentRole == "raw" then lookup "format" fields else baseFmt

891− annotate :: [String] -> [String]

893+ annotate :: [Text] -> [Text]

892894 annotate = maybe id (:) $

893895 if baseRole == "code"

894896 then lookup "language" fields

@@ -904,7 +906,7 @@ addNewRole roleString fields = do

904906 pos

905907 "format" -> when (baseRole /= "raw") $ logMessage $

906908 SkippedContent ":format: [because parent of role is not :raw:]" pos

907− _ -> logMessage $ SkippedContent (":" ++ key ++ ":") pos

909+ _ -> logMessage $ SkippedContent (":" <> key <> ":") pos

908910 when (parentRole == "raw" && countKeys "format" > 1) $

909911 logMessage $ SkippedContent

910912 ":format: [after first in definition of role]"

@@ -930,30 +932,29 @@ addNewRole roleString fields = do

930932-- hexadecimal numbers, prefixed by 0x, x, \x, U+, u, or \u

931933-- or as XML-style hexadecimal character entities, e.g. ᨫ

932934-- or text, which is used as-is. Comments start with ..

933−unicodeTransform :: String -> String

934−unicodeTransform t =

935− case t of

936− ('.':'.':xs) -> unicodeTransform $ dropWhile (/='\n') xs -- comment

937− ('0':'x':xs) -> go "0x" xs

938− ('x':xs) -> go "x" xs

939− ('\\':'x':xs) -> go "\\x" xs

940− ('U':'+':xs) -> go "U+" xs

941− ('u':xs) -> go "u" xs

942− ('\\':'u':xs) -> go "\\u" xs

943− ('&':'#':'x':xs) -> maybe ("&#x" ++ unicodeTransform xs)

944− -- drop semicolon

945− (\(c,s) -> c : unicodeTransform (drop 1 s))

946− $ extractUnicodeChar xs

947− (x:xs) -> x : unicodeTransform xs

948− [] -> []

949− where go pref zs = maybe (pref ++ unicodeTransform zs)

950− (\(c,s) -> c : unicodeTransform s)

951− $ extractUnicodeChar zs

952−

953−extractUnicodeChar :: String -> Maybe (Char, String)

935+unicodeTransform :: Text -> Text

936+unicodeTransform t

937+ | Just xs <- T.stripPrefix ".." t = unicodeTransform $ T.dropWhile (/= '\n') xs -- comment

938+ | Just xs <- T.stripPrefix "0x" t = go "0x" xs

939+ | Just xs <- T.stripPrefix "x" t = go "x" xs

940+ | Just xs <- T.stripPrefix "\\x" t = go "\\x" xs

941+ | Just xs <- T.stripPrefix "U+" t = go "U+" xs

942+ | Just xs <- T.stripPrefix "u" t = go "u" xs

943+ | Just xs <- T.stripPrefix "\\u" t = go "\\u" xs

944+ | Just xs <- T.stripPrefix "&#x" t = maybe ("&#x" <> unicodeTransform xs)

945+ -- drop semicolon

946+ (\(c,s) -> T.cons c $ unicodeTransform $ T.drop 1 s)

947+ $ extractUnicodeChar xs

948+ | Just (x, xs) <- T.uncons t = T.cons x $ unicodeTransform xs

949+ | otherwise = ""

950+ where go pref zs = maybe (pref <> unicodeTransform zs)

951+ (\(c,s) -> T.cons c $ unicodeTransform s)

952+ $ extractUnicodeChar zs

953+

954+extractUnicodeChar :: Text -> Maybe (Char, Text)

954955extractUnicodeChar s = maybe Nothing (\c -> Just (c,rest)) mbc

955− where (ds,rest) = span isHexDigit s

956− mbc = safeRead ('\'':'\\':'x':ds ++ "'")

956+ where (ds,rest) = T.span isHexDigit s

957+ mbc = safeRead ("'\\x" <> ds <> "'")

957958

958959extractCaption :: PandocMonad m => RSTParser m (Inlines, Blocks)

959960extractCaption = do

@@ -963,16 +964,16 @@ extractCaption = do

963964

964965-- divide string by blanklines, and surround with

965966-- \begin{aligned}...\end{aligned} if needed.

966−toChunks :: String -> [String]

967−toChunks = dropWhile null

968− . map (addAligned . trim . unlines)

969− . splitBy (all (`elem` (" \t" :: String))) . lines

967+toChunks :: Text -> [Text]

968+toChunks = dropWhile T.null

969+ . map (addAligned . trim . T.unlines)

970+ . splitBy (T.all (`elem` (" \t" :: String))) . T.lines

970971 -- we put this in an aligned environment if it contains \\, see #4254

971− where addAligned s = if "\\\\" `isInfixOf` s

972− then "\\begin{aligned}\n" ++ s ++ "\n\\end{aligned}"

972+ where addAligned s = if "\\\\" `T.isInfixOf` s

973+ then "\\begin{aligned}\n" <> s <> "\n\\end{aligned}"

973974 else s

974975

975−codeblock :: String -> [String] -> Maybe String -> String -> String -> Bool

976+codeblock :: Text -> [Text] -> Maybe Text -> Text -> Text -> Bool

976977 -> RSTParser m Blocks

977978codeblock ident classes numberLines lang body rmTrailingNewlines =

978979 return $ B.codeBlockWith attribs $ stripTrailingNewlines' body

@@ -984,7 +985,7 @@ codeblock ident classes numberLines lang body rmTrailingNewlines =

984985 : maybe [] (const ["numberLines"]) numberLines

985986 ++ classes

986987 kvs = maybe [] (\n -> case trimr n of

987− [] -> []

988+ "" -> []

988989 xs -> [("startFrom", xs)])

989990 numberLines

990991

@@ -992,25 +993,25 @@ codeblock ident classes numberLines lang body rmTrailingNewlines =

992993--- note block

993994---

994995

995−noteBlock :: Monad m => RSTParser m [Char]

996+noteBlock :: Monad m => RSTParser m Text

996997noteBlock = try $ do

997998 (ref, raw, replacement) <- noteBlock' noteMarker

998999 updateState $ \s -> s { stateNotes = (ref, raw) : stateNotes s }

9991000 -- return blanks so line count isn't affected

10001001 return replacement

10011002

1002−citationBlock :: Monad m => RSTParser m [Char]

1003+citationBlock :: Monad m => RSTParser m Text

10031004citationBlock = try $ do

10041005 (ref, raw, replacement) <- noteBlock' citationMarker

10051006 updateState $ \s ->

10061007 s { stateCitations = M.insert ref raw (stateCitations s),

1007− stateKeys = M.insert (toKey ref) (('#':ref,""), ("",["citation"],[]))

1008+ stateKeys = M.insert (toKey ref) (("#" <> ref,""), ("",["citation"],[]))

10081009 (stateKeys s) }

10091010 -- return blanks so line count isn't affected

10101011 return replacement

10111012

10121013noteBlock' :: Monad m

1013− => RSTParser m String -> RSTParser m (String, String, String)

1014+ => RSTParser m Text -> RSTParser m (Text, Text, Text)

10141015noteBlock' marker = try $ do

10151016 startPos <- getPosition

10161017 string ".."

@@ -1021,24 +1022,24 @@ noteBlock' marker = try $ do

10211022 blanks <- option "" blanklines

10221023 rest <- option "" indentedBlock

10231024 endPos <- getPosition

1024− let raw = first ++ "\n" ++ blanks ++ rest ++ "\n"

1025− let replacement =replicate (sourceLine endPos - sourceLine startPos) '\n'

1025+ let raw = first <> "\n" <> blanks <> rest <> "\n"

1026+ let replacement = T.replicate (sourceLine endPos - sourceLine startPos) "\n"

10261027 return (ref, raw, replacement)

10271028

1028−citationMarker :: Monad m => RSTParser m [Char]

1029+citationMarker :: Monad m => RSTParser m Text

10291030citationMarker = do

10301031 char '['

10311032 res <- simpleReferenceName

10321033 char ']'

10331034 return res

10341035

1035−noteMarker :: Monad m => RSTParser m [Char]

1036+noteMarker :: Monad m => RSTParser m Text

10361037noteMarker = do

10371038 char '['

1038− res <- many1 digit

1039+ res <- many1Char digit

10391040 <|>

1040− try (char '#' >> liftM ('#':) simpleReferenceName)

1041− <|> count 1 (oneOf "#*")

1041+ try (char '#' >> liftM ("#" <>) simpleReferenceName)

1042+ <|> countChar 1 (oneOf "#*")

10421043 char ']'

10431044 return res

10441045

@@ -1046,47 +1047,48 @@ noteMarker = do

10461047-- reference key

10471048--

10481049

1049−quotedReferenceName :: PandocMonad m => RSTParser m String

1050+quotedReferenceName :: PandocMonad m => RSTParser m Text

10501051quotedReferenceName = try $ do

10511052 char '`' >> notFollowedBy (char '`') -- `` means inline code!

1052− manyTill anyChar (char '`')

1053+ manyTillChar anyChar (char '`')

10531054

10541055-- Simple reference names are single words consisting of alphanumerics

10551056-- plus isolated (no two adjacent) internal hyphens, underscores,

10561057-- periods, colons and plus signs; no whitespace or other characters

10571058-- are allowed.

1058−simpleReferenceName :: Monad m => ParserT [Char] st m String

1059+simpleReferenceName :: Monad m => ParserT Text st m Text

10591060simpleReferenceName = do

10601061 x <- alphaNum

10611062 xs <- many $ alphaNum

10621063 <|> try (oneOf "-_:+." <* lookAhead alphaNum)

1063− return (x:xs)

1064+ return $ T.pack (x:xs)

10641065

1065−referenceName :: PandocMonad m => RSTParser m String

1066+referenceName :: PandocMonad m => RSTParser m Text

10661067referenceName = quotedReferenceName <|> simpleReferenceName

10671068

1068−referenceKey :: PandocMonad m => RSTParser m [Char]

1069+referenceKey :: PandocMonad m => RSTParser m Text

10691070referenceKey = do

10701071 startPos <- getPosition

10711072 choice [substKey, anonymousKey, regularKey]

10721073 optional blanklines

10731074 endPos <- getPosition

10741075 -- return enough blanks to replace key

1075− return $ replicate (sourceLine endPos - sourceLine startPos) '\n'

1076+ return $ T.replicate (sourceLine endPos - sourceLine startPos) "\n"

10761077

1077−targetURI :: Monad m => ParserT [Char] st m [Char]

1078+targetURI :: Monad m => ParserT Text st m Text

10781079targetURI = do

10791080 skipSpaces

10801081 optional $ try $ newline >> notFollowedBy blankline

10811082 contents <- trim <$>

1082− many1 (satisfy (/='\n')

1083+ many1Char (satisfy (/='\n')

10831084 <|> try (newline >> many1 spaceChar >> noneOf " \t\n"))

10841085 blanklines

1085− case reverse contents of

1086− -- strip backticks

1087− '_':'`':xs -> return (dropWhile (=='`') (reverse xs) ++ "_")

1088− '_':_ -> return contents

1089− _ -> return (escapeURI contents)

1086+ return $ stripBackticks contents

1087+ where

1088+ stripBackticks t

1089+ | Just xs <- T.stripSuffix "`_" t = T.dropWhile (=='`') xs <> "_"

1090+ | Just _ <- T.stripSuffix "_" t = t

1091+ | otherwise = escapeURI t

10901092

10911093substKey :: PandocMonad m => RSTParser m ()

10921094substKey = try $ do

@@ -1112,21 +1114,21 @@ anonymousKey = try $ do

11121114 oneOfStrings [".. __:", "__"]

11131115 src <- targetURI

11141116 pos <- getPosition

1115− let key = toKey $ "_" ++ printf "%09d" (sourceLine pos)

1117+ let key = toKey $ "_" <> T.pack (printf "%09d" (sourceLine pos))

11161118 updateState $ \s -> s { stateKeys = M.insert key ((src,""), nullAttr) $

11171119 stateKeys s }

11181120

1119−referenceNames :: PandocMonad m => RSTParser m [String]

1121+referenceNames :: PandocMonad m => RSTParser m [Text]

11201122referenceNames = do

11211123 let rn = try $ do

11221124 string ".. _"

11231125 ref <- quotedReferenceName

1124− <|> many ( noneOf ":\n"

1125− <|> try (char '\n' <*

1126− string " " <*

1127− notFollowedBy blankline)

1128− <|> try (char ':' <* lookAhead alphaNum)

1129− )

1126+ <|> manyChar ( noneOf ":\n"

1127+ <|> try (char '\n' <*

1128+ string " " <*

1129+ notFollowedBy blankline)

1130+ <|> try (char ':' <* lookAhead alphaNum)

1131+ )

11301132 char ':'

11311133 return ref

11321134 first <- rn

@@ -1140,18 +1142,18 @@ regularKey = try $ do

11401142 -- .. _goodbye: url.com

11411143 refs <- referenceNames

11421144 src <- targetURI

1143− guard $ not (null src)

1145+ guard $ not (T.null src)

11441146 let keys = map toKey refs

11451147 forM_ keys $ \key ->

11461148 updateState $ \s -> s { stateKeys = M.insert key ((src,""), nullAttr) $

11471149 stateKeys s }

11481150

1149−anchorDef :: PandocMonad m => RSTParser m [Char]

1151+anchorDef :: PandocMonad m => RSTParser m Text

11501152anchorDef = try $ do

11511153 (refs, raw) <- withRaw $ try (referenceNames <* blanklines)

11521154 forM_ refs $ \rawkey ->

11531155 updateState $ \s -> s { stateKeys =

1154− M.insert (toKey rawkey) (('#':rawkey,""), nullAttr) $ stateKeys s }

1156+ M.insert (toKey rawkey) (("#" <> rawkey,""), nullAttr) $ stateKeys s }

11551157 -- keep this for 2nd round of parsing, where we'll add the divs (anchor)

11561158 return raw

11571159

@@ -1174,12 +1176,12 @@ anchor = try $ do

11741176 -- because it hides them from promoteHeader, see #4240

11751177 _ -> return $ foldr addDiv b refs

11761178

1177−headerBlock :: PandocMonad m => RSTParser m [Char]

1179+headerBlock :: PandocMonad m => RSTParser m Text

11781180headerBlock = do

11791181 ((txt, _), raw) <- withRaw (doubleHeader' <|> singleHeader')

11801182 (ident,_,_) <- registerHeader nullAttr txt

11811183 let key = toKey (stringify txt)

1182− updateState $ \s -> s { stateKeys = M.insert key (('#':ident,""), nullAttr)

1184+ updateState $ \s -> s { stateKeys = M.insert key (("#" <> ident,""), nullAttr)

11831185 $ stateKeys s }

11841186 return raw

11851187

@@ -1201,13 +1203,13 @@ headerBlock = do

12011203-- Grid tables TODO:

12021204-- - column spans

12031205

1204−dashedLine :: Monad m => Char -> ParserT [Char] st m (Int, Int)

1206+dashedLine :: Monad m => Char -> ParserT Text st m (Int, Int)

12051207dashedLine ch = do

12061208 dashes <- many1 (char ch)

12071209 sp <- many (char ' ')

12081210 return (length dashes, length $ dashes ++ sp)

12091211

1210−simpleDashedLines :: Monad m => Char -> ParserT [Char] st m [(Int,Int)]

1212+simpleDashedLines :: Monad m => Char -> ParserT Text st m [(Int,Int)]

12111213simpleDashedLines ch = try $ many1 (dashedLine ch)

12121214

12131215-- Parse a table row separator

@@ -1215,17 +1217,17 @@ simpleTableSep :: Monad m => Char -> RSTParser m Char

12151217simpleTableSep ch = try $ simpleDashedLines ch >> newline

12161218

12171219-- Parse a table footer

1218−simpleTableFooter :: Monad m => RSTParser m [Char]

1220+simpleTableFooter :: Monad m => RSTParser m Text

12191221simpleTableFooter = try $ simpleTableSep '=' >> blanklines

12201222

12211223-- Parse a raw line and split it into chunks by indices.

1222−simpleTableRawLine :: Monad m => [Int] -> RSTParser m [String]

1224+simpleTableRawLine :: Monad m => [Int] -> RSTParser m [Text]

12231225simpleTableRawLine indices = simpleTableSplitLine indices <$> anyLine

12241226

1225−simpleTableRawLineWithEmptyCell :: Monad m => [Int] -> RSTParser m [String]

1227+simpleTableRawLineWithEmptyCell :: Monad m => [Int] -> RSTParser m [Text]

12261228simpleTableRawLineWithEmptyCell indices = try $ do

12271229 cs <- simpleTableRawLine indices

1228− let isEmptyCell = all (\c -> c == ' ' || c == '\t')

1230+ let isEmptyCell = T.all (\c -> c == ' ' || c == '\t')

12291231 guard $ any isEmptyCell cs

12301232 return cs

12311233

@@ -1235,15 +1237,15 @@ simpleTableRow indices = do

12351237 notFollowedBy' simpleTableFooter

12361238 firstLine <- simpleTableRawLine indices

12371239 conLines <- many $ simpleTableRawLineWithEmptyCell indices

1238− let cols = map unlines . transpose $ firstLine : conLines ++

1239− [replicate (length indices) ""

1240− | not (null conLines)]

1240+ let cols = map T.unlines . transpose $ firstLine : conLines ++

1241+ [replicate (length indices) ""

1242+ | not (null conLines)]

12411243 mapM (parseFromString' parseBlocks) cols

12421244

1243−simpleTableSplitLine :: [Int] -> String -> [String]

1245+simpleTableSplitLine :: [Int] -> Text -> [Text]

12441246simpleTableSplitLine indices line =

12451247 map trimr

1246− $ tail $ splitByIndices (init indices) line

1248+ $ tail $ splitTextByIndices (init indices) line

12471249

12481250simpleTableHeader :: PandocMonad m

12491251 => Bool -- ^ Headerless table

@@ -1322,35 +1324,35 @@ inlineContent = choice [ whitespace

13221324 , escapedChar

13231325 , symbol ] <?> "inline content"

13241326

1325−parseInlineFromString :: PandocMonad m => String -> RSTParser m Inlines

1326−parseInlineFromString = parseFromString' (trimInlines . mconcat <$> many inline)

1327+parseInlineFromText :: PandocMonad m => Text -> RSTParser m Inlines

1328+parseInlineFromText = parseFromString' (trimInlines . mconcat <$> many inline)

13271329

13281330hyphens :: Monad m => RSTParser m Inlines

13291331hyphens = do

1330− result <- many1 (char '-')

1332+ result <- many1Char (char '-')

13311333 optional endline

13321334 -- don't want to treat endline after hyphen or dash as a space

13331335 return $ B.str result

13341336

1335−escapedChar :: Monad m => ParserT [Char] st m Inlines

1337+escapedChar :: Monad m => ParserT Text st m Inlines

13361338escapedChar = do c <- escaped anyChar

13371339 return $ if c == ' ' || c == '\n' || c == '\r'

13381340 -- '\ ' is null in RST

13391341 then mempty

1340− else B.str [c]

1342+ else B.str $ T.singleton c

13411343

13421344symbol :: Monad m => RSTParser m Inlines

13431345symbol = do

13441346 result <- oneOf specialChars

1345− return $ B.str [result]

1347+ return $ B.str $ T.singleton result

13461348

13471349-- parses inline code, between codeStart and codeEnd

13481350code :: Monad m => RSTParser m Inlines

13491351code = try $ do

13501352 string "``"

1351− result <- manyTill anyChar (try (string "``"))

1353+ result <- manyTillChar anyChar (try (string "``"))

13521354 return $ B.code

1353− $ trim $ unwords $ lines result

1355+ $ trim $ T.unwords $ T.lines result

13541356

13551357-- succeeds only if we're not right after a str (ie. in middle of word)

13561358atStart :: Monad m => RSTParser m a -> RSTParser m a

@@ -1382,7 +1384,7 @@ interpretedRole = try $ do

13821384 renderRole contents Nothing role nullAttr

13831385

13841386renderRole :: PandocMonad m

1385− => String -> Maybe String -> String -> Attr -> RSTParser m Inlines

1387+ => Text -> Maybe Text -> Text -> Attr -> RSTParser m Inlines

13861388renderRole contents fmt role attr = case role of

13871389 "sup" -> return $ B.superscript $ treatAsText contents

13881390 "superscript" -> return $ B.superscript $ treatAsText contents

@@ -1412,36 +1414,36 @@ renderRole contents fmt role attr = case role of

14121414 contents

14131415 where

14141416 titleRef ref = return $ B.spanWith ("",["title-ref"],[]) $ treatAsText ref

1415− rfcLink rfcNo = B.link rfcUrl ("RFC " ++ rfcNo) $ B.str ("RFC " ++ rfcNo)

1416− where rfcUrl = "http://www.faqs.org/rfcs/rfc" ++ rfcNo ++ ".html"

1417− pepLink pepNo = B.link pepUrl ("PEP " ++ pepNo) $ B.str ("PEP " ++ pepNo)

1418− where padNo = replicate (4 - length pepNo) '0' ++ pepNo

1419− pepUrl = "http://www.python.org/dev/peps/pep-" ++ padNo ++ "/"

1417+ rfcLink rfcNo = B.link rfcUrl ("RFC " <> rfcNo) $ B.str ("RFC " <> rfcNo)

1418+ where rfcUrl = "http://www.faqs.org/rfcs/rfc" <> rfcNo <> ".html"

1419+ pepLink pepNo = B.link pepUrl ("PEP " <> pepNo) $ B.str ("PEP " <> pepNo)

1420+ where padNo = T.replicate (4 - T.length pepNo) "0" <> pepNo

1421+ pepUrl = "http://www.python.org/dev/peps/pep-" <> padNo <> "/"

14201422 treatAsText = B.text . handleEscapes

1421− handleEscapes [] = []

1422− handleEscapes ('\\':' ':cs) = handleEscapes cs

1423− handleEscapes ('\\':c:cs) = c : handleEscapes cs

1424− handleEscapes (c:cs) = c : handleEscapes cs

1423+ handleEscapes = T.concat . removeSpace . T.splitOn "\\"

1424+ where headSpace t = fromMaybe t $ T.stripPrefix " " t

1425+ removeSpace (x:xs) = x : map headSpace xs

1426+ removeSpace [] = []

14251427

1426−roleName :: PandocMonad m => RSTParser m String

1427−roleName = many1 (letter <|> char '-')

1428+roleName :: PandocMonad m => RSTParser m Text

1429+roleName = many1Char (letter <|> char '-')

14281430

1429−roleMarker :: PandocMonad m => RSTParser m String

1431+roleMarker :: PandocMonad m => RSTParser m Text

14301432roleMarker = char ':' *> roleName <* char ':'

14311433

1432−roleBefore :: PandocMonad m => RSTParser m (String,String)

1434+roleBefore :: PandocMonad m => RSTParser m (Text,Text)

14331435roleBefore = try $ do

14341436 role <- roleMarker

14351437 contents <- unmarkedInterpretedText

14361438 return (role,contents)

14371439

1438−roleAfter :: PandocMonad m => RSTParser m (String,String)

1440+roleAfter :: PandocMonad m => RSTParser m (Text,Text)

14391441roleAfter = try $ do

14401442 contents <- unmarkedInterpretedText

14411443 role <- roleMarker <|> (stateRstDefaultRole <$> getState)

14421444 return (role,contents)

14431445

1444−unmarkedInterpretedText :: PandocMonad m => RSTParser m [Char]

1446+unmarkedInterpretedText :: PandocMonad m => RSTParser m Text

14451447unmarkedInterpretedText = try $ do

14461448 atStart (char '`')

14471449 contents <- mconcat <$> (many1

@@ -1453,7 +1455,7 @@ unmarkedInterpretedText = try $ do

14531455 lookAhead (satisfy isAlphaNum))

14541456 ))

14551457 char '`'

1456− return contents

1458+ return $ T.pack contents

14571459

14581460whitespace :: PandocMonad m => RSTParser m Inlines

14591461whitespace = B.space <$ skipMany1 spaceChar <?> "whitespace"

@@ -1461,7 +1463,7 @@ whitespace = B.space <$ skipMany1 spaceChar <?> "whitespace"

14611463str :: Monad m => RSTParser m Inlines

14621464str = do

14631465 let strChar = noneOf ("\t\n " ++ specialChars)

1464− result <- many1 strChar

1466+ result <- many1Char strChar

14651467 updateLastStrPos

14661468 return $ B.str result

14671469

@@ -1489,7 +1491,7 @@ explicitLink = try $ do

14891491 notFollowedBy (char '`') -- `` marks start of inline code

14901492 label' <- trimInlines . mconcat <$>

14911493 manyTill (notFollowedBy (char '`') >> inlineContent) (char '<')

1492− src <- trim <$> manyTill (noneOf ">\n") (char '>')

1494+ src <- trim <$> manyTillChar (noneOf ">\n") (char '>')

14931495 skipSpaces

14941496 string "`_"

14951497 optional $ char '_' -- anonymous form

@@ -1501,22 +1503,22 @@ explicitLink = try $ do

15011503 if isURI src

15021504 then return ((src, ""), nullAttr)

15031505 else

1504− case reverse src of

1505− '_':xs -> lookupKey [] (toKey (reverse xs))

1506− _ -> return ((src, ""), nullAttr)

1506+ case T.unsnoc src of

1507+ Just (xs, '_') -> lookupKey [] (toKey xs)

1508+ _ -> return ((src, ""), nullAttr)

15071509 return $ B.linkWith attr (escapeURI src') tit label''

15081510

1509−citationName :: PandocMonad m => RSTParser m String

1511+citationName :: PandocMonad m => RSTParser m Text

15101512citationName = do

15111513 raw <- citationMarker

1512− return $ "[" ++ raw ++ "]"

1514+ return $ "[" <> raw <> "]"

15131515

15141516referenceLink :: PandocMonad m => RSTParser m Inlines

15151517referenceLink = try $ do

15161518 ref <- (referenceName <|> citationName) <* char '_'

15171519 let label' = B.text ref

1518− let isAnonKey (Key ('_':_)) = True

1519− isAnonKey _ = False

1520+ let isAnonKey (Key (T.uncons -> Just ('_',_))) = True

1521+ isAnonKey _ = False

15201522 state <- getState

15211523 let keyTable = stateKeys state

15221524 key <- option (toKey ref) $

@@ -1533,7 +1535,7 @@ referenceLink = try $ do

15331535

15341536-- We keep a list of oldkeys so we can detect lookup loops.

15351537lookupKey :: PandocMonad m

1536− => [Key] -> Key -> RSTParser m ((String, String), Attr)

1538+ => [Key] -> Key -> RSTParser m ((Text, Text), Attr)

15371539lookupKey oldkeys key = do

15381540 pos <- getPosition

15391541 state <- getState

@@ -1544,8 +1546,8 @@ lookupKey oldkeys key = do

15441546 logMessage $ ReferenceNotFound key' pos

15451547 return (("",""),nullAttr)

15461548 -- check for keys of the form link_, which need to be resolved:

1547− Just ((u@(c:_),""),_) | last u == '_', c /= '#' -> do

1548− let rawkey = init u

1549+ Just ((u, ""),_) | T.length u > 1, T.last u == '_', T.head u /= '#' -> do

1550+ let rawkey = T.init u

15491551 let newkey = toKey rawkey

15501552 if newkey `elem` oldkeys

15511553 then do

@@ -1576,7 +1578,7 @@ subst = try $ do

15761578 case M.lookup key substTable of

15771579 Nothing -> do

15781580 pos <- getPosition

1579− logMessage $ ReferenceNotFound (show key) pos

1581+ logMessage $ ReferenceNotFound (tshow key) pos

15801582 return mempty

15811583 Just target -> return target

15821584

src/Text/Pandoc/Readers/Roff.hs+92−93

@@ -1,5 +1,7 @@

11{-# LANGUAGE FlexibleContexts #-}

22{-# LANGUAGE GeneralizedNewtypeDeriving #-}

3+{-# LANGUAGE OverloadedStrings #-}

4+{-# LANGUAGE ViewPatterns #-}

35{- |

46 Module : Text.Pandoc.Readers.Roff

@@ -21,7 +23,7 @@ module Text.Pandoc.Readers.Roff

2123 , TableRow

2224 , RoffToken(..)

2325 , RoffTokens(..)

24− , linePartsToString

26+ , linePartsToText

2527 , lexRoff

2628 )

2729where

@@ -40,7 +42,7 @@ import qualified Data.Text as T

4042import Text.Pandoc.Logging (LogMessage(..))

4143import Text.Pandoc.Options

4244import Text.Pandoc.Parsing

43−import Text.Pandoc.Shared (safeRead, substitute)

45+import Text.Pandoc.Shared (safeRead)

4446import Text.Parsec hiding (tokenPrim)

4547import Text.Pandoc.RoffChar (characterCodes, combiningAccents)

4648import qualified Data.Sequence as Seq

@@ -60,28 +62,28 @@ data FontSpec = FontSpec{ fontBold :: Bool

6062defaultFontSpec :: FontSpec

6163defaultFontSpec = FontSpec False False False

6264

63−data LinePart = RoffStr String

65+data LinePart = RoffStr T.Text

6466 | Font FontSpec

6567 | MacroArg Int

6668 deriving Show

6769

6870type Arg = [LinePart]

6971

70−type TableOption = (String, String)

72+type TableOption = (T.Text, T.Text)

7173

7274data CellFormat =

7375 CellFormat

7476 { columnType :: Char

7577 , pipePrefix :: Bool

7678 , pipeSuffix :: Bool

77− , columnSuffixes :: [String]

79+ , columnSuffixes :: [T.Text]

7880 } deriving (Show, Eq, Ord)

7981

8082type TableRow = ([CellFormat], [RoffTokens])

8183

8284data RoffToken = TextLine [LinePart]

8385 | EmptyLine

84− | ControlLine String [Arg] SourcePos

86+ | ControlLine T.Text [Arg] SourcePos

8587 | Tbl [TableOption] [TableRow] SourcePos

8688 deriving Show

8789

@@ -95,7 +97,7 @@ data RoffMode = NormalMode

9597 | CopyMode

9698 deriving Show

9799

98−data RoffState = RoffState { customMacros :: M.Map String RoffTokens

100+data RoffState = RoffState { customMacros :: M.Map T.Text RoffTokens

99101 , prevFont :: FontSpec

100102 , currentFont :: FontSpec

101103 , tableTabChar :: Char

@@ -121,10 +123,10 @@ instance Default RoffState where

121123 , afterConditional = False

122124 }

123125

124−type RoffLexer m = ParserT [Char] RoffState m

126+type RoffLexer m = ParserT T.Text RoffState m

125127

126128--

127−-- Lexer: String -> RoffToken

129+-- Lexer: T.Text -> RoffToken

128130--

129131

130132eofline :: Stream s m Char => ParsecT s u m ()

@@ -133,11 +135,11 @@ eofline = void newline <|> eof <|> () <$ lookAhead (string "\\}")

133135spacetab :: Stream s m Char => ParsecT s u m Char

134136spacetab = char ' ' <|> char '\t'

135137

136−characterCodeMap :: M.Map String Char

138+characterCodeMap :: M.Map T.Text Char

137139characterCodeMap =

138140 M.fromList $ map (\(x,y) -> (y,x)) characterCodes

139141

140−combiningAccentsMap :: M.Map String Char

142+combiningAccentsMap :: M.Map T.Text Char

141143combiningAccentsMap =

142144 M.fromList $ map (\(x,y) -> (y,x)) combiningAccents

143145

@@ -151,43 +153,40 @@ escapeGlyph = do

151153 c <- lookAhead (oneOf ['[','('])

152154 escapeArg >>= resolveGlyph c

153155

154−resolveGlyph :: PandocMonad m => Char -> String -> RoffLexer m [LinePart]

156+resolveGlyph :: PandocMonad m => Char -> T.Text -> RoffLexer m [LinePart]

155157resolveGlyph delimChar glyph = do

156− let cs = substitute "_u" " u" glyph -- unicode glyphs separated by _

157− (case words cs of

158+ let cs = T.replace "_u" " u" glyph -- unicode glyphs separated by _

159+ (case T.words cs of

158160 [] -> mzero

159161 [s] -> case M.lookup s characterCodeMap `mplus` readUnicodeChar s of

160162 Nothing -> mzero

161− Just c -> return [RoffStr [c]]

163+ Just c -> return [RoffStr $ T.singleton c]

162164 (s:ss) -> do

163165 basechar <- case M.lookup s characterCodeMap `mplus`

164166 readUnicodeChar s of

165167 Nothing ->

166− case s of

168+ case T.unpack s of

167169 [ch] | isAscii ch && isAlphaNum ch ->

168170 return ch

169171 _ -> mzero

170172 Just c -> return c

171− let addAccents [] xs = return $ T.unpack .

172− Normalize.normalize Normalize.NFC .

173− T.pack $ reverse xs

173+ let addAccents [] xs = return $ Normalize.normalize Normalize.NFC $

174+ T.reverse xs

174175 addAccents (a:as) xs =

175176 case M.lookup a combiningAccentsMap `mplus` readUnicodeChar a of

176− Just x -> addAccents as (x:xs)

177+ Just x -> addAccents as $ T.cons x xs

177178 Nothing -> mzero

178− addAccents ss [basechar] >>= \xs -> return [RoffStr xs])

179+ addAccents ss (T.singleton basechar) >>= \xs -> return [RoffStr xs])

179180 <|> case delimChar of

180− '[' -> escUnknown ("\\[" ++ glyph ++ "]")

181− '(' -> escUnknown ("\\(" ++ glyph)

182− '\'' -> escUnknown ("\\C'" ++ glyph ++ "'")

181+ '[' -> escUnknown ("\\[" <> glyph <> "]")

182+ '(' -> escUnknown ("\\(" <> glyph)

183+ '\'' -> escUnknown ("\\C'" <> glyph <> "'")

183184 _ -> Prelude.fail "resolveGlyph: unknown glyph delimiter"

184185

185−readUnicodeChar :: String -> Maybe Char

186−readUnicodeChar ('u':cs@(_:_:_:_:_)) =

187− case safeRead ('0':'x':cs) of

188− Just i -> Just (chr i)

189− Nothing -> Nothing

190−readUnicodeChar _ = Nothing

186+readUnicodeChar :: T.Text -> Maybe Char

187+readUnicodeChar t = case T.uncons t of

188+ Just ('u', cs) | T.length cs > 3 -> chr <$> safeRead ("0x" <> cs)

189+ _ -> Nothing

191190

192191escapeNormal :: PandocMonad m => RoffLexer m [LinePart]

193192escapeNormal = do

@@ -218,14 +217,14 @@ escapeNormal = do

218217 NormalMode -> return [RoffStr "\\"]

219218 'H' -> escIgnore 'H' [quoteArg]

220219 'L' -> escIgnore 'L' [quoteArg]

221− 'M' -> escIgnore 'M' [escapeArg, count 1 (satisfy (/='\n'))]

220+ 'M' -> escIgnore 'M' [escapeArg, countChar 1 (satisfy (/='\n'))]

222221 'N' -> escIgnore 'N' [quoteArg]

223− 'O' -> escIgnore 'O' [count 1 (oneOf ['0','1'])]

222+ 'O' -> escIgnore 'O' [countChar 1 (oneOf ['0','1'])]

224223 'R' -> escIgnore 'R' [quoteArg]

225224 'S' -> escIgnore 'S' [quoteArg]

226− 'V' -> escIgnore 'V' [escapeArg, count 1 alphaNum]

225+ 'V' -> escIgnore 'V' [escapeArg, countChar 1 alphaNum]

227226 'X' -> escIgnore 'X' [quoteArg]

228− 'Y' -> escIgnore 'Y' [escapeArg, count 1 (satisfy (/='\n'))]

227+ 'Y' -> escIgnore 'Y' [escapeArg, countChar 1 (satisfy (/='\n'))]

229228 'Z' -> escIgnore 'Z' [quoteArg]

230229 '\'' -> return [RoffStr "`"]

231230 '\n' -> return mempty -- line continuation

@@ -238,12 +237,12 @@ escapeNormal = do

238237 'd' -> escIgnore 'd' [] -- forward down 1/2em

239238 'e' -> return [RoffStr "\\"]

240239 'f' -> escFont

241− 'g' -> escIgnore 'g' [escapeArg, count 1 (satisfy (/='\n'))]

240+ 'g' -> escIgnore 'g' [escapeArg, countChar 1 (satisfy (/='\n'))]

242241 'h' -> escIgnore 'h' [quoteArg]

243− 'k' -> escIgnore 'k' [escapeArg, count 1 (satisfy (/='\n'))]

242+ 'k' -> escIgnore 'k' [escapeArg, countChar 1 (satisfy (/='\n'))]

244243 'l' -> escIgnore 'l' [quoteArg]

245− 'm' -> escIgnore 'm' [escapeArg, count 1 (satisfy (/='\n'))]

246− 'n' -> escIgnore 'm' [escapeArg, count 1 (satisfy (/='\n'))]

244+ 'm' -> escIgnore 'm' [escapeArg, countChar 1 (satisfy (/='\n'))]

245+ 'n' -> escIgnore 'm' [escapeArg, countChar 1 (satisfy (/='\n'))]

247246 'o' -> escIgnore 'o' [quoteArg]

248247 'p' -> escIgnore 'p' []

249248 'r' -> escIgnore 'r' []

@@ -253,7 +252,7 @@ escapeNormal = do

253252 'v' -> escIgnore 'v' [quoteArg]

254253 'w' -> escIgnore 'w' [quoteArg]

255254 'x' -> escIgnore 'x' [quoteArg]

256− 'z' -> escIgnore 'z' [count 1 anyChar]

255+ 'z' -> escIgnore 'z' [countChar 1 anyChar]

257256 '|' -> return [RoffStr "\x2006"] --1/6 em space

258257 '~' -> return [RoffStr "\160"] -- nonbreaking space

259258 '\\' -> do

@@ -262,40 +261,40 @@ escapeNormal = do

262261 CopyMode -> char '\\'

263262 NormalMode -> return '\\'

264263 return [RoffStr "\\"]

265− _ -> return [RoffStr [c]]

264+ _ -> return [RoffStr $ T.singleton c]

266265 -- man 7 groff: "If a backslash is followed by a character that

267266 -- does not constitute a defined escape sequence, the backslash

268267 -- is silently ignored and the character maps to itself."

269268

270269escIgnore :: PandocMonad m

271270 => Char

272− -> [RoffLexer m String]

271+ -> [RoffLexer m T.Text]

273272 -> RoffLexer m [LinePart]

274273escIgnore c argparsers = do

275274 pos <- getPosition

276275 arg <- snd <$> withRaw (choice argparsers) <|> return ""

277− report $ SkippedContent ('\\':c:arg) pos

276+ report $ SkippedContent ("\\" <> T.cons c arg) pos

278277 return mempty

279278

280−escUnknown :: PandocMonad m => String -> RoffLexer m [LinePart]

279+escUnknown :: PandocMonad m => T.Text -> RoffLexer m [LinePart]

281280escUnknown s = do

282281 pos <- getPosition

283282 report $ SkippedContent s pos

284283 return [RoffStr "\xFFFD"]

285284

286−signedNumber :: PandocMonad m => RoffLexer m String

285+signedNumber :: PandocMonad m => RoffLexer m T.Text

287286signedNumber = try $ do

288287 sign <- option "" ("-" <$ char '-' <|> "" <$ char '+')

289− ds <- many1 digit

290− return (sign ++ ds)

288+ ds <- many1Char digit

289+ return (sign <> ds)

291290

292291-- Parses: [..] or (..

293−escapeArg :: PandocMonad m => RoffLexer m String

292+escapeArg :: PandocMonad m => RoffLexer m T.Text

294293escapeArg = choice

295294 [ char '[' *> optional expandString *>

296− manyTill (noneOf ['\n',']']) (char ']')

295+ manyTillChar (noneOf ['\n',']']) (char ']')

297296 , char '(' *> optional expandString *>

298− count 2 (satisfy (/='\n'))

297+ countChar 2 (satisfy (/='\n'))

299298 ]

300299

301300expandString :: PandocMonad m => RoffLexer m ()

@@ -303,21 +302,21 @@ expandString = try $ do

303302 pos <- getPosition

304303 char '\\'

305304 char '*'

306− cs <- escapeArg <|> count 1 anyChar

307− s <- linePartsToString <$> resolveString cs pos

308− getInput >>= setInput . (s ++)

305+ cs <- escapeArg <|> countChar 1 anyChar

306+ s <- linePartsToText <$> resolveText cs pos

307+ getInput >>= setInput . (s <>)

309308 return ()

310309

311310-- Parses: '..'

312−quoteArg :: PandocMonad m => RoffLexer m String

313−quoteArg = char '\'' *> manyTill (noneOf ['\n','\'']) (char '\'')

311+quoteArg :: PandocMonad m => RoffLexer m T.Text

312+quoteArg = char '\'' *> manyTillChar (noneOf ['\n','\'']) (char '\'')

314313

315314escFont :: PandocMonad m => RoffLexer m [LinePart]

316315escFont = do

317− font <- escapeArg <|> count 1 alphaNum

318− font' <- if null font || font == "P"

316+ font <- escapeArg <|> countChar 1 alphaNum

317+ font' <- if T.null font || font == "P"

319318 then prevFont <$> getState

320− else return $ foldr processFontLetter defaultFontSpec font

319+ else return $ foldr processFontLetter defaultFontSpec $ T.unpack font

321320 modifyState $ \st -> st{ prevFont = currentFont st

322321 , currentFont = font' }

323322 return [Font font']

@@ -345,7 +344,7 @@ lexMacro = do

345344 guard $ sourceColumn pos == 1 || afterConditional st

346345 char '.' <|> char '\''

347346 skipMany spacetab

348− macroName <- many (satisfy isAlphaNum)

347+ macroName <- manyChar (satisfy isAlphaNum)

349348 case macroName of

350349 "nop" -> return mempty

351350 "ie" -> lexConditional "ie"

@@ -374,8 +373,8 @@ lexTable pos = do

374373 spaces

375374 opts <- try tableOptions <|> [] <$ optional (char ';')

376375 case lookup "tab" opts of

377− Just (c:_) -> modifyState $ \st -> st{ tableTabChar = c }

378− _ -> modifyState $ \st -> st{ tableTabChar = '\t' }

376+ Just (T.uncons -> Just (c, _)) -> modifyState $ \st -> st{ tableTabChar = c }

377+ _ -> modifyState $ \st -> st{ tableTabChar = '\t' }

379378 spaces

380379 skipMany lexComment

381380 spaces

@@ -388,7 +387,7 @@ lexTable pos = do

388387 string ".TE"

389388 skipMany spacetab

390389 eofline

391− return $ singleTok $ Tbl opts (rows ++ concat morerows) pos

390+ return $ singleTok $ Tbl opts (rows <> concat morerows) pos

392391

393392lexTableRows :: PandocMonad m => RoffLexer m [TableRow]

394393lexTableRows = do

@@ -428,11 +427,11 @@ tableOptions = many1 tableOption <* spaces <* char ';'

428427

429428tableOption :: PandocMonad m => RoffLexer m TableOption

430429tableOption = do

431− k <- many1 letter

430+ k <- many1Char letter

432431 v <- option "" $ try $ do

433432 skipMany spacetab

434433 char '('

435− manyTill anyChar (char ')')

434+ manyTillChar anyChar (char ')')

436435 skipMany spacetab

437436 optional (char ',' >> skipMany spacetab)

438437 return (k,v)

@@ -444,7 +443,7 @@ tableFormatSpec = do

444443 let speclines = first:rest

445444 spaces

446445 char '.'

447− return $ speclines ++ repeat (lastDef [] speclines) -- last line is default

446+ return $ speclines <> repeat (lastDef [] speclines) -- last line is default

448447

449448tableFormatSpecLine :: PandocMonad m => RoffLexer m [CellFormat]

450449tableFormatSpecLine =

@@ -456,19 +455,19 @@ tableColFormat = do

456455 $ True <$ try (string "|" <* notFollowedBy spacetab)

457456 c <- oneOf ['a','A','c','C','l','L','n','N','r','R','s','S','^','_','-',

458457 '=','|']

459− suffixes <- many $ try (skipMany spacetab *> count 1 digit) <|>

458+ suffixes <- many $ try (skipMany spacetab *> countChar 1 digit) <|>

460459 (do x <- oneOf ['b','B','d','D','e','E','f','F','i','I','m','M',

461460 'p','P','t','T','u','U','v','V','w','W','x','X', 'z','Z']

462461 num <- case toLower x of

463462 'w' -> many1 digit <|>

464463 (do char '('

465464 xs <- manyTill anyChar (char ')')

466− return ("(" ++ xs ++ ")")) <|>

465+ return ("(" <> xs <> ")")) <|>

467466 return ""

468467 'f' -> count 1 alphaNum <* skipMany spacetab

469468 'm' -> count 1 alphaNum <* skipMany spacetab

470469 _ -> return ""

471− return $ x : num)

470+ return $ T.pack $ x : num)

472471 pipeSuffix' <- option False $ True <$ string "|"

473472 return $ CellFormat

474473 { columnType = c

@@ -479,7 +478,7 @@ tableColFormat = do

479478-- We don't fully handle the conditional. But we do

480479-- include everything under '.ie n', which occurs commonly

481480-- in man pages.

482−lexConditional :: PandocMonad m => String -> RoffLexer m RoffTokens

481+lexConditional :: PandocMonad m => T.Text -> RoffLexer m RoffTokens

483482lexConditional mname = do

484483 pos <- getPosition

485484 skipMany spacetab

@@ -498,7 +497,7 @@ lexConditional mname = do

498497 case mbtest of

499498 Nothing -> do

500499 putState st -- reset state, so we don't record macros in skipped section

501− report $ SkippedContent ('.':mname) pos

500+ report $ SkippedContent (T.cons '.' mname) pos

502501 return mempty

503502 Just True -> return ifPart

504503 Just False -> do

@@ -508,7 +507,7 @@ lexConditional mname = do

508507expression :: PandocMonad m => RoffLexer m (Maybe Bool)

509508expression = do

510509 raw <- charsInBalanced '(' ')' (satisfy (/= '\n'))

511− <|> many1 nonspaceChar

510+ <|> many1Char nonspaceChar

512511 returnValue $

513512 case raw of

514513 "1" -> Just True

@@ -533,17 +532,17 @@ lexIncludeFile args = do

533532 pos <- getPosition

534533 case args of

535534 (f:_) -> do

536− let fp = linePartsToString f

535+ let fp = linePartsToText f

537536 dirs <- getResourcePath

538− result <- readFileFromDirs dirs fp

537+ result <- readFileFromDirs dirs $ T.unpack fp

539538 case result of

540539 Nothing -> report $ CouldNotLoadIncludeFile fp pos

541− Just s -> getInput >>= setInput . (s ++)

540+ Just s -> getInput >>= setInput . (s <>)

542541 return mempty

543542 [] -> return mempty

544543

545544resolveMacro :: PandocMonad m

546− => String -> [Arg] -> SourcePos -> RoffLexer m RoffTokens

545+ => T.Text -> [Arg] -> SourcePos -> RoffLexer m RoffTokens

547546resolveMacro macroName args pos = do

548547 macros <- customMacros <$> getState

549548 case M.lookup macroName macros of

@@ -552,7 +551,7 @@ resolveMacro macroName args pos = do

552551 let fillLP (MacroArg i) zs =

553552 case drop (i - 1) args of

554553 [] -> zs

555− (ys:_) -> ys ++ zs

554+ (ys:_) -> ys <> zs

556555 fillLP z zs = z : zs

557556 let fillMacroArg (TextLine lineparts) =

558557 TextLine (foldr fillLP [] lineparts)

@@ -565,7 +564,7 @@ lexStringDef args = do -- string definition

565564 [] -> Prelude.fail "No argument to .ds"

566565 (x:ys) -> do

567566 let ts = singleTok $ TextLine (intercalate [RoffStr " " ] ys)

568− let stringName = linePartsToString x

567+ let stringName = linePartsToText x

569568 modifyState $ \st ->

570569 st{ customMacros = M.insert stringName ts (customMacros st) }

571570 return mempty

@@ -575,14 +574,14 @@ lexMacroDef args = do -- macro definition

575574 modifyState $ \st -> st{ roffMode = CopyMode }

576575 (macroName, stopMacro) <-

577576 case args of

578− (x : y : _) -> return (linePartsToString x, linePartsToString y)

577+ (x : y : _) -> return (linePartsToText x, linePartsToText y)

579578 -- optional second arg

580− (x:_) -> return (linePartsToString x, ".")

579+ (x:_) -> return (linePartsToText x, ".")

581580 [] -> Prelude.fail "No argument to .de"

582581 let stop = try $ do

583582 char '.' <|> char '\''

584583 skipMany spacetab

585− string stopMacro

584+ textStr stopMacro

586585 _ <- lexArgs

587586 return ()

588587 ts <- mconcat <$> manyTill manToken stop

@@ -628,7 +627,7 @@ lexArgs = do

628627 char '"'

629628 return [RoffStr "\""]

630629

631−checkDefined :: PandocMonad m => String -> RoffLexer m [LinePart]

630+checkDefined :: PandocMonad m => T.Text -> RoffLexer m [LinePart]

632631checkDefined name = do

633632 macros <- customMacros <$> getState

634633 case M.lookup name macros of

@@ -638,19 +637,19 @@ checkDefined name = do

638637escString :: PandocMonad m => RoffLexer m [LinePart]

639638escString = try $ do

640639 pos <- getPosition

641− (do cs <- escapeArg <|> count 1 anyChar

642− resolveString cs pos)

640+ (do cs <- escapeArg <|> countChar 1 anyChar

641+ resolveText cs pos)

643642 <|> mempty <$ char 'S'

644643

645644-- strings and macros share namespace

646−resolveString :: PandocMonad m

647− => String -> SourcePos -> RoffLexer m [LinePart]

648−resolveString stringname pos = do

645+resolveText :: PandocMonad m

646+ => T.Text -> SourcePos -> RoffLexer m [LinePart]

647+resolveText stringname pos = do

649648 RoffTokens ts <- resolveMacro stringname [] pos

650649 case Foldable.toList ts of

651650 [TextLine xs] -> return xs

652651 _ -> do

653− report $ SkippedContent ("unknown string " ++ stringname) pos

652+ report $ SkippedContent ("unknown string " <> stringname) pos

654653 return mempty

655654

656655lexLine :: PandocMonad m => RoffLexer m RoffTokens

@@ -688,16 +687,16 @@ macroArg = try $ do

688687 pos <- getPosition

689688 backslash

690689 char '$'

691− x <- escapeArg <|> count 1 digit

690+ x <- escapeArg <|> countChar 1 digit

692691 case safeRead x of

693692 Just i -> return [MacroArg i]

694693 Nothing -> do

695− report $ SkippedContent ("illegal macro argument " ++ x) pos

694+ report $ SkippedContent ("illegal macro argument " <> x) pos

696695 return []

697696

698697regularText :: PandocMonad m => RoffLexer m [LinePart]

699698regularText = do

700− s <- many1 $ noneOf "\n\r\t \\\""

699+ s <- many1Char $ noneOf "\n\r\t \\\""

701700 return [RoffStr s]

702701

703702quoteChar :: PandocMonad m => RoffLexer m [LinePart]

@@ -708,7 +707,7 @@ quoteChar = do

708707spaceTabChar :: PandocMonad m => RoffLexer m [LinePart]

709708spaceTabChar = do

710709 c <- spacetab

711− return [RoffStr [c]]

710+ return [RoffStr $ T.singleton c]

712711

713712lexEmptyLine :: PandocMonad m => RoffLexer m RoffTokens

714713lexEmptyLine = newline >> return (singleTok EmptyLine)

@@ -716,8 +715,8 @@ lexEmptyLine = newline >> return (singleTok EmptyLine)

716715manToken :: PandocMonad m => RoffLexer m RoffTokens

717716manToken = lexComment <|> lexMacro <|> lexLine <|> lexEmptyLine

718717

719−linePartsToString :: [LinePart] -> String

720−linePartsToString = mconcat . map go

718+linePartsToText :: [LinePart] -> T.Text

719+linePartsToText = mconcat . map go

721720 where

722721 go (RoffStr s) = s

723722 go _ = mempty

@@ -726,7 +725,7 @@ linePartsToString = mconcat . map go

726725lexRoff :: PandocMonad m => SourcePos -> T.Text -> m RoffTokens

727726lexRoff pos txt = do

728727 eithertokens <- readWithM (do setPosition pos

729− mconcat <$> many manToken) def (T.unpack txt)

728+ mconcat <$> many manToken) def txt

730729 case eithertokens of

731730 Left e -> throwError e

732731 Right tokenz -> return tokenz

src/Text/Pandoc/Readers/TWiki.hs+62−55

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleContexts #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{-# LANGUAGE RelaxedPolyRec #-}

45-- RelaxedPolyRec needed for inlinesBetween on GHC < 7

56{- |

@@ -31,7 +32,7 @@ import Text.Pandoc.Definition

3132import Text.Pandoc.Options

3233import Text.Pandoc.Parsing hiding (enclosed, nested)

3334import Text.Pandoc.Readers.HTML (htmlTag, isCommentTag)

34−import Text.Pandoc.Shared (crFilter)

35+import Text.Pandoc.Shared (crFilter, tshow)

3536import Text.Pandoc.XML (fromEntities)

3637

3738-- | Read twiki from an input string and return a Pandoc document.

@@ -41,19 +42,19 @@ readTWiki :: PandocMonad m

4142 -> m Pandoc

4243readTWiki opts s = do

4344 res <- readWithM parseTWiki def{ stateOptions = opts }

44− (T.unpack (crFilter s) ++ "\n\n")

45+ (crFilter s <> "\n\n")

4546 case res of

4647 Left e -> throwError e

4748 Right d -> return d

4849

49−type TWParser = ParserT [Char] ParserState

50+type TWParser = ParserT Text ParserState

5051

5152--

5253-- utility functions

5354--

5455

55−tryMsg :: String -> TWParser m a -> TWParser m a

56−tryMsg msg p = try p <?> msg

56+tryMsg :: Text -> TWParser m a -> TWParser m a

57+tryMsg msg p = try p <?> T.unpack msg

5758

5859nested :: PandocMonad m => TWParser m a -> TWParser m a

5960nested p = do

@@ -64,25 +65,25 @@ nested p = do

6465 updateState $ \st -> st{ stateMaxNestingLevel = nestlevel }

6566 return res

6667

67−htmlElement :: PandocMonad m => String -> TWParser m (Attr, String)

68+htmlElement :: PandocMonad m => Text -> TWParser m (Attr, Text)

6869htmlElement tag = tryMsg tag $ do

6970 (TagOpen _ attr, _) <- htmlTag (~== TagOpen tag [])

70− content <- manyTill anyChar (endtag <|> endofinput)

71+ content <- T.pack <$> manyTill anyChar (endtag <|> endofinput)

7172 return (htmlAttrToPandoc attr, trim content)

7273 where

7374 endtag = void $ htmlTag (~== TagClose tag)

7475 endofinput = lookAhead $ try $ skipMany blankline >> skipSpaces >> eof

75− trim = dropWhile (=='\n') . reverse . dropWhile (=='\n') . reverse

76+ trim = T.dropAround (=='\n')

7677

77−htmlAttrToPandoc :: [Attribute String] -> Attr

78+htmlAttrToPandoc :: [Attribute Text] -> Attr

7879htmlAttrToPandoc attrs = (ident, classes, keyvals)

7980 where

8081 ident = fromMaybe "" $ lookup "id" attrs

81− classes = maybe [] words $ lookup "class" attrs

82+ classes = maybe [] T.words $ lookup "class" attrs

8283 keyvals = [(k,v) | (k,v) <- attrs, k /= "id" && k /= "class"]

8384

8485parseHtmlContentWithAttrs :: PandocMonad m

85− => String -> TWParser m a -> TWParser m (Attr, [a])

86+ => Text -> TWParser m a -> TWParser m (Attr, [a])

8687parseHtmlContentWithAttrs tag parser = do

8788 (attr, content) <- htmlElement tag

8889 parsedContent <- try $ parseContent content

@@ -91,7 +92,13 @@ parseHtmlContentWithAttrs tag parser = do

9192 parseContent = parseFromString' $ nested $ manyTill parser endOfContent

9293 endOfContent = try $ skipMany blankline >> skipSpaces >> eof

9394

94−parseHtmlContent :: PandocMonad m => String -> TWParser m a -> TWParser m [a]

95+parseCharHtmlContentWithAttrs :: PandocMonad m

96+ => Text -> TWParser m Char -> TWParser m (Attr, Text)

97+parseCharHtmlContentWithAttrs tag = fmap go . parseHtmlContentWithAttrs tag

98+ where

99+ go (x, y) = (x, T.pack y)

100+

101+parseHtmlContent :: PandocMonad m => Text -> TWParser m a -> TWParser m [a]

95102parseHtmlContent tag p = snd <$> parseHtmlContentWithAttrs tag p

96103

97104--

@@ -113,7 +120,7 @@ block = do

113120 <|> blockElements

114121 <|> para

115122 skipMany blankline

116− trace (take 60 $ show $ B.toList res)

123+ trace (T.take 60 $ tshow $ B.toList res)

117124 return res

118125

119126blockElements :: PandocMonad m => TWParser m B.Blocks

@@ -150,38 +157,38 @@ literal = rawBlock <$> htmlElement "literal"

150157 format (_, _, kvs) = fromMaybe "html" $ lookup "format" kvs

151158 rawBlock (attrs, content) = B.rawBlock (format attrs) content

152159

153−list :: PandocMonad m => String -> TWParser m B.Blocks

160+list :: PandocMonad m => Text -> TWParser m B.Blocks

154161list prefix = choice [ bulletList prefix

155162 , orderedList prefix

156163 , definitionList prefix]

157164

158−definitionList :: PandocMonad m => String -> TWParser m B.Blocks

165+definitionList :: PandocMonad m => Text -> TWParser m B.Blocks

159166definitionList prefix = tryMsg "definitionList" $ do

160− indent <- lookAhead $ string prefix *> many1 (string " ") <* string "$ "

161− elements <- many $ parseDefinitionListItem (prefix ++ concat indent)

167+ indent <- lookAhead $ textStr prefix *> many1 (textStr " ") <* textStr "$ "

168+ elements <- many $ parseDefinitionListItem (prefix <> T.concat indent)

162169 return $ B.definitionList elements

163170 where

164171 parseDefinitionListItem :: PandocMonad m

165− => String -> TWParser m (B.Inlines, [B.Blocks])

172+ => Text -> TWParser m (B.Inlines, [B.Blocks])

166173 parseDefinitionListItem indent = do

167− string (indent ++ "$ ") >> skipSpaces

174+ textStr (indent <> "$ ") >> skipSpaces

168175 term <- many1Till inline $ string ": "

169176 line <- listItemLine indent $ string "$ "

170177 return (mconcat term, [line])

171178

172−bulletList :: PandocMonad m => String -> TWParser m B.Blocks

179+bulletList :: PandocMonad m => Text -> TWParser m B.Blocks

173180bulletList prefix = tryMsg "bulletList" $

174181 parseList prefix (char '*') (char ' ')

175182

176−orderedList :: PandocMonad m => String -> TWParser m B.Blocks

183+orderedList :: PandocMonad m => Text -> TWParser m B.Blocks

177184orderedList prefix = tryMsg "orderedList" $

178185 parseList prefix (oneOf "1iIaA") (string ". ")

179186

180187parseList :: PandocMonad m

181− => String -> TWParser m Char -> TWParser m a -> TWParser m B.Blocks

188+ => Text -> TWParser m Char -> TWParser m a -> TWParser m B.Blocks

182189parseList prefix marker delim = do

183− (indent, style) <- lookAhead $ string prefix *> listStyle <* delim

184− blocks <- many $ parseListItem (prefix ++ indent) (char style <* delim)

190+ (indent, style) <- lookAhead $ textStr prefix *> listStyle <* delim

191+ blocks <- many $ parseListItem (prefix <> indent) (char style <* delim)

185192 return $ case style of

186193 '1' -> B.orderedListWith (1, DefaultStyle, DefaultDelim) blocks

187194 'i' -> B.orderedListWith (1, LowerRoman, DefaultDelim) blocks

@@ -191,24 +198,24 @@ parseList prefix marker delim = do

191198 _ -> B.bulletList blocks

192199 where

193200 listStyle = do

194− indent <- many1 $ string " "

201+ indent <- many1 $ textStr " "

195202 style <- marker

196− return (concat indent, style)

203+ return (T.concat indent, style)

197204

198205parseListItem :: (PandocMonad m, Show a)

199− => String -> TWParser m a -> TWParser m B.Blocks

200−parseListItem prefix marker = string prefix >> marker >> listItemLine prefix marker

206+ => Text -> TWParser m a -> TWParser m B.Blocks

207+parseListItem prefix marker = textStr prefix >> marker >> listItemLine prefix marker

201208

202209listItemLine :: (PandocMonad m, Show a)

203− => String -> TWParser m a -> TWParser m B.Blocks

210+ => Text -> TWParser m a -> TWParser m B.Blocks

204211listItemLine prefix marker = mconcat <$> (lineContent >>= parseContent)

205212 where

206213 lineContent = do

207214 content <- anyLine

208215 continuation <- optionMaybe listContinuation

209− return $ filterSpaces content ++ "\n" ++ maybe "" (" " ++) continuation

210− filterSpaces = reverse . dropWhile (== ' ') . reverse

211− listContinuation = notFollowedBy (string prefix >> marker) >>

216+ return $ filterSpaces content <> "\n" <> maybe "" (" " <>) continuation

217+ filterSpaces = T.dropWhileEnd (== ' ')

218+ listContinuation = notFollowedBy (textStr prefix >> marker) >>

212219 string " " >> lineContent

213220 parseContent = parseFromString' $ many1 $ nestedList <|> parseInline

214221 parseInline = (B.plain . mconcat) <$> many1Till inline (lastNewline <|> newlineBeforeNestedList)

@@ -352,29 +359,29 @@ macroWithParameters = try $ do

352359 char '%'

353360 return $ buildSpan name kvs $ B.str content

354361

355−buildSpan :: String -> [(String, String)] -> B.Inlines -> B.Inlines

362+buildSpan :: Text -> [(Text, Text)] -> B.Inlines -> B.Inlines

356363buildSpan className kvs = B.spanWith attrs

357364 where

358365 attrs = ("", ["twiki-macro", className] ++ additionalClasses, kvsWithoutClasses)

359− additionalClasses = maybe [] words $ lookup "class" kvs

366+ additionalClasses = maybe [] T.words $ lookup "class" kvs

360367 kvsWithoutClasses = [(k,v) | (k,v) <- kvs, k /= "class"]

361368

362−macroName :: PandocMonad m => TWParser m String

369+macroName :: PandocMonad m => TWParser m Text

363370macroName = do

364371 first <- letter

365372 rest <- many $ alphaNum <|> char '_'

366− return (first:rest)

373+ return $ T.pack $ first:rest

367374

368−attributes :: PandocMonad m => TWParser m (String, [(String, String)])

369−attributes = foldr (either mkContent mkKvs) ([], [])

375+attributes :: PandocMonad m => TWParser m (Text, [(Text, Text)])

376+attributes = foldr (either mkContent mkKvs) ("", [])

370377 <$> (char '{' *> spnl *> many (attribute <* spnl) <* char '}')

371378 where

372379 spnl = skipMany (spaceChar <|> newline)

373− mkContent c ([], kvs) = (c, kvs)

374− mkContent c (rest, kvs) = (c ++ " " ++ rest, kvs)

380+ mkContent c ("", kvs) = (c, kvs)

381+ mkContent c (rest, kvs) = (c <> " " <> rest, kvs)

375382 mkKvs kv (cont, rest) = (cont, kv : rest)

376383

377−attribute :: PandocMonad m => TWParser m (Either String (String, String))

384+attribute :: PandocMonad m => TWParser m (Either Text (Text, Text))

378385attribute = withKey <|> withoutKey

379386 where

380387 withKey = try $ do

@@ -383,10 +390,10 @@ attribute = withKey <|> withoutKey

383390 curry Right key <$> parseValue False

384391 withoutKey = try $ Left <$> parseValue True

385392 parseValue allowSpaces = fromEntities <$> (withQuotes <|> withoutQuotes allowSpaces)

386− withQuotes = between (char '"') (char '"') (\_ -> count 1 $ noneOf ['"'])

393+ withQuotes = between (char '"') (char '"') (\_ -> countChar 1 $ noneOf ['"'])

387394 withoutQuotes allowSpaces

388− | allowSpaces = many1 $ noneOf "}"

389− | otherwise = many1 $ noneOf " }"

395+ | allowSpaces = many1Char $ noneOf "}"

396+ | otherwise = many1Char $ noneOf " }"

390397

391398nestedInlines :: (Show a, PandocMonad m)

392399 => TWParser m a -> TWParser m B.Inlines

@@ -413,10 +420,10 @@ emphHtml :: PandocMonad m => TWParser m B.Inlines

413420emphHtml = B.emph . mconcat <$> (parseHtmlContent "em" inline <|> parseHtmlContent "i" inline)

414421

415422nestedString :: (Show a, PandocMonad m)

416− => TWParser m a -> TWParser m String

417−nestedString end = innerSpace <|> count 1 nonspaceChar

423+ => TWParser m a -> TWParser m Text

424+nestedString end = innerSpace <|> countChar 1 nonspaceChar

418425 where

419− innerSpace = try $ many1 spaceChar <* notFollowedBy end

426+ innerSpace = try $ many1Char spaceChar <* notFollowedBy end

420427

421428boldCode :: PandocMonad m => TWParser m B.Inlines

422429boldCode = try $ (B.strong . B.code . fromEntities) <$> enclosed (string "==") nestedString

@@ -429,7 +436,7 @@ code = try $ (B.code . fromEntities) <$> enclosed (char '=') nestedString

429436

430437codeHtml :: PandocMonad m => TWParser m B.Inlines

431438codeHtml = do

432− (attrs, content) <- parseHtmlContentWithAttrs "code" anyChar

439+ (attrs, content) <- parseCharHtmlContentWithAttrs "code" anyChar

433440 return $ B.codeWith attrs $ fromEntities content

434441

435442autoLink :: PandocMonad m => TWParser m B.Inlines

@@ -437,7 +444,7 @@ autoLink = try $ do

437444 state <- getState

438445 guard $ stateAllowLinks state

439446 (text, url) <- parseLink

440− guard $ checkLink (last url)

447+ guard $ checkLink (T.last url)

441448 return $ makeLink (text, url)

442449 where

443450 parseLink = notFollowedBy nop >> (uri <|> emailAddress)

@@ -447,17 +454,17 @@ autoLink = try $ do

447454 | otherwise = isAlphaNum c

448455

449456str :: PandocMonad m => TWParser m B.Inlines

450−str = B.str <$> (many1 alphaNum <|> count 1 characterReference)

457+str = B.str <$> (many1Char alphaNum <|> countChar 1 characterReference)

451458

452459nop :: PandocMonad m => TWParser m B.Inlines

453460nop = try $ (void exclamation <|> void nopTag) >> followContent

454461 where

455462 exclamation = char '!'

456463 nopTag = stringAnyCase "<nop>"

457− followContent = B.str . fromEntities <$> many1 nonspaceChar

464+ followContent = B.str . fromEntities <$> many1Char nonspaceChar

458465

459466symbol :: PandocMonad m => TWParser m B.Inlines

460−symbol = B.str <$> count 1 nonspaceChar

467+symbol = B.str <$> countChar 1 nonspaceChar

461468

462469smart :: PandocMonad m => TWParser m B.Inlines

463470smart = do

@@ -491,13 +498,13 @@ link = try $ do

491498 setState $ st{ stateAllowLinks = True }

492499 return $ B.link url title content

493500

494−linkText :: PandocMonad m => TWParser m (String, String, B.Inlines)

501+linkText :: PandocMonad m => TWParser m (Text, Text, B.Inlines)

495502linkText = do

496503 string "[["

497− url <- many1Till anyChar (char ']')

504+ url <- T.pack <$> many1Till anyChar (char ']')

498505 content <- option (B.str url) (mconcat <$> linkContent)

499506 char ']'

500507 return (url, "", content)

501508 where

502− linkContent = char '[' >> many1Till anyChar (char ']') >>= parseLinkContent

509+ linkContent = char '[' >> many1Till anyChar (char ']') >>= parseLinkContent . T.pack

503510 parseLinkContent = parseFromString' $ many1 inline

src/Text/Pandoc/Readers/Textile.hs+136−137

@@ -1,4 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

3+{-# LANGUAGE ViewPatterns #-}

24{- |

35 Module : Text.Pandoc.Readers.Textile

@@ -38,7 +40,7 @@ import Prelude

3840import Control.Monad (guard, liftM)

3941import Control.Monad.Except (throwError)

4042import Data.Char (digitToInt, isUpper)

41−import Data.List (intercalate, intersperse, transpose)

43+import Data.List (intersperse, transpose)

4244import Data.Text (Text)

4345import qualified Data.Text as T

4446import Text.HTML.TagSoup (Tag (..), fromAttrib)

@@ -52,7 +54,7 @@ import Text.Pandoc.Options

5254import Text.Pandoc.Parsing

5355import Text.Pandoc.Readers.HTML (htmlTag, isBlockTag, isInlineTag)

5456import Text.Pandoc.Readers.LaTeX (rawLaTeXBlock, rawLaTeXInline)

55−import Text.Pandoc.Shared (crFilter, trim, underlineSpan)

57+import Text.Pandoc.Shared (crFilter, trim, underlineSpan, tshow)

5658

5759-- | Parse a Textile text and return a Pandoc document.

5860readTextile :: PandocMonad m

@@ -61,21 +63,21 @@ readTextile :: PandocMonad m

6163 -> m Pandoc

6264readTextile opts s = do

6365 parsed <- readWithM parseTextile def{ stateOptions = opts }

64− (T.unpack (crFilter s) ++ "\n\n")

66+ (crFilter s <> "\n\n")

6567 case parsed of

6668 Right result -> return result

6769 Left e -> throwError e

6870

6971

7072-- | Generate a Pandoc ADT from a textile document

71−parseTextile :: PandocMonad m => ParserT [Char] ParserState m Pandoc

73+parseTextile :: PandocMonad m => ParserT Text ParserState m Pandoc

7274parseTextile = do

7375 many blankline

7476 startPos <- getPosition

7577 -- go through once just to get list of reference keys and notes

7678 -- docMinusKeys is the raw document with blanks where the keys/notes were...

7779 let firstPassParser = noteBlock <|> lineClump

78− manyTill firstPassParser eof >>= setInput . concat

80+ manyTill firstPassParser eof >>= setInput . T.concat

7981 setPosition startPos

8082 st' <- getState

8183 let reversedNotes = stateNotes st'

@@ -84,29 +86,29 @@ parseTextile = do

8486 blocks <- parseBlocks

8587 return $ Pandoc nullMeta (B.toList blocks) -- FIXME

8688

87−noteMarker :: PandocMonad m => ParserT [Char] ParserState m [Char]

88−noteMarker = skipMany spaceChar >> string "fn" >> manyTill digit (char '.')

89+noteMarker :: PandocMonad m => ParserT Text ParserState m Text

90+noteMarker = skipMany spaceChar >> string "fn" >> T.pack <$> manyTill digit (char '.')

8991

90−noteBlock :: PandocMonad m => ParserT [Char] ParserState m [Char]

92+noteBlock :: PandocMonad m => ParserT Text ParserState m Text

9193noteBlock = try $ do

9294 startPos <- getPosition

9395 ref <- noteMarker

9496 optional blankline

95− contents <- unlines <$> many1Till anyLine (blanklines <|> noteBlock)

97+ contents <- T.unlines <$> many1Till anyLine (blanklines <|> noteBlock)

9698 endPos <- getPosition

97− let newnote = (ref, contents ++ "\n")

99+ let newnote = (ref, contents <> "\n")

98100 st <- getState

99101 let oldnotes = stateNotes st

100102 updateState $ \s -> s { stateNotes = newnote : oldnotes }

101103 -- return blanks so line count isn't affected

102− return $ replicate (sourceLine endPos - sourceLine startPos) '\n'

104+ return $ T.replicate (sourceLine endPos - sourceLine startPos) "\n"

103105

104106-- | Parse document blocks

105−parseBlocks :: PandocMonad m => ParserT [Char] ParserState m Blocks

107+parseBlocks :: PandocMonad m => ParserT Text ParserState m Blocks

106108parseBlocks = mconcat <$> manyTill block eof

107109

108110-- | Block parsers list tried in definition order

109−blockParsers :: PandocMonad m => [ParserT [Char] ParserState m Blocks]

111+blockParsers :: PandocMonad m => [ParserT Text ParserState m Blocks]

110112blockParsers = [ codeBlock

111113 , header

112114 , blockQuote

@@ -121,22 +123,22 @@ blockParsers = [ codeBlock

121123 ]

122124

123125-- | Any block in the order of definition of blockParsers

124−block :: PandocMonad m => ParserT [Char] ParserState m Blocks

126+block :: PandocMonad m => ParserT Text ParserState m Blocks

125127block = do

126128 res <- choice blockParsers <?> "block"

127− trace (take 60 $ show $ B.toList res)

129+ trace (T.take 60 $ tshow $ B.toList res)

128130 return res

129131

130−commentBlock :: PandocMonad m => ParserT [Char] ParserState m Blocks

132+commentBlock :: PandocMonad m => ParserT Text ParserState m Blocks

131133commentBlock = try $ do

132134 string "###."

133135 manyTill anyLine blanklines

134136 return mempty

135137

136−codeBlock :: PandocMonad m => ParserT [Char] ParserState m Blocks

138+codeBlock :: PandocMonad m => ParserT Text ParserState m Blocks

137139codeBlock = codeBlockBc <|> codeBlockPre

138140

139−codeBlockBc :: PandocMonad m => ParserT [Char] ParserState m Blocks

141+codeBlockBc :: PandocMonad m => ParserT Text ParserState m Blocks

140142codeBlockBc = try $ do

141143 string "bc."

142144 extended <- option False (True <$ char '.')

@@ -150,31 +152,31 @@ codeBlockBc = try $ do

150152 rest <- many (notFollowedBy ender *> anyLine)

151153 return (f:rest)

152154 else manyTill anyLine blanklines

153− return $ B.codeBlock (trimTrailingNewlines (unlines contents))

155+ return $ B.codeBlock (trimTrailingNewlines (T.unlines contents))

154156

155−trimTrailingNewlines :: String -> String

156−trimTrailingNewlines = reverse . dropWhile (=='\n') . reverse

157+trimTrailingNewlines :: Text -> Text

158+trimTrailingNewlines = T.dropWhileEnd (=='\n')

157159

158160-- | Code Blocks in Textile are between <pre> and </pre>

159−codeBlockPre :: PandocMonad m => ParserT [Char] ParserState m Blocks

161+codeBlockPre :: PandocMonad m => ParserT Text ParserState m Blocks

160162codeBlockPre = try $ do

161163 (t@(TagOpen _ attrs),_) <- htmlTag (tagOpen (=="pre") (const True))

162− result' <- manyTill anyChar (htmlTag (tagClose (=="pre")))

164+ result' <- T.pack <$> manyTill anyChar (htmlTag (tagClose (=="pre")))

163165 -- drop leading newline if any

164− let result'' = case result' of

165− '\n':xs -> xs

166− _ -> result'

166+ let result'' = case T.uncons result' of

167+ Just ('\n', xs) -> xs

168+ _ -> result'

167169 -- drop trailing newline if any

168− let result''' = case reverse result'' of

169− '\n':_ -> init result''

170− _ -> result''

171− let classes = words $ fromAttrib "class" t

170+ let result''' = case T.unsnoc result'' of

171+ Just (xs, '\n') -> xs

172+ _ -> result''

173+ let classes = T.words $ fromAttrib "class" t

172174 let ident = fromAttrib "id" t

173175 let kvs = [(k,v) | (k,v) <- attrs, k /= "id" && k /= "class"]

174176 return $ B.codeBlockWith (ident,classes,kvs) result'''

175177

176178-- | Header of the form "hN. content" with N in 1..6

177−header :: PandocMonad m => ParserT [Char] ParserState m Blocks

179+header :: PandocMonad m => ParserT Text ParserState m Blocks

178180header = try $ do

179181 char 'h'

180182 level <- digitToInt <$> oneOf "123456"

@@ -186,14 +188,14 @@ header = try $ do

186188 return $ B.headerWith attr' level name

187189

188190-- | Blockquote of the form "bq. content"

189−blockQuote :: PandocMonad m => ParserT [Char] ParserState m Blocks

191+blockQuote :: PandocMonad m => ParserT Text ParserState m Blocks

190192blockQuote = try $ do

191193 string "bq" >> attributes >> char '.' >> whitespace

192194 B.blockQuote <$> para

193195

194196-- Horizontal rule

195197

196−hrule :: PandocMonad m => ParserT [Char] st m Blocks

198+hrule :: PandocMonad m => ParserT Text st m Blocks

197199hrule = try $ do

198200 skipSpaces

199201 start <- oneOf "-*"

@@ -208,39 +210,39 @@ hrule = try $ do

208210-- | Can be a bullet list or an ordered list. This implementation is

209211-- strict in the nesting, sublist must start at exactly "parent depth

210212-- plus one"

211−anyList :: PandocMonad m => ParserT [Char] ParserState m Blocks

213+anyList :: PandocMonad m => ParserT Text ParserState m Blocks

212214anyList = try $ anyListAtDepth 1 <* blanklines

213215

214216-- | This allow one type of list to be nested into an other type,

215217-- provided correct nesting

216−anyListAtDepth :: PandocMonad m => Int -> ParserT [Char] ParserState m Blocks

218+anyListAtDepth :: PandocMonad m => Int -> ParserT Text ParserState m Blocks

217219anyListAtDepth depth = choice [ bulletListAtDepth depth,

218220 orderedListAtDepth depth,

219221 definitionList ]

220222

221223-- | Bullet List of given depth, depth being the number of leading '*'

222−bulletListAtDepth :: PandocMonad m => Int -> ParserT [Char] ParserState m Blocks

224+bulletListAtDepth :: PandocMonad m => Int -> ParserT Text ParserState m Blocks

223225bulletListAtDepth depth = try $ B.bulletList <$> many1 (bulletListItemAtDepth depth)

224226

225227-- | Bullet List Item of given depth, depth being the number of

226228-- leading '*'

227−bulletListItemAtDepth :: PandocMonad m => Int -> ParserT [Char] ParserState m Blocks

229+bulletListItemAtDepth :: PandocMonad m => Int -> ParserT Text ParserState m Blocks

228230bulletListItemAtDepth = genericListItemAtDepth '*'

229231

230232-- | Ordered List of given depth, depth being the number of

231233-- leading '#'

232−orderedListAtDepth :: PandocMonad m => Int -> ParserT [Char] ParserState m Blocks

234+orderedListAtDepth :: PandocMonad m => Int -> ParserT Text ParserState m Blocks

233235orderedListAtDepth depth = try $ do

234236 items <- many1 (orderedListItemAtDepth depth)

235237 return $ B.orderedList items

236238

237239-- | Ordered List Item of given depth, depth being the number of

238240-- leading '#'

239−orderedListItemAtDepth :: PandocMonad m => Int -> ParserT [Char] ParserState m Blocks

241+orderedListItemAtDepth :: PandocMonad m => Int -> ParserT Text ParserState m Blocks

240242orderedListItemAtDepth = genericListItemAtDepth '#'

241243

242244-- | Common implementation of list items

243−genericListItemAtDepth :: PandocMonad m => Char -> Int -> ParserT [Char] ParserState m Blocks

245+genericListItemAtDepth :: PandocMonad m => Char -> Int -> ParserT Text ParserState m Blocks

244246genericListItemAtDepth c depth = try $ do

245247 count depth (char c) >> attributes >> whitespace

246248 contents <- mconcat <$> many ((B.plain . mconcat <$> many1 inline) <|>

@@ -250,25 +252,25 @@ genericListItemAtDepth c depth = try $ do

250252 return $ contents <> sublist

251253

252254-- | A definition list is a set of consecutive definition items

253−definitionList :: PandocMonad m => ParserT [Char] ParserState m Blocks

255+definitionList :: PandocMonad m => ParserT Text ParserState m Blocks

254256definitionList = try $ B.definitionList <$> many1 definitionListItem

255257

256258-- | List start character.

257−listStart :: PandocMonad m => ParserT [Char] ParserState m ()

259+listStart :: PandocMonad m => ParserT Text ParserState m ()

258260listStart = genericListStart '*'

259261 <|> () <$ genericListStart '#'

260262 <|> () <$ definitionListStart

261263

262−genericListStart :: PandocMonad m => Char -> ParserT [Char] st m ()

264+genericListStart :: PandocMonad m => Char -> ParserT Text st m ()

263265genericListStart c = () <$ try (many1 (char c) >> whitespace)

264266

265−basicDLStart :: PandocMonad m => ParserT [Char] ParserState m ()

267+basicDLStart :: PandocMonad m => ParserT Text ParserState m ()

266268basicDLStart = do

267269 char '-'

268270 whitespace

269271 notFollowedBy newline

270272

271−definitionListStart :: PandocMonad m => ParserT [Char] ParserState m Inlines

273+definitionListStart :: PandocMonad m => ParserT Text ParserState m Inlines

272274definitionListStart = try $ do

273275 basicDLStart

274276 trimInlines . mconcat <$>

@@ -281,26 +283,26 @@ definitionListStart = try $ do

281283-- the term defined, then spaces and ":=". The definition follows, on

282284-- the same single line, or spaned on multiple line, after a line

283285-- break.

284−definitionListItem :: PandocMonad m => ParserT [Char] ParserState m (Inlines, [Blocks])

286+definitionListItem :: PandocMonad m => ParserT Text ParserState m (Inlines, [Blocks])

285287definitionListItem = try $ do

286288 term <- (mconcat . intersperse B.linebreak) <$> many1 definitionListStart

287289 def' <- string ":=" *> optional whitespace *> (multilineDef <|> inlineDef)

288290 return (term, def')

289− where inlineDef :: PandocMonad m => ParserT [Char] ParserState m [Blocks]

291+ where inlineDef :: PandocMonad m => ParserT Text ParserState m [Blocks]

290292 inlineDef = liftM (\d -> [B.plain d])

291293 $ optional whitespace >> (trimInlines . mconcat <$> many inline) <* newline

292− multilineDef :: PandocMonad m => ParserT [Char] ParserState m [Blocks]

294+ multilineDef :: PandocMonad m => ParserT Text ParserState m [Blocks]

293295 multilineDef = try $ do

294296 optional whitespace >> newline

295− s <- many1Till anyChar (try (string "=:" >> newline))

296− -- this ++ "\n\n" does not look very good

297− ds <- parseFromString' parseBlocks (s ++ "\n\n")

297+ s <- T.pack <$> many1Till anyChar (try (string "=:" >> newline))

298+ -- this <> "\n\n" does not look very good

299+ ds <- parseFromString' parseBlocks (s <> "\n\n")

298300 return [ds]

299301

300302-- raw content

301303

302304-- | A raw Html Block, optionally followed by blanklines

303−rawHtmlBlock :: PandocMonad m => ParserT [Char] ParserState m Blocks

305+rawHtmlBlock :: PandocMonad m => ParserT Text ParserState m Blocks

304306rawHtmlBlock = try $ do

305307 skipMany spaceChar

306308 (_,b) <- htmlTag isBlockTag

@@ -308,14 +310,14 @@ rawHtmlBlock = try $ do

308310 return $ B.rawBlock "html" b

309311

310312-- | Raw block of LaTeX content

311−rawLaTeXBlock' :: PandocMonad m => ParserT [Char] ParserState m Blocks

313+rawLaTeXBlock' :: PandocMonad m => ParserT Text ParserState m Blocks

312314rawLaTeXBlock' = do

313315 guardEnabled Ext_raw_tex

314316 B.rawBlock "latex" <$> (rawLaTeXBlock <* spaces)

315317

316318

317319-- | In textile, paragraphs are separated by blank lines.

318−para :: PandocMonad m => ParserT [Char] ParserState m Blocks

320+para :: PandocMonad m => ParserT Text ParserState m Blocks

319321para = B.para . trimInlines . mconcat <$> many1 inline

320322

321323-- Tables

@@ -326,7 +328,7 @@ toAlignment '>' = AlignRight

326328toAlignment '=' = AlignCenter

327329toAlignment _ = AlignDefault

328330

329−cellAttributes :: PandocMonad m => ParserT [Char] ParserState m (Bool, Alignment)

331+cellAttributes :: PandocMonad m => ParserT Text ParserState m (Bool, Alignment)

330332cellAttributes = try $ do

331333 isHeader <- option False (True <$ char '_')

332334 -- we just ignore colspan and rowspan markers:

@@ -339,18 +341,18 @@ cellAttributes = try $ do

339341 return (isHeader, alignment)

340342

341343-- | A table cell spans until a pipe |

342−tableCell :: PandocMonad m => ParserT [Char] ParserState m ((Bool, Alignment), Blocks)

344+tableCell :: PandocMonad m => ParserT Text ParserState m ((Bool, Alignment), Blocks)

343345tableCell = try $ do

344346 char '|'

345347 (isHeader, alignment) <- option (False, AlignDefault) cellAttributes

346348 notFollowedBy blankline

347− raw <- trim <$>

349+ raw <- trim . T.pack <$>

348350 many (noneOf "|\n" <|> try (char '\n' <* notFollowedBy blankline))

349351 content <- mconcat <$> parseFromString' (many inline) raw

350352 return ((isHeader, alignment), B.plain content)

351353

352354-- | A table row is made of many table cells

353−tableRow :: PandocMonad m => ParserT [Char] ParserState m [((Bool, Alignment), Blocks)]

355+tableRow :: PandocMonad m => ParserT Text ParserState m [((Bool, Alignment), Blocks)]

354356tableRow = try $ do

355357 -- skip optional row attributes

356358 optional $ try $ do

@@ -360,7 +362,7 @@ tableRow = try $ do

360362 many1 tableCell <* char '|' <* blankline

361363

362364-- | A table with an optional header.

363−table :: PandocMonad m => ParserT [Char] ParserState m Blocks

365+table :: PandocMonad m => ParserT Text ParserState m Blocks

364366table = try $ do

365367 -- ignore table attributes

366368 caption <- option mempty $ try $ do

@@ -384,7 +386,7 @@ table = try $ do

384386 (map (map snd) rows)

385387

386388-- | Ignore markers for cols, thead, tfoot.

387−ignorableRow :: PandocMonad m => ParserT [Char] ParserState m ()

389+ignorableRow :: PandocMonad m => ParserT Text ParserState m ()

388390ignorableRow = try $ do

389391 char '|'

390392 oneOf ":^-~"

@@ -393,9 +395,9 @@ ignorableRow = try $ do

393395 _ <- anyLine

394396 return ()

395397

396−explicitBlockStart :: PandocMonad m => String -> ParserT [Char] ParserState m ()

398+explicitBlockStart :: PandocMonad m => Text -> ParserT Text ParserState m ()

397399explicitBlockStart name = try $ do

398− string name

400+ string (T.unpack name)

399401 attributes

400402 char '.'

401403 optional whitespace

@@ -404,9 +406,9 @@ explicitBlockStart name = try $ do

404406-- | Blocks like 'p' and 'table' do not need explicit block tag.

405407-- However, they can be used to set HTML/CSS attributes when needed.

406408maybeExplicitBlock :: PandocMonad m

407− => String -- ^ block tag name

408− -> ParserT [Char] ParserState m Blocks -- ^ implicit block

409− -> ParserT [Char] ParserState m Blocks

409+ => Text -- ^ block tag name

410+ -> ParserT Text ParserState m Blocks -- ^ implicit block

411+ -> ParserT Text ParserState m Blocks

410412maybeExplicitBlock name blk = try $ do

411413 optional $ explicitBlockStart name

412414 blk

@@ -419,11 +421,11 @@ maybeExplicitBlock name blk = try $ do

419421

420422

421423-- | Any inline element

422−inline :: PandocMonad m => ParserT [Char] ParserState m Inlines

424+inline :: PandocMonad m => ParserT Text ParserState m Inlines

423425inline = choice inlineParsers <?> "inline"

424426

425427-- | Inline parsers tried in order

426−inlineParsers :: PandocMonad m => [ParserT [Char] ParserState m Inlines]

428+inlineParsers :: PandocMonad m => [ParserT Text ParserState m Inlines]

427429inlineParsers = [ str

428430 , whitespace

429431 , endline

@@ -437,13 +439,13 @@ inlineParsers = [ str

437439 , link

438440 , image

439441 , mark

440− , (B.str . (:[])) <$> characterReference

442+ , (B.str . T.singleton) <$> characterReference

441443 , smartPunctuation inline

442444 , symbol

443445 ]

444446

445447-- | Inline markups

446−inlineMarkup :: PandocMonad m => ParserT [Char] ParserState m Inlines

448+inlineMarkup :: PandocMonad m => ParserT Text ParserState m Inlines

447449inlineMarkup = choice [ simpleInline (string "??") (B.cite [])

448450 , simpleInline (string "**") B.strong

449451 , simpleInline (string "__") B.emph

@@ -457,33 +459,33 @@ inlineMarkup = choice [ simpleInline (string "??") (B.cite [])

457459 ]

458460

459461-- | Trademark, registered, copyright

460−mark :: PandocMonad m => ParserT [Char] st m Inlines

462+mark :: PandocMonad m => ParserT Text st m Inlines

461463mark = try $ char '(' >> (try tm <|> try reg <|> copy)

462464

463−reg :: PandocMonad m => ParserT [Char] st m Inlines

465+reg :: PandocMonad m => ParserT Text st m Inlines

464466reg = do

465467 oneOf "Rr"

466468 char ')'

467469 return $ B.str "\174"

468470

469−tm :: PandocMonad m => ParserT [Char] st m Inlines

471+tm :: PandocMonad m => ParserT Text st m Inlines

470472tm = do

471473 oneOf "Tt"

472474 oneOf "Mm"

473475 char ')'

474476 return $ B.str "\8482"

475477

476−copy :: PandocMonad m => ParserT [Char] st m Inlines

478+copy :: PandocMonad m => ParserT Text st m Inlines

477479copy = do

478480 oneOf "Cc"

479481 char ')'

480482 return $ B.str "\169"

481483

482−note :: PandocMonad m => ParserT [Char] ParserState m Inlines

484+note :: PandocMonad m => ParserT Text ParserState m Inlines

483485note = try $ do

484486 ref <- char '[' *> many1 digit <* char ']'

485487 notes <- stateNotes <$> getState

486− case lookup ref notes of

488+ case lookup (T.pack ref) notes of

487489 Nothing -> Prelude.fail "note not found"

488490 Just raw -> B.note <$> parseFromString' parseBlocks raw

489491

@@ -500,42 +502,42 @@ stringBreakers :: [Char]

500502stringBreakers = " \t\n\r.,\"'?!;:<>«»„“”‚‘’()[]"

501503

502504wordBoundaries :: [Char]

503−wordBoundaries = markupChars ++ stringBreakers

505+wordBoundaries = markupChars <> stringBreakers

504506

505507-- | Parse a hyphened sequence of words

506−hyphenedWords :: PandocMonad m => ParserT [Char] ParserState m String

508+hyphenedWords :: PandocMonad m => ParserT Text ParserState m Text

507509hyphenedWords = do

508510 x <- wordChunk

509511 xs <- many (try $ char '-' >> wordChunk)

510− return $ intercalate "-" (x:xs)

512+ return $ T.intercalate "-" (x:xs)

511513

512−wordChunk :: PandocMonad m => ParserT [Char] ParserState m String

514+wordChunk :: PandocMonad m => ParserT Text ParserState m Text

513515wordChunk = try $ do

514516 hd <- noneOf wordBoundaries

515517 tl <- many ( noneOf wordBoundaries <|>

516518 try (notFollowedBy' note *> oneOf markupChars

517519 <* lookAhead (noneOf wordBoundaries) ) )

518− return $ hd:tl

520+ return $ T.pack $ hd:tl

519521

520522-- | Any string

521−str :: PandocMonad m => ParserT [Char] ParserState m Inlines

523+str :: PandocMonad m => ParserT Text ParserState m Inlines

522524str = do

523525 baseStr <- hyphenedWords

524526 -- RedCloth compliance : if parsed word is uppercase and immediately

525527 -- followed by parens, parens content is unconditionally word acronym

526528 fullStr <- option baseStr $ try $ do

527− guard $ all isUpper baseStr

528− acro <- enclosed (char '(') (char ')') anyChar'

529− return $ concat [baseStr, " (", acro, ")"]

529+ guard $ T.all isUpper baseStr

530+ acro <- T.pack <$> enclosed (char '(') (char ')') anyChar'

531+ return $ T.concat [baseStr, " (", acro, ")"]

530532 updateLastStrPos

531533 return $ B.str fullStr

532534

533535-- | Some number of space chars

534−whitespace :: PandocMonad m => ParserT [Char] st m Inlines

536+whitespace :: PandocMonad m => ParserT Text st m Inlines

535537whitespace = many1 spaceChar >> return B.space <?> "whitespace"

536538

537539-- | In Textile, an isolated endline character is a line break

538−endline :: PandocMonad m => ParserT [Char] ParserState m Inlines

540+endline :: PandocMonad m => ParserT Text ParserState m Inlines

539541endline = try $ do

540542 newline

541543 notFollowedBy blankline

@@ -543,18 +545,18 @@ endline = try $ do

543545 notFollowedBy rawHtmlBlock

544546 return B.linebreak

545547

546−rawHtmlInline :: PandocMonad m => ParserT [Char] ParserState m Inlines

548+rawHtmlInline :: PandocMonad m => ParserT Text ParserState m Inlines

547549rawHtmlInline = B.rawInline "html" . snd <$> htmlTag isInlineTag

548550

549551-- | Raw LaTeX Inline

550−rawLaTeXInline' :: PandocMonad m => ParserT [Char] ParserState m Inlines

552+rawLaTeXInline' :: PandocMonad m => ParserT Text ParserState m Inlines

551553rawLaTeXInline' = try $ do

552554 guardEnabled Ext_raw_tex

553555 B.rawInline "latex" <$> rawLaTeXInline

554556

555557-- | Textile standard link syntax is "label":target. But we

556558-- can also have ["label":target].

557−link :: PandocMonad m => ParserT [Char] ParserState m Inlines

559+link :: PandocMonad m => ParserT Text ParserState m Inlines

558560link = try $ do

559561 bracketed <- (True <$ char '[') <|> return False

560562 char '"' *> notFollowedBy (oneOf " \t\n\r")

@@ -567,121 +569,122 @@ link = try $ do

567569 else lookAhead $ space <|> eof' <|>

568570 try (oneOf "!.,;:" *>

569571 (space <|> newline <|> eof'))

570− url <- many1Till nonspaceChar stop

572+ url <- T.pack <$> many1Till nonspaceChar stop

571573 let name' = if B.toList name == [Str "$"] then B.str url else name

572574 return $ if attr == nullAttr

573575 then B.link url "" name'

574576 else B.spanWith attr $ B.link url "" name'

575577

576578-- | image embedding

577−image :: PandocMonad m => ParserT [Char] ParserState m Inlines

579+image :: PandocMonad m => ParserT Text ParserState m Inlines

578580image = try $ do

579581 char '!' >> notFollowedBy space

580582 (ident, cls, kvs) <- attributes

581583 let attr = case lookup "style" kvs of

582584 Just stls -> (ident, cls, pickStylesToKVs ["width", "height"] stls)

583585 Nothing -> (ident, cls, kvs)

584− src <- many1 (noneOf " \t\n\r!(")

585− alt <- option "" $ try $ char '(' *> manyTill anyChar (char ')')

586+ src <- T.pack <$> many1 (noneOf " \t\n\r!(")

587+ alt <- fmap T.pack $ option "" $ try $ char '(' *> manyTill anyChar (char ')')

586588 char '!'

587589 return $ B.imageWith attr src alt (B.str alt)

588590

589−escapedInline :: PandocMonad m => ParserT [Char] ParserState m Inlines

591+escapedInline :: PandocMonad m => ParserT Text ParserState m Inlines

590592escapedInline = escapedEqs <|> escapedTag

591593

592−escapedEqs :: PandocMonad m => ParserT [Char] ParserState m Inlines

593−escapedEqs = B.str <$>

594+escapedEqs :: PandocMonad m => ParserT Text ParserState m Inlines

595+escapedEqs = B.str . T.pack <$>

594596 try (string "==" *> manyTill anyChar' (try $ string "=="))

595597

596598-- | literal text escaped btw <notextile> tags

597−escapedTag :: PandocMonad m => ParserT [Char] ParserState m Inlines

598−escapedTag = B.str <$>

599+escapedTag :: PandocMonad m => ParserT Text ParserState m Inlines

600+escapedTag = B.str . T.pack <$>

599601 try (string "<notextile>" *>

600602 manyTill anyChar' (try $ string "</notextile>"))

601603

602604-- | Any special symbol defined in wordBoundaries

603−symbol :: PandocMonad m => ParserT [Char] ParserState m Inlines

604−symbol = B.str . singleton <$> (notFollowedBy newline *>

605− notFollowedBy rawHtmlBlock *>

606− oneOf wordBoundaries)

605+symbol :: PandocMonad m => ParserT Text ParserState m Inlines

606+symbol = B.str . T.singleton <$> (notFollowedBy newline *>

607+ notFollowedBy rawHtmlBlock *>

608+ oneOf wordBoundaries)

607609

608610-- | Inline code

609−code :: PandocMonad m => ParserT [Char] ParserState m Inlines

611+code :: PandocMonad m => ParserT Text ParserState m Inlines

610612code = code1 <|> code2

611613

612614-- any character except a newline before a blank line

613−anyChar' :: PandocMonad m => ParserT [Char] ParserState m Char

615+anyChar' :: PandocMonad m => ParserT Text ParserState m Char

614616anyChar' =

615617 satisfy (/='\n') <|>

616618 try (char '\n' <* notFollowedBy blankline)

617619

618−code1 :: PandocMonad m => ParserT [Char] ParserState m Inlines

619−code1 = B.code <$> surrounded (char '@') anyChar'

620+code1 :: PandocMonad m => ParserT Text ParserState m Inlines

621+code1 = B.code . T.pack <$> surrounded (char '@') anyChar'

620622

621−code2 :: PandocMonad m => ParserT [Char] ParserState m Inlines

623+code2 :: PandocMonad m => ParserT Text ParserState m Inlines

622624code2 = do

623625 htmlTag (tagOpen (=="tt") null)

624− B.code <$> manyTill anyChar' (try $ htmlTag $ tagClose (=="tt"))

626+ B.code . T.pack <$> manyTill anyChar' (try $ htmlTag $ tagClose (=="tt"))

625627

626628-- | Html / CSS attributes

627−attributes :: PandocMonad m => ParserT [Char] ParserState m Attr

629+attributes :: PandocMonad m => ParserT Text ParserState m Attr

628630attributes = foldl (flip ($)) ("",[],[]) <$>

629631 try (do special <- option id specialAttribute

630632 attrs <- many attribute

631633 return (special : attrs))

632634

633−specialAttribute :: PandocMonad m => ParserT [Char] ParserState m (Attr -> Attr)

635+specialAttribute :: PandocMonad m => ParserT Text ParserState m (Attr -> Attr)

634636specialAttribute = do

635637 alignStr <- ("center" <$ char '=') <|>

636638 ("justify" <$ try (string "<>")) <|>

637639 ("right" <$ char '>') <|>

638640 ("left" <$ char '<')

639641 notFollowedBy spaceChar

640− return $ addStyle ("text-align:" ++ alignStr)

642+ return $ addStyle $ T.pack $ "text-align:" ++ alignStr

641643

642−attribute :: PandocMonad m => ParserT [Char] ParserState m (Attr -> Attr)

644+attribute :: PandocMonad m => ParserT Text ParserState m (Attr -> Attr)

643645attribute = try $

644646 (classIdAttr <|> styleAttr <|> langAttr) <* notFollowedBy spaceChar

645647

646−classIdAttr :: PandocMonad m => ParserT [Char] ParserState m (Attr -> Attr)

648+classIdAttr :: PandocMonad m => ParserT Text ParserState m (Attr -> Attr)

647649classIdAttr = try $ do -- (class class #id)

648650 char '('

649− ws <- words `fmap` manyTill anyChar' (char ')')

651+ ws <- T.words `fmap` T.pack <$> manyTill anyChar' (char ')')

650652 case reverse ws of

651− [] -> return $ \(_,_,keyvals) -> ("",[],keyvals)

652− (('#':ident'):classes') -> return $ \(_,_,keyvals) ->

653− (ident',classes',keyvals)

654− classes' -> return $ \(_,_,keyvals) ->

655− ("",classes',keyvals)

656−

657−styleAttr :: PandocMonad m => ParserT [Char] ParserState m (Attr -> Attr)

653+ []

654+ -> return $ \(_,_,keyvals) -> ("",[],keyvals)

655+ ((T.uncons -> Just ('#', ident')):classes')

656+ -> return $ \(_,_,keyvals) -> (ident',classes',keyvals)

657+ classes'

658+ -> return $ \(_,_,keyvals) -> ("",classes',keyvals)

659+

660+styleAttr :: PandocMonad m => ParserT Text ParserState m (Attr -> Attr)

658661styleAttr = do

659662 style <- try $ enclosed (char '{') (char '}') anyChar'

660− return $ addStyle style

663+ return $ addStyle $ T.pack style

661664

662−addStyle :: String -> Attr -> Attr

665+addStyle :: Text -> Attr -> Attr

663666addStyle style (id',classes,keyvals) =

664667 (id',classes,keyvals')

665668 where keyvals' = ("style", style') : [(k,v) | (k,v) <- keyvals, k /= "style"]

666− style' = style ++ ";" ++ concat [v | ("style",v) <- keyvals]

669+ style' = style <> ";" <> T.concat [v | ("style",v) <- keyvals]

667670

668−langAttr :: PandocMonad m => ParserT [Char] ParserState m (Attr -> Attr)

671+langAttr :: PandocMonad m => ParserT Text ParserState m (Attr -> Attr)

669672langAttr = do

670673 lang <- try $ enclosed (char '[') (char ']') alphaNum

671− return $ \(id',classes,keyvals) -> (id',classes,("lang",lang):keyvals)

674+ return $ \(id',classes,keyvals) -> (id',classes,("lang",T.pack lang):keyvals)

672675

673676-- | Parses material surrounded by a parser.

674677surrounded :: (PandocMonad m, Show t)

675− => ParserT [Char] st m t -- ^ surrounding parser

676− -> ParserT [Char] st m a -- ^ content parser (to be used repeatedly)

677− -> ParserT [Char] st m [a]

678+ => ParserT Text st m t -- ^ surrounding parser

679+ -> ParserT Text st m a -- ^ content parser (to be used repeatedly)

680+ -> ParserT Text st m [a]

678681surrounded border =

679682 enclosed (border *> notFollowedBy (oneOf " \t\n\r")) (try border)

680683

681684simpleInline :: PandocMonad m

682− => ParserT [Char] ParserState m t -- ^ surrounding parser

685+ => ParserT Text ParserState m t -- ^ surrounding parser

683686 -> (Inlines -> Inlines) -- ^ Inline constructor

684− -> ParserT [Char] ParserState m Inlines -- ^ content parser (to be used repeatedly)

687+ -> ParserT Text ParserState m Inlines -- ^ content parser (to be used repeatedly)

685688simpleInline border construct = try $ do

686689 notAfterString

687690 border *> notFollowedBy (oneOf " \t\n\r")

@@ -695,7 +698,7 @@ simpleInline border construct = try $ do

695698 then body

696699 else B.spanWith attr body

697700

698−groupedInlineMarkup :: PandocMonad m => ParserT [Char] ParserState m Inlines

701+groupedInlineMarkup :: PandocMonad m => ParserT Text ParserState m Inlines

699702groupedInlineMarkup = try $ do

700703 char '['

701704 sp1 <- option mempty $ B.space <$ whitespace

@@ -704,9 +707,5 @@ groupedInlineMarkup = try $ do

704707 char ']'

705708 return $ sp1 <> result <> sp2

706709

707−-- | Create a singleton list

708−singleton :: a -> [a]

709−singleton x = [x]

710−

711−eof' :: Monad m => ParserT [Char] s m Char

710+eof' :: Monad m => ParserT Text s m Char

712711eof' = '\n' <$ eof

src/Text/Pandoc/Readers/TikiWiki.hs+47−47

@@ -43,19 +43,19 @@ readTikiWiki :: PandocMonad m

4343 -> m Pandoc

4444readTikiWiki opts s = do

4545 res <- readWithM parseTikiWiki def{ stateOptions = opts }

46− (T.unpack (crFilter s) ++ "\n\n")

46+ (crFilter s <> "\n\n")

4747 case res of

4848 Left e -> throwError e

4949 Right d -> return d

5050

51−type TikiWikiParser = ParserT [Char] ParserState

51+type TikiWikiParser = ParserT Text ParserState

5252

5353--

5454-- utility functions

5555--

5656

57−tryMsg :: String -> TikiWikiParser m a -> TikiWikiParser m a

58−tryMsg msg p = try p <?> msg

57+tryMsg :: Text -> TikiWikiParser m a -> TikiWikiParser m a

58+tryMsg msg p = try p <?> (T.unpack msg)

5959

6060skip :: TikiWikiParser m a -> TikiWikiParser m ()

6161skip parser = Control.Monad.void parser

@@ -89,7 +89,7 @@ block = do

8989 <|> para

9090 skipMany blankline

9191 when (verbosity >= INFO) $

92− trace (printf "line %d: %s" (sourceLine pos) (take 60 $ show $ B.toList res))

92+ trace (T.pack $ printf "line %d: %s" (sourceLine pos) (take 60 $ show $ B.toList res))

9393 return res

9494

9595blockElements :: PandocMonad m => TikiWikiParser m B.Blocks

@@ -133,7 +133,7 @@ tableRow :: PandocMonad m => TikiWikiParser m [B.Blocks]

133133tableRow = try $ do

134134-- row <- sepBy1 (many1Till inline $ oneOf "\n|") (try $ string "|" <* notFollowedBy (oneOf "|\n"))

135135-- return $ map (B.plain . mconcat) row

136− row <- sepBy1 (many1 (noneOf "\n|") >>= parseColumn) (try $ string "|" <* notFollowedBy (oneOf "|\n"))

136+ row <- sepBy1 (many1 (noneOf "\n|") >>= parseColumn . T.pack) (try $ string "|" <* notFollowedBy (oneOf "|\n"))

137137 return $ map B.plain row

138138 where

139139 parseColumn x = do

@@ -342,15 +342,15 @@ listItemLine nest = lineContent >>= parseContent

342342 lineContent = do

343343 content <- anyLine

344344 continuation <- optionMaybe listContinuation

345− return $ filterSpaces content ++ "\n" ++ Data.Maybe.fromMaybe "" continuation

346− filterSpaces = reverse . dropWhile (== ' ') . reverse

345+ return $ filterSpaces content <> "\n" <> Data.Maybe.fromMaybe "" continuation

346+ filterSpaces = T.dropWhileEnd (== ' ')

347347 listContinuation = string (replicate nest '+') >> lineContent

348348 parseContent x = do

349349 parsed <- parseFromString (many1 inline) x

350350 return $ mconcat $ dropWhileEnd (== B.space) parsed

351351

352352-- Turn the CODE macro attributes into Pandoc code block attributes.

353−mungeAttrs :: [(String, String)] -> (String, [String], [(String, String)])

353+mungeAttrs :: [(Text, Text)] -> (Text, [Text], [(Text, Text)])

354354mungeAttrs rawAttrs = ("", classes, rawAttrs)

355355 where

356356 -- "colors" is TikiWiki CODE macro for "name of language to do

@@ -370,7 +370,7 @@ codeMacro = try $ do

370370 string "{CODE("

371371 rawAttrs <- macroAttrs

372372 string ")}"

373− body <- manyTill anyChar (try (string "{CODE}"))

373+ body <- T.pack <$> manyTill anyChar (try (string "{CODE}"))

374374 newline

375375 if not (null rawAttrs)

376376 then

@@ -428,9 +428,9 @@ nbsp = try $ do

428428htmlComment :: PandocMonad m => TikiWikiParser m B.Inlines

429429htmlComment = try $ do

430430 string "~hc~"

431− inner <- many1 $ noneOf "~"

431+ inner <- fmap T.pack $ many1 $ noneOf "~"

432432 string "~/hc~"

433− return $ B.str $ " NOT SUPPORTED: ~hc~ (html comment opener) BEGIN: " ++ inner ++ " ~/hc~ :END "

433+ return $ B.str $ " NOT SUPPORTED: ~hc~ (html comment opener) BEGIN: " <> inner <> " ~/hc~ :END "

434434

435435linebreak :: PandocMonad m => TikiWikiParser m B.Inlines

436436linebreak = newline >> notFollowedBy newline >> (lastNewline <|> innerNewline)

@@ -469,15 +469,15 @@ image = try $ do

469469 let title = fromMaybe src $ lookup "desc" rawAttrs

470470 let alt = fromMaybe title $ lookup "alt" rawAttrs

471471 let classes = map fst $ filter (\(_,b) -> b == "" || b == "y") rawAttrs

472− if not (null src)

472+ if not (T.null src)

473473 then

474474 return $ B.imageWith ("", classes, rawAttrs) src title (B.str alt)

475475 else

476− return $ B.str $ " NOT SUPPORTED: image without src attribute BEGIN: {img " ++ printAttrs rawAttrs ++ "} :END "

476+ return $ B.str $ " NOT SUPPORTED: image without src attribute BEGIN: {img " <> printAttrs rawAttrs <> "} :END "

477477 where

478− printAttrs attrs = unwords $ map (\(a, b) -> a ++ "=\"" ++ b ++ "\"") attrs

478+ printAttrs attrs = T.unwords $ map (\(a, b) -> a <> "=\"" <> b <> "\"") attrs

479479

480−imageAttr :: PandocMonad m => TikiWikiParser m (String, String)

480+imageAttr :: PandocMonad m => TikiWikiParser m (Text, Text)

481481imageAttr = try $ do

482482 key <- many1 (noneOf "=} \t\n")

483483 char '='

@@ -485,7 +485,7 @@ imageAttr = try $ do

485485 value <- many1 (noneOf "}\"\n")

486486 optional $ char '"'

487487 optional $ char ','

488− return (key, value)

488+ return (T.pack key, T.pack value)

489489

490490

491491-- __strong__

@@ -500,57 +500,57 @@ emph = try $ fmap B.emph (enclosed (string "''") nestedInlines)

500500escapedChar :: PandocMonad m => TikiWikiParser m B.Inlines

501501escapedChar = try $ do

502502 string "~"

503− mNumber <- safeRead <$> many1 digit

503+ mNumber <- safeRead . T.pack <$> many1 digit

504504 string "~"

505505 return $ B.str $

506506 case mNumber of

507− Just number -> [toEnum (number :: Int) :: Char]

508− Nothing -> []

507+ Just number -> T.singleton $ toEnum (number :: Int)

508+ Nothing -> ""

509509

510510-- UNSUPPORTED, as there doesn't seem to be any facility in calibre

511511-- for this

512512centered :: PandocMonad m => TikiWikiParser m B.Inlines

513513centered = try $ do

514514 string "::"

515− inner <- many1 $ noneOf ":\n"

515+ inner <- fmap T.pack $ many1 $ noneOf ":\n"

516516 string "::"

517− return $ B.str $ " NOT SUPPORTED: :: (centered) BEGIN: ::" ++ inner ++ ":: :END "

517+ return $ B.str $ " NOT SUPPORTED: :: (centered) BEGIN: ::" <> inner <> ":: :END "

518518

519519-- UNSUPPORTED, as there doesn't seem to be any facility in calibre

520520-- for this

521521colored :: PandocMonad m => TikiWikiParser m B.Inlines

522522colored = try $ do

523523 string "~~"

524− inner <- many1 $ noneOf "~\n"

524+ inner <- fmap T.pack $ many1 $ noneOf "~\n"

525525 string "~~"

526− return $ B.str $ " NOT SUPPORTED: ~~ (colored) BEGIN: ~~" ++ inner ++ "~~ :END "

526+ return $ B.str $ " NOT SUPPORTED: ~~ (colored) BEGIN: ~~" <> inner <> "~~ :END "

527527

528528-- UNSUPPORTED, as there doesn't seem to be any facility in calibre

529529-- for this

530530underlined :: PandocMonad m => TikiWikiParser m B.Inlines

531531underlined = try $ do

532532 string "==="

533− inner <- many1 $ noneOf "=\n"

533+ inner <- fmap T.pack $ many1 $ noneOf "=\n"

534534 string "==="

535− return $ B.str $ " NOT SUPPORTED: ==== (underlined) BEGIN: ===" ++ inner ++ "=== :END "

535+ return $ B.str $ " NOT SUPPORTED: ==== (underlined) BEGIN: ===" <> inner <> "=== :END "

536536

537537-- UNSUPPORTED, as there doesn't seem to be any facility in calibre

538538-- for this

539539boxed :: PandocMonad m => TikiWikiParser m B.Inlines

540540boxed = try $ do

541541 string "^"

542− inner <- many1 $ noneOf "^\n"

542+ inner <- fmap T.pack $ many1 $ noneOf "^\n"

543543 string "^"

544− return $ B.str $ " NOT SUPPORTED: ^ (boxed) BEGIN: ^" ++ inner ++ "^ :END "

544+ return $ B.str $ " NOT SUPPORTED: ^ (boxed) BEGIN: ^" <> inner <> "^ :END "

545545

546546-- --text--

547547strikeout :: PandocMonad m => TikiWikiParser m B.Inlines

548548strikeout = try $ fmap B.strikeout (enclosed (string "--") nestedInlines)

549549

550−nestedString :: (Show a, PandocMonad m) => TikiWikiParser m a -> TikiWikiParser m String

551−nestedString end = innerSpace <|> count 1 nonspaceChar

550+nestedString :: (Show a, PandocMonad m) => TikiWikiParser m a -> TikiWikiParser m Text

551+nestedString end = innerSpace <|> countChar 1 nonspaceChar

552552 where

553− innerSpace = try $ many1 spaceChar <* notFollowedBy end

553+ innerSpace = try $ T.pack <$> many1 spaceChar <* notFollowedBy end

554554

555555breakChars :: PandocMonad m => TikiWikiParser m B.Inlines

556556breakChars = try $ string "%%%" >> return B.linebreak

@@ -564,7 +564,7 @@ superMacro = try $ do

564564 string "{SUP("

565565 manyTill anyChar (string ")}")

566566 body <- manyTill anyChar (string "{SUP}")

567− return $ B.superscript $ B.text body

567+ return $ B.superscript $ B.text $ T.pack body

568568

569569-- subscript: baz{TAG(tag=>sub)}sub{TAG}qux / qux{SUB()}sub2{SUB}qux

570570subTag :: PandocMonad m => TikiWikiParser m B.Inlines

@@ -575,22 +575,22 @@ subMacro = try $ do

575575 string "{SUB("

576576 manyTill anyChar (string ")}")

577577 body <- manyTill anyChar (string "{SUB}")

578− return $ B.subscript $ B.text body

578+ return $ B.subscript $ B.text $ T.pack body

579579

580580-- -+text+-

581581code :: PandocMonad m => TikiWikiParser m B.Inlines

582582code = try $ fmap (B.code . fromEntities) ( between (string "-+") (string "+-") nestedString)

583583

584−macroAttr :: PandocMonad m => TikiWikiParser m (String, String)

584+macroAttr :: PandocMonad m => TikiWikiParser m (Text, Text)

585585macroAttr = try $ do

586586 key <- many1 (noneOf "=)")

587587 char '='

588588 optional $ char '"'

589589 value <- many1 (noneOf " )\"")

590590 optional $ char '"'

591− return (key, value)

591+ return (T.pack key, T.pack value)

592592

593−macroAttrs :: PandocMonad m => TikiWikiParser m [(String, String)]

593+macroAttrs :: PandocMonad m => TikiWikiParser m [(Text, Text)]

594594macroAttrs = try $ sepEndBy macroAttr spaces

595595

596596-- ~np~ __not bold__ ~/np~

@@ -598,13 +598,13 @@ noparse :: PandocMonad m => TikiWikiParser m B.Inlines

598598noparse = try $ do

599599 string "~np~"

600600 body <- manyTill anyChar (string "~/np~")

601− return $ B.str body

601+ return $ B.str $ T.pack body

602602

603603str :: PandocMonad m => TikiWikiParser m B.Inlines

604−str = fmap B.str (many1 alphaNum <|> count 1 characterReference)

604+str = fmap B.str (T.pack <$> many1 alphaNum <|> countChar 1 characterReference)

605605

606606symbol :: PandocMonad m => TikiWikiParser m B.Inlines

607−symbol = fmap B.str (count 1 nonspaceChar)

607+symbol = fmap B.str (countChar 1 nonspaceChar)

608608

609609-- [[not a link]

610610notExternalLink :: PandocMonad m => TikiWikiParser m B.Inlines

@@ -612,14 +612,14 @@ notExternalLink = try $ do

612612 start <- string "[["

613613 body <- many (noneOf "\n[]")

614614 end <- string "]"

615− return $ B.text (start ++ body ++ end)

615+ return $ B.text $ T.pack $ start ++ body ++ end

616616

617617-- [http://www.somesite.org url|Some Site title]

618618-- ((internal link))

619619--

620620-- The ((...)) wiki links and [...] external links are handled

621621-- exactly the same; this abstracts that out

622−makeLink :: PandocMonad m => String -> String -> String -> TikiWikiParser m B.Inlines

622+makeLink :: PandocMonad m => Text -> Text -> Text -> TikiWikiParser m B.Inlines

623623makeLink start middle end = try $ do

624624 st <- getState

625625 guard $ stateAllowLinks st

@@ -627,15 +627,15 @@ makeLink start middle end = try $ do

627627 (url, title, anchor) <- wikiLinkText start middle end

628628 parsedTitle <- parseFromString (many1 inline) title

629629 setState $ st{ stateAllowLinks = True }

630− return $ B.link (url++anchor) "" $mconcat parsedTitle

630+ return $ B.link (url <> anchor) "" $ mconcat parsedTitle

631631

632−wikiLinkText :: PandocMonad m => String -> String -> String -> TikiWikiParser m (String, String, String)

632+wikiLinkText :: PandocMonad m => Text -> Text -> Text -> TikiWikiParser m (Text, Text, Text)

633633wikiLinkText start middle end = do

634− string start

635− url <- many1 (noneOf $ middle ++ "\n")

634+ string (T.unpack start)

635+ url <- T.pack <$> many1 (noneOf $ T.unpack middle ++ "\n")

636636 seg1 <- option url linkContent

637637 seg2 <- option "" linkContent

638− string end

638+ string (T.unpack end)

639639 if seg2 /= ""

640640 then

641641 return (url, seg2, seg1)

@@ -644,7 +644,7 @@ wikiLinkText start middle end = do

644644 where

645645 linkContent = do

646646 char '|'

647− many (noneOf middle)

647+ T.pack <$> many (noneOf $ T.unpack middle)

648648

649649externalLink :: PandocMonad m => TikiWikiParser m B.Inlines

650650externalLink = makeLink "[" "]|" "]"

src/Text/Pandoc/Readers/Txt2Tags.hs+66−66

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Readers.Txt2Tags

@@ -18,7 +19,6 @@ import Prelude

1819import Control.Monad (guard, void, when)

1920import Control.Monad.Except (catchError, throwError)

2021import Control.Monad.Reader (Reader, asks, runReader)

21−import Data.Char (toLower)

2222import Data.Default

2323import Data.List (intercalate, transpose)

2424import Data.Maybe (fromMaybe)

@@ -36,13 +36,13 @@ import Text.Pandoc.Parsing hiding (space, spaces, uri)

3636import Text.Pandoc.Shared (compactify, compactifyDL, crFilter, escapeURI,

3737 underlineSpan)

3838

39−type T2T = ParserT String ParserState (Reader T2TMeta)

39+type T2T = ParserT Text ParserState (Reader T2TMeta)

4040

4141-- | An object for the T2T macros meta information

4242-- the contents of each field is simply substituted verbatim into the file

4343data T2TMeta = T2TMeta {

44− date :: String -- ^ Current date

45− , mtime :: String -- ^ Last modification time of infile

44+ date :: Text -- ^ Current date

45+ , mtime :: Text -- ^ Last modification time of infile

4646 , infile :: FilePath -- ^ Input file

4747 , outfile :: FilePath -- ^ Output file

4848 } deriving Show

@@ -63,7 +63,7 @@ getT2TMeta = do

6363 _ -> catchError

6464 (maximum <$> mapM getModTime inps)

6565 (const (return ""))

66− return $ T2TMeta curDate curMtime (intercalate ", " inps) outp

66+ return $ T2TMeta (T.pack curDate) (T.pack curMtime) (intercalate ", " inps) outp

6767

6868-- | Read Txt2Tags from an input string returning a Pandoc document

6969readTxt2Tags :: PandocMonad m

@@ -74,14 +74,14 @@ readTxt2Tags opts s = do

7474 meta <- getT2TMeta

7575 let parsed = flip runReader meta $

7676 readWithM parseT2T (def {stateOptions = opts}) $

77− T.unpack (crFilter s) ++ "\n\n"

77+ crFilter s <> "\n\n"

7878 case parsed of

7979 Right result -> return result

8080 Left e -> throwError e

8181

8282-- | Read Txt2Tags (ignoring all macros) from an input string returning

8383-- a Pandoc document

84−-- readTxt2TagsNoMacros :: PandocMonad m => ReaderOptions -> String -> m Pandoc

84+-- readTxt2TagsNoMacros :: PandocMonad m => ReaderOptions -> Text -> m Pandoc

8585-- readTxt2TagsNoMacros = readTxt2Tags

8686

8787parseT2T :: T2T Pandoc

@@ -106,7 +106,7 @@ parseHeader = do

106106header :: T2T ()

107107header = titleline >> authorline >> dateline

108108

109−headerline :: B.ToMetaValue a => String -> T2T a -> T2T ()

109+headerline :: B.ToMetaValue a => Text -> T2T a -> T2T ()

110110headerline field p = (() <$ try blankline)

111111 <|> (p >>= updateState . B.setMeta field)

112112

@@ -123,15 +123,15 @@ authorline =

123123dateline :: T2T ()

124124dateline = headerline "date" (trimInlines . mconcat <$> manyTill inline newline)

125125

126−type Keyword = String

127−type Value = String

126+type Keyword = Text

127+type Value = Text

128128

129129setting :: T2T (Keyword, Value)

130130setting = do

131131 string "%!"

132− keyword <- ignoreSpacesCap (many1 alphaNum)

132+ keyword <- ignoreSpacesCap (many1Char alphaNum)

133133 char ':'

134− value <- ignoreSpacesCap (manyTill anyChar newline)

134+ value <- ignoreSpacesCap (manyTillChar anyChar newline)

135135 return (keyword, value)

136136

137137-- Blocks

@@ -163,10 +163,10 @@ balancedTitle c = try $ do

163163 spaces

164164 level <- length <$> many1 (char c)

165165 guard (level <= 5) -- Max header level 5

166− heading <- manyTill (noneOf "\n\r") (count level (char c))

166+ heading <- manyTillChar (noneOf "\n\r") (count level (char c))

167167 label <- optionMaybe (enclosed (char '[') (char ']') (alphaNum <|> oneOf "_-"))

168168 many spaceChar *> newline

169− let attr = maybe nullAttr (\x -> (x, [], [])) label

169+ let attr = maybe nullAttr (\x -> (T.pack x, [], [])) label

170170 return $ B.headerWith attr level (trimInlines $ B.text heading)

171171

172172para :: T2T Blocks

@@ -192,7 +192,7 @@ quote :: T2T Blocks

192192quote = try $ do

193193 lookAhead tab

194194 rawQuote <- many1 (tab *> optional spaces *> anyLine)

195− contents <- parseFromString' parseBlocks (intercalate "\n" rawQuote ++ "\n\n")

195+ contents <- parseFromString' parseBlocks (T.intercalate "\n" rawQuote <> "\n\n")

196196 return $ B.blockQuote contents

197197

198198commentLine :: T2T Inlines

@@ -243,17 +243,17 @@ listItem start end = try $ do

243243 markerLength <- try start

244244 firstLine <- anyLineNewline

245245 blank <- option "" ("\n" <$ blankline)

246− rest <- concat <$> many (listContinuation markerLength)

247− parseFromString' end $ firstLine ++ blank ++ rest

246+ rest <- T.concat <$> many (listContinuation markerLength)

247+ parseFromString' end $ firstLine <> blank <> rest

248248

249249-- continuation of a list item - indented and separated by blankline or endline.

250250-- Note: nested lists are parsed as continuations.

251251listContinuation :: Int

252− -> T2T String

252+ -> T2T Text

253253listContinuation markerLength = try $

254254 notFollowedBy' (blankline >> blankline)

255− *> (mappend <$> (concat <$> many1 listLine)

256− <*> many blankline)

255+ *> (mappend <$> (T.concat <$> many1 listLine)

256+ <*> manyChar blankline)

257257 where listLine = try $ indentWith markerLength *> anyLineNewline

258258

259259-- Table

@@ -327,16 +327,16 @@ taggedBlock = do

327327

328328-- Generic

329329

330−genericBlock :: Monoid a => T2T a -> (a -> Blocks) -> String -> T2T Blocks

330+genericBlock :: Monoid a => T2T a -> (a -> Blocks) -> Text -> T2T Blocks

331331genericBlock p f s = blockMarkupArea p f s <|> blockMarkupLine p f s

332332

333−blockMarkupArea :: Monoid a => T2T a -> (a -> Blocks) -> String -> T2T Blocks

333+blockMarkupArea :: Monoid a => T2T a -> (a -> Blocks) -> Text -> T2T Blocks

334334blockMarkupArea p f s = try (do

335− string s *> blankline

336− f . mconcat <$> manyTill p (eof <|> void (string s *> blankline)))

335+ textStr s *> blankline

336+ f . mconcat <$> manyTill p (eof <|> void (textStr s *> blankline)))

337337

338−blockMarkupLine :: T2T a -> (a -> Blocks) -> String -> T2T Blocks

339−blockMarkupLine p f s = try (f <$> (string s *> space *> p))

338+blockMarkupLine :: T2T a -> (a -> Blocks) -> Text -> T2T Blocks

339+blockMarkupLine p f s = try (f <$> (textStr s *> space *> p))

340340

341341-- Can be in either block or inline position

342342comment :: Monoid a => T2T a

@@ -385,15 +385,15 @@ italic :: T2T Inlines

385385italic = inlineMarkup inline B.emph '/' B.str

386386

387387code :: T2T Inlines

388−code = inlineMarkup ((:[]) <$> anyChar) B.code '`' id

388+code = inlineMarkup (T.singleton <$> anyChar) B.code '`' id

389389

390390raw :: T2T Inlines

391−raw = inlineMarkup ((:[]) <$> anyChar) B.text '"' id

391+raw = inlineMarkup (T.singleton <$> anyChar) B.text '"' id

392392

393393tagged :: T2T Inlines

394394tagged = do

395395 target <- getTarget

396− inlineMarkup ((:[]) <$> anyChar) (B.rawInline target) '\'' id

396+ inlineMarkup (T.singleton <$> anyChar) (B.rawInline target) '\'' id

397397

398398-- Parser for markup indicated by a double character.

399399-- Inline markup is greedy and glued

@@ -404,33 +404,33 @@ inlineMarkup :: Monoid a

404404 => T2T a -- Content parser

405405 -> (a -> Inlines) -- Constructor

406406 -> Char -- Fence

407− -> (String -> a) -- Special Case to handle ******

407+ -> (Text -> a) -- Special Case to handle ******

408408 -> T2T Inlines

409409inlineMarkup p f c special = try $ do

410− start <- many1 (char c)

411− let l = length start

410+ start <- many1Char (char c)

411+ let l = T.length start

412412 guard (l >= 2)

413413 when (l == 2) (void $ notFollowedBy space)

414414 -- We must make sure that there is no space before the start of the

415415 -- closing tags

416− body <- optionMaybe (try $ manyTill (noneOf "\n\r")

416+ body <- optionMaybe (try $ manyTillChar (noneOf "\n\r")

417417 (try $ lookAhead (noneOf " " >> string [c,c] )))

418418 case body of

419419 Just middle -> do

420420 lastChar <- anyChar

421− end <- many1 (char c)

421+ end <- many1Char (char c)

422422 let parser inp = parseFromString' (mconcat <$> many p) inp

423− let start' = case drop 2 start of

423+ let start' = case T.drop 2 start of

424424 "" -> mempty

425425 xs -> special xs

426− body' <- parser (middle ++ [lastChar])

427− let end' = case drop 2 end of

426+ body' <- parser (middle <> T.singleton lastChar)

427+ let end' = case T.drop 2 end of

428428 "" -> mempty

429429 xs -> special xs

430430 return $ f (start' `mappend` body' `mappend` end')

431431 Nothing -> do -- Either bad or case such as *****

432432 guard (l >= 5)

433− let body' = replicate (l - 4) c

433+ let body' = T.replicate (l - 4) $ T.singleton c

434434 return $ f (special body')

435435

436436link :: T2T Inlines

@@ -441,12 +441,12 @@ titleLink :: T2T Inlines

441441titleLink = try $ do

442442 char '['

443443 notFollowedBy space

444− tokens <- sepBy1 (many $ noneOf " ]") space

444+ tokens <- sepBy1 (manyChar $ noneOf " ]") space

445445 guard (length tokens >= 2)

446446 char ']'

447447 let link' = last tokens

448− guard $ not $ null link'

449− let tit = unwords (init tokens)

448+ guard $ not $ T.null link'

449+ let tit = T.unwords (init tokens)

450450 return $ B.link link' "" (B.text tit)

451451

452452-- Link with image

@@ -455,7 +455,7 @@ imageLink = try $ do

455455 char '['

456456 body <- image

457457 many1 space

458− l <- manyTill (noneOf "\n\r ") (char ']')

458+ l <- manyTillChar (noneOf "\n\r ") (char ']')

459459 return (B.link l "" body)

460460

461461macro :: T2T Inlines

@@ -466,7 +466,7 @@ macro = try $ do

466466 maybe (return mempty) (\f -> B.str <$> asks f) (lookup name commands)

467467 where

468468 commands = [ ("date", date), ("mtime", mtime)

469− , ("infile", infile), ("outfile", outfile)]

469+ , ("infile", T.pack . infile), ("outfile", T.pack . outfile)]

470470

471471-- raw URLs in text are automatically linked

472472url :: T2T Inlines

@@ -474,7 +474,7 @@ url = try $ do

474474 (rawUrl, escapedUrl) <- try uri <|> emailAddress

475475 return $ B.link rawUrl "" (B.str escapedUrl)

476476

477−uri :: T2T (String, String)

477+uri :: T2T (Text, Text)

478478uri = try $ do

479479 address <- t2tURI

480480 return (address, escapeURI address)

@@ -486,25 +486,25 @@ uri = try $ do

486486--isT2TURI (parse t2tURI "" -> Right _) = True

487487--isT2TURI _ = False

488488

489−t2tURI :: T2T String

489+t2tURI :: T2T Text

490490t2tURI = do

491− start <- try ((++) <$> proto <*> urlLogin) <|> guess

492− domain <- many1 chars

493− sep <- many (char '/')

494− form' <- option mempty ((:) <$> char '?' <*> many1 form)

495− anchor' <- option mempty ((:) <$> char '#' <*> many anchor)

496− return (start ++ domain ++ sep ++ form' ++ anchor')

491+ start <- try ((<>) <$> proto <*> urlLogin) <|> guess

492+ domain <- many1Char chars

493+ sep <- manyChar (char '/')

494+ form' <- option mempty (T.cons <$> char '?' <*> many1Char form)

495+ anchor' <- option mempty (T.cons <$> char '#' <*> manyChar anchor)

496+ return (start <> domain <> sep <> form' <> anchor')

497497 where

498498 protos = ["http", "https", "ftp", "telnet", "gopher", "wais"]

499− proto = (++) <$> oneOfStrings protos <*> string "://"

500− guess = (++) <$> (((++) <$> stringAnyCase "www" <*> option mempty ((:[]) <$> oneOf "23"))

501− <|> stringAnyCase "ftp") <*> ((:[]) <$> char '.')

499+ proto = (<>) <$> oneOfStrings protos <*> textStr "://"

500+ guess = (<>) <$> (((<>) <$> stringAnyCase "www" <*> option mempty (T.singleton <$> oneOf "23"))

501+ <|> stringAnyCase "ftp") <*> (T.singleton <$> char '.')

502502 login = alphaNum <|> oneOf "_.-"

503− pass = many (noneOf " @")

503+ pass = manyChar (noneOf " @")

504504 chars = alphaNum <|> oneOf "%._/~:,=$@&+-"

505505 anchor = alphaNum <|> oneOf "%._0"

506506 form = chars <|> oneOf ";*"

507− urlLogin = option mempty $ try ((\x y z -> x ++ y ++ [z]) <$> many1 login <*> option mempty ((:) <$> char ':' <*> pass) <*> char '@')

507+ urlLogin = option mempty $ try ((\x y z -> x <> y <> T.singleton z) <$> many1Char login <*> option mempty (T.cons <$> char ':' <*> pass) <*> char '@')

508508

509509

510510image :: T2T Inlines

@@ -512,12 +512,12 @@ image = try $ do

512512 -- List taken from txt2tags source

513513 let extensions = [".jpg", ".jpeg", ".gif", ".png", ".eps", ".bmp"]

514514 char '['

515− (path, ext) <- manyUntil (noneOf "\n\t\r ") (oneOfStrings extensions)

515+ (path, ext) <- manyUntilChar (noneOf "\n\t\r ") (oneOfStrings extensions)

516516 char ']'

517− return $ B.image (path ++ ext) "" mempty

517+ return $ B.image (path <> ext) "" mempty

518518

519519-- Characters used in markup

520−specialChars :: String

520+specialChars :: [Char]

521521specialChars = "%*-_/|:+;"

522522

523523tab :: T2T Char

@@ -526,8 +526,8 @@ tab = char '\t'

526526space :: T2T Char

527527space = char ' '

528528

529−spaces :: T2T String

530−spaces = many space

529+spaces :: T2T Text

530+spaces = manyChar space

531531

532532endline :: T2T Inlines

533533endline = try $ do

@@ -544,17 +544,17 @@ endline = try $ do

544544 return B.softbreak

545545

546546str :: T2T Inlines

547−str = try $ B.str <$> many1 (noneOf $ specialChars ++ "\n\r ")

547+str = try $ B.str <$> many1Char (noneOf $ specialChars ++ "\n\r ")

548548

549549whitespace :: T2T Inlines

550550whitespace = try $ B.space <$ spaceChar

551551

552552symbol :: T2T Inlines

553−symbol = B.str . (:[]) <$> oneOf specialChars

553+symbol = B.str . T.singleton <$> oneOf specialChars

554554

555555-- Utility

556556

557−getTarget :: T2T String

557+getTarget :: T2T Text

558558getTarget = do

559559 mv <- lookupMeta "target" . stateMeta <$> getState

560560 return $ case mv of

@@ -565,5 +565,5 @@ getTarget = do

565565atStart :: T2T ()

566566atStart = (sourceColumn <$> getPosition) >>= guard . (== 1)

567567

568−ignoreSpacesCap :: T2T String -> T2T String

569−ignoreSpacesCap p = map toLower <$> (spaces *> p <* spaces)

568+ignoreSpacesCap :: T2T Text -> T2T Text

569+ignoreSpacesCap p = T.toLower <$> (spaces *> p <* spaces)

src/Text/Pandoc/Readers/Vimwiki.hs+65−62

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{-# LANGUAGE CPP #-}

34{- |

45 Module : Text.Pandoc.Readers.Vimwiki

@@ -51,9 +52,10 @@ import Prelude

5152import Control.Monad (guard)

5253import Control.Monad.Except (throwError)

5354import Data.Default

54−import Data.List (isInfixOf, isPrefixOf)

55+import Data.List (isInfixOf)

5556import Data.Maybe

5657import Data.Text (Text)

58+import qualified Data.Text as T

5759import Text.Pandoc.Builder (Blocks, Inlines, fromList, toList, trimInlines)

5860import qualified Text.Pandoc.Builder as B (blockQuote, bulletList, code,

5961 codeBlockWith, definitionList,

@@ -73,12 +75,13 @@ import Text.Pandoc.Options (ReaderOptions)

7375import Text.Pandoc.Parsing (ParserState, ParserT, blanklines, emailAddress,

7476 many1Till, orderedListMarker, readWithM,

7577 registerHeader, spaceChar, stateMeta,

76− stateOptions, uri)

77−import Text.Pandoc.Shared (crFilter, splitBy, stringify, stripFirstAndLast,

78− isURI)

78+ stateOptions, uri, manyTillChar, manyChar, textStr,

79+ many1Char, countChar, many1TillChar)

80+import Text.Pandoc.Shared (crFilter, splitTextBy, stringify, stripFirstAndLast,

81+ isURI, tshow)

7982import Text.Parsec.Char (alphaNum, anyChar, char, newline, noneOf, oneOf, space,

8083 spaces, string)

81−import Text.Parsec.Combinator (between, choice, count, eof, lookAhead, many1,

84+import Text.Parsec.Combinator (between, choice, eof, lookAhead, many1,

8285 manyTill, notFollowedBy, option, skipMany1)

8386import Text.Parsec.Prim (getState, many, try, updateState, (<|>))

8487

@@ -128,7 +131,7 @@ block = do

128131 , definitionList

129132 , para

130133 ]

131− trace (take 60 $ show $ toList res)

134+ trace (T.take 60 $ tshow $ toList res)

132135 return res

133136

134137blockML :: PandocMonad m => VwParser m Blocks

@@ -218,32 +221,32 @@ defMarkerM = string "::" >> spaceChar

218221defMarkerE :: PandocMonad m => VwParser m Char

219222defMarkerE = string "::" >> newline

220223

221−hasDefMarkerM :: PandocMonad m => VwParser m String

222−hasDefMarkerM = manyTill (noneOf "\n") (try defMarkerM)

224+hasDefMarkerM :: PandocMonad m => VwParser m Text

225+hasDefMarkerM = manyTillChar (noneOf "\n") (try defMarkerM)

223226

224227preformatted :: PandocMonad m => VwParser m Blocks

225228preformatted = try $ do

226229 many spaceChar >> string "{{{"

227− attrText <- many (noneOf "\n")

230+ attrText <- manyChar (noneOf "\n")

228231 lookAhead newline

229− contents <- manyTill anyChar (try (char '\n' >> many spaceChar >> string "}}}"

232+ contents <- manyTillChar anyChar (try (char '\n' >> many spaceChar >> string "}}}"

230233 >> many spaceChar >> newline))

231− if (contents /= "") && (head contents == '\n')

232− then return $ B.codeBlockWith (makeAttr attrText) (tail contents)

234+ if (contents /= "") && (T.head contents == '\n')

235+ then return $ B.codeBlockWith (makeAttr attrText) (T.tail contents)

233236 else return $ B.codeBlockWith (makeAttr attrText) contents

234237

235−makeAttr :: String -> Attr

238+makeAttr :: Text -> Attr

236239makeAttr s =

237− let xs = splitBy (`elem` " \t") s in

240+ let xs = splitTextBy (`elem` (" \t" :: String)) s in

238241 ("", [], mapMaybe nameValue xs)

239242

240−nameValue :: String -> Maybe (String, String)

243+nameValue :: Text -> Maybe (Text, Text)

241244nameValue s =

242− let t = splitBy (== '=') s in

245+ let t = splitTextBy (== '=') s in

243246 if length t /= 2

244247 then Nothing

245248 else let (a, b) = (head t, last t) in

246− if (length b < 2) || ((head b, last b) /= ('"', '"'))

249+ if (T.length b < 2) || ((T.head b, T.last b) /= ('"', '"'))

247250 then Nothing

248251 else Just (a, stripFirstAndLast b)

249252

@@ -253,16 +256,16 @@ displayMath = try $ do

253256 many spaceChar >> string "{{$"

254257 mathTag <- option "" mathTagParser

255258 many space

256− contents <- manyTill anyChar (try (char '\n' >> many spaceChar >> string "}}$"

259+ contents <- manyTillChar anyChar (try (char '\n' >> many spaceChar >> string "}}$"

257260 >> many spaceChar >> newline))

258261 let contentsWithTags

259262 | mathTag == "" = contents

260− | otherwise = "\\begin{" ++ mathTag ++ "}\n" ++ contents

261− ++ "\n\\end{" ++ mathTag ++ "}"

263+ | otherwise = "\\begin{" <> mathTag <> "}\n" <> contents

264+ <> "\n\\end{" <> mathTag <> "}"

262265 return $ B.para $ B.displayMath contentsWithTags

263266

264267

265−mathTagLaTeX :: String -> String

268+mathTagLaTeX :: Text -> Text

266269mathTagLaTeX s = case s of

267270 "equation" -> ""

268271 "equation*" -> ""

@@ -360,17 +363,17 @@ combineList x [y] = case toList y of

360363 _ -> x:[y]

361364combineList x xs = x:xs

362365

363−listStart :: PandocMonad m => VwParser m (Int, String)

366+listStart :: PandocMonad m => VwParser m (Int, Text)

364367listStart = try $ do

365368 s <- many spaceChar

366369 listType <- bulletListMarkers <|> orderedListMarkers

367370 spaceChar

368371 return (length s, listType)

369372

370−bulletListMarkers :: PandocMonad m => VwParser m String

373+bulletListMarkers :: PandocMonad m => VwParser m Text

371374bulletListMarkers = "ul" <$ (char '*' <|> char '-')

372375

373−orderedListMarkers :: PandocMonad m => VwParser m String

376+orderedListMarkers :: PandocMonad m => VwParser m Text

374377orderedListMarkers =

375378 ("ol" <$choice (orderedListMarker Decimal Period:(($OneParen) . orderedListMarker <$> [Decimal, LowerRoman, UpperRoman, LowerAlpha, UpperAlpha])))

376379 <|> ("ol" <$ char '#')

@@ -421,9 +424,9 @@ placeholder :: PandocMonad m => VwParser m ()

421424placeholder = try $

422425 choice (ph <$> ["title", "date"]) <|> noHtmlPh <|> templatePh

423426

424−ph :: PandocMonad m => String -> VwParser m ()

427+ph :: PandocMonad m => Text -> VwParser m ()

425428ph s = try $ do

426− many spaceChar >>string ('%':s) >> spaceChar

429+ many spaceChar >> textStr (T.cons '%' s) >> spaceChar

427430 contents <- trimInlines . mconcat <$> manyTill inline (lookAhead newline)

428431 --use lookAhead because of placeholder in the whitespace parser

429432 let meta' = B.setMeta s contents nullMeta

@@ -476,7 +479,7 @@ inlineML :: PandocMonad m => VwParser m Inlines

476479inlineML = choice $ whitespace endlineML:inlineList

477480

478481str :: PandocMonad m => VwParser m Inlines

479−str = B.str <$>many1 (noneOf $ spaceChars ++ specialChars)

482+str = B.str <$> many1Char (noneOf $ spaceChars ++ specialChars)

480483

481484whitespace :: PandocMonad m => VwParser m () -> VwParser m Inlines

482485whitespace endline = B.space <$ (skipMany1 spaceChar <|>

@@ -487,7 +490,7 @@ whitespace' :: PandocMonad m => VwParser m Inlines

487490whitespace' = B.space <$ skipMany1 spaceChar

488491

489492special :: PandocMonad m => VwParser m Inlines

490−special = B.str <$> count 1 (oneOf specialChars)

493+special = B.str <$> countChar 1 (oneOf specialChars)

491494

492495bareURL :: PandocMonad m => VwParser m Inlines

493496bareURL = try $ do

@@ -505,8 +508,8 @@ strong = try $ do

505508 return $ B.spanWith (makeId contents, [], []) mempty

506509 <> B.strong contents

507510

508−makeId :: Inlines -> String

509−makeId i = concat (stringify <$> toList i)

511+makeId :: Inlines -> Text

512+makeId i = T.concat (stringify <$> toList i)

510513

511514emph :: PandocMonad m => VwParser m Inlines

512515emph = try $ do

@@ -527,7 +530,7 @@ strikeout = try $ do

527530code :: PandocMonad m => VwParser m Inlines

528531code = try $ do

529532 char '`'

530− contents <- many1Till (noneOf "\n") (char '`')

533+ contents <- many1TillChar (noneOf "\n") (char '`')

531534 return $ B.code contents

532535

533536superscript :: PandocMonad m => VwParser m Inlines

@@ -542,8 +545,8 @@ subscript = try $

542545link :: PandocMonad m => VwParser m Inlines

543546link = try $ do

544547 string "[["

545− contents <- lookAhead $ manyTill anyChar (string "]]")

546− case '|' `elem` contents of

548+ contents <- lookAhead $ manyTillChar anyChar (string "]]")

549+ case T.any (== '|') contents of

547550 False -> do

548551 manyTill anyChar (string "]]")

549552-- not using try here because [[hell]o]] is not rendered as a link in vimwiki

@@ -552,7 +555,7 @@ link = try $ do

552555 else "wikilink"

553556 return $ B.link (procLink contents) tit (B.str contents)

554557 True -> do

555− url <- manyTill anyChar $ char '|'

558+ url <- manyTillChar anyChar $ char '|'

556559 lab <- mconcat <$> manyTill inline (string "]]")

557560 let tit = if isURI url

558561 then ""

@@ -568,52 +571,52 @@ image = try $ do

568571images :: PandocMonad m => Int -> VwParser m Inlines

569572images k

570573 | k == 0 = do

571− imgurl <- manyTill anyChar (try $ string "}}")

574+ imgurl <- manyTillChar anyChar (try $ string "}}")

572575 return $ B.image (procImgurl imgurl) "" (B.str "")

573576 | k == 1 = do

574− imgurl <- manyTill anyChar (char '|')

577+ imgurl <- manyTillChar anyChar (char '|')

575578 alt <- mconcat <$> manyTill inline (try $ string "}}")

576579 return $ B.image (procImgurl imgurl) "" alt

577580 | k == 2 = do

578− imgurl <- manyTill anyChar (char '|')

579− alt <- mconcat <$>manyTill inline (char '|')

580− attrText <- manyTill anyChar (try $ string "}}")

581+ imgurl <- manyTillChar anyChar (char '|')

582+ alt <- mconcat <$> manyTill inline (char '|')

583+ attrText <- manyTillChar anyChar (try $ string "}}")

581584 return $ B.imageWith (makeAttr attrText) (procImgurl imgurl) "" alt

582585 | otherwise = do

583− imgurl <- manyTill anyChar (char '|')

584− alt <- mconcat <$>manyTill inline (char '|')

585− attrText <- manyTill anyChar (char '|')

586+ imgurl <- manyTillChar anyChar (char '|')

587+ alt <- mconcat <$> manyTill inline (char '|')

588+ attrText <- manyTillChar anyChar (char '|')

586589 manyTill anyChar (try $ string "}}")

587590 return $ B.imageWith (makeAttr attrText) (procImgurl imgurl) "" alt

588591

589−procLink' :: String -> String

592+procLink' :: Text -> Text

590593procLink' s

591− | take 6 s == "local:" = "file" ++ drop 5 s

592− | take 6 s == "diary:" = "diary/" ++ drop 6 s

593− | or ((`isPrefixOf` s) <$> [ "http:", "https:", "ftp:", "file:", "mailto:",

594+ | T.take 6 s == "local:" = "file" <> T.drop 5 s

595+ | T.take 6 s == "diary:" = "diary/" <> T.drop 6 s

596+ | or ((`T.isPrefixOf` s) <$> [ "http:", "https:", "ftp:", "file:", "mailto:",

594597 "news:", "telnet:" ])

595598 = s

596599 | s == "" = ""

597− | last s == '/' = s

600+ | T.last s == '/' = s

598601 | otherwise = s

599602

600−procLink :: String -> String

601−procLink s = procLink' x ++ y

602− where (x, y) = break (=='#') s

603+procLink :: Text -> Text

604+procLink s = procLink' x <> y

605+ where (x, y) = T.break (=='#') s

603606

604−procImgurl :: String -> String

605−procImgurl s = if take 6 s == "local:" then "file" ++ drop 5 s else s

607+procImgurl :: Text -> Text

608+procImgurl s = if T.take 6 s == "local:" then "file" <> T.drop 5 s else s

606609

607610inlineMath :: PandocMonad m => VwParser m Inlines

608611inlineMath = try $

609− B.math <$ char '$' <*> many1Till (noneOf "\n") (char '$')

612+ B.math <$ char '$' <*> many1TillChar (noneOf "\n") (char '$')

610613

611614tag :: PandocMonad m => VwParser m Inlines

612615tag = try $ do

613616 char ':'

614− s <- manyTill (noneOf spaceChars) (try (char ':' >> lookAhead space))

615− guard $ not $ "::" `isInfixOf` (":" ++ s ++ ":")

616− let ss = splitBy (==':') s

617+ s <- manyTillChar (noneOf spaceChars) (try (char ':' >> lookAhead space))

618+ guard $ not $ "::" `T.isInfixOf` (":" <> s <> ":")

619+ let ss = splitTextBy (==':') s

617620 return $ mconcat $ makeTagSpan' (head ss):(makeTagSpan <$> tail ss)

618621

619622todoMark :: PandocMonad m => VwParser m Inlines

@@ -646,16 +649,16 @@ nFBTTBSB =

646649hasDefMarker :: PandocMonad m => VwParser m ()

647650hasDefMarker = () <$ manyTill (noneOf "\n") (string "::" >> oneOf spaceChars)

648651

649−makeTagSpan' :: String -> Inlines

650−makeTagSpan' s = B.spanWith ('-' : s, [], []) (B.str "") <>

652+makeTagSpan' :: Text -> Inlines

653+makeTagSpan' s = B.spanWith (T.cons '-' s, [], []) (B.str "") <>

651654 B.spanWith (s, ["tag"], []) (B.str s)

652655

653−makeTagSpan :: String -> Inlines

656+makeTagSpan :: Text -> Inlines

654657makeTagSpan s = B.space <> makeTagSpan' s

655658

656−mathTagParser :: PandocMonad m => VwParser m String

659+mathTagParser :: PandocMonad m => VwParser m Text

657660mathTagParser = do

658− s <- try $ lookAhead (char '%' >> manyTill (noneOf spaceChars)

661+ s <- try $ lookAhead (char '%' >> manyTillChar (noneOf spaceChars)

659662 (try $ char '%' >> many (noneOf $ '%':spaceChars) >> space))

660− char '%' >> string s >> char '%'

663+ char '%' >> textStr s >> char '%'

661664 return $ mathTagLaTeX s

src/Text/Pandoc/RoffChar.hs+4−3

@@ -18,10 +18,11 @@ module Text.Pandoc.RoffChar (

1818 , combiningAccents

1919 ) where

2020import Prelude

21+import qualified Data.Text as T

2122

2223-- | These are the escapes specifically mentioned in groff_man(7),

2324-- plus @ and ellipsis.

24−standardEscapes :: [(Char, String)]

25+standardEscapes :: [(Char, T.Text)]

2526standardEscapes =

2627 [ ('\160', "\\ ")

2728 , ('\'', "\\[aq]")

@@ -40,7 +41,7 @@ standardEscapes =

4041 , ('\x2026', "\\&...") -- because u2026 doesn't render on tty

4142 ]

4243

43−characterCodes :: [(Char, String)]

44+characterCodes :: [(Char, T.Text)]

4445characterCodes =

4546 [ ('Ð', "-D")

4647 , ('ð', "Sd")

@@ -402,7 +403,7 @@ characterCodes =

402403 ]

403404

404405-- use like: \\[E a^ aa]

405−combiningAccents :: [(Char, String)]

406+combiningAccents :: [(Char, T.Text)]

406407combiningAccents =

407408 [ ('\779' , "a\"")

408409 , ('\772', "a-")

src/Text/Pandoc/SelfContained.hs+61−60

@@ -24,8 +24,8 @@ import Data.ByteString (ByteString)

2424import Data.ByteString.Base64

2525import qualified Data.ByteString.Char8 as B

2626import qualified Data.ByteString.Lazy as L

27−import Data.Char (isAlphaNum, isAscii, toLower)

28−import Data.List (isPrefixOf)

27+import qualified Data.Text as T

28+import Data.Char (isAlphaNum, isAscii)

2929import Network.URI (escapeURIString)

3030import System.FilePath (takeDirectory, takeExtension, (</>))

3131import Text.HTML.TagSoup

@@ -35,24 +35,24 @@ import Text.Pandoc.Error

3535import Text.Pandoc.Logging

3636import Text.Pandoc.MIME (MimeType)

3737import Text.Pandoc.Shared (isURI, renderTags', trim)

38−import Text.Pandoc.UTF8 (toString)

38+import Text.Pandoc.UTF8 (toString, toText, fromText)

3939import Text.Parsec (ParsecT, runParserT)

4040import qualified Text.Parsec as P

4141

4242isOk :: Char -> Bool

4343isOk c = isAscii c && isAlphaNum c

4444

45−makeDataURI :: (MimeType, ByteString) -> String

45+makeDataURI :: (MimeType, ByteString) -> T.Text

4646makeDataURI (mime, raw) =

4747 if textual

48− then "data:" ++ mime' ++ "," ++ escapeURIString isOk (toString raw)

49− else "data:" ++ mime' ++ ";base64," ++ toString (encode raw)

50− where textual = "text/" `Data.List.isPrefixOf` mime

51− mime' = if textual && ';' `notElem` mime

52− then mime ++ ";charset=utf-8"

48+ then "data:" <> mime' <> "," <> T.pack (escapeURIString isOk (toString raw))

49+ else "data:" <> mime' <> ";base64," <> toText (encode raw)

50+ where textual = "text/" `T.isPrefixOf` mime

51+ mime' = if textual && T.any (== ';') mime

52+ then mime <> ";charset=utf-8"

5353 else mime -- mime type already has charset

5454

55−convertTags :: PandocMonad m => [Tag String] -> m [Tag String]

55+convertTags :: PandocMonad m => [Tag T.Text] -> m [Tag T.Text]

5656convertTags [] = return []

5757convertTags (t@TagOpen{}:ts)

5858 | fromAttrib "data-external" t == "1" = (t:) <$> convertTags ts

@@ -69,10 +69,10 @@ convertTags (t@(TagOpen tagname as):ts)

6969 enc <- getDataURI (fromAttrib "type" t) y

7070 return (x, enc)

7171 else return (x,y)

72−convertTags (t@(TagOpen "script" as):TagClose "script":ts) =

72+convertTags (t@(TagOpen "script" as):TagClose "script":ts) =

7373 case fromAttrib "src" t of

74− [] -> (t:) <$> convertTags ts

75− src -> do

74+ "" -> (t:) <$> convertTags ts

75+ src -> do

7676 let typeAttr = fromAttrib "type" t

7777 res <- getData typeAttr src

7878 rest <- convertTags ts

@@ -81,13 +81,13 @@ convertTags (t@(TagOpen "script" as):TagClose "script":ts) =

8181 (("src",dataUri) : [(x,y) | (x,y) <- as, x /= "src"]) :

8282 TagClose "script" : rest

8383 Right (mime, bs)

84− | ("text/javascript" `isPrefixOf` mime ||

85− "application/javascript" `isPrefixOf` mime ||

86− "application/x-javascript" `isPrefixOf` mime) &&

84+ | ("text/javascript" `T.isPrefixOf` mime ||

85+ "application/javascript" `T.isPrefixOf` mime ||

86+ "application/x-javascript" `T.isPrefixOf` mime) &&

8787 not ("</script" `B.isInfixOf` bs) ->

8888 return $

89− TagOpen "script" [("type", typeAttr)|not (null typeAttr)]

90− : TagText (toString bs)

89+ TagOpen "script" [("type", typeAttr)|not (T.null typeAttr)]

90+ : TagText (toText bs)

9191 : TagClose "script"

9292 : rest

9393 | otherwise ->

@@ -97,7 +97,7 @@ convertTags (t@(TagOpen "script" as):TagClose "script":ts) =

9797 TagClose "script" : rest

9898convertTags (t@(TagOpen "link" as):ts) =

9999 case fromAttrib "href" t of

100− [] -> (t:) <$> convertTags ts

100+ "" -> (t:) <$> convertTags ts

101101 src -> do

102102 res <- getData (fromAttrib "type" t) src

103103 case res of

@@ -107,14 +107,14 @@ convertTags (t@(TagOpen "link" as):ts) =

107107 (("href",dataUri) : [(x,y) | (x,y) <- as, x /= "href"]) :

108108 rest

109109 Right (mime, bs)

110− | "text/css" `isPrefixOf` mime

111− && null (fromAttrib "media" t)

110+ | "text/css" `T.isPrefixOf` mime

111+ && T.null (fromAttrib "media" t)

112112 && not ("</" `B.isInfixOf` bs) -> do

113113 rest <- convertTags $

114114 dropWhile (==TagClose "link") ts

115115 return $

116116 TagOpen "style" [("type", "text/css")] -- see #5725

117− : TagText (toString bs)

117+ : TagText (toText bs)

118118 : TagClose "style"

119119 : rest

120120 | otherwise -> do

@@ -130,7 +130,7 @@ cssURLs d orig = do

130130 res <- runParserT (parseCSSUrls d) () "css" orig

131131 case res of

132132 Left e -> do

133− report $ CouldNotParseCSS (show e)

133+ report $ CouldNotParseCSS $ T.pack $ show e

134134 return orig

135135 Right bs -> return bs

136136

@@ -176,52 +176,52 @@ pCSSUrl d = P.try $ do

176176 Left b -> return b

177177 Right (mt,b) -> do

178178 let enc = makeDataURI (mt, b)

179− return (B.pack $ "url(" ++ enc ++ ")")

179+ return $ fromText $ "url(" <> enc <> ")"

180180

181181pQuoted :: PandocMonad m

182− => ParsecT ByteString () m (String, ByteString)

182+ => ParsecT ByteString () m (T.Text, ByteString)

183183pQuoted = P.try $ do

184184 quote <- P.oneOf "\"'"

185− url <- P.manyTill P.anyChar (P.char quote)

186− let fallback = B.pack ([quote] ++ trim url ++ [quote])

185+ url <- T.pack <$> P.manyTill P.anyChar (P.char quote)

186+ let fallback = fromText $ T.singleton quote <> trim url <> T.singleton quote

187187 return (url, fallback)

188188

189189pUrl :: PandocMonad m

190− => ParsecT ByteString () m (String, ByteString)

190+ => ParsecT ByteString () m (T.Text, ByteString)

191191pUrl = P.try $ do

192192 P.string "url("

193193 P.spaces

194194 quote <- P.option Nothing (Just <$> P.oneOf "\"'")

195− url <- P.manyTill P.anyChar (maybe (P.lookAhead (P.char ')')) P.char quote)

195+ url <- T.pack <$> P.manyTill P.anyChar (maybe (P.lookAhead (P.char ')')) P.char quote)

196196 P.spaces

197197 P.char ')'

198− let fallback = B.pack ("url(" ++ maybe "" (:[]) quote ++ trim url ++

199− maybe "" (:[]) quote ++ ")")

198+ let fallback = fromText ("url(" <> maybe "" T.singleton quote <> trim url <>

199+ maybe "" T.singleton quote <> ")")

200200 return (url, fallback)

201201

202202handleCSSUrl :: PandocMonad m

203− => FilePath -> (String, ByteString)

203+ => FilePath -> (T.Text, ByteString)

204204 -> ParsecT ByteString () m

205205 (Either ByteString (MimeType, ByteString))

206206handleCSSUrl d (url, fallback) =

207− case escapeURIString (/='|') (trim url) of

207+ case escapeURIString (/='|') (T.unpack $ trim url) of

208208 '#':_ -> return $ Left fallback

209209 'd':'a':'t':'a':':':_ -> return $ Left fallback

210− u -> do let url' = if isURI u then u else d </> u

210+ u -> do let url' = if isURI (T.pack u) then T.pack u else T.pack (d </> u)

211211 res <- lift $ getData "" url'

212212 case res of

213− Left uri -> return $ Left (B.pack $ "url(" ++ uri ++ ")")

213+ Left uri -> return $ Left (fromText $ "url(" <> uri <> ")")

214214 Right (mt', raw) -> do

215215 -- note that the downloaded CSS may

216216 -- itself contain url(...).

217− (mt, b) <- if "text/css" `isPrefixOf` mt'

217+ (mt, b) <- if "text/css" `T.isPrefixOf` mt'

218218 -- see #5725: in HTML5, content type

219219 -- isn't allowed on style type attribute

220220 then ("text/css",) <$> cssURLs d raw

221221 else return (mt', raw)

222222 return $ Right (mt, b)

223223

224−getDataURI :: PandocMonad m => MimeType -> String -> m String

224+getDataURI :: PandocMonad m => MimeType -> T.Text -> m T.Text

225225getDataURI mimetype src = do

226226 res <- getData mimetype src

227227 case res of

@@ -229,35 +229,36 @@ getDataURI mimetype src = do

229229 Right x -> return $ makeDataURI x

230230

231231getData :: PandocMonad m

232− => MimeType -> String

233− -> m (Either String (MimeType, ByteString))

234−getData _ src@('d':'a':'t':'a':':':_) = return $ Left src-- already data: uri

235−getData mimetype src = do

236− let ext = map toLower $ takeExtension src

237− (raw, respMime) <- fetchItem src

238− let raw' = if ext `elem` [".gz", ".svgz"]

239− then B.concat $ L.toChunks $ Gzip.decompress $ L.fromChunks [raw]

240− else raw

241− mime <- case (mimetype, respMime) of

242− ("",Nothing) -> throwError $ PandocSomeError

243− $ "Could not determine mime type for `" ++ src ++ "'"

244− (x, Nothing) -> return x

245− (_, Just x ) -> return x

246− result <- if "text/css" `isPrefixOf` mime

247− then do

248− oldInputs <- getInputFiles

249− setInputFiles [src]

250− res <- cssURLs (takeDirectory src) raw'

251− setInputFiles oldInputs

252− return res

232+ => MimeType -> T.Text

233+ -> m (Either T.Text (MimeType, ByteString))

234+getData mimetype src

235+ | "data:" `T.isPrefixOf` src = return $ Left src -- already data: uri

236+ | otherwise = do

237+ let ext = T.toLower $ T.pack $ takeExtension $ T.unpack src

238+ (raw, respMime) <- fetchItem src

239+ let raw' = if ext `elem` [".gz", ".svgz"]

240+ then B.concat $ L.toChunks $ Gzip.decompress $ L.fromChunks [raw]

241+ else raw

242+ mime <- case (mimetype, respMime) of

243+ ("",Nothing) -> throwError $ PandocSomeError

244+ $ "Could not determine mime type for `" <> src <> "'"

245+ (x, Nothing) -> return x

246+ (_, Just x ) -> return x

247+ result <- if "text/css" `T.isPrefixOf` mime

248+ then do

249+ oldInputs <- getInputFiles

250+ setInputFiles [T.unpack src]

251+ res <- cssURLs (takeDirectory $ T.unpack src) raw'

252+ setInputFiles oldInputs

253+ return res

253254 else return raw'

254− return $ Right (mime, result)

255+ return $ Right (mime, result)

255256

256257

257258

258259-- | Convert HTML into self-contained HTML, incorporating images,

259260-- scripts, and CSS using data: URIs.

260−makeSelfContained :: PandocMonad m => String -> m String

261+makeSelfContained :: PandocMonad m => T.Text -> m T.Text

261262makeSelfContained inp = do

262263 let tags = parseTags inp

263264 out' <- convertTags tags

src/Text/Pandoc/Shared.hs+175−118

@@ -8,6 +8,7 @@

88{-# LANGUAGE FlexibleInstances #-}

99{-# LANGUAGE TypeSynonymInstances #-}

1010{-# LANGUAGE LambdaCase #-}

11+{-# LANGUAGE OverloadedStrings #-}

1112{- |

1213 Module : Text.Pandoc.Shared

@@ -22,14 +23,20 @@ Utility functions and definitions used by the various Pandoc modules.

2223module Text.Pandoc.Shared (

2324 -- * List processing

2425 splitBy,

26+ splitTextBy,

2527 splitByIndices,

2628 splitStringByIndices,

29+ splitTextByIndices,

2730 substitute,

2831 ordNub,

2932 -- * Text processing

3033 ToString (..),

34+ ToText (..),

35+ tshow,

3136 backslashEscapes,

3237 escapeStringUsing,

38+ elemText,

39+ notElemText,

3340 stripTrailingNewlines,

3441 trim,

3542 triml,

@@ -37,6 +44,7 @@ module Text.Pandoc.Shared (

3744 trimMath,

3845 stripFirstAndLast,

3946 camelCaseToHyphenated,

47+ camelCaseStrToHyphenated,

4048 toRomanNumeral,

4149 escapeURI,

4250 tabFilter,

@@ -90,6 +98,7 @@ module Text.Pandoc.Shared (

9098 defaultBlocksSeparator,

9199 -- * Safe read

92100 safeRead,

101+ safeStrRead,

93102 -- * User data directory

94103 defaultUserDataDirs,

95104 -- * Version

@@ -133,8 +142,8 @@ import Text.DocLayout (charWidth)

133142import Text.Pandoc.Walk

134143

135144-- | Version number of pandoc library.

136−pandocVersion :: String

137−pandocVersion = showVersion version

145+pandocVersion :: T.Text

146+pandocVersion = T.pack $ showVersion version

138147

139148--

140149-- List processing

@@ -148,6 +157,13 @@ splitBy isSep lst =

148157 rest' = dropWhile isSep rest

149158 in first:splitBy isSep rest'

150159

160+splitTextBy :: (Char -> Bool) -> T.Text -> [T.Text]

161+splitTextBy isSep t

162+ | T.null t = []

163+ | otherwise = let (first, rest) = T.break isSep t

164+ rest' = T.dropWhile isSep rest

165+ in first : splitTextBy isSep rest'

166+

151167splitByIndices :: [Int] -> [a] -> [[a]]

152168splitByIndices [] lst = [lst]

153169splitByIndices (x:xs) lst = first:splitByIndices (map (\y -> y - x) xs) rest

@@ -160,6 +176,9 @@ splitStringByIndices (x:xs) lst =

160176 let (first, rest) = splitAt' x lst in

161177 first : splitStringByIndices (map (\y -> y - x) xs) rest

162178

179+splitTextByIndices :: [Int] -> T.Text -> [T.Text]

180+splitTextByIndices ns = fmap T.pack . splitStringByIndices ns . T.unpack

181+

163182splitAt' :: Int -> [Char] -> ([Char],[Char])

164183splitAt' _ [] = ([],[])

165184splitAt' n xs | n <= 0 = ([],xs)

@@ -195,89 +214,115 @@ instance ToString String where

195214instance ToString T.Text where

196215 toString = T.unpack

197216

217+class ToText a where

218+ toText :: a -> T.Text

219+

220+instance ToText String where

221+ toText = T.pack

222+

223+instance ToText T.Text where

224+ toText = id

225+

226+tshow :: Show a => a -> T.Text

227+tshow = T.pack . show

228+

198229-- | Returns an association list of backslash escapes for the

199230-- designated characters.

200231backslashEscapes :: [Char] -- ^ list of special characters to escape

201− -> [(Char, String)]

202−backslashEscapes = map (\ch -> (ch, ['\\',ch]))

232+ -> [(Char, T.Text)]

233+backslashEscapes = map (\ch -> (ch, T.pack ['\\',ch]))

203234

204235-- | Escape a string of characters, using an association list of

205236-- characters and strings.

206−escapeStringUsing :: [(Char, String)] -> String -> String

207−escapeStringUsing _ [] = ""

208−escapeStringUsing escapeTable (x:xs) =

209− case lookup x escapeTable of

210− Just str -> str ++ rest

211− Nothing -> x:rest

212− where rest = escapeStringUsing escapeTable xs

237+escapeStringUsing :: [(Char, T.Text)] -> T.Text -> T.Text

238+escapeStringUsing tbl = T.concatMap $ \c -> fromMaybe (T.singleton c) $ lookup c tbl

239+

240+-- | @True@ exactly when the @Char@ appears in the @Text@.

241+elemText :: Char -> T.Text -> Bool

242+elemText c = T.any (== c)

243+

244+-- | @True@ exactly when the @Char@ does not appear in the @Text@.

245+notElemText :: Char -> T.Text -> Bool

246+notElemText c = T.all (/= c)

213247

214248-- | Strip trailing newlines from string.

215−stripTrailingNewlines :: String -> String

216−stripTrailingNewlines = reverse . dropWhile (== '\n') . reverse

249+stripTrailingNewlines :: T.Text -> T.Text

250+stripTrailingNewlines = T.dropWhileEnd (== '\n')

217251

218252-- | Remove leading and trailing space (including newlines) from string.

219−trim :: String -> String

220−trim = triml . trimr

253+trim :: T.Text -> T.Text

254+trim = T.dropAround (`elemText` " \r\n\t")

221255

222256-- | Remove leading space (including newlines) from string.

223−triml :: String -> String

224−triml = dropWhile (`elem` " \r\n\t")

257+triml :: T.Text -> T.Text

258+triml = T.dropWhile (`elemText` " \r\n\t")

225259

226260-- | Remove trailing space (including newlines) from string.

227−trimr :: String -> String

228−trimr = reverse . triml . reverse

261+trimr :: T.Text -> T.Text

262+trimr = T.dropWhileEnd (`elemText` " \r\n\t")

229263

230264-- | Trim leading space and trailing space unless after \.

231−trimMath :: String -> String

232−trimMath = triml . reverse . stripspace . reverse

265+trimMath :: T.Text -> T.Text

266+trimMath = triml . T.reverse . stripBeginSpace . T.reverse -- no Text.spanEnd

233267 where

234− stripspace (c1:c2:cs)

235− | c1 `elem` [' ','\t','\n','\r']

236− , c2 /= '\\' = stripspace (c2:cs)

237− stripspace cs = cs

268+ stripBeginSpace t

269+ | T.null pref = t

270+ | Just ('\\', _) <- T.uncons suff = T.cons (T.last pref) suff

271+ | otherwise = suff

272+ where

273+ (pref, suff) = T.span (`elemText` " \t\n\r") t

238274

239275-- | Strip leading and trailing characters from string

240−stripFirstAndLast :: String -> String

241−stripFirstAndLast str =

242− drop 1 $ take (length str - 1) str

276+stripFirstAndLast :: T.Text -> T.Text

277+stripFirstAndLast t = case T.uncons t of

278+ Just (_, t') -> case T.unsnoc t' of

279+ Just (t'', _) -> t''

280+ _ -> t'

281+ _ -> ""

243282

244283-- | Change CamelCase word to hyphenated lowercase (e.g., camel-case).

245−camelCaseToHyphenated :: String -> String

246−camelCaseToHyphenated [] = ""

247−camelCaseToHyphenated (a:b:rest)

284+camelCaseToHyphenated :: T.Text -> T.Text

285+camelCaseToHyphenated = T.pack . camelCaseStrToHyphenated . T.unpack

286+

287+-- This may not work as expected on general Unicode, if it contains

288+-- letters with a longer lower case form than upper case. I don't know

289+-- what the camel case practices of affected scripts are, though.

290+camelCaseStrToHyphenated :: String -> String

291+camelCaseStrToHyphenated [] = ""

292+camelCaseStrToHyphenated (a:b:rest)

248293 | isLower a

249− , isUpper b = a:'-':toLower b:camelCaseToHyphenated rest

294+ , isUpper b = a:'-':toLower b:camelCaseStrToHyphenated rest

250295-- handle ABCDef = abc-def

251−camelCaseToHyphenated (a:b:c:rest)

296+camelCaseStrToHyphenated (a:b:c:rest)

252297 | isUpper a

253298 , isUpper b

254− , isLower c = toLower a:'-':toLower b:camelCaseToHyphenated (c:rest)

255−camelCaseToHyphenated (a:rest) = toLower a:camelCaseToHyphenated rest

299+ , isLower c = toLower a:'-':toLower b:camelCaseStrToHyphenated (c:rest)

300+camelCaseStrToHyphenated (a:rest) = toLower a:camelCaseStrToHyphenated rest

256301

257302-- | Convert number < 4000 to uppercase roman numeral.

258−toRomanNumeral :: Int -> String

303+toRomanNumeral :: Int -> T.Text

259304toRomanNumeral x

260305 | x >= 4000 || x < 0 = "?"

261− | x >= 1000 = "M" ++ toRomanNumeral (x - 1000)

262− | x >= 900 = "CM" ++ toRomanNumeral (x - 900)

263− | x >= 500 = "D" ++ toRomanNumeral (x - 500)

264− | x >= 400 = "CD" ++ toRomanNumeral (x - 400)

265− | x >= 100 = "C" ++ toRomanNumeral (x - 100)

266− | x >= 90 = "XC" ++ toRomanNumeral (x - 90)

267− | x >= 50 = "L" ++ toRomanNumeral (x - 50)

268− | x >= 40 = "XL" ++ toRomanNumeral (x - 40)

269− | x >= 10 = "X" ++ toRomanNumeral (x - 10)

306+ | x >= 1000 = "M" <> toRomanNumeral (x - 1000)

307+ | x >= 900 = "CM" <> toRomanNumeral (x - 900)

308+ | x >= 500 = "D" <> toRomanNumeral (x - 500)

309+ | x >= 400 = "CD" <> toRomanNumeral (x - 400)

310+ | x >= 100 = "C" <> toRomanNumeral (x - 100)

311+ | x >= 90 = "XC" <> toRomanNumeral (x - 90)

312+ | x >= 50 = "L" <> toRomanNumeral (x - 50)

313+ | x >= 40 = "XL" <> toRomanNumeral (x - 40)

314+ | x >= 10 = "X" <> toRomanNumeral (x - 10)

270315 | x == 9 = "IX"

271− | x >= 5 = "V" ++ toRomanNumeral (x - 5)

316+ | x >= 5 = "V" <> toRomanNumeral (x - 5)

272317 | x == 4 = "IV"

273− | x >= 1 = "I" ++ toRomanNumeral (x - 1)

318+ | x >= 1 = "I" <> toRomanNumeral (x - 1)

274319 | otherwise = ""

275320

276321-- | Escape whitespace and some punctuation characters in URI.

277−escapeURI :: String -> String

278−escapeURI = escapeURIString (not . needsEscaping)

279− where needsEscaping c = isSpace c || c `elem`

280− ['<','>','|','"','{','}','[',']','^', '`']

322+escapeURI :: T.Text -> T.Text

323+escapeURI = T.pack . escapeURIString (not . needsEscaping) . T.unpack

324+ where needsEscaping c = isSpace c || c `elemText` "<>|\"{}[]^`"

325+

281326

282327-- | Convert tabs to spaces. Tabs will be preserved if tab stop is set to 0.

283328tabFilter :: Int -- ^ Tab stop

@@ -304,8 +349,11 @@ crFilter = T.filter (/= '\r')

304349-- | Parse a date and convert (if possible) to "YYYY-MM-DD" format. We

305350-- limit years to the range 1601-9999 (ISO 8601 accepts greater than

306351-- or equal to 1583, but MS Word only accepts dates starting 1601).

307−normalizeDate :: String -> Maybe String

308−normalizeDate s = fmap (formatTime defaultTimeLocale "%F")

352+normalizeDate :: T.Text -> Maybe T.Text

353+normalizeDate = fmap T.pack . normalizeDate' . T.unpack

354+

355+normalizeDate' :: String -> Maybe String

356+normalizeDate' s = fmap (formatTime defaultTimeLocale "%F")

309357 (msum $ map (\fs -> parsetimeWith fs s >>= rejectBadYear) formats :: Maybe Day)

310358 where rejectBadYear day = case toGregorian day of

311359 (y, _, _) | y >= 1601 && y <= 9999 -> Just day

@@ -321,26 +369,26 @@ normalizeDate s = fmap (formatTime defaultTimeLocale "%F")

321369

322370-- | Generate infinite lazy list of markers for an ordered list,

323371-- depending on list attributes.

324−orderedListMarkers :: (Int, ListNumberStyle, ListNumberDelim) -> [String]

372+orderedListMarkers :: (Int, ListNumberStyle, ListNumberDelim) -> [T.Text]

325373orderedListMarkers (start, numstyle, numdelim) =

326− let singleton c = [c]

327− nums = case numstyle of

328− DefaultStyle -> map show [start..]

329− Example -> map show [start..]

330− Decimal -> map show [start..]

374+ let nums = case numstyle of

375+ DefaultStyle -> map tshow [start..]

376+ Example -> map tshow [start..]

377+ Decimal -> map tshow [start..]

331378 UpperAlpha -> drop (start - 1) $ cycle $

332− map singleton ['A'..'Z']

379+ map T.singleton ['A'..'Z']

333380 LowerAlpha -> drop (start - 1) $ cycle $

334− map singleton ['a'..'z']

381+ map T.singleton ['a'..'z']

335382 UpperRoman -> map toRomanNumeral [start..]

336− LowerRoman -> map (map toLower . toRomanNumeral) [start..]

383+ LowerRoman -> map (T.toLower . toRomanNumeral) [start..]

337384 inDelim str = case numdelim of

338− DefaultDelim -> str ++ "."

339− Period -> str ++ "."

340− OneParen -> str ++ ")"

341− TwoParens -> "(" ++ str ++ ")"

385+ DefaultDelim -> str <> "."

386+ Period -> str <> "."

387+ OneParen -> str <> ")"

388+ TwoParens -> "(" <> str <> ")"

342389 in map inDelim nums

343390

391+

344392-- | Extract the leading and trailing spaces from inside an inline element

345393-- and place them outside the element. SoftBreaks count as Spaces for

346394-- these purposes.

@@ -387,15 +435,16 @@ deQuote x = x

387435-- | Convert pandoc structure to a string with formatting removed.

388436-- Footnotes are skipped (since we don't want their contents in link

389437-- labels).

390−stringify :: Walkable Inline a => a -> String

438+stringify :: Walkable Inline a => a -> T.Text

391439stringify = query go . walk (deNote . deQuote)

392− where go :: Inline -> [Char]

440+ where go :: Inline -> T.Text

393441 go Space = " "

394442 go SoftBreak = " "

395443 go (Str x) = x

396444 go (Code _ x) = x

397445 go (Math _ x) = x

398− go (RawInline (Format "html") ('<':'b':'r':_)) = " " -- see #2105

446+ go (RawInline (Format "html") (T.unpack -> ('<':'b':'r':_)))

447+ = " " -- see #2105

399448 go LineBreak = " "

400449 go _ = ""

401450

@@ -407,7 +456,7 @@ stringify = query go . walk (deNote . deQuote)

407456capitalize :: Walkable Inline a => a -> a

408457capitalize = walk go

409458 where go :: Inline -> Inline

410− go (Str s) = Str (T.unpack $ T.toUpper $ T.pack s)

459+ go (Str s) = Str $ T.toUpper s

411460 go x = x

412461

413462-- | Change final list item from @Para@ to @Plain@ if the list contains

@@ -463,7 +512,7 @@ isPara _ = False

463512-- | Convert Pandoc inline list to plain text identifier. HTML

464513-- identifiers must start with a letter, and may contain only

465514-- letters, digits, and the characters _-.

466−inlineListToIdentifier :: Extensions -> [Inline] -> String

515+inlineListToIdentifier :: Extensions -> [Inline] -> T.Text

467516inlineListToIdentifier exts =

468517 dropNonLetter . filterAscii . toIdent . stringify . walk unEmojify

469518 where

@@ -476,23 +525,23 @@ inlineListToIdentifier exts =

476525 unEmoji x = x

477526 dropNonLetter

478527 | extensionEnabled Ext_gfm_auto_identifiers exts = id

479− | otherwise = dropWhile (not . isAlpha)

528+ | otherwise = T.dropWhile (not . isAlpha)

480529 filterAscii

481530 | extensionEnabled Ext_ascii_identifiers exts

482− = mapMaybe toAsciiChar

531+ = T.pack . mapMaybe toAsciiChar . T.unpack

483532 | otherwise = id

484533 toIdent

485534 | extensionEnabled Ext_gfm_auto_identifiers exts =

486− filterPunct . spaceToDash . map toLower

487− | otherwise = intercalate "-" . words . filterPunct . map toLower

488− filterPunct = filter (\c -> isSpace c || isAlphaNum c || isAllowedPunct c)

535+ filterPunct . spaceToDash . T.toLower

536+ | otherwise = T.intercalate "-" . T.words . filterPunct . T.toLower

537+ filterPunct = T.filter (\c -> isSpace c || isAlphaNum c || isAllowedPunct c)

489538 isAllowedPunct c

490539 | extensionEnabled Ext_gfm_auto_identifiers exts

491540 = c == '-' || c == '_' ||

492541 generalCategory c `elem` [NonSpacingMark, SpacingCombiningMark,

493542 EnclosingMark, ConnectorPunctuation]

494543 | otherwise = c == '_' || c == '-' || c == '.'

495− spaceToDash = map (\c -> if isSpace c then '-' else c)

544+ spaceToDash = T.map (\c -> if isSpace c then '-' else c)

496545

497546

498547-- | Put a list of Pandoc blocks into a hierarchical structure:

@@ -529,7 +578,7 @@ makeSections numbering mbBaseLevel bs =

529578 -- don't touch number if already present

530579 case lookup "number" kvs of

531580 Nothing | numbering ->

532− ("number", intercalate "." (map show newnum)) : kvs

581+ ("number", T.intercalate "." (map tshow newnum)) : kvs

533582 _ -> kvs)

534583 return $

535584 Div divattr (Header level' attr title' : sectionContents') : rest'

@@ -542,7 +591,7 @@ makeSections numbering mbBaseLevel bs =

542591 let inner' =

543592 case inner of

544593 (Div (dident',dclasses',dkvs') zs@(Header{}:zs') : ws)

545− | null dident ->

594+ | T.null dident ->

546595 Div (dident',dclasses' ++ dclasses,dkvs' ++ dkvs) zs : ws

547596 | otherwise -> -- keep id on header so we don't lose anchor

548597 Div (dident,dclasses ++ dclasses',dkvs ++ dkvs')

@@ -564,7 +613,7 @@ headerLtEq _ _ = False

564613

565614-- | Generate a unique identifier from a list of inlines.

566615-- Second argument is a list of already used identifiers.

567−uniqueIdent :: Extensions -> [Inline] -> Set.Set String -> String

616+uniqueIdent :: Extensions -> [Inline] -> Set.Set T.Text -> T.Text

568617uniqueIdent exts title' usedIdents =

569618 if baseIdent `Set.member` usedIdents

570619 then case find (\x -> not $ numIdent x `Set.member` usedIdents)

@@ -577,7 +626,7 @@ uniqueIdent exts title' usedIdents =

577626 baseIdent = case inlineListToIdentifier exts title' of

578627 "" -> "section"

579628 x -> x

580− numIdent n = baseIdent ++ "-" ++ show n

629+ numIdent n = baseIdent <> "-" <> tshow n

581630

582631-- | True if block is a Header block.

583632isHeaderBlock :: Block -> Bool

@@ -664,7 +713,7 @@ handleTaskListItem handleInlines exts bls =

664713-- | Set a field of a 'Meta' object. If the field already has a value,

665714-- convert it into a list with the new value appended to the old value(s).

666715addMetaField :: ToMetaValue a

667− => String

716+ => T.Text

668717 -> a

669718 -> Meta

670719 -> Meta

@@ -686,12 +735,16 @@ makeMeta title authors date =

686735-- | Remove soft breaks between East Asian characters.

687736eastAsianLineBreakFilter :: Pandoc -> Pandoc

688737eastAsianLineBreakFilter = bottomUp go

689− where go (x:SoftBreak:y:zs) =

690− case (stringify x, stringify y) of

691− (xs@(_:_), c:_)

692− | charWidth (last xs) == 2 && charWidth c == 2 -> x:y:zs

693− _ -> x:SoftBreak:y:zs

694− go xs = xs

738+ where go (x:SoftBreak:y:zs)

739+ | Just (_, b) <- T.unsnoc $ stringify x

740+ , Just (c, _) <- T.uncons $ stringify y

741+ , charWidth b == 2

742+ , charWidth c == 2

743+ = x:y:zs

744+ | otherwise

745+ = x:SoftBreak:y:zs

746+ go xs

747+ = xs

695748

696749-- | Builder for underline.

697750-- This probably belongs in Builder.hs in pandoc-types.

@@ -702,27 +755,28 @@ underlineSpan = B.spanWith ("", ["underline"], [])

702755-- | Set of HTML elements that are represented as Span with a class equal as

703756-- the element tag itself.

704757htmlSpanLikeElements :: Set.Set T.Text

705−htmlSpanLikeElements = Set.fromList [T.pack "kbd", T.pack "mark", T.pack "dfn"]

758+htmlSpanLikeElements = Set.fromList ["kbd", "mark", "dfn"]

706759

707760-- | Returns the first sentence in a list of inlines, and the rest.

708761breakSentence :: [Inline] -> ([Inline], [Inline])

709762breakSentence [] = ([],[])

710763breakSentence xs =

711− let isSentenceEndInline (Str ys@(_:_)) | last ys == '.' = True

712− isSentenceEndInline (Str ys@(_:_)) | last ys == '?' = True

713− isSentenceEndInline LineBreak = True

714− isSentenceEndInline _ = False

764+ let isSentenceEndInline (Str ys)

765+ | Just (_, c) <- T.unsnoc ys = c == '.' || c == '?'

766+ isSentenceEndInline LineBreak = True

767+ isSentenceEndInline _ = False

715768 (as, bs) = break isSentenceEndInline xs

716769 in case bs of

717− [] -> (as, [])

718− [c] -> (as ++ [c], [])

719− (c:Space:cs) -> (as ++ [c], cs)

720− (c:SoftBreak:cs) -> (as ++ [c], cs)

721− (Str ".":Str (')':ys):cs) -> (as ++ [Str ".", Str (')':ys)], cs)

722− (x@(Str ('.':')':_)):cs) -> (as ++ [x], cs)

723− (LineBreak:x@(Str ('.':_)):cs) -> (as ++[LineBreak], x:cs)

724− (c:cs) -> (as ++ [c] ++ ds, es)

725− where (ds, es) = breakSentence cs

770+ [] -> (as, [])

771+ [c] -> (as ++ [c], [])

772+ (c:Space:cs) -> (as ++ [c], cs)

773+ (c:SoftBreak:cs) -> (as ++ [c], cs)

774+ (Str ".":Str s@(T.uncons -> Just (')',_)):cs)

775+ -> (as ++ [Str ".", Str s], cs)

776+ (x@(Str (T.stripPrefix ".)" -> Just _)):cs) -> (as ++ [x], cs)

777+ (LineBreak:x@(Str (T.uncons -> Just ('.',_))):cs) -> (as ++[LineBreak], x:cs)

778+ (c:cs) -> (as ++ [c] ++ ds, es)

779+ where (ds, es) = breakSentence cs

726780

727781-- | Split a list of inlines into sentences.

728782splitSentences :: [Inline] -> [[Inline]]

@@ -763,10 +817,11 @@ filterIpynbOutput mode = walk go

763817 removeANSI (CodeBlock attr code) =

764818 CodeBlock attr (removeANSIEscapes code)

765819 removeANSI x = x

766− removeANSIEscapes [] = []

767− removeANSIEscapes ('\x1b':'[':cs) =

768− removeANSIEscapes (drop 1 $ dropWhile (/='m') cs)

769− removeANSIEscapes (c:cs) = c : removeANSIEscapes cs

820+ removeANSIEscapes t

821+ | Just cs <- T.stripPrefix "\x1b[" t =

822+ removeANSIEscapes $ T.drop 1 $ T.dropWhile (/='m') cs

823+ | Just (c, cs) <- T.uncons t = T.cons c $ removeANSIEscapes cs

824+ | otherwise = ""

770825 go x = x

771826

772827--

@@ -774,12 +829,12 @@ filterIpynbOutput mode = walk go

774829--

775830

776831-- | Render HTML tags.

777−renderTags' :: [Tag String] -> String

832+renderTags' :: [Tag T.Text] -> T.Text

778833renderTags' = renderTagsOptions

779834 renderOptions{ optMinimize = matchTags ["hr", "br", "img",

780835 "meta", "link"]

781836 , optRawTag = matchTags ["script", "style"] }

782− where matchTags tags = flip elem tags . map toLower

837+ where matchTags tags = flip elem tags . T.toLower

783838

784839--

785840-- File handling

@@ -826,8 +881,8 @@ collapseFilePath = Posix.joinPath . reverse . foldl go [] . splitDirectories

826881-- Convert the path part of a file: URI to a regular path.

827882-- On windows, @/c:/foo@ should be @c:/foo@.

828883-- On linux, @/foo@ should be @/foo@.

829−uriPathToPath :: String -> FilePath

830−uriPathToPath path =

884+uriPathToPath :: T.Text -> FilePath

885+uriPathToPath (T.unpack -> path) =

831886#ifdef _WINDOWS

832887 case path of

833888 '/':ps -> ps

@@ -853,7 +908,7 @@ filteredFilesFromArchive zf f =

853908

854909-- | Schemes from http://www.iana.org/assignments/uri-schemes.html plus

855910-- the unofficial schemes doi, javascript, isbn, pmid.

856−schemes :: Set.Set String

911+schemes :: Set.Set T.Text

857912schemes = Set.fromList

858913 -- Official IANA schemes

859914 [ "aaa", "aaas", "about", "acap", "acct", "acr", "adiumxtra", "afp", "afs"

@@ -905,11 +960,11 @@ schemes = Set.fromList

905960

906961-- | Check if the string is a valid URL with a IANA or frequently used but

907962-- unofficial scheme (see @schemes@).

908−isURI :: String -> Bool

909−isURI = maybe False hasKnownScheme . parseURI

963+isURI :: T.Text -> Bool

964+isURI = maybe False hasKnownScheme . parseURI . T.unpack

910965 where

911− hasKnownScheme = (`Set.member` schemes) . map toLower .

912− filter (/= ':') . uriScheme

966+ hasKnownScheme = (`Set.member` schemes) . T.toLower .

967+ T.filter (/= ':') . T.pack . uriScheme

913968

914969---

915970--- Squash blocks into inlines

@@ -962,12 +1017,14 @@ defaultBlocksSeparator =

9621017-- Safe read

9631018--

9641019

965−safeRead :: (MonadPlus m, Read a) => String -> m a

966−safeRead s = case reads s of

1020+safeRead :: (MonadPlus m, Read a) => T.Text -> m a

1021+safeRead = safeStrRead . T.unpack

1022+

1023+safeStrRead :: (MonadPlus m, Read a) => String -> m a

1024+safeStrRead s = case reads s of

9671025 (d,x):_

9681026 | all isSpace x -> return d

9691027 _ -> mzero

970−

9711028--

9721029-- User data directory

9731030--

src/Text/Pandoc/Slides.hs+1

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Slides

src/Text/Pandoc/Templates.hs+4−3

@@ -25,13 +25,14 @@ import Text.DocTemplates (Template, compileTemplate, renderTemplate)

2525import Text.Pandoc.Class (PandocMonad, readDataFile)

2626import qualified Text.Pandoc.UTF8 as UTF8

2727import Data.Text (Text)

28+import qualified Data.Text as T

2829

2930-- | Get default template for the specified writer.

3031getDefaultTemplate :: PandocMonad m

31− => String -- ^ Name of writer

32+ => Text -- ^ Name of writer

3233 -> m Text

3334getDefaultTemplate writer = do

34− let format = takeWhile (`notElem` ("+-" :: String)) writer -- strip off extensions

35+ let format = T.takeWhile (`notElem` ("+-" :: String)) writer -- strip off extensions

3536 case format of

3637 "native" -> return ""

3738 "json" -> return ""

@@ -51,7 +52,7 @@ getDefaultTemplate writer = do

5152 "markdown_phpextra" -> getDefaultTemplate "markdown"

5253 "gfm" -> getDefaultTemplate "commonmark"

5354 _ -> do

54− let fname = "templates" </> "default" <.> format

55+ let fname = "templates" </> "default" <.> T.unpack format

5556 UTF8.toText <$> readDataFile fname

5657

5758

src/Text/Pandoc/Translations.hs+14−13

@@ -1,7 +1,8 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE CPP #-}

33{-# LANGUAGE DeriveGeneric #-}

44{-# LANGUAGE GeneralizedNewtypeDeriving #-}

5+{-# LANGUAGE OverloadedStrings #-}

56{- |

67 Module : Text.Pandoc.Translations

@@ -34,7 +35,7 @@ import Data.Aeson.Types (Value(..), FromJSON(..))

3435import qualified Data.Aeson.Types as Aeson

3536import qualified Data.HashMap.Strict as HM

3637import qualified Data.Map as M

37−import Data.Text as T

38+import qualified Data.Text as T

3839import qualified Data.YAML as YAML

3940import GHC.Generics (Generic)

4041import Text.Pandoc.Shared (safeRead)

@@ -65,11 +66,11 @@ data Term =

6566 | To

6667 deriving (Show, Eq, Ord, Generic, Enum, Read)

6768

68−newtype Translations = Translations (M.Map Term String)

69+newtype Translations = Translations (M.Map Term T.Text)

6970 deriving (Show, Generic, Semigroup, Monoid)

7071

7172instance FromJSON Term where

72− parseJSON (String t) = case safeRead (T.unpack t) of

73+ parseJSON (String t) = case safeRead t of

7374 Just t' -> pure t'

7475 Nothing -> Prelude.fail $ "Invalid Term name " ++

7576 show t

@@ -77,7 +78,7 @@ instance FromJSON Term where

7778

7879instance YAML.FromYAML Term where

7980 parseYAML (YAML.Scalar _ (YAML.SStr t)) =

80− case safeRead (T.unpack t) of

81+ case safeRead t of

8182 Just t' -> pure t'

8283 Nothing -> Prelude.fail $ "Invalid Term name " ++

8384 show t

@@ -88,11 +89,11 @@ instance FromJSON Translations where

8889 xs <- mapM addItem (HM.toList hm)

8990 return $ Translations (M.fromList xs)

9091 where addItem (k,v) =

91− case safeRead (T.unpack k) of

92+ case safeRead k of

9293 Nothing -> Prelude.fail $ "Invalid Term name " ++ show k

9394 Just t ->

9495 case v of

95− (String s) -> return (t, T.unpack $ T.strip s)

96+ (String s) -> return (t, T.strip s)

9697 inv -> Aeson.typeMismatch "String" inv

9798 parseJSON invalid = Aeson.typeMismatch "Translations" invalid

9899

@@ -100,22 +101,22 @@ instance YAML.FromYAML Translations where

100101 parseYAML = YAML.withMap "Translations" $

101102 \tr -> Translations .M.fromList <$> mapM addItem (M.toList tr)

102103 where addItem (n@(YAML.Scalar _ (YAML.SStr k)), v) =

103− case safeRead (T.unpack k) of

104+ case safeRead k of

104105 Nothing -> YAML.typeMismatch "Term" n

105106 Just t ->

106107 case v of

107108 (YAML.Scalar _ (YAML.SStr s)) ->

108− return (t, T.unpack (T.strip s))

109+ return (t, T.strip s)

109110 n' -> YAML.typeMismatch "String" n'

110111 addItem (n, _) = YAML.typeMismatch "String" n

111112

112−lookupTerm :: Term -> Translations -> Maybe String

113+lookupTerm :: Term -> Translations -> Maybe T.Text

113114lookupTerm t (Translations tm) = M.lookup t tm

114115

115−readTranslations :: String -> Either String Translations

116+readTranslations :: T.Text -> Either T.Text Translations

116117readTranslations s =

117− case YAML.decodeStrict $ UTF8.fromString s of

118− Left (pos,err') -> Left $ err' ++

118+ case YAML.decodeStrict $ UTF8.fromText s of

119+ Left (pos,err') -> Left $ T.pack $ err' ++

119120 " (line " ++ show (YAML.posLine pos) ++ " column " ++

120121 show (YAML.posColumn pos) ++ ")"

121122 Right (t:_) -> Right t

src/Text/Pandoc/Writers.hs+7−6

@@ -1,7 +1,8 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE FlexibleInstances #-}

33{-# LANGUAGE GADTs #-}

44{-# LANGUAGE ScopedTypeVariables #-}

5+{-# LANGUAGE OverloadedStrings #-}

56{- |

67 Module : Text.Pandoc

@@ -74,8 +75,8 @@ import Control.Monad.Except (throwError)

7475import Control.Monad (unless)

7576import Data.Aeson

7677import qualified Data.ByteString.Lazy as BL

77−import Data.List (intercalate)

7878import Data.Text (Text)

79+import qualified Data.Text as T

7980import Text.Pandoc.Class

8081import Text.Pandoc.Definition

8182import Text.Pandoc.Options

@@ -121,7 +122,7 @@ data Writer m = TextWriter (WriterOptions -> Pandoc -> m Text)

121122 | ByteStringWriter (WriterOptions -> Pandoc -> m BL.ByteString)

122123

123124-- | Association list of formats and writers.

124−writers :: PandocMonad m => [ ( String, Writer m) ]

125+writers :: PandocMonad m => [ (Text, Writer m) ]

125126writers = [

126127 ("native" , TextWriter writeNative)

127128 ,("json" , TextWriter $ \o d -> writeJSON o d)

@@ -179,11 +180,11 @@ writers = [

179180 ]

180181

181182-- | Retrieve writer, extensions based on formatSpec (format+extensions).

182−getWriter :: PandocMonad m => String -> m (Writer m, Extensions)

183+getWriter :: PandocMonad m => Text -> m (Writer m, Extensions)

183184getWriter s =

184185 case parseFormatSpec s of

185186 Left e -> throwError $ PandocAppError

186− $ intercalate "\n" [m | Message m <- errorMessages e]

187+ $ T.intercalate "\n" [T.pack m | Message m <- errorMessages e]

187188 Right (writerName, extsToEnable, extsToDisable) ->

188189 case lookup writerName writers of

189190 Nothing -> throwError $

@@ -198,7 +199,7 @@ getWriter s =

198199 unless (extensionEnabled ext allExts) $

199200 throwError $

200201 PandocUnsupportedExtensionError

201− (drop 4 $ show ext) writerName)

202+ (T.drop 4 $ T.pack $ show ext) writerName)

202203 (extsToEnable ++ extsToDisable)

203204 return (w, exts)

204205

src/Text/Pandoc/Writers/AsciiDoc.hs+55−50

@@ -22,9 +22,9 @@ AsciiDoc: <http://www.methods.co.nz/asciidoc/>

2222module Text.Pandoc.Writers.AsciiDoc (writeAsciiDoc, writeAsciiDoctor) where

2323import Prelude

2424import Control.Monad.State.Strict

25−import Data.Char (isPunctuation, isSpace, toLower, toUpper)

26−import Data.List (intercalate, intersperse, stripPrefix)

27−import Data.Maybe (fromMaybe, isJust, listToMaybe)

25+import Data.Char (isPunctuation, isSpace)

26+import Data.List (intercalate, intersperse)

27+import Data.Maybe (fromMaybe, isJust)

2828import qualified Data.Set as Set

2929import qualified Data.Text as T

3030import Data.Text (Text)

@@ -39,11 +39,11 @@ import Text.Pandoc.Shared

3939import Text.Pandoc.Templates (renderTemplate)

4040import Text.Pandoc.Writers.Shared

4141

42−data WriterState = WriterState { defListMarker :: String

42+data WriterState = WriterState { defListMarker :: Text

4343 , orderedListLevel :: Int

4444 , bulletListLevel :: Int

4545 , intraword :: Bool

46− , autoIds :: Set.Set String

46+ , autoIds :: Set.Set Text

4747 , asciidoctorVariant :: Bool

4848 , inList :: Bool

4949 , hasMath :: Bool

@@ -98,12 +98,12 @@ pandocToAsciiDoc opts (Pandoc meta blocks) = do

9898 Just tpl -> renderTemplate tpl context

9999

100100-- | Escape special characters for AsciiDoc.

101−escapeString :: String -> String

101+escapeString :: Text -> Text

102102escapeString = escapeStringUsing escs

103103 where escs = backslashEscapes "{"

104104

105105-- | Ordered list start parser for use in Para below.

106−olMarker :: Parser [Char] ParserState Char

106+olMarker :: Parser Text ParserState Char

107107olMarker = do (start, style', delim) <- anyOrderedListMarker

108108 if delim == Period &&

109109 (style' == UpperAlpha || (style' == UpperRoman &&

@@ -113,15 +113,18 @@ olMarker = do (start, style', delim) <- anyOrderedListMarker

113113

114114-- | True if string begins with an ordered list marker

115115-- or would be interpreted as an AsciiDoc option command

116−needsEscaping :: String -> Bool

116+needsEscaping :: Text -> Bool

117117needsEscaping s = beginsWithOrderedListMarker s || isBracketed s

118118 where

119119 beginsWithOrderedListMarker str =

120− case runParser olMarker defaultParserState "para start" (take 10 str) of

120+ case runParser olMarker defaultParserState "para start" (T.take 10 str) of

121121 Left _ -> False

122122 Right _ -> True

123− isBracketed ('[':cs) = listToMaybe (reverse cs) == Just ']'

124− isBracketed _ = False

123+ isBracketed t

124+ | Just ('[', t') <- T.uncons t

125+ , Just (_, ']') <- T.unsnoc t'

126+ = True

127+ | otherwise = False

125128

126129-- | Convert Pandoc block element to asciidoc.

127130blockToAsciiDoc :: PandocMonad m

@@ -137,12 +140,13 @@ blockToAsciiDoc opts (Div (id',"section":_,_)

137140blockToAsciiDoc opts (Plain inlines) = do

138141 contents <- inlineListToAsciiDoc opts inlines

139142 return $ contents <> blankline

140−blockToAsciiDoc opts (Para [Image attr alt (src,'f':'i':'g':':':tit)]) =

141− blockToAsciiDoc opts (Para [Image attr alt (src,tit)])

143+blockToAsciiDoc opts (Para [Image attr alt (src,tgt)])

144+ | Just tit <- T.stripPrefix "fig:" tgt

145+ = blockToAsciiDoc opts (Para [Image attr alt (src,tit)])

142146blockToAsciiDoc opts (Para inlines) = do

143147 contents <- inlineListToAsciiDoc opts inlines

144148 -- escape if para starts with ordered list marker

145− let esc = if needsEscaping (T.unpack $ render Nothing contents)

149+ let esc = if needsEscaping (render Nothing contents)

146150 then text "{empty}"

147151 else empty

148152 return $ esc <> contents <> blankline

@@ -154,7 +158,7 @@ blockToAsciiDoc opts (LineBlock lns) = do

154158 contents <- joinWithLinefeeds <$> mapM docify lns

155159 return $ "[verse]" $$ text "--" $$ contents $$ text "--" $$ blankline

156160blockToAsciiDoc _ b@(RawBlock f s)

157− | f == "asciidoc" = return $ text s

161+ | f == "asciidoc" = return $ literal s

158162 | otherwise = do

159163 report $ BlockNotRendered b

160164 return empty

@@ -165,20 +169,20 @@ blockToAsciiDoc opts (Header level (ident,_,_) inlines) = do

165169 ids <- gets autoIds

166170 let autoId = uniqueIdent (writerExtensions opts) inlines ids

167171 modify $ \st -> st{ autoIds = Set.insert autoId ids }

168− let identifier = if null ident ||

172+ let identifier = if T.null ident ||

169173 (isEnabled Ext_auto_identifiers opts && ident == autoId)

170174 then empty

171− else "[[" <> text ident <> "]]"

175+ else "[[" <> literal ident <> "]]"

172176 return $ identifier $$

173177 nowrap (text (replicate (level + 1) '=') <> space <> contents) <>

174178 blankline

175179

176180blockToAsciiDoc _ (CodeBlock (_,classes,_) str) = return $ flush (

177181 if null classes

178− then "...." $$ text str $$ "...."

179− else attrs $$ "----" $$ text str $$ "----")

182+ then "...." $$ literal str $$ "...."

183+ else attrs $$ "----" $$ literal str $$ "----")

180184 <> blankline

181− where attrs = "[" <> text (intercalate "," ("source" : classes)) <> "]"

185+ where attrs = "[" <> literal (T.intercalate "," ("source" : classes)) <> "]"

182186blockToAsciiDoc opts (BlockQuote blocks) = do

183187 contents <- blockListToAsciiDoc opts blocks

184188 let isBlock (BlockQuote _) = True

@@ -258,11 +262,11 @@ blockToAsciiDoc opts (OrderedList (start, sty, _delim) items) = do

258262 DefaultStyle -> []

259263 Decimal -> ["arabic"]

260264 Example -> []

261− _ -> [map toLower (show sty)]

262− let listStart = if start == 1 then [] else ["start=" ++ show start]

263− let listoptions = case intercalate ", " (listStyle ++ listStart) of

264− [] -> empty

265− x -> brackets (text x)

265+ _ -> [T.toLower (tshow sty)]

266+ let listStart = if start == 1 then [] else ["start=" <> tshow start]

267+ let listoptions = case T.intercalate ", " (listStyle ++ listStart) of

268+ "" -> empty

269+ x -> brackets (literal x)

266270 inlist <- gets inList

267271 modify $ \st -> st{ inList = True }

268272 contents <- mapM (orderedListItemToAsciiDoc opts) items

@@ -275,7 +279,7 @@ blockToAsciiDoc opts (DefinitionList items) = do

275279 modify $ \st -> st{ inList = inlist }

276280 return $ mconcat contents <> blankline

277281blockToAsciiDoc opts (Div (ident,classes,_) bs) = do

278− let identifier = if null ident then empty else "[[" <> text ident <> "]]"

282+ let identifier = if T.null ident then empty else "[[" <> literal ident <> "]]"

279283 let admonitions = ["attention","caution","danger","error","hint",

280284 "important","note","tip","warning"]

281285 contents <-

@@ -290,7 +294,7 @@ blockToAsciiDoc opts (Div (ident,classes,_) bs) = do

290294 else ("." <>) <$>

291295 blockListToAsciiDoc opts titleBs

292296 admonitionBody <- blockListToAsciiDoc opts bodyBs

293− return $ "[" <> text (map toUpper l) <> "]" $$

297+ return $ "[" <> literal (T.toUpper l) <> "]" $$

294298 chomp admonitionTitle $$

295299 "====" $$

296300 chomp admonitionBody $$

@@ -365,7 +369,7 @@ definitionListItemToAsciiDoc opts (label, defs) = do

365369 defs' <- mapM defsToAsciiDoc defs

366370 modify (\st -> st{ defListMarker = marker })

367371 let contents = nest 2 $ vcat $ intersperse divider $ map chomp defs'

368− return $ labelText <> text marker <> cr <> contents <> cr

372+ return $ labelText <> literal marker <> cr <> contents <> cr

369373

370374-- | Convert list of Pandoc block elements to asciidoc.

371375blockListToAsciiDoc :: PandocMonad m

@@ -408,10 +412,11 @@ inlineListToAsciiDoc opts lst = do

408412 isSpacy _ SoftBreak = True

409413 -- Note that \W characters count as spacy in AsciiDoc

410414 -- for purposes of determining interword:

411− isSpacy End (Str xs) = case reverse xs of

412− c:_ -> isPunctuation c || isSpace c

413− _ -> False

414− isSpacy Start (Str (c:_)) = isPunctuation c || isSpace c

415+ isSpacy End (Str xs) = case T.unsnoc xs of

416+ Just (_, c) -> isPunctuation c || isSpace c

417+ _ -> False

418+ isSpacy Start (Str xs)

419+ | Just (c, _) <- T.uncons xs = isPunctuation c || isSpace c

415420 isSpacy _ _ = False

416421

417422setIntraword :: PandocMonad m => Bool -> ADW m ()

@@ -456,25 +461,25 @@ inlineToAsciiDoc opts (Quoted qt lst) = do

456461 | otherwise -> [Str "``"] ++ lst ++ [Str "''"]

457462inlineToAsciiDoc _ (Code _ str) = do

458463 isAsciidoctor <- gets asciidoctorVariant

459− let contents = text (escapeStringUsing (backslashEscapes "`") str)

464+ let contents = literal (escapeStringUsing (backslashEscapes "`") str)

460465 return $

461466 if isAsciidoctor

462467 then text "`+" <> contents <> "+`"

463468 else text "`" <> contents <> "`"

464−inlineToAsciiDoc _ (Str str) = return $ text $ escapeString str

469+inlineToAsciiDoc _ (Str str) = return $ literal $ escapeString str

465470inlineToAsciiDoc _ (Math InlineMath str) = do

466471 isAsciidoctor <- gets asciidoctorVariant

467472 modify $ \st -> st{ hasMath = True }

468473 let content = if isAsciidoctor

469− then text str

470− else "$" <> text str <> "$"

474+ then literal str

475+ else "$" <> literal str <> "$"

471476 return $ "latexmath:[" <> content <> "]"

472477inlineToAsciiDoc _ (Math DisplayMath str) = do

473478 isAsciidoctor <- gets asciidoctorVariant

474479 modify $ \st -> st{ hasMath = True }

475480 let content = if isAsciidoctor

476− then text str

477− else "\\[" <> text str <> "\\]"

481+ then literal str

482+ else "\\[" <> literal str <> "\\]"

478483 inlist <- gets inList

479484 let sepline = if inlist

480485 then text "+"

@@ -483,7 +488,7 @@ inlineToAsciiDoc _ (Math DisplayMath str) = do

483488 (cr <> sepline) $$ "[latexmath]" $$ "++++" $$

484489 content $$ "++++" <> cr

485490inlineToAsciiDoc _ il@(RawInline f s)

486− | f == "asciidoc" = return $ text s

491+ | f == "asciidoc" = return $ literal s

487492 | otherwise = do

488493 report $ InlineNotRendered il

489494 return empty

@@ -501,38 +506,38 @@ inlineToAsciiDoc opts (Link _ txt (src, _tit)) = do

501506-- abs: http://google.cod[Google]

502507-- or my@email.com[email john]

503508 linktext <- inlineListToAsciiDoc opts txt

504− let isRelative = ':' `notElem` src

509+ let isRelative = T.all (/= ':') src

505510 let prefix = if isRelative

506511 then text "link:"

507512 else empty

508− let srcSuffix = fromMaybe src (stripPrefix "mailto:" src)

513+ let srcSuffix = fromMaybe src (T.stripPrefix "mailto:" src)

509514 let useAuto = case txt of

510515 [Str s] | escapeURI s == srcSuffix -> True

511516 _ -> False

512517 return $ if useAuto

513− then text srcSuffix

514− else prefix <> text src <> "[" <> linktext <> "]"

518+ then literal srcSuffix

519+ else prefix <> literal src <> "[" <> linktext <> "]"

515520inlineToAsciiDoc opts (Image attr alternate (src, tit)) = do

516521-- image:images/logo.png[Company logo, title="blah"]

517522 let txt = if null alternate || (alternate == [Str ""])

518523 then [Str "image"]

519524 else alternate

520525 linktext <- inlineListToAsciiDoc opts txt

521− let linktitle = if null tit

526+ let linktitle = if T.null tit

522527 then empty

523− else ",title=\"" <> text tit <> "\""

528+ else ",title=\"" <> literal tit <> "\""

524529 showDim dir = case dimension dir attr of

525530 Just (Percent a) ->

526531 ["scaledwidth=" <> text (show (Percent a))]

527532 Just dim ->

528− [text (show dir) <> "=" <> text (showInPixel opts dim)]

533+ [text (show dir) <> "=" <> literal (showInPixel opts dim)]

529534 Nothing ->

530535 []

531536 dimList = showDim Width ++ showDim Height

532537 dims = if null dimList

533538 then empty

534539 else "," <> mconcat (intersperse "," dimList)

535− return $ "image:" <> text src <> "[" <> linktext <> linktitle <> dims <> "]"

540+ return $ "image:" <> literal src <> "[" <> linktext <> linktitle <> dims <> "]"

536541inlineToAsciiDoc opts (Note [Para inlines]) =

537542 inlineToAsciiDoc opts (Note [Plain inlines])

538543inlineToAsciiDoc opts (Note [Plain inlines]) = do

@@ -544,9 +549,9 @@ inlineToAsciiDoc opts (Span (ident,classes,_) ils) = do

544549 contents <- inlineListToAsciiDoc opts ils

545550 isIntraword <- gets intraword

546551 let marker = if isIntraword then "##" else "#"

547− if null ident && null classes

552+ if T.null ident && null classes

548553 then return contents

549554 else do

550− let modifier = brackets $ text $ unwords $

551− [ '#':ident | not (null ident)] ++ map ('.':) classes

555+ let modifier = brackets $ literal $ T.unwords $

556+ [ "#" <> ident | not (T.null ident)] ++ map ("." <>) classes

552557 return $ modifier <> marker <> contents <> marker

src/Text/Pandoc/Writers/CommonMark.hs+30−29

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE OverloadedStrings #-}

3+{-# LANGUAGE ViewPatterns #-}

34{- |

45 Module : Text.Pandoc.Writers.CommonMark

@@ -28,7 +29,7 @@ import Text.Pandoc.Class (PandocMonad)

2829import Text.Pandoc.Definition

2930import Text.Pandoc.Options

3031import Text.Pandoc.Shared (capitalize, isHeaderBlock, isTightList,

31− linesToPara, onlySimpleTableCells, substitute, taskListItemToAscii)

32+ linesToPara, onlySimpleTableCells, taskListItemToAscii, tshow)

3233import Text.Pandoc.Templates (renderTemplate)

3334import Text.Pandoc.Walk (walk, walkM)

3435import Text.Pandoc.Writers.HTML (writeHtml5String, tagWithAttributes)

@@ -73,7 +74,7 @@ processNotes :: Inline -> State [[Block]] Inline

7374processNotes (Note bs) = do

7475 modify (bs :)

7576 notes <- get

76− return $ Str $ "[" ++ show (length notes) ++ "]"

77+ return $ Str $ "[" <> tshow (length notes) <> "]"

7778processNotes x = return x

7879

7980node :: NodeType -> [Node] -> Node

@@ -109,14 +110,14 @@ blockToNodes opts (Para xs) ns =

109110 return (node PARAGRAPH (inlinesToNodes opts xs) : ns)

110111blockToNodes opts (LineBlock lns) ns = blockToNodes opts (linesToPara lns) ns

111112blockToNodes _ (CodeBlock (_,classes,_) xs) ns = return

112− (node (CODE_BLOCK (T.pack (unwords classes)) (T.pack xs)) [] : ns)

113+ (node (CODE_BLOCK (T.unwords classes) xs) [] : ns)

113114blockToNodes opts (RawBlock (Format f) xs) ns

114115 | f == "html" && isEnabled Ext_raw_html opts

115− = return (node (HTML_BLOCK (T.pack xs)) [] : ns)

116+ = return (node (HTML_BLOCK xs) [] : ns)

116117 | (f == "latex" || f == "tex") && isEnabled Ext_raw_tex opts

117− = return (node (CUSTOM_BLOCK (T.pack xs) T.empty) [] : ns)

118+ = return (node (CUSTOM_BLOCK xs T.empty) [] : ns)

118119 | f == "markdown"

119− = return (node (CUSTOM_BLOCK (T.pack xs) T.empty) [] : ns)

120+ = return (node (CUSTOM_BLOCK xs T.empty) [] : ns)

120121 | otherwise = return ns

121122blockToNodes opts (BlockQuote bs) ns = do

122123 nodes <- blocksToNodes opts bs

@@ -169,9 +170,9 @@ blockToNodes opts t@(Table capt aligns _widths headers rows) ns = do

169170 let capt' = node PARAGRAPH (inlinesToNodes opts capt)

170171 -- backslash | in code and raw:

171172 let fixPipe (Code attr xs) =

172− Code attr (substitute "|" "\\|" xs)

173+ Code attr (T.replace "|" "\\|" xs)

173174 fixPipe (RawInline format xs) =

174− RawInline format (substitute "|" "\\|" xs)

175+ RawInline format (T.replace "|" "\\|" xs)

175176 fixPipe x = x

176177 let toCell [Plain ils] = T.strip

177178 $ nodeToCommonmark [] Nothing

@@ -276,19 +277,19 @@ inlineToNodes opts (SmallCaps xs) =

276277 [node (HTML_INLINE (T.pack "")) []]) ++ )

277278 else (inlinesToNodes opts (capitalize xs) ++)

278279inlineToNodes opts (Link _ ils (url,tit)) =

279− (node (LINK (T.pack url) (T.pack tit)) (inlinesToNodes opts ils) :)

280+ (node (LINK url tit) (inlinesToNodes opts ils) :)

280281-- title beginning with fig: indicates implicit figure

281−inlineToNodes opts (Image alt ils (url,'f':'i':'g':':':tit)) =

282+inlineToNodes opts (Image alt ils (url,T.stripPrefix "fig:" -> Just tit)) =

282283 inlineToNodes opts (Image alt ils (url,tit))

283284inlineToNodes opts (Image _ ils (url,tit)) =

284− (node (IMAGE (T.pack url) (T.pack tit)) (inlinesToNodes opts ils) :)

285+ (node (IMAGE url tit) (inlinesToNodes opts ils) :)

285286inlineToNodes opts (RawInline (Format f) xs)

286287 | f == "html" && isEnabled Ext_raw_html opts

287− = (node (HTML_INLINE (T.pack xs)) [] :)

288+ = (node (HTML_INLINE xs) [] :)

288289 | (f == "latex" || f == "tex") && isEnabled Ext_raw_tex opts

289− = (node (CUSTOM_INLINE (T.pack xs) T.empty) [] :)

290+ = (node (CUSTOM_INLINE xs T.empty) [] :)

290291 | f == "markdown"

291− = (node (CUSTOM_INLINE (T.pack xs) T.empty) [] :)

292+ = (node (CUSTOM_INLINE xs T.empty) [] :)

292293 | otherwise = id

293294inlineToNodes opts (Quoted qt ils) =

294295 ((node (HTML_INLINE start) [] :

@@ -304,12 +305,12 @@ inlineToNodes opts (Quoted qt ils) =

304305 | writerPreferAscii opts ->

305306 ("“", "”")

306307 | otherwise -> ("“", "”")

307−inlineToNodes _ (Code _ str) = (node (CODE (T.pack str)) [] :)

308+inlineToNodes _ (Code _ str) = (node (CODE str) [] :)

308309inlineToNodes opts (Math mt str) =

309310 case writerHTMLMathMethod opts of

310311 WebTeX url ->

311312 let core = inlineToNodes opts

312− (Image nullAttr [Str str] (url ++ urlEncode str, str))

313+ (Image nullAttr [Str str] (url <> T.pack (urlEncode $ T.unpack str), str))

313314 sep = if mt == DisplayMath

314315 then (node LINEBREAK [] :)

315316 else id

@@ -317,14 +318,14 @@ inlineToNodes opts (Math mt str) =

317318 _ ->

318319 case mt of

319320 InlineMath ->

320− (node (HTML_INLINE (T.pack ("\$" ++ str ++ "\$"))) [] :)

321+ (node (HTML_INLINE ("\$" <> str <> "\$")) [] :)

321322 DisplayMath ->

322− (node (HTML_INLINE (T.pack ("\\[" ++ str ++ "\\]"))) [] :)

323+ (node (HTML_INLINE ("\\[" <> str <> "\\]")) [] :)

323324inlineToNodes opts (Span ("",["emoji"],kvs) [Str s]) = do

324325 case lookup "data-emoji" kvs of

325326 Just emojiname | isEnabled Ext_emoji opts ->

326− (node (TEXT (":" <> T.pack emojiname <> ":")) [] :)

327− _ -> (node (TEXT (T.pack s)) [] :)

327+ (node (TEXT (":" <> emojiname <> ":")) [] :)

328+ _ -> (node (TEXT s) [] :)

328329inlineToNodes opts (Span attr ils) =

329330 let nodes = inlinesToNodes opts ils

330331 op = tagWithAttributes opts True False "span" attr

@@ -336,17 +337,17 @@ inlineToNodes opts (Cite _ ils) = (inlinesToNodes opts ils ++)

336337inlineToNodes _ (Note _) = id -- should not occur

337338-- we remove Note elements in preprocessing

338339

339−stringToNodes :: WriterOptions -> String -> [Node] -> [Node]

340+stringToNodes :: WriterOptions -> Text -> [Node] -> [Node]

340341stringToNodes opts s

341− | not (writerPreferAscii opts) = (node (TEXT (T.pack s)) [] :)

342+ | not (writerPreferAscii opts) = (node (TEXT s) [] :)

342343 | otherwise = step s

343344 where

344345 step input =

345− let (ascii, rest) = span isAscii input

346− this = node (TEXT (T.pack ascii)) []

347− nodes = case rest of

348− [] -> id

349− (nonAscii : rest') ->

346+ let (ascii, rest) = T.span isAscii input

347+ this = node (TEXT ascii) []

348+ nodes = case T.uncons rest of

349+ Nothing -> id

350+ Just (nonAscii, rest') ->

350351 let escaped = toHtml5Entities (T.singleton nonAscii)

351352 in (node (HTML_INLINE escaped) [] :) . step rest'

352353 in (this :) . nodes

@@ -354,7 +355,7 @@ stringToNodes opts s

354355toSubscriptInline :: Inline -> Maybe Inline

355356toSubscriptInline Space = Just Space

356357toSubscriptInline (Span attr ils) = Span attr <$> traverse toSubscriptInline ils

357−toSubscriptInline (Str s) = Str <$> traverse toSubscript s

358+toSubscriptInline (Str s) = Str . T.pack <$> traverse toSubscript (T.unpack s)

358359toSubscriptInline LineBreak = Just LineBreak

359360toSubscriptInline SoftBreak = Just SoftBreak

360361toSubscriptInline _ = Nothing

@@ -362,7 +363,7 @@ toSubscriptInline _ = Nothing

362363toSuperscriptInline :: Inline -> Maybe Inline

363364toSuperscriptInline Space = Just Space

364365toSuperscriptInline (Span attr ils) = Span attr <$> traverse toSuperscriptInline ils

365−toSuperscriptInline (Str s) = Str <$> traverse toSuperscript s

366+toSuperscriptInline (Str s) = Str . T.pack <$> traverse toSuperscript (T.unpack s)

366367toSuperscriptInline LineBreak = Just LineBreak

367368toSuperscriptInline SoftBreak = Just SoftBreak

368369toSuperscriptInline _ = Nothing

src/Text/Pandoc/Writers/ConTeXt.hs+77−74

@@ -1,6 +1,7 @@

1−{-# LANGUAGE NoImplicitPrelude #-}

1+{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE OverloadedStrings #-}

33{-# LANGUAGE ScopedTypeVariables #-}

4+{-# LANGUAGE ViewPatterns #-}

45{- |

56 Module : Text.Pandoc.Writers.ConTeXt

@@ -15,8 +16,8 @@ Conversion of 'Pandoc' format into ConTeXt.

1516module Text.Pandoc.Writers.ConTeXt ( writeConTeXt ) where

1617import Prelude

1718import Control.Monad.State.Strict

18−import Data.Char (ord, isDigit, toLower)

19−import Data.List (intercalate, intersperse)

19+import Data.Char (ord, isDigit)

20+import Data.List (intersperse)

2021import Data.Maybe (mapMaybe)

2122import Data.Text (Text)

2223import qualified Data.Text as T

@@ -89,14 +90,14 @@ pandocToConTeXt options (Pandoc meta blocks) = do

8990 $ defField "layout" layoutFromMargins

9091 $ defField "number-sections" (writerNumberSections options)

9192 $ maybe id (\l ->

92− defField "context-lang" (text l :: Doc Text)) mblang

93+ defField "context-lang" (literal l :: Doc Text)) mblang

9394 $ (case T.unpack . render Nothing <$>

9495 getField "papersize" metadata of

9596 Just (('a':d:ds) :: String)

9697 | all isDigit (d:ds) -> resetField "papersize"

9798 (T.pack ('A':d:ds))

9899 _ -> id)

99− $ (case toLower <$> lookupMetaString "pdfa" meta of

100+ $ (case T.toLower $ lookupMetaString "pdfa" meta of

100101 "true" -> resetField "pdfa" (T.pack "1b:2005")

101102 _ -> id) metadata

102103 let context' = defField "context-dir" (maybe mempty toContextDir

@@ -114,7 +115,7 @@ toContextDir = fmap (\t -> case t of

114115 _ -> t)

115116

116117-- | escape things as needed for ConTeXt

117−escapeCharForConTeXt :: WriterOptions -> Char -> String

118+escapeCharForConTeXt :: WriterOptions -> Char -> Text

118119escapeCharForConTeXt opts ch =

119120 let ligatures = isEnabled Ext_smart opts in

120121 case ch of

@@ -133,18 +134,18 @@ escapeCharForConTeXt opts ch =

133134 '\x2013' | ligatures -> "--"

134135 '\x2019' | ligatures -> "'"

135136 '\x2026' -> "\\ldots{}"

136− x -> [x]

137+ x -> T.singleton x

137138

138139-- | Escape string for ConTeXt

139−stringToConTeXt :: WriterOptions -> String -> String

140−stringToConTeXt opts = concatMap (escapeCharForConTeXt opts)

140+stringToConTeXt :: WriterOptions -> Text -> Text

141+stringToConTeXt opts = T.concatMap (escapeCharForConTeXt opts)

141142

142143-- | Sanitize labels

143−toLabel :: String -> String

144−toLabel z = concatMap go z

144+toLabel :: Text -> Text

145+toLabel z = T.concatMap go z

145146 where go x

146− | x `elem` ("\\#[]\",{}%()|=" :: String) = "ux" ++ printf "%x" (ord x)

147− | otherwise = [x]

147+ | x `elem` ("\\#[]\",{}%()|=" :: String) = "ux" <> T.pack (printf "%x" (ord x))

148+ | otherwise = T.singleton x

148149

149150-- | Convert Pandoc block element to ConTeXt.

150151blockToConTeXt :: PandocMonad m => Block -> WM m (Doc Text)

@@ -157,14 +158,16 @@ blockToConTeXt (Div attr@(_,"section":_,_)

157158 return $ header' $$ innerContents $$ footer'

158159blockToConTeXt (Plain lst) = inlineListToConTeXt lst

159160-- title beginning with fig: indicates that the image is a figure

160−blockToConTeXt (Para [Image attr txt (src,'f':'i':'g':':':_)]) = do

161− capt <- inlineListToConTeXt txt

162− img <- inlineToConTeXt (Image attr txt (src, ""))

163− let (ident, _, _) = attr

164− label = if null ident

165− then empty

166− else "[]" <> brackets (text $ toLabel ident)

167− return $ blankline $$ "\\placefigure" <> label <> braces capt <> img <> blankline

161+blockToConTeXt (Para [Image attr txt (src,tgt)])

162+ | Just _ <- T.stripPrefix "fig:" tgt

163+ = do

164+ capt <- inlineListToConTeXt txt

165+ img <- inlineToConTeXt (Image attr txt (src, ""))

166+ let (ident, _, _) = attr

167+ label = if T.null ident

168+ then empty

169+ else "[]" <> brackets (literal $ toLabel ident)

170+ return $ blankline $$ "\\placefigure" <> label <> braces capt <> img <> blankline

168171blockToConTeXt (Para lst) = do

169172 contents <- inlineListToConTeXt lst

170173 return $ contents <> blankline

@@ -175,17 +178,17 @@ blockToConTeXt (BlockQuote lst) = do

175178 contents <- blockListToConTeXt lst

176179 return $ "\\startblockquote" $$ nest 0 contents $$ "\\stopblockquote" <> blankline

177180blockToConTeXt (CodeBlock _ str) =

178− return $ flush ("\\starttyping" <> cr <> text str <> cr <> "\\stoptyping") $$ blankline

181+ return $ flush ("\\starttyping" <> cr <> literal str <> cr <> "\\stoptyping") $$ blankline

179182 -- blankline because \stoptyping can't have anything after it, inc. '}'

180183blockToConTeXt b@(RawBlock f str)

181− | f == Format "context" || f == Format "tex" = return $ text str <> blankline

184+ | f == Format "context" || f == Format "tex" = return $ literal str <> blankline

182185 | otherwise = empty <$ report (BlockNotRendered b)

183186blockToConTeXt (Div (ident,_,kvs) bs) = do

184187 let align dir txt = "\\startalignment[" <> dir <> "]" $$ txt $$ "\\stopalignment"

185188 mblang <- fromBCP47 (lookup "lang" kvs)

186− let wrapRef txt = if null ident

189+ let wrapRef txt = if T.null ident

187190 then txt

188− else ("\\reference" <> brackets (text $ toLabel ident) <>

191+ else ("\\reference" <> brackets (literal $ toLabel ident) <>

189192 braces empty <> "%") $$ txt

190193 wrapDir = case lookup "dir" kvs of

191194 Just "rtl" -> align "righttoleft"

@@ -193,7 +196,7 @@ blockToConTeXt (Div (ident,_,kvs) bs) = do

193196 _ -> id

194197 wrapLang txt = case mblang of

195198 Just lng -> "\\start\\language["

196− <> text lng <> "]" $$ txt $$ "\\stop"

199+ <> literal lng <> "]" $$ txt $$ "\\stop"

197200 Nothing -> txt

198201 wrapBlank txt = blankline <> txt <> blankline

199202 (wrapBlank . wrapLang . wrapDir . wrapRef) <$> blockListToConTeXt bs

@@ -202,29 +205,29 @@ blockToConTeXt (BulletList lst) = do

202205 return $ ("\\startitemize" <> if isTightList lst

203206 then brackets "packed"

204207 else empty) $$

205− vcat contents $$ text "\\stopitemize" <> blankline

208+ vcat contents $$ literal "\\stopitemize" <> blankline

206209blockToConTeXt (OrderedList (start, style', delim) lst) = do

207210 st <- get

208211 let level = stOrderedListLevel st

209212 put st {stOrderedListLevel = level + 1}

210213 contents <- mapM listItemToConTeXt lst

211214 put st {stOrderedListLevel = level}

212− let start' = if start == 1 then "" else "start=" ++ show start

215+ let start' = if start == 1 then "" else "start=" <> tshow start

213216 let delim' = case delim of

214217 DefaultDelim -> ""

215218 Period -> "stopper=."

216219 OneParen -> "stopper=)"

217220 TwoParens -> "left=(,stopper=)"

218− let width = maximum $ map length $ take (length contents)

221+ let width = maximum $ map T.length $ take (length contents)

219222 (orderedListMarkers (start, style', delim))

220223 let width' = (toEnum width + 1) / 2

221224 let width'' = if width' > (1.5 :: Double)

222− then "width=" ++ show width' ++ "em"

225+ then "width=" <> tshow width' <> "em"

223226 else ""

224− let specs2Items = filter (not . null) [start', delim', width'']

227+ let specs2Items = filter (not . T.null) [start', delim', width'']

225228 let specs2 = if null specs2Items

226229 then ""

227− else "[" ++ intercalate "," specs2Items ++ "]"

230+ else "[" <> T.intercalate "," specs2Items <> "]"

228231 let style'' = '[': (case style' of

229232 DefaultStyle -> orderedListStyles !! level

230233 Decimal -> 'n'

@@ -234,8 +237,8 @@ blockToConTeXt (OrderedList (start, style', delim) lst) = do

234237 LowerAlpha -> 'a'

235238 UpperAlpha -> 'A') :

236239 if isTightList lst then ",packed]" else "]"

237− let specs = style'' ++ specs2

238− return $ "\\startitemize" <> text specs $$ vcat contents $$

240+ let specs = T.pack style'' <> specs2

241+ return $ "\\startitemize" <> literal specs $$ vcat contents $$

239242 "\\stopitemize" <> blankline

240243blockToConTeXt (DefinitionList lst) =

241244 liftM vcat $ mapM defListItemToConTeXt lst

@@ -343,9 +346,9 @@ inlineListToConTeXt lst = liftM hcat $ mapM inlineToConTeXt $ addStruts lst

343346 addStruts xs

344347 addStruts (x:xs) = x : addStruts xs

345348 addStruts [] = []

346− isSpacey Space = True

347− isSpacey (Str ('\160':_)) = True

348− isSpacey _ = False

349+ isSpacey Space = True

350+ isSpacey (Str (T.uncons -> Just ('\160',_))) = True

351+ isSpacey _ = False

349352

350353-- | Convert inline element to ConTeXt

351354inlineToConTeXt :: PandocMonad m

@@ -369,11 +372,11 @@ inlineToConTeXt (Subscript lst) = do

369372inlineToConTeXt (SmallCaps lst) = do

370373 contents <- inlineListToConTeXt lst

371374 return $ braces $ "\\sc " <> contents

372−inlineToConTeXt (Code _ str) | not ('{' `elem` str || '}' `elem` str) =

373− return $ "\\type" <> braces (text str)

375+inlineToConTeXt (Code _ str) | not ('{' `elemText` str || '}' `elemText` str) =

376+ return $ "\\type" <> braces (literal str)

374377inlineToConTeXt (Code _ str) = do

375378 opts <- gets stOptions

376− return $ "\\mono" <> braces (text $ stringToConTeXt opts str)

379+ return $ "\\mono" <> braces (literal $ stringToConTeXt opts str)

377380inlineToConTeXt (Quoted SingleQuote lst) = do

378381 contents <- inlineListToConTeXt lst

379382 return $ "\\quote" <> braces contents

@@ -383,15 +386,15 @@ inlineToConTeXt (Quoted DoubleQuote lst) = do

383386inlineToConTeXt (Cite _ lst) = inlineListToConTeXt lst

384387inlineToConTeXt (Str str) = do

385388 opts <- gets stOptions

386− return $ text $ stringToConTeXt opts str

389+ return $ literal $ stringToConTeXt opts str

387390inlineToConTeXt (Math InlineMath str) =

388− return $ char '$' <> text str <> char '$'

391+ return $ char '$' <> literal str <> char '$'

389392inlineToConTeXt (Math DisplayMath str) =

390− return $ text "\\startformula " <> text str <> text " \\stopformula" <> space

393+ return $ literal "\\startformula " <> literal str <> literal " \\stopformula" <> space

391394inlineToConTeXt il@(RawInline f str)

392− | f == Format "tex" || f == Format "context" = return $ text str

395+ | f == Format "tex" || f == Format "context" = return $ literal str

393396 | otherwise = empty <$ report (InlineNotRendered il)

394−inlineToConTeXt LineBreak = return $ text "\\crlf" <> cr

397+inlineToConTeXt LineBreak = return $ literal "\\crlf" <> cr

395398inlineToConTeXt SoftBreak = do

396399 wrapText <- gets (writerWrapText . stOptions)

397400 return $ case wrapText of

@@ -400,39 +403,39 @@ inlineToConTeXt SoftBreak = do

400403 WrapPreserve -> cr

401404inlineToConTeXt Space = return space

402405-- Handle HTML-like internal document references to sections

403−inlineToConTeXt (Link _ txt ('#' : ref, _)) = do

406+inlineToConTeXt (Link _ txt (T.uncons -> Just ('#', ref), _)) = do

404407 opts <- gets stOptions

405408 contents <- inlineListToConTeXt txt

406409 let ref' = toLabel $ stringToConTeXt opts ref

407− return $ text "\\goto"

410+ return $ literal "\\goto"

408411 <> braces contents

409− <> brackets (text ref')

412+ <> brackets (literal ref')

410413

411414inlineToConTeXt (Link _ txt (src, _)) = do

412− let isAutolink = txt == [Str (unEscapeString src)]

415+ let isAutolink = txt == [Str (T.pack $ unEscapeString $ T.unpack src)]

413416 st <- get

414417 let next = stNextRef st

415418 put $ st {stNextRef = next + 1}

416− let ref = "url" ++ show next

419+ let ref = "url" <> tshow next

417420 contents <- inlineListToConTeXt txt

418421 return $ "\\useURL"

419− <> brackets (text ref)

420− <> brackets (text $ escapeStringUsing [('#',"\\#"),('%',"\\%")] src)

422+ <> brackets (literal ref)

423+ <> brackets (literal $ escapeStringUsing [('#',"\\#"),('%',"\\%")] src)

421424 <> (if isAutolink

422425 then empty

423426 else brackets empty <> brackets contents)

424427 <> "\\from"

425− <> brackets (text ref)

428+ <> brackets (literal ref)

426429inlineToConTeXt (Image attr@(_,cls,_) _ (src, _)) = do

427430 opts <- gets stOptions

428− let showDim dir = let d = text (show dir) <> "="

431+ let showDim dir = let d = literal (tshow dir) <> "="

429432 in case dimension dir attr of

430433 Just (Pixel a) ->

431− [d <> text (showInInch opts (Pixel a)) <> "in"]

434+ [d <> literal (showInInch opts (Pixel a)) <> "in"]

432435 Just (Percent a) ->

433− [d <> text (showFl (a / 100)) <> "\\textwidth"]

436+ [d <> literal (showFl (a / 100)) <> "\\textwidth"]

434437 Just dim ->

435− [d <> text (show dim)]

438+ [d <> literal (tshow dim)]

436439 Nothing ->

437440 []

438441 dimList = showDim Width ++ showDim Height

@@ -441,25 +444,25 @@ inlineToConTeXt (Image attr@(_,cls,_) _ (src, _)) = do

441444 else brackets $ mconcat (intersperse "," dimList)

442445 clas = if null cls

443446 then empty

444− else brackets $ text $ toLabel $ head cls

447+ else brackets $ literal $ toLabel $ head cls

445448 -- Use / for path separators on Windows; see #4918

446− fixPathSeparators = map $ \c -> case c of

447− '\\' -> '/'

448− _ -> c

449+ fixPathSeparators = T.map $ \c -> case c of

450+ '\\' -> '/'

451+ _ -> c

449452 src' = fixPathSeparators $

450453 if isURI src

451454 then src

452− else unEscapeString src

453− return $ braces $ "\\externalfigure" <> brackets (text src') <> dims <> clas

455+ else T.pack $ unEscapeString $ T.unpack src

456+ return $ braces $ "\\externalfigure" <> brackets (literal src') <> dims <> clas

454457inlineToConTeXt (Note contents) = do

455458 contents' <- blockListToConTeXt contents

456459 let codeBlock x@(CodeBlock _ _) = [x]

457460 codeBlock _ = []

458461 let codeBlocks = query codeBlock contents

459462 return $ if null codeBlocks

460− then text "\\footnote{" <> nest 2 (chomp contents') <> char '}'

461− else text "\\startbuffer " <> nest 2 (chomp contents') <>

462− text "\\stopbuffer\\footnote{\\getbuffer}"

463+ then literal "\\footnote{" <> nest 2 (chomp contents') <> char '}'

464+ else literal "\\startbuffer " <> nest 2 (chomp contents') <>

465+ literal "\\stopbuffer\\footnote{\\getbuffer}"

463466inlineToConTeXt (Span (_,_,kvs) ils) = do

464467 mblang <- fromBCP47 (lookup "lang" kvs)

465468 let wrapDir txt = case lookup "dir" kvs of

@@ -467,7 +470,7 @@ inlineToConTeXt (Span (_,_,kvs) ils) = do

467470 Just "ltr" -> braces $ "\\lefttoright " <> txt

468471 _ -> txt

469472 wrapLang txt = case mblang of

470− Just lng -> "\\start\\language[" <> text lng

473+ Just lng -> "\\start\\language[" <> literal lng

471474 <> "]" <> txt <> "\\stop "

472475 Nothing -> txt

473476 (wrapLang . wrapDir) <$> inlineListToConTeXt ils

@@ -482,9 +485,9 @@ sectionHeader (ident,classes,kvs) hdrLevel lst = do

482485 opts <- gets stOptions

483486 contents <- inlineListToConTeXt lst

484487 levelText <- sectionLevelToText opts (ident,classes,kvs) hdrLevel

485− let ident' = if null ident

488+ let ident' = if T.null ident

486489 then empty

487− else "reference=" <> braces (text (toLabel ident))

490+ else "reference=" <> braces (literal (toLabel ident))

488491 let contents' = if isEmpty contents

489492 then empty

490493 else "title=" <> braces contents

@@ -515,23 +518,23 @@ sectionLevelToText opts (_,classes,_) hdrLevel = do

515518 TopLevelSection -> hdrLevel

516519 TopLevelDefault -> hdrLevel

517520 let (section, chapter) = if "unnumbered" `elem` classes

518− then (text "subject", text "title")

519− else (text "section", text "chapter")

521+ then (literal "subject", literal "title")

522+ else (literal "section", literal "chapter")

520523 return $ case level' of

521− -1 -> text "part"

524+ -1 -> literal "part"

522525 0 -> chapter

523526 n | n >= 1 -> text (concat (replicate (n - 1) "sub"))

524527 <> section

525528 _ -> empty -- cannot happen

526529

527−fromBCP47 :: PandocMonad m => Maybe String -> WM m (Maybe String)

530+fromBCP47 :: PandocMonad m => Maybe Text -> WM m (Maybe Text)

528531fromBCP47 mbs = fromBCP47' <$> toLang mbs

529532

530533-- Takes a list of the constituents of a BCP 47 language code

531534-- and irons out ConTeXt's exceptions

532535-- https://tools.ietf.org/html/bcp47#section-2.1

533536-- http://wiki.contextgarden.net/Language_Codes

534−fromBCP47' :: Maybe Lang -> Maybe String

537+fromBCP47' :: Maybe Lang -> Maybe Text

535538fromBCP47' (Just (Lang "ar" _ "SY" _) ) = Just "ar-sy"

536539fromBCP47' (Just (Lang "ar" _ "IQ" _) ) = Just "ar-iq"

537540fromBCP47' (Just (Lang "ar" _ "JO" _) ) = Just "ar-jo"

src/Text/Pandoc/Writers/Custom.hs+7−6

@@ -1,5 +1,6 @@

11{-# LANGUAGE FlexibleInstances #-}

22{-# LANGUAGE NoImplicitPrelude #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45 Module : Text.Pandoc.Writers.Custom

@@ -17,9 +18,9 @@ import Prelude

1718import Control.Arrow ((***))

1819import Control.Exception

1920import Control.Monad (when)

20−import Data.Char (toLower)

2121import Data.List (intersperse)

2222import qualified Data.Map as M

23+import qualified Data.Text as T

2324import Data.Text (Text, pack)

2425import Data.Typeable

2526import Foreign.Lua (Lua, Pushable)

@@ -36,16 +37,16 @@ import Text.Pandoc.Writers.Shared

3637

3738import qualified Foreign.Lua as Lua

3839

39−attrToMap :: Attr -> M.Map String String

40+attrToMap :: Attr -> M.Map T.Text T.Text

4041attrToMap (id',classes,keyvals) = M.fromList

4142 $ ("id", id')

42− : ("class", unwords classes)

43+ : ("class", T.unwords classes)

4344 : keyvals

4445

4546newtype Stringify a = Stringify a

4647

4748instance Pushable (Stringify Format) where

48− push (Stringify (Format f)) = Lua.push (map toLower f)

49+ push (Stringify (Format f)) = Lua.push (T.toLower f)

4950

5051instance Pushable (Stringify [Inline]) where

5152 push (Stringify ils) = Lua.push =<< inlineListToCustom ils

@@ -82,7 +83,7 @@ instance (Pushable a, Pushable b) => Pushable (KeyValue a b) where

8283 Lua.push v

8384 Lua.rawset (Lua.nthFromTop 3)

8485

85−data PandocLuaException = PandocLuaException String

86+data PandocLuaException = PandocLuaException Text

8687 deriving (Show, Typeable)

8788

8889instance Exception PandocLuaException

@@ -99,7 +100,7 @@ writeCustom luaFile opts doc@(Pandoc meta _) = do

99100 -- check for error in lua script (later we'll change the return type

100101 -- to handle this more gracefully):

101102 when (stat /= Lua.OK) $

102− Lua.tostring' (-1) >>= throw . PandocLuaException . UTF8.toString

103+ Lua.tostring' (-1) >>= throw . PandocLuaException . UTF8.toText

103104 rendered <- docToCustom opts doc

104105 context <- metaToContext opts

105106 (fmap (literal . pack) . blockListToCustom)

src/Text/Pandoc/Writers/Docbook.hs+49−50

@@ -1,6 +1,7 @@

11{-# LANGUAGE NoImplicitPrelude #-}

22{-# LANGUAGE OverloadedStrings #-}

33{-# LANGUAGE PatternGuards #-}

4+{-# LANGUAGE ViewPatterns #-}

45{- |

56 Module : Text.Pandoc.Writers.Docbook

@@ -15,9 +16,7 @@ Conversion of 'Pandoc' documents to Docbook XML.

1516module Text.Pandoc.Writers.Docbook ( writeDocbook4, writeDocbook5 ) where

1617import Prelude

1718import Control.Monad.Reader

18−import Data.Char (toLower)

1919import Data.Generics (everywhere, mkT)

20−import Data.List (isPrefixOf, stripPrefix)

2120import Data.Monoid (Any (..))

2221import Data.Text (Text)

2322import qualified Data.Text as T

@@ -46,26 +45,26 @@ type DB = ReaderT DocBookVersion

4645-- | Convert list of authors to a docbook <author> section

4746authorToDocbook :: PandocMonad m => WriterOptions -> [Inline] -> DB m B.Inlines

4847authorToDocbook opts name' = do

49− name <- T.unpack . render Nothing <$> inlinesToDocbook opts name'

48+ name <- render Nothing <$> inlinesToDocbook opts name'

5049 let colwidth = if writerWrapText opts == WrapAuto

5150 then Just $ writerColumns opts

5251 else Nothing

5352 return $ B.rawInline "docbook" $ render colwidth $

54− if ',' `elem` name

53+ if T.any (== ',') name

5554 then -- last name first

56− let (lastname, rest) = break (==',') name

55+ let (lastname, rest) = T.break (==',') name

5756 firstname = triml rest in

58− inTagsSimple "firstname" (text $ escapeStringForXML firstname) <>

59− inTagsSimple "surname" (text $ escapeStringForXML lastname)

57+ inTagsSimple "firstname" (literal $ escapeStringForXML firstname) <>

58+ inTagsSimple "surname" (literal $ escapeStringForXML lastname)

6059 else -- last name last

61− let namewords = words name

60+ let namewords = T.words name

6261 lengthname = length namewords

6362 (firstname, lastname) = case lengthname of

6463 0 -> ("","")

6564 1 -> ("", name)

66− n -> (unwords (take (n-1) namewords), last namewords)

67− in inTagsSimple "firstname" (text $ escapeStringForXML firstname) $$

68− inTagsSimple "surname" (text $ escapeStringForXML lastname)

65+ n -> (T.unwords (take (n-1) namewords), last namewords)

66+ in inTagsSimple "firstname" (literal $ escapeStringForXML firstname) $$

67+ inTagsSimple "surname" (literal $ escapeStringForXML lastname)

6968

7069writeDocbook4 :: PandocMonad m => WriterOptions -> Pandoc -> m Text

7170writeDocbook4 opts d =

@@ -141,13 +140,13 @@ listItemToDocbook :: PandocMonad m => WriterOptions -> [Block] -> DB m (Doc Text

141140listItemToDocbook opts item =

142141 inTagsIndented "listitem" <$> blocksToDocbook opts (map plainToPara item)

143142

144−imageToDocbook :: WriterOptions -> Attr -> String -> Doc Text

143+imageToDocbook :: WriterOptions -> Attr -> Text -> Doc Text

145144imageToDocbook _ attr src = selfClosingTag "imagedata" $

146− ("fileref", src) : idAndRole attr ++ dims

145+ ("fileref", src) : idAndRole attr <> dims

147146 where

148− dims = go Width "width" ++ go Height "depth"

147+ dims = go Width "width" <> go Height "depth"

149148 go dir dstr = case dimension dir attr of

150− Just a -> [(dstr, show a)]

149+ Just a -> [(dstr, tshow a)]

151150 Nothing -> []

152151

153152-- | Convert a Pandoc block element to Docbook.

@@ -166,20 +165,20 @@ blockToDocbook opts (Div (id',"section":_,_) (Header lvl _ ils : xs)) = do

166165 0 -> "chapter"

167166 n | n >= 1 && n <= 5 -> if version == DocBook5

168167 then "section"

169− else "sect" ++ show n

168+ else "sect" <> tshow n

170169 _ -> "simplesect"

171170 idName = if version == DocBook5

172171 then "xml:id"

173172 else "id"

174− idAttr = [(idName, writerIdentifierPrefix opts ++ id') | not (null id')]

173+ idAttr = [(idName, writerIdentifierPrefix opts <> id') | not (T.null id')]

175174 nsAttr = if version == DocBook5 && lvl == 0 then [("xmlns", "http://docbook.org/ns/docbook"),("xmlns:xlink", "http://www.w3.org/1999/xlink")]

176175 else []

177− attribs = nsAttr ++ idAttr

176+ attribs = nsAttr <> idAttr

178177 title' <- inlinesToDocbook opts ils

179178 contents <- blocksToDocbook opts bs

180179 return $ inTags True tag attribs $ inTagsSimple "title" title' $$ contents

181180blockToDocbook opts (Div (ident,_,_) [Para lst]) =

182− let attribs = [("id", ident) | not (null ident)] in

181+ let attribs = [("id", ident) | not (T.null ident)] in

183182 if hasLineBreaks lst

184183 then (flush . nowrap . inTags False "literallayout" attribs)

185184 <$> inlinesToDocbook opts lst

@@ -187,7 +186,7 @@ blockToDocbook opts (Div (ident,_,_) [Para lst]) =

187186blockToDocbook opts (Div (ident,_,_) bs) = do

188187 contents <- blocksToDocbook opts (map plainToPara bs)

189188 return $

190− (if null ident

189+ (if T.null ident

191190 then mempty

192191 else selfClosingTag "anchor" [("id", ident)]) $$ contents

193192blockToDocbook _ h@Header{} = do

@@ -196,7 +195,7 @@ blockToDocbook _ h@Header{} = do

196195 return empty

197196blockToDocbook opts (Plain lst) = inlinesToDocbook opts lst

198197-- title beginning with fig: indicates that the image is a figure

199−blockToDocbook opts (Para [Image attr txt (src,'f':'i':'g':':':_)]) = do

198+blockToDocbook opts (Para [Image attr txt (src,T.stripPrefix "fig:" -> Just _)]) = do

200199 alt <- inlinesToDocbook opts txt

201200 let capt = if null txt

202201 then empty

@@ -216,16 +215,16 @@ blockToDocbook opts (LineBlock lns) =

216215blockToDocbook opts (BlockQuote blocks) =

217216 inTagsIndented "blockquote" <$> blocksToDocbook opts blocks

218217blockToDocbook _ (CodeBlock (_,classes,_) str) = return $

219− text ("<programlisting" ++ lang ++ ">") <> cr <>

220− flush (text (escapeStringForXML str) <> cr <> text "</programlisting>")

218+ literal ("<programlisting" <> lang <> ">") <> cr <>

219+ flush (literal (escapeStringForXML str) <> cr <> literal "</programlisting>")

221220 where lang = if null langs

222221 then ""

223− else " language=\"" ++ escapeStringForXML (head langs) ++

222+ else " language=\"" <> escapeStringForXML (head langs) <>

224223 "\""

225− isLang l = map toLower l `elem` map (map toLower) languages

224+ isLang l = T.toLower l `elem` map T.toLower languages

226225 langsFrom s = if isLang s

227226 then [s]

228− else languagesByExtension . map toLower $ s

227+ else languagesByExtension . T.toLower $ s

229228 langs = concatMap langsFrom classes

230229blockToDocbook opts (BulletList lst) = do

231230 let attribs = [("spacing", "compact") | isTightList lst]

@@ -241,26 +240,26 @@ blockToDocbook opts (OrderedList (start, numstyle, _) (first:rest)) = do

241240 UpperRoman -> [("numeration", "upperroman")]

242241 LowerRoman -> [("numeration", "lowerroman")]

243242 spacing = [("spacing", "compact") | isTightList (first:rest)]

244− attribs = numeration ++ spacing

243+ attribs = numeration <> spacing

245244 items <- if start == 1

246245 then listItemsToDocbook opts (first:rest)

247246 else do

248247 first' <- blocksToDocbook opts (map plainToPara first)

249248 rest' <- listItemsToDocbook opts rest

250249 return $

251− inTags True "listitem" [("override",show start)] first' $$

250+ inTags True "listitem" [("override",tshow start)] first' $$

252251 rest'

253252 return $ inTags True "orderedlist" attribs items

254253blockToDocbook opts (DefinitionList lst) = do

255254 let attribs = [("spacing", "compact") | isTightList $ concatMap snd lst]

256255 inTags True "variablelist" attribs <$> deflistItemsToDocbook opts lst

257256blockToDocbook _ b@(RawBlock f str)

258− | f == "docbook" = return $ text str -- raw XML block

257+ | f == "docbook" = return $ literal str -- raw XML block

259258 | f == "html" = do

260259 version <- ask

261260 if version == DocBook5

262261 then return empty -- No html in Docbook5

263− else return $ text str -- allow html for backwards compatibility

262+ else return $ literal str -- allow html for backwards compatibility

264263 | otherwise = do

265264 report $ BlockNotRendered b

266265 return empty

@@ -271,9 +270,9 @@ blockToDocbook opts (Table caption aligns widths headers rows) = do

271270 else inTagsIndented "title" <$>

272271 inlinesToDocbook opts caption

273272 let tableType = if isEmpty captionDoc then "informaltable" else "table"

274− percent w = show (truncate (100*w) :: Integer) ++ "*"

273+ percent w = tshow (truncate (100*w) :: Integer) <> "*"

275274 coltags = vcat $ zipWith (\w al -> selfClosingTag "colspec"

276− ([("colwidth", percent w) | w > 0] ++

275+ ([("colwidth", percent w) | w > 0] <>

277276 [("align", alignmentToString al)])) widths aligns

278277 head' <- if all null headers

279278 then return empty

@@ -281,7 +280,7 @@ blockToDocbook opts (Table caption aligns widths headers rows) = do

281280 body' <- (inTagsIndented "tbody" . vcat) <$>

282281 mapM (tableRowToDocbook opts) rows

283282 return $ inTagsIndented tableType $ captionDoc $$

284− inTags True "tgroup" [("cols", show (length headers))] (

283+ inTags True "tgroup" [("cols", tshow (length headers))] (

285284 coltags $$ head' $$ body')

286285

287286hasLineBreaks :: [Inline] -> Bool

@@ -294,7 +293,7 @@ hasLineBreaks = getAny . query isLineBreak . walk removeNote

294293 isLineBreak LineBreak = Any True

295294 isLineBreak _ = Any False

296295

297−alignmentToString :: Alignment -> [Char]

296+alignmentToString :: Alignment -> Text

298297alignmentToString alignment = case alignment of

299298 AlignLeft -> "left"

300299 AlignRight -> "right"

@@ -321,7 +320,7 @@ inlinesToDocbook opts lst = hcat <$> mapM (inlineToDocbook opts) lst

321320

322321-- | Convert an inline element to Docbook.

323322inlineToDocbook :: PandocMonad m => WriterOptions -> Inline -> DB m (Doc Text)

324−inlineToDocbook _ (Str str) = return $ text $ escapeStringForXML str

323+inlineToDocbook _ (Str str) = return $ literal $ escapeStringForXML str

325324inlineToDocbook opts (Emph lst) =

326325 inTagsSimple "emphasis" <$> inlinesToDocbook opts lst

327326inlineToDocbook opts (Strong lst) =

@@ -341,18 +340,18 @@ inlineToDocbook opts (Quoted _ lst) =

341340inlineToDocbook opts (Cite _ lst) =

342341 inlinesToDocbook opts lst

343342inlineToDocbook opts (Span (ident,_,_) ils) =

344− ((if null ident

343+ ((if T.null ident

345344 then mempty

346345 else selfClosingTag "anchor" [("id", ident)]) <>) <$>

347346 inlinesToDocbook opts ils

348347inlineToDocbook _ (Code _ str) =

349− return $ inTagsSimple "literal" $ text (escapeStringForXML str)

348+ return $ inTagsSimple "literal" $ literal (escapeStringForXML str)

350349inlineToDocbook opts (Math t str)

351350 | isMathML (writerHTMLMathMethod opts) = do

352351 res <- convertMath writeMathML t str

353352 case res of

354353 Right r -> return $ inTagsSimple tagtype

355− $ text $ Xml.ppcElement conf

354+ $ literal $ T.pack $ Xml.ppcElement conf

356355 $ fixNS

357356 $ removeAttr r

358357 Left il -> inlineToDocbook opts il

@@ -366,19 +365,19 @@ inlineToDocbook opts (Math t str)

366365 fixNS' qname = qname{ Xml.qPrefix = Just "mml" }

367366 fixNS = everywhere (mkT fixNS')

368367inlineToDocbook _ il@(RawInline f x)

369− | f == "html" || f == "docbook" = return $ text x

368+ | f == "html" || f == "docbook" = return $ literal x

370369 | otherwise = do

371370 report $ InlineNotRendered il

372371 return empty

373−inlineToDocbook _ LineBreak = return $ text "\n"

372+inlineToDocbook _ LineBreak = return $ literal "\n"

374373-- currently ignore, would require the option to add custom

375374-- styles to the document

376375inlineToDocbook _ Space = return space

377376-- because we use \n for LineBreak, we can't do soft breaks:

378377inlineToDocbook _ SoftBreak = return space

379378inlineToDocbook opts (Link attr txt (src, _))

380− | Just email <- stripPrefix "mailto:" src =

381− let emailLink = inTagsSimple "email" $ text $

379+ | Just email <- T.stripPrefix "mailto:" src =

380+ let emailLink = inTagsSimple "email" $ literal $

382381 escapeStringForXML email

383382 in case txt of

384383 [Str s] | escapeURI s == email -> return emailLink

@@ -387,17 +386,17 @@ inlineToDocbook opts (Link attr txt (src, _))

387386 char '(' <> emailLink <> char ')'

388387 | otherwise = do

389388 version <- ask

390− (if "#" `isPrefixOf` src

391− then inTags False "link" $ ("linkend", writerIdentifierPrefix opts ++ drop 1 src) : idAndRole attr

389+ (if "#" `T.isPrefixOf` src

390+ then inTags False "link" $ ("linkend", writerIdentifierPrefix opts <> T.drop 1 src) : idAndRole attr

392391 else if version == DocBook5

393392 then inTags False "link" $ ("xlink:href", src) : idAndRole attr

394393 else inTags False "ulink" $ ("url", src) : idAndRole attr )

395394 <$> inlinesToDocbook opts txt

396395inlineToDocbook opts (Image attr _ (src, tit)) = return $

397− let titleDoc = if null tit

396+ let titleDoc = if T.null tit

398397 then empty

399398 else inTagsIndented "objectinfo" $

400− inTagsIndented "title" (text $ escapeStringForXML tit)

399+ inTagsIndented "title" (literal $ escapeStringForXML tit)

401400 in inTagsIndented "inlinemediaobject" $ inTagsIndented "imageobject" $

402401 titleDoc $$ imageToDocbook opts attr src

403402inlineToDocbook opts (Note contents) =

@@ -407,12 +406,12 @@ isMathML :: HTMLMathMethod -> Bool

407406isMathML MathML = True

408407isMathML _ = False

409408

410−idAndRole :: Attr -> [(String, String)]

411−idAndRole (id',cls,_) = ident ++ role

409+idAndRole :: Attr -> [(Text, Text)]

410+idAndRole (id',cls,_) = ident <> role

412411 where

413− ident = if null id'

412+ ident = if T.null id'

414413 then []

415414 else [("id", id')]

416415 role = if null cls

417416 then []

418− else [("role", unwords cls)]

417+ else [("role", T.unwords cls)]

src/Text/Pandoc/Writers/Docx.hs+87−82

@@ -32,6 +32,7 @@ import qualified Data.Map as M

3232import Data.Maybe (fromMaybe, isNothing, mapMaybe, maybeToList)

3333import qualified Data.Set as Set

3434import qualified Data.Text as T

35+import qualified Data.Text.Lazy as TL

3536import Data.Time.Clock.POSIX

3637import Data.Digest.Pure.SHA (sha1, showDigest)

3738import Skylighting

@@ -40,7 +41,7 @@ import Text.Pandoc.BCP47 (getLang, renderLang)

4041import Text.Pandoc.Class (PandocMonad, report, toLang)

4142import qualified Text.Pandoc.Class as P

4243import Data.Time

43−import Text.Pandoc.UTF8 (fromStringLazy)

44+import Text.Pandoc.UTF8 (fromTextLazy)

4445import Text.Pandoc.Definition

4546import Text.Pandoc.Generic

4647import Text.Pandoc.Highlighting (highlight)

@@ -107,8 +108,8 @@ data WriterEnv = WriterEnv{ envTextProperties :: EnvProps

107108 , envListLevel :: Int

108109 , envListNumId :: Int

109110 , envInDel :: Bool

110− , envChangesAuthor :: String

111− , envChangesDate :: String

111+ , envChangesAuthor :: T.Text

112+ , envChangesDate :: T.Text

112113 , envPrintWidth :: Integer

113114 }

114115

@@ -126,8 +127,8 @@ defaultWriterEnv = WriterEnv{ envTextProperties = mempty

126127

127128data WriterState = WriterState{

128129 stFootnotes :: [Element]

129− , stComments :: [([(String,String)], [Inline])]

130− , stSectionIds :: Set.Set String

130+ , stComments :: [([(T.Text, T.Text)], [Inline])]

131+ , stSectionIds :: Set.Set T.Text

131132 , stExternalLinks :: M.Map String String

132133 , stImages :: M.Map FilePath (String, String, Maybe MimeType, B.ByteString)

133134 , stLists :: [ListMarker]

@@ -163,7 +164,6 @@ defaultWriterState = WriterState{

163164

164165type WS m = ReaderT WriterEnv (StateT WriterState m)

165166

166−

167167renumIdMap :: Int -> [Element] -> M.Map String String

168168renumIdMap _ [] = M.empty

169169renumIdMap n (e:es)

@@ -189,10 +189,16 @@ renumId f renumMap e

189189renumIds :: (QName -> Bool) -> M.Map String String -> [Element] -> [Element]

190190renumIds f renumMap = map (renumId f renumMap)

191191

192+findAttrTextBy :: (QName -> Bool) -> Element -> Maybe T.Text

193+findAttrTextBy x = fmap T.pack . findAttrBy x

194+

195+lookupAttrTextBy :: (QName -> Bool) -> [XML.Attr] -> Maybe T.Text

196+lookupAttrTextBy x = fmap T.pack . lookupAttrBy x

197+

192198-- | Certain characters are invalid in XML even if escaped.

193199-- See #1992

194−stripInvalidChars :: String -> String

195−stripInvalidChars = filter isValidChar

200+stripInvalidChars :: T.Text -> T.Text

201+stripInvalidChars = T.filter isValidChar

196202

197203-- | See XML reference

198204isValidChar :: Char -> Bool

@@ -230,11 +236,11 @@ writeDocx opts doc@(Pandoc meta _) = do

230236

231237 -- Gets the template size

232238 let mbpgsz = mbsectpr >>= filterElementName (wname (=="pgSz"))

233− let mbAttrSzWidth = (elAttribs <$> mbpgsz) >>= lookupAttrBy ((=="w") . qName)

239+ let mbAttrSzWidth = (elAttribs <$> mbpgsz) >>= lookupAttrTextBy ((=="w") . qName)

234240

235241 let mbpgmar = mbsectpr >>= filterElementName (wname (=="pgMar"))

236− let mbAttrMarLeft = (elAttribs <$> mbpgmar) >>= lookupAttrBy ((=="left") . qName)

237− let mbAttrMarRight = (elAttribs <$> mbpgmar) >>= lookupAttrBy ((=="right") . qName)

242+ let mbAttrMarLeft = (elAttribs <$> mbpgmar) >>= lookupAttrTextBy ((=="left") . qName)

243+ let mbAttrMarRight = (elAttribs <$> mbpgmar) >>= lookupAttrTextBy ((=="right") . qName)

238244

239245 -- Get the available area (converting the size and the margins to int and

240246 -- doing the difference

@@ -248,7 +254,7 @@ writeDocx opts doc@(Pandoc meta _) = do

248254 mblang <- toLang $ getLang opts meta

249255 let addLang :: Element -> Element

250256 addLang e = case mblang >>= \l ->

251− (return . XMLC.toTree . go (renderLang l)

257+ (return . XMLC.toTree . go (T.unpack $ renderLang l)

252258 . XMLC.fromElement) e of

253259 Just (Elem e') -> e'

254260 _ -> e -- return original

@@ -289,7 +295,7 @@ writeDocx opts doc@(Pandoc meta _) = do

289295 let env = defaultWriterEnv {

290296 envRTL = isRTLmeta

291297 , envChangesAuthor = fromMaybe "unknown" username

292− , envChangesDate = formatTime defaultTimeLocale "%FT%XZ" utctime

298+ , envChangesDate = T.pack $ formatTime defaultTimeLocale "%FT%XZ" utctime

293299 , envPrintWidth = maybe 420 (`quot` 20) pgContentWidth

294300 }

295301

@@ -337,9 +343,9 @@ writeDocx opts doc@(Pandoc meta _) = do

337343 [("PartName",part'),("ContentType",contentType')] ()

338344 let mkImageOverride (_, imgpath, mbMimeType, _) =

339345 mkOverrideNode ("/word/" ++ imgpath,

340− fromMaybe "application/octet-stream" mbMimeType)

346+ maybe "application/octet-stream" T.unpack mbMimeType)

341347 let mkMediaOverride imgpath =

342− mkOverrideNode ('/':imgpath, getMimeTypeDef imgpath)

348+ mkOverrideNode ('/':imgpath, T.unpack $ getMimeTypeDef imgpath)

343349 let overrides = map mkOverrideNode (

344350 [("/word/webSettings.xml",

345351 "application/vnd.openxmlformats-officedocument.wordprocessingml.webSettings+xml")

@@ -488,10 +494,10 @@ writeDocx opts doc@(Pandoc meta _) = do

488494 numbering <- parseXml refArchive distArchive numpath

489495 newNumElts <- mkNumbering (stLists st)

490496 let pandocAdded e =

491− case findAttrBy ((== "abstractNumId") . qName) e >>= safeRead of

497+ case findAttrTextBy ((== "abstractNumId") . qName) e >>= safeRead of

492498 Just numid -> numid >= (990 :: Int)

493499 Nothing ->

494− case findAttrBy ((== "numId") . qName) e >>= safeRead of

500+ case findAttrTextBy ((== "numId") . qName) e >>= safeRead of

495501 Just numid -> numid >= (1000 :: Int)

496502 Nothing -> False

497503 let oldElts = filter (not . pandocAdded) $ onlyElems (elContent numbering)

@@ -513,11 +519,11 @@ writeDocx opts doc@(Pandoc meta _) = do

513519 let extraCoreProps = ["subject","lang","category","description"]

514520 let extraCorePropsMap = M.fromList $ zip extraCoreProps

515521 ["dc:subject","dc:language","cp:category","dc:description"]

516− let lookupMetaString' :: String -> Meta -> String

522+ let lookupMetaString' :: T.Text -> Meta -> T.Text

517523 lookupMetaString' key' meta' =

518524 case key' of

519− "description" -> intercalate "_x000d_\n" (map stringify $ lookupMetaBlocks "description" meta')

520− _ -> lookupMetaString key' meta'

525+ "description" -> T.intercalate "_x000d_\n" (map stringify $ lookupMetaBlocks "description" meta')

526+ key'' -> lookupMetaString key'' meta'

521527

522528 let docProps = mknode "cp:coreProperties"

523529 [("xmlns:cp","http://schemas.openxmlformats.org/package/2006/metadata/core-properties")

@@ -525,11 +531,11 @@ writeDocx opts doc@(Pandoc meta _) = do

525531 ,("xmlns:dcterms","http://purl.org/dc/terms/")

526532 ,("xmlns:dcmitype","http://purl.org/dc/dcmitype/")

527533 ,("xmlns:xsi","http://www.w3.org/2001/XMLSchema-instance")]

528− $ mknode "dc:title" [] (stringify $ docTitle meta)

529− : mknode "dc:creator" [] (intercalate "; " (map stringify $ docAuthors meta))

530− : [ mknode (M.findWithDefault "" k extraCorePropsMap) [] (lookupMetaString' k meta)

534+ $ mktnode "dc:title" [] (stringify $ docTitle meta)

535+ : mktnode "dc:creator" [] (T.intercalate "; " (map stringify $ docAuthors meta))

536+ : [ mktnode (M.findWithDefault "" k extraCorePropsMap) [] (lookupMetaString' k meta)

531537 | k <- M.keys (unMeta meta), k `elem` extraCoreProps]

532− ++ mknode "cp:keywords" [] (intercalate ", " keywords)

538+ ++ mknode "cp:keywords" [] (T.unpack $ T.intercalate ", " keywords)

533539 : (\x -> [ mknode "dcterms:created" [("xsi:type","dcterms:W3CDTF")] x

534540 , mknode "dcterms:modified" [("xsi:type","dcterms:W3CDTF")] x

535541 ]) (formatTime defaultTimeLocale "%FT%XZ" utctime)

@@ -537,7 +543,7 @@ writeDocx opts doc@(Pandoc meta _) = do

537543

538544 -- docProps/custom.xml

539545 let customProperties :: [(String, String)]

540− customProperties = [(k, lookupMetaString k meta) | k <- M.keys (unMeta meta)

546+ customProperties = [(T.unpack k, T.unpack $ lookupMetaString k meta) | k <- M.keys (unMeta meta)

541547 , k `notElem` (["title", "author", "keywords"]

542548 ++ extraCoreProps)]

543549 let mkCustomProp (k, v) pid = mknode "property"

@@ -584,7 +590,7 @@ writeDocx opts doc@(Pandoc meta _) = do

584590

585591 let entryFromArchive arch path =

586592 maybe (throwError $ PandocSomeError

587− $ path ++ " missing in reference docx")

593+ $ T.pack $ path ++ " missing in reference docx")

588594 return

589595 (findEntryByPath path arch `mplus` findEntryByPath path distArchive)

590596 docPropsAppEntry <- entryFromArchive refArchive "docProps/app.xml"

@@ -614,25 +620,24 @@ writeDocx opts doc@(Pandoc meta _) = do

614620 miscRelEntries ++ otherMediaEntries

615621 return $ fromArchive archive

616622

617−

618623newParaPropToOpenXml :: ParaStyleName -> Element

619624newParaPropToOpenXml (fromStyleName -> s) =

620− let styleId = filter (not . isSpace) s

625+ let styleId = T.filter (not . isSpace) s

621626 in mknode "w:style" [ ("w:type", "paragraph")

622627 , ("w:customStyle", "1")

623− , ("w:styleId", styleId)]

624− [ mknode "w:name" [("w:val", s)] ()

628+ , ("w:styleId", T.unpack styleId)]

629+ [ mknode "w:name" [("w:val", T.unpack s)] ()

625630 , mknode "w:basedOn" [("w:val","BodyText")] ()

626631 , mknode "w:qFormat" [] ()

627632 ]

628633

629634newTextPropToOpenXml :: CharStyleName -> Element

630635newTextPropToOpenXml (fromStyleName -> s) =

631− let styleId = filter (not . isSpace) s

636+ let styleId = T.filter (not . isSpace) s

632637 in mknode "w:style" [ ("w:type", "character")

633638 , ("w:customStyle", "1")

634− , ("w:styleId", styleId)]

635− [ mknode "w:name" [("w:val", s)] ()

639+ , ("w:styleId", T.unpack styleId)]

640+ [ mknode "w:name" [("w:val", T.unpack s)] ()

636641 , mknode "w:basedOn" [("w:val","BodyTextChar")] ()

637642 ]

638643

@@ -821,8 +826,8 @@ writeOpenXML opts (Pandoc meta blocks) = do

821826 abstract <- if null abstract'

822827 then return []

823828 else withParaPropM (pStyleM "Abstract") $ blocksToOpenXML opts abstract'

824− let convertSpace (Str x : Space : Str y : xs) = Str (x ++ " " ++ y) : xs

825− convertSpace (Str x : Str y : xs) = Str (x ++ y) : xs

829+ let convertSpace (Str x : Space : Str y : xs) = Str (x <> " " <> y) : xs

830+ convertSpace (Str x : Str y : xs) = Str (x <> y) : xs

826831 convertSpace xs = xs

827832 let blocks' = bottomUp convertSpace blocks

828833 doc' <- setFirstPara >> blocksToOpenXML opts blocks'

@@ -831,7 +836,7 @@ writeOpenXML opts (Pandoc meta blocks) = do

831836 let toComment (kvs, ils) = do

832837 annotation <- inlinesToOpenXML opts ils

833838 return $

834− mknode "w:comment" [('w':':':k,v) | (k,v) <- kvs]

839+ mknode "w:comment" [('w':':':T.unpack k,T.unpack v) | (k,v) <- kvs]

835840 [ mknode "w:p" [] $

836841 [ mknode "w:pPr" []

837842 [ mknode "w:pStyle" [("w:val", "CommentText")] () ]

@@ -858,13 +863,13 @@ pStyleM :: (PandocMonad m) => ParaStyleName -> WS m XML.Element

858863pStyleM styleName = do

859864 pStyleMap <- gets (smParaStyle . stStyleMaps)

860865 let sty' = getStyleIdFromName styleName pStyleMap

861− return $ mknode "w:pStyle" [("w:val", fromStyleId sty')] ()

866+ return $ mknode "w:pStyle" [("w:val", T.unpack $ fromStyleId sty')] ()

862867

863868rStyleM :: (PandocMonad m) => CharStyleName -> WS m XML.Element

864869rStyleM styleName = do

865870 cStyleMap <- gets (smCharStyle . stStyleMaps)

866871 let sty' = getStyleIdFromName styleName cStyleMap

867− return $ mknode "w:rStyle" [("w:val", fromStyleId sty')] ()

872+ return $ mknode "w:rStyle" [("w:val", T.unpack $ fromStyleId sty')] ()

868873

869874getUniqueId :: (PandocMonad m) => WS m String

870875-- the + 20 is to ensure that there are no clashes with the rIds

@@ -875,7 +880,7 @@ getUniqueId = do

875880 return $ show n

876881

877882-- | Key for specifying user-defined docx styles.

878−dynamicStyleKey :: String

883+dynamicStyleKey :: T.Text

879884dynamicStyleKey = "custom-style"

880885

881886-- | Convert a Pandoc block element to OpenXML.

@@ -886,7 +891,7 @@ blockToOpenXML' :: (PandocMonad m) => WriterOptions -> Block -> WS m [Element]

886891blockToOpenXML' _ Null = return []

887892blockToOpenXML' opts (Div (ident,_classes,kvs) bs) = do

888893 stylemod <- case lookup dynamicStyleKey kvs of

889− Just (fromString -> sty) -> do

894+ Just (fromString . T.unpack -> sty) -> do

890895 modify $ \s ->

891896 s{stDynamicParaProps = Set.insert sty

892897 (stDynamicParaProps s)}

@@ -904,14 +909,14 @@ blockToOpenXML' opts (Div (ident,_classes,kvs) bs) = do

904909 else id

905910 header <- dirmod $ stylemod $ blocksToOpenXML opts hs

906911 contents <- dirmod $ bibmod $ stylemod $ blocksToOpenXML opts bs'

907− wrapBookmark ident $ header ++ contents

912+ wrapBookmark ident $ header <> contents

908913blockToOpenXML' opts (Header lev (ident,_,_) lst) = do

909914 setFirstPara

910915 paraProps <- withParaPropM (pStyleM (fromString $ "Heading "++show lev)) $

911916 getParaProps False

912917 contents <- inlinesToOpenXML opts lst

913− if null ident

914− then return [mknode "w:p" [] (paraProps ++contents)]

918+ if T.null ident

919+ then return [mknode "w:p" [] (paraProps ++ contents)]

915920 else do

916921 let bookmarkName = ident

917922 modify $ \s -> s{ stSectionIds = Set.insert bookmarkName

@@ -924,7 +929,7 @@ blockToOpenXML' opts (Plain lst) = do

924929 prop <- pStyleM "Compact"

925930 if isInTable then withParaProp prop block else block

926931-- title beginning with fig: indicates that the image is a figure

927−blockToOpenXML' opts (Para [Image attr alt (src,'f':'i':'g':':':tit)]) = do

932+blockToOpenXML' opts (Para [Image attr alt (src,T.stripPrefix "fig:" -> Just tit)]) = do

928933 setFirstPara

929934 prop <- pStyleM $

930935 if null alt

@@ -1021,7 +1026,7 @@ blockToOpenXML' opts (Table caption aligns widths headers rows) = do

10211026 ( mknode "w:tblStyle" [("w:val","Table")] () :

10221027 mknode "w:tblW" [("w:type", "pct"), ("w:w", show rowwidth)] () :

10231028 mknode "w:tblLook" [("w:firstRow",if hasHeader then "1" else "0") ] () :

1024− [ mknode "w:tblCaption" [("w:val", captionStr)] ()

1029+ [ mknode "w:tblCaption" [("w:val", T.unpack captionStr)] ()

10251030 | not (null caption) ] )

10261031 : mknode "w:tblGrid" []

10271032 (if all (==0) widths

@@ -1122,19 +1127,19 @@ withParaProp d p =

11221127withParaPropM :: PandocMonad m => WS m Element -> WS m a -> WS m a

11231128withParaPropM = (. flip withParaProp) . (>>=)

11241129

1125−formattedString :: PandocMonad m => String -> WS m [Element]

1130+formattedString :: PandocMonad m => T.Text -> WS m [Element]

11261131formattedString str =

11271132 -- properly handle soft hyphens

1128− case splitBy (=='\173') str of

1133+ case splitTextBy (=='\173') str of

11291134 [w] -> formattedString' w

11301135 ws -> do

11311136 sh <- formattedRun [mknode "w:softHyphen" [] ()]

11321137 intercalate sh <$> mapM formattedString' ws

11331138

1134−formattedString' :: PandocMonad m => String -> WS m [Element]

1139+formattedString' :: PandocMonad m => T.Text -> WS m [Element]

11351140formattedString' str = do

11361141 inDel <- asks envInDel

1137− formattedRun [ mknode (if inDel then "w:delText" else "w:t")

1142+ formattedRun [ mktnode (if inDel then "w:delText" else "w:t")

11381143 [("xml:space","preserve")] (stripInvalidChars str) ]

11391144

11401145formattedRun :: PandocMonad m => [Element] -> WS m [Element]

@@ -1163,21 +1168,21 @@ inlineToOpenXML' _ (Span (ident,["comment-start"],kvs) ils) = do

11631168 let ident' = fromMaybe ident (lookup "id" kvs)

11641169 kvs' = filter (("id" /=) . fst) kvs

11651170 modify $ \st -> st{ stComments = (("id",ident'):kvs', ils) : stComments st }

1166− return [ mknode "w:commentRangeStart" [("w:id", ident')] () ]

1171+ return [ mknode "w:commentRangeStart" [("w:id", T.unpack ident')] () ]

11671172inlineToOpenXML' _ (Span (ident,["comment-end"],kvs) _) =

11681173 -- prefer the "id" in kvs, since that is the one produced by the docx

11691174 -- reader.

11701175 let ident' = fromMaybe ident (lookup "id" kvs)

11711176 in

1172− return [ mknode "w:commentRangeEnd" [("w:id", ident')] ()

1177+ return [ mknode "w:commentRangeEnd" [("w:id", T.unpack ident')] ()

11731178 , mknode "w:r" []

11741179 [ mknode "w:rPr" []

11751180 [ mknode "w:rStyle" [("w:val", "CommentReference")] () ]

1176− , mknode "w:commentReference" [("w:id", ident')] () ]

1181+ , mknode "w:commentReference" [("w:id", T.unpack ident')] () ]

11771182 ]

11781183inlineToOpenXML' opts (Span (ident,classes,kvs) ils) = do

11791184 stylemod <- case lookup dynamicStyleKey kvs of

1180− Just (fromString -> sty) -> do

1185+ Just (fromString . T.unpack -> sty) -> do

11811186 modify $ \s ->

11821187 s{stDynamicTextProps = Set.insert sty

11831188 (stDynamicTextProps s)}

@@ -1208,8 +1213,8 @@ inlineToOpenXML' opts (Span (ident,classes,kvs) ils) = do

12081213 x <- f

12091214 return [ mknode "w:ins"

12101215 [("w:id", show insId),

1211− ("w:author", author),

1212− ("w:date", date)] x ]

1216+ ("w:author", T.unpack author),

1217+ ("w:date", T.unpack date)] x ]

12131218 else return id

12141219 delmod <- if "deletion" `elem` classes

12151220 then do

@@ -1220,8 +1225,8 @@ inlineToOpenXML' opts (Span (ident,classes,kvs) ils) = do

12201225 x <- f

12211226 return [mknode "w:del"

12221227 [("w:id", show delId),

1223− ("w:author", author),

1224− ("w:date", date)] x]

1228+ ("w:author", T.unpack author),

1229+ ("w:date", T.unpack date)] x]

12251230 else return id

12261231 contents <- insmod $ delmod $ dirmod $ stylemod $ pmod

12271232 $ inlinesToOpenXML opts ils

@@ -1264,7 +1269,7 @@ inlineToOpenXML' opts (Code attrs str) = do

12641269 let alltoktypes = [KeywordTok ..]

12651270 tokTypesMap <- mapM (\tt -> (,) tt <$> rStyleM (fromString $ show tt)) alltoktypes

12661271 let unhighlighted = intercalate [br] `fmap`

1267− mapM formattedString (lines str)

1272+ mapM formattedString (T.lines str)

12681273 formatOpenXML _fmtOpts = intercalate [br] . map (map toHlTok)

12691274 toHlTok (toktype,tok) =

12701275 mknode "w:r" []

@@ -1278,7 +1283,7 @@ inlineToOpenXML' opts (Code attrs str) = do

12781283 formatOpenXML attrs str of

12791284 Right h -> return h

12801285 Left msg -> do

1281− unless (null msg) $ report $ CouldNotHighlight msg

1286+ unless (T.null msg) $ report $ CouldNotHighlight msg

12821287 unhighlighted

12831288inlineToOpenXML' opts (Note bs) = do

12841289 notes <- gets stFootnotes

@@ -1287,7 +1292,7 @@ inlineToOpenXML' opts (Note bs) = do

12871292 let notemarker = mknode "w:r" []

12881293 [ mknode "w:rPr" [] footnoteStyle

12891294 , mknode "w:footnoteRef" [] () ]

1290− let notemarkerXml = RawInline (Format "openxml") $ ppElement notemarker

1295+ let notemarkerXml = RawInline (Format "openxml") $ T.pack $ ppElement notemarker

12911296 let insertNoteRef (Plain ils : xs) = Plain (notemarkerXml : Space : ils) : xs

12921297 insertNoteRef (Para ils : xs) = Para (notemarkerXml : Space : ils) : xs

12931298 insertNoteRef xs = Para [notemarkerXml] : xs

@@ -1303,27 +1308,27 @@ inlineToOpenXML' opts (Note bs) = do

13031308 [ mknode "w:rPr" [] footnoteStyle

13041309 , mknode "w:footnoteReference" [("w:id", notenum)] () ] ]

13051310-- internal link:

1306−inlineToOpenXML' opts (Link _ txt ('#':xs,_)) = do

1311+inlineToOpenXML' opts (Link _ txt (T.uncons -> Just ('#', xs),_)) = do

13071312 contents <- withTextPropM (rStyleM "Hyperlink") $ inlinesToOpenXML opts txt

13081313 return

1309− [ mknode "w:hyperlink" [("w:anchor", toBookmarkName xs)] contents ]

1314+ [ mknode "w:hyperlink" [("w:anchor", T.unpack $ toBookmarkName xs)] contents ]

13101315-- external link:

13111316inlineToOpenXML' opts (Link _ txt (src,_)) = do

13121317 contents <- withTextPropM (rStyleM "Hyperlink") $ inlinesToOpenXML opts txt

13131318 extlinks <- gets stExternalLinks

1314− id' <- case M.lookup src extlinks of

1319+ id' <- case M.lookup (T.unpack src) extlinks of

13151320 Just i -> return i

13161321 Nothing -> do

13171322 i <- ("rId"++) `fmap` getUniqueId

13181323 modify $ \st -> st{ stExternalLinks =

1319− M.insert src i extlinks }

1324+ M.insert (T.unpack src) i extlinks }

13201325 return i

13211326 return [ mknode "w:hyperlink" [("r:id",id')] contents ]

13221327inlineToOpenXML' opts (Image attr@(imgident, _, _) alt (src, title)) = do

13231328 pageWidth <- asks envPrintWidth

13241329 imgs <- gets stImages

13251330 let

1326− stImage = M.lookup src imgs

1331+ stImage = M.lookup (T.unpack src) imgs

13271332 generateImgElt (ident, _, _, img) =

13281333 let

13291334 (xpt,ypt) = desiredSizeInPoints opts attr

@@ -1336,7 +1341,7 @@ inlineToOpenXML' opts (Image attr@(imgident, _, _) alt (src, title)) = do

13361341 ,("noChangeAspect","1")] ()

13371342 nvPicPr = mknode "pic:nvPicPr" []

13381343 [ mknode "pic:cNvPr"

1339− [("descr",src),("id","0"),("name","Picture")] ()

1344+ [("descr",T.unpack src),("id","0"),("name","Picture")] ()

13401345 , cNvPicPr ]

13411346 blipFill = mknode "pic:blipFill" []

13421347 [ mknode "a:blip" [("r:embed",ident)] ()

@@ -1371,8 +1376,8 @@ inlineToOpenXML' opts (Image attr@(imgident, _, _) alt (src, title)) = do

13711376 , mknode "wp:effectExtent"

13721377 [("b","0"),("l","0"),("r","0"),("t","0")] ()

13731378 , mknode "wp:docPr"

1374− [ ("descr", stringify alt)

1375− , ("title", title)

1379+ [ ("descr", T.unpack $ stringify alt)

1380+ , ("title", T.unpack title)

13761381 , ("id","1")

13771382 , ("name","Picture")

13781383 ] ()

@@ -1389,7 +1394,7 @@ inlineToOpenXML' opts (Image attr@(imgident, _, _) alt (src, title)) = do

13891394

13901395 let

13911396 imgext = case mt >>= extensionFromMimeType of

1392− Just x -> '.':x

1397+ Just x -> "." <> x

13931398 Nothing -> case imageType img of

13941399 Just Png -> ".png"

13951400 Just Jpeg -> ".jpeg"

@@ -1399,21 +1404,21 @@ inlineToOpenXML' opts (Image attr@(imgident, _, _) alt (src, title)) = do

13991404 Just Svg -> ".svg"

14001405 Just Emf -> ".emf"

14011406 Nothing -> ""

1402− imgpath = "media/" ++ ident ++ imgext

1407+ imgpath = "media/" <> ident <> T.unpack imgext

14031408 mbMimeType = mt <|> getMimeType imgpath

14041409

14051410 imgData = (ident, imgpath, mbMimeType, img)

14061411

1407− if null imgext

1412+ if T.null imgext

14081413 then -- without an extension there is no rule for content type

14091414 inlinesToOpenXML opts alt -- return alt to avoid corrupted docx

14101415 else do

14111416 -- insert mime type to use in constructing [Content_Types].xml

1412− modify $ \st -> st { stImages = M.insert src imgData $ stImages st }

1417+ modify $ \st -> st { stImages = M.insert (T.unpack src) imgData $ stImages st }

14131418 return [generateImgElt imgData]

14141419 )

14151420 `catchError` ( \e -> do

1416− report $ CouldNotFetchResource src (show e)

1421+ report $ CouldNotFetchResource src $ T.pack (show e)

14171422 -- emit alt text

14181423 inlinesToOpenXML opts alt

14191424 )

@@ -1460,22 +1465,22 @@ withDirection x = do

14601465 , envTextProperties = EnvProps textStyle textProps'

14611466 }

14621467

1463−wrapBookmark :: (PandocMonad m) => String -> [Element] -> WS m [Element]

1464−wrapBookmark [] contents = return contents

1468+wrapBookmark :: (PandocMonad m) => T.Text -> [Element] -> WS m [Element]

1469+wrapBookmark "" contents = return contents

14651470wrapBookmark ident contents = do

14661471 id' <- getUniqueId

14671472 let bookmarkStart = mknode "w:bookmarkStart"

14681473 [("w:id", id')

1469− ,("w:name", toBookmarkName ident)] ()

1474+ ,("w:name", T.unpack $ toBookmarkName ident)] ()

14701475 bookmarkEnd = mknode "w:bookmarkEnd" [("w:id", id')] ()

14711476 return $ bookmarkStart : contents ++ [bookmarkEnd]

14721477

14731478-- Word imposes a 40 character limit on bookmark names and requires

14741479-- that they begin with a letter. So we just use a hash of the

14751480-- identifier when otherwise we'd have an illegal bookmark name.

1476−toBookmarkName :: String -> String

1477−toBookmarkName s =

1478− case s of

1479− (c:_) | isLetter c

1480− , length s <= 40 -> s

1481− _ -> 'X' : drop 1 (showDigest (sha1 (fromStringLazy s)))

1481+toBookmarkName :: T.Text -> T.Text

1482+toBookmarkName s

1483+ | Just (c, _) <- T.uncons s

1484+ , isLetter c

1485+ , T.length s <= 40 = s

1486+ | otherwise = T.pack $ 'X' : drop 1 (showDigest (sha1 (fromTextLazy $ TL.fromStrict s)))

src/Text/Pandoc/Writers/Docx/StyleMap.hs+2−1

@@ -27,6 +27,7 @@ module Text.Pandoc.Writers.Docx.StyleMap ( StyleMaps(..)

2727import Text.Pandoc.Readers.Docx.Parse.Styles

2828import Codec.Archive.Zip

2929import qualified Data.Map as M

30+import qualified Data.Text as T

3031import Data.String

3132import Data.Char (isSpace)

3233import Prelude

@@ -38,7 +39,7 @@ type CharStyleNameMap = M.Map CharStyleName CharStyle

3839getStyleIdFromName :: (Ord sn, FromStyleName sn, IsString (StyleId sty), HasStyleId sty)

3940 => sn -> M.Map sn sty -> StyleId sty

4041getStyleIdFromName s = maybe (fallback s) getStyleId . M.lookup s

41− where fallback = fromString . filter (not . isSpace) . fromStyleName

42+ where fallback = fromString . T.unpack . T.filter (not . isSpace) . fromStyleName

4243

4344hasStyleName :: (Ord sn, HasStyleId sty)

4445 => sn -> M.Map sn sty -> Bool

src/Text/Pandoc/Writers/DokuWiki.hs+113−110

@@ -1,4 +1,5 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2+{-# LANGUAGE OverloadedStrings #-}

23{- |

34 Module : Text.Pandoc.Writers.DokuWiki

@@ -27,15 +28,16 @@ import Control.Monad (zipWithM)

2728import Control.Monad.Reader (ReaderT, asks, local, runReaderT)

2829import Control.Monad.State.Strict (StateT, evalStateT)

2930import Data.Default (Default (..))

30−import Data.List (intercalate, intersect, isPrefixOf, transpose)

31−import Data.Text (Text, pack)

31+import Data.List (intersect, transpose)

32+import Data.Text (Text)

33+import qualified Data.Text as T

3234import Text.Pandoc.Class (PandocMonad, report)

3335import Text.Pandoc.Definition

3436import Text.Pandoc.ImageSize

3537import Text.Pandoc.Logging

3638import Text.Pandoc.Options (WrapOption (..), WriterOptions (writerTableOfContents, writerTemplate, writerWrapText))

3739import Text.Pandoc.Shared (camelCaseToHyphenated, escapeURI, isURI, linesToPara,

38− removeFormatting, substitute, trimr)

40+ removeFormatting, trimr, tshow)

3941import Text.Pandoc.Templates (renderTemplate)

4042import Text.DocLayout (render, literal)

4143import Text.Pandoc.Writers.Shared (defField, metaToContext)

@@ -44,7 +46,7 @@ data WriterState = WriterState {

4446 }

4547

4648data WriterEnvironment = WriterEnvironment {

47− stIndent :: String -- Indent after the marker at the beginning of list items

49+ stIndent :: Text -- Indent after the marker at the beginning of list items

4850 , stUseTags :: Bool -- True if we should use HTML tags because we're in a complex list

4951 , stBackSlashLB :: Bool -- True if we should produce formatted strings with newlines (as in a table cell)

5052 }

@@ -72,57 +74,58 @@ pandocToDokuWiki :: PandocMonad m

7274 => WriterOptions -> Pandoc -> DokuWiki m Text

7375pandocToDokuWiki opts (Pandoc meta blocks) = do

7476 metadata <- metaToContext opts

75− (fmap (literal . pack . trimr) . blockListToDokuWiki opts)

76− (fmap (literal . pack . trimr) . inlineListToDokuWiki opts)

77+ (fmap (literal . trimr) . blockListToDokuWiki opts)

78+ (fmap (literal . trimr) . inlineListToDokuWiki opts)

7779 meta

7880 body <- blockListToDokuWiki opts blocks

79− let main = pack body

80− let context = defField "body" main

81+ let context = defField "body" body

8182 $ defField "toc" (writerTableOfContents opts) metadata

8283 return $

8384 case writerTemplate opts of

84− Nothing -> main

85+ Nothing -> body

8586 Just tpl -> render Nothing $ renderTemplate tpl context

8687

8788-- | Escape special characters for DokuWiki.

88−escapeString :: String -> String

89−escapeString = substitute "__" "%%__%%" .

90− substitute "**" "%%**%%" .

91− substitute "//" "%%//%%"

89+escapeString :: Text -> Text

90+escapeString = T.replace "__" "%%__%%" .

91+ T.replace "**" "%%**%%" .

92+ T.replace "//" "%%//%%"

9293

9394-- | Convert Pandoc block element to DokuWiki.

9495blockToDokuWiki :: PandocMonad m

9596 => WriterOptions -- ^ Options

9697 -> Block -- ^ Block element

97− -> DokuWiki m String

98+ -> DokuWiki m Text

9899

99100blockToDokuWiki _ Null = return ""

100101

101102blockToDokuWiki opts (Div _attrs bs) = do

102103 contents <- blockListToDokuWiki opts bs

103− return $ contents ++ "\n"

104+ return $ contents <> "\n"

104105

105106blockToDokuWiki opts (Plain inlines) =

106107 inlineListToDokuWiki opts inlines

107108

108109-- title beginning with fig: indicates that the image is a figure

109110-- dokuwiki doesn't support captions - so combine together alt and caption into alt

110−blockToDokuWiki opts (Para [Image attr txt (src,'f':'i':'g':':':tit)]) = do

111− capt <- if null txt

112− then return ""

113− else (" " ++) `fmap` inlineListToDokuWiki opts txt

114− let opt = if null txt

115− then ""

116− else "|" ++ if null tit then capt else tit ++ capt

117− return $ "{{" ++ src ++ imageDims opts attr ++ opt ++ "}}\n"

111+blockToDokuWiki opts (Para [Image attr txt (src,tgt)])

112+ | Just tit <- T.stripPrefix "fig:" tgt

113+ = do

114+ capt <- if null txt

115+ then return ""

116+ else (" " <>) `fmap` inlineListToDokuWiki opts txt

117+ let opt = if null txt

118+ then ""

119+ else "|" <> if T.null tit then capt else tit <> capt

120+ return $ "{{" <> src <> imageDims opts attr <> opt <> "}}\n"

118121

119122blockToDokuWiki opts (Para inlines) = do

120123 indent <- asks stIndent

121124 useTags <- asks stUseTags

122125 contents <- inlineListToDokuWiki opts inlines

123126 return $ if useTags

124− then "<HTML></HTML>" ++ contents ++ "<HTML></HTML>"

125− else contents ++ if null indent then "\n" else ""

127+ then "<HTML></HTML>" <> contents <> "<HTML></HTML>"

128+ else contents <> if T.null indent then "\n" else ""

126129

127130blockToDokuWiki opts (LineBlock lns) =

128131 blockToDokuWiki opts $ linesToPara lns

@@ -131,7 +134,7 @@ blockToDokuWiki _ b@(RawBlock f str)

131134 | f == Format "dokuwiki" = return str

132135 -- See https://www.dokuwiki.org/wiki:syntax

133136 -- use uppercase HTML tag for block-level content:

134− | f == Format "html" = return $ "<HTML>\n" ++ str ++ "\n</HTML>"

137+ | f == Format "html" = return $ "<HTML>\n" <> str <> "\n</HTML>"

135138 | otherwise = "" <$

136139 report (BlockNotRendered b)

137140

@@ -141,8 +144,8 @@ blockToDokuWiki opts (Header level _ inlines) = do

141144 -- emphasis, links etc. not allowed in headers, apparently,

142145 -- so we remove formatting:

143146 contents <- inlineListToDokuWiki opts $ removeFormatting inlines

144− let eqs = replicate ( 7 - level ) '='

145− return $ eqs ++ " " ++ contents ++ " " ++ eqs ++ "\n"

147+ let eqs = T.replicate ( 7 - level ) "="

148+ return $ eqs <> " " <> contents <> " " <> eqs <> "\n"

146149

147150blockToDokuWiki _ (CodeBlock (_,classes,_) str) = do

148151 let at = classes `intersect` ["actionscript", "ada", "apache", "applescript", "asm", "asp",

@@ -154,43 +157,43 @@ blockToDokuWiki _ (CodeBlock (_,classes,_) str) = do

154157 "python", "qbasic", "rails", "reg", "robots", "ruby", "sas", "scheme", "sdlbasic",

155158 "smalltalk", "smarty", "sql", "tcl", "", "thinbasic", "tsql", "vb", "vbnet", "vhdl",

156159 "visualfoxpro", "winbatch", "xml", "xpp", "z80"]

157− return $ "<code" ++

160+ return $ "<code" <>

158161 (case at of

159162 [] -> ">\n"

160− (x:_) -> " " ++ x ++ ">\n") ++ str ++ "\n</code>"

163+ (x:_) -> " " <> x <> ">\n") <> str <> "\n</code>"

161164

162165blockToDokuWiki opts (BlockQuote blocks) = do

163166 contents <- blockListToDokuWiki opts blocks

164167 if isSimpleBlockQuote blocks

165− then return $ unlines $ map ("> " ++) $ lines contents

166− else return $ "<HTML><blockquote>\n" ++ contents ++ "</blockquote></HTML>"

168+ then return $ T.unlines $ map ("> " <>) $ T.lines contents

169+ else return $ "<HTML><blockquote>\n" <> contents <> "</blockquote></HTML>"

167170

168171blockToDokuWiki opts (Table capt aligns _ headers rows) = do

169172 captionDoc <- if null capt

170173 then return ""

171174 else do

172175 c <- inlineListToDokuWiki opts capt

173− return $ "" ++ c ++ "\n"

176+ return $ "" <> c <> "\n"

174177 headers' <- if all null headers

175178 then return []

176179 else zipWithM (tableItemToDokuWiki opts) aligns headers

177180 rows' <- mapM (zipWithM (tableItemToDokuWiki opts) aligns) rows

178− let widths = map (maximum . map length) $ transpose (headers':rows')

181+ let widths = map (maximum . map T.length) $ transpose (headers':rows')

179182 let padTo (width, al) s =

180− case width - length s of

183+ case width - T.length s of

181184 x | x > 0 ->

182185 if al == AlignLeft || al == AlignDefault

183− then s ++ replicate x ' '

186+ then s <> T.replicate x " "

184187 else if al == AlignRight

185− then replicate x ' ' ++ s

186− else replicate (x `div` 2) ' ' ++

187− s ++ replicate (x - x `div` 2) ' '

188+ then T.replicate x " " <> s

189+ else T.replicate (x `div` 2) " " <>

190+ s <> T.replicate (x - x `div` 2) " "

188191 | otherwise -> s

189− let renderRow sep cells = sep ++

190− intercalate sep (zipWith padTo (zip widths aligns) cells) ++ sep

191− return $ captionDoc ++

192− (if null headers' then "" else renderRow "^" headers' ++ "\n") ++

193− unlines (map (renderRow "|") rows')

192+ let renderRow sep cells = sep <>

193+ T.intercalate sep (zipWith padTo (zip widths aligns) cells) <> sep

194+ return $ captionDoc <>

195+ (if null headers' then "" else renderRow "^" headers' <> "\n") <>

196+ T.unlines (map (renderRow "|") rows')

194197

195198blockToDokuWiki opts x@(BulletList items) = do

196199 oldUseTags <- asks stUseTags

@@ -201,12 +204,12 @@ blockToDokuWiki opts x@(BulletList items) = do

201204 then do

202205 contents <- local (\s -> s { stUseTags = True })

203206 (mapM (listItemToDokuWiki opts) items)

204− return $ "<HTML><ul></HTML>\n" ++ vcat contents ++ "<HTML></ul></HTML>\n"

207+ return $ "<HTML><ul></HTML>\n" <> vcat contents <> "<HTML></ul></HTML>\n"

205208 else do

206− contents <- local (\s -> s { stIndent = stIndent s ++ " "

209+ contents <- local (\s -> s { stIndent = stIndent s <> " "

207210 , stBackSlashLB = backSlash})

208211 (mapM (listItemToDokuWiki opts) items)

209− return $ vcat contents ++ if null indent then "\n" else ""

212+ return $ vcat contents <> if T.null indent then "\n" else ""

210213

211214blockToDokuWiki opts x@(OrderedList attribs items) = do

212215 oldUseTags <- asks stUseTags

@@ -217,12 +220,12 @@ blockToDokuWiki opts x@(OrderedList attribs items) = do

217220 then do

218221 contents <- local (\s -> s { stUseTags = True })

219222 (mapM (orderedListItemToDokuWiki opts) items)

220− return $ "<HTML><ol" ++ listAttribsToString attribs ++ "></HTML>\n" ++ vcat contents ++ "<HTML></ol></HTML>\n"

223+ return $ "<HTML><ol" <> listAttribsToString attribs <> "></HTML>\n" <> vcat contents <> "<HTML></ol></HTML>\n"

221224 else do

222− contents <- local (\s -> s { stIndent = stIndent s ++ " "

225+ contents <- local (\s -> s { stIndent = stIndent s <> " "

223226 , stBackSlashLB = backSlash})

224227 (mapM (orderedListItemToDokuWiki opts) items)

225− return $ vcat contents ++ if null indent then "\n" else ""

228+ return $ vcat contents <> if T.null indent then "\n" else ""

226229

227230-- TODO Need to decide how to make definition lists work on dokuwiki - I don't think there

228231-- is a specific representation of them.

@@ -236,76 +239,76 @@ blockToDokuWiki opts x@(DefinitionList items) = do

236239 then do

237240 contents <- local (\s -> s { stUseTags = True })

238241 (mapM (definitionListItemToDokuWiki opts) items)

239− return $ "<HTML><dl></HTML>\n" ++ vcat contents ++ "<HTML></dl></HTML>\n"

242+ return $ "<HTML><dl></HTML>\n" <> vcat contents <> "<HTML></dl></HTML>\n"

240243 else do

241− contents <- local (\s -> s { stIndent = stIndent s ++ " "

244+ contents <- local (\s -> s { stIndent = stIndent s <> " "

242245 , stBackSlashLB = backSlash})

243246 (mapM (definitionListItemToDokuWiki opts) items)

244− return $ vcat contents ++ if null indent then "\n" else ""

247+ return $ vcat contents <> if T.null indent then "\n" else ""

245248

246249-- Auxiliary functions for lists:

247250

248251-- | Convert ordered list attributes to HTML attribute string

249−listAttribsToString :: ListAttributes -> String

252+listAttribsToString :: ListAttributes -> Text

250253listAttribsToString (startnum, numstyle, _) =

251− let numstyle' = camelCaseToHyphenated $ show numstyle

254+ let numstyle' = camelCaseToHyphenated $ tshow numstyle

252255 in (if startnum /= 1

253− then " start=\"" ++ show startnum ++ "\""

254− else "") ++

256+ then " start=\"" <> tshow startnum <> "\""

257+ else "") <>

255258 (if numstyle /= DefaultStyle

256− then " style=\"list-style-type: " ++ numstyle' ++ ";\""

259+ then " style=\"list-style-type: " <> numstyle' <> ";\""

257260 else "")

258261

259262-- | Convert bullet list item (list of blocks) to DokuWiki.

260263listItemToDokuWiki :: PandocMonad m

261− => WriterOptions -> [Block] -> DokuWiki m String

264+ => WriterOptions -> [Block] -> DokuWiki m Text

262265listItemToDokuWiki opts items = do

263266 useTags <- asks stUseTags

264267 if useTags

265268 then do

266269 contents <- blockListToDokuWiki opts items

267− return $ "<HTML><li></HTML>" ++ contents ++ "<HTML></li></HTML>"

270+ return $ "<HTML><li></HTML>" <> contents <> "<HTML></li></HTML>"

268271 else do

269272 bs <- mapM (blockToDokuWiki opts) items

270273 let contents = case items of

271− [_, CodeBlock _ _] -> concat bs

274+ [_, CodeBlock _ _] -> T.concat bs

272275 _ -> vcat bs

273276 indent <- asks stIndent

274277 backSlash <- asks stBackSlashLB

275− let indent' = if backSlash then drop 2 indent else indent

276− return $ indent' ++ "* " ++ contents

278+ let indent' = if backSlash then T.drop 2 indent else indent

279+ return $ indent' <> "* " <> contents

277280

278281-- | Convert ordered list item (list of blocks) to DokuWiki.

279282-- | TODO Emiminate dreadful duplication of text from listItemToDokuWiki

280−orderedListItemToDokuWiki :: PandocMonad m => WriterOptions -> [Block] -> DokuWiki m String

283+orderedListItemToDokuWiki :: PandocMonad m => WriterOptions -> [Block] -> DokuWiki m Text

281284orderedListItemToDokuWiki opts items = do

282285 contents <- blockListToDokuWiki opts items

283286 useTags <- asks stUseTags

284287 if useTags

285− then return $ "<HTML><li></HTML>" ++ contents ++ "<HTML></li></HTML>"

288+ then return $ "<HTML><li></HTML>" <> contents <> "<HTML></li></HTML>"

286289 else do

287290 indent <- asks stIndent

288291 backSlash <- asks stBackSlashLB

289− let indent' = if backSlash then drop 2 indent else indent

290− return $ indent' ++ "- " ++ contents

292+ let indent' = if backSlash then T.drop 2 indent else indent

293+ return $ indent' <> "- " <> contents

291294

292295-- | Convert definition list item (label, list of blocks) to DokuWiki.

293296definitionListItemToDokuWiki :: PandocMonad m

294297 => WriterOptions

295298 -> ([Inline],[[Block]])

296− -> DokuWiki m String

299+ -> DokuWiki m Text

297300definitionListItemToDokuWiki opts (label, items) = do

298301 labelText <- inlineListToDokuWiki opts label

299302 contents <- mapM (blockListToDokuWiki opts) items

300303 useTags <- asks stUseTags

301304 if useTags

302− then return $ "<HTML><dt></HTML>" ++ labelText ++ "<HTML></dt></HTML>\n" ++

303− intercalate "\n" (map (\d -> "<HTML><dd></HTML>" ++ d ++ "<HTML></dd></HTML>") contents)

305+ then return $ "<HTML><dt></HTML>" <> labelText <> "<HTML></dt></HTML>\n" <>

306+ T.intercalate "\n" (map (\d -> "<HTML><dd></HTML>" <> d <> "<HTML></dd></HTML>") contents)

304307 else do

305308 indent <- asks stIndent

306309 backSlash <- asks stBackSlashLB

307− let indent' = if backSlash then drop 2 indent else indent

308− return $ indent' ++ "* **" ++ labelText ++ "** " ++ concat contents

310+ let indent' = if backSlash then T.drop 2 indent else indent

311+ return $ indent' <> "* **" <> labelText <> "** " <> T.concat contents

309312

310313-- | True if the list can be handled by simple wiki markup, False if HTML tags will be needed.

311314isSimpleList :: Block -> Bool

@@ -333,17 +336,17 @@ isSimpleBlockQuote :: [Block] -> Bool

333336isSimpleBlockQuote bs = all isPlainOrPara bs

334337

335338-- | Concatenates strings with line breaks between them.

336−vcat :: [String] -> String

337−vcat = intercalate "\n"

339+vcat :: [Text] -> Text

340+vcat = T.intercalate "\n"

338341

339342-- | For each string in the input list, convert all newlines to

340343-- dokuwiki escaped newlines. Then concat the list using double linebreaks.

341−backSlashLineBreaks :: [String] -> String

342−backSlashLineBreaks ls = vcatBackSlash $ map escape ls

344+backSlashLineBreaks :: [Text] -> Text

345+backSlashLineBreaks ls = vcatBackSlash $ map (T.pack . escape . T.unpack) ls

343346 where

344− vcatBackSlash = intercalate "\\\\ \\\\ " -- simulate paragraphs.

345− escape ['\n'] = "" -- remove trailing newlines

346− escape ('\n':cs) = "\\\\ " ++ escape cs

347+ vcatBackSlash = T.intercalate "\\\\ \\\\ " -- simulate paragraphs.

348+ escape ['\n'] = "" -- remove trailing newlines

349+ escape ('\n':cs) = "\\\\ " <> escape cs

347350 escape (c:cs) = c : escape cs

348351 escape [] = []

349352

@@ -353,11 +356,11 @@ tableItemToDokuWiki :: PandocMonad m

353356 => WriterOptions

354357 -> Alignment

355358 -> [Block]

356− -> DokuWiki m String

359+ -> DokuWiki m Text

357360tableItemToDokuWiki opts align' item = do

358361 let mkcell x = (if align' == AlignRight || align' == AlignCenter

359362 then " "

360− else "") ++ x ++

363+ else "") <> x <>

361364 (if align' == AlignLeft || align' == AlignCenter

362365 then " "

363366 else "")

@@ -369,7 +372,7 @@ tableItemToDokuWiki opts align' item = do

369372blockListToDokuWiki :: PandocMonad m

370373 => WriterOptions -- ^ Options

371374 -> [Block] -- ^ List of block elements

372− -> DokuWiki m String

375+ -> DokuWiki m Text

373376blockListToDokuWiki opts blocks = do

374377 backSlash <- asks stBackSlashLB

375378 let blocks' = consolidateRawBlocks blocks

@@ -380,51 +383,51 @@ blockListToDokuWiki opts blocks = do

380383consolidateRawBlocks :: [Block] -> [Block]

381384consolidateRawBlocks [] = []

382385consolidateRawBlocks (RawBlock f1 b1 : RawBlock f2 b2 : xs)

383− | f1 == f2 = consolidateRawBlocks (RawBlock f1 (b1 ++ "\n" ++ b2) : xs)

386+ | f1 == f2 = consolidateRawBlocks (RawBlock f1 (b1 <> "\n" <> b2) : xs)

384387consolidateRawBlocks (x:xs) = x : consolidateRawBlocks xs

385388

386389-- | Convert list of Pandoc inline elements to DokuWiki.

387390inlineListToDokuWiki :: PandocMonad m

388− => WriterOptions -> [Inline] -> DokuWiki m String

391+ => WriterOptions -> [Inline] -> DokuWiki m Text

389392inlineListToDokuWiki opts lst =

390− concat <$> mapM (inlineToDokuWiki opts) lst

393+ T.concat <$> mapM (inlineToDokuWiki opts) lst

391394

392395-- | Convert Pandoc inline element to DokuWiki.

393396inlineToDokuWiki :: PandocMonad m

394− => WriterOptions -> Inline -> DokuWiki m String

397+ => WriterOptions -> Inline -> DokuWiki m Text

395398

396399inlineToDokuWiki opts (Span _attrs ils) =

397400 inlineListToDokuWiki opts ils

398401

399402inlineToDokuWiki opts (Emph lst) = do

400403 contents <- inlineListToDokuWiki opts lst

401− return $ "//" ++ contents ++ "//"

404+ return $ "//" <> contents <> "//"

402405

403406inlineToDokuWiki opts (Strong lst) = do

404407 contents <- inlineListToDokuWiki opts lst

405− return $ "**" ++ contents ++ "**"

408+ return $ "**" <> contents <> "**"

406409

407410inlineToDokuWiki opts (Strikeout lst) = do

408411 contents <- inlineListToDokuWiki opts lst

409− return $ "<del>" ++ contents ++ "</del>"

412+ return $ "<del>" <> contents <> "</del>"

410413

411414inlineToDokuWiki opts (Superscript lst) = do

412415 contents <- inlineListToDokuWiki opts lst

413− return $ "" ++ contents ++ ""

416+ return $ "" <> contents <> ""

414417

415418inlineToDokuWiki opts (Subscript lst) = do

416419 contents <- inlineListToDokuWiki opts lst

417− return $ "" ++ contents ++ ""

420+ return $ "" <> contents <> ""

418421

419422inlineToDokuWiki opts (SmallCaps lst) = inlineListToDokuWiki opts lst

420423

421424inlineToDokuWiki opts (Quoted SingleQuote lst) = do

422425 contents <- inlineListToDokuWiki opts lst

423− return $ "\8216" ++ contents ++ "\8217"

426+ return $ "\8216" <> contents <> "\8217"

424427

425428inlineToDokuWiki opts (Quoted DoubleQuote lst) = do

426429 contents <- inlineListToDokuWiki opts lst

427− return $ "\8220" ++ contents ++ "\8221"

430+ return $ "\8220" <> contents <> "\8221"

428431

429432inlineToDokuWiki opts (Cite _ lst) = inlineListToDokuWiki opts lst

430433

@@ -438,11 +441,11 @@ inlineToDokuWiki _ (Code _ str) =

438441 -- characters.

439442 -- It does mean that if pandoc could ever read dokuwiki, and so round-trip the format,

440443 -- any formatting inside inlined code blocks would be lost, or presented incorrectly.

441− return $ "''%%" ++ str ++ "%%''"

444+ return $ "''%%" <> str <> "%%''"

442445

443446inlineToDokuWiki _ (Str str) = return $ escapeString str

444447

445−inlineToDokuWiki _ (Math mathType str) = return $ delim ++ str ++ delim

448+inlineToDokuWiki _ (Math mathType str) = return $ delim <> str <> delim

446449 -- note: str should NOT be escaped

447450 where delim = case mathType of

448451 DisplayMath -> "$$"

@@ -450,7 +453,7 @@ inlineToDokuWiki _ (Math mathType str) = return $ delim ++ str ++ delim

450453

451454inlineToDokuWiki _ il@(RawInline f str)

452455 | f == Format "dokuwiki" = return str

453− | f == Format "html" = return $ "<html>" ++ str ++ "</html>"

456+ | f == Format "html" = return $ "<html>" <> str <> "</html>"

454457 | otherwise = "" <$ report (InlineNotRendered il)

455458

456459inlineToDokuWiki _ LineBreak = do

@@ -470,34 +473,34 @@ inlineToDokuWiki _ Space = return " "

470473inlineToDokuWiki opts (Link _ txt (src, _)) = do

471474 label <- inlineListToDokuWiki opts txt

472475 case txt of

473− [Str s] | "mailto:" `isPrefixOf` src -> return $ "<" ++ s ++ ">"

476+ [Str s] | "mailto:" `T.isPrefixOf` src -> return $ "<" <> s <> ">"

474477 | escapeURI s == src -> return src

475478 _ -> if isURI src

476− then return $ "[[" ++ src ++ "|" ++ label ++ "]]"

477− else return $ "[[" ++ src' ++ "|" ++ label ++ "]]"

478− where src' = case src of

479− '/':xs -> xs -- with leading / it's a

480− _ -> src -- link to a help page

479+ then return $ "[[" <> src <> "|" <> label <> "]]"

480+ else return $ "[[" <> src' <> "|" <> label <> "]]"

481+ where src' = case T.uncons src of

482+ Just ('/',xs) -> xs -- with leading / it's a

483+ _ -> src -- link to a help page

481484inlineToDokuWiki opts (Image attr alt (source, tit)) = do

482485 alt' <- inlineListToDokuWiki opts alt

483486 let txt = case (tit, alt) of

484487 ("", []) -> ""

485− ("", _ ) -> "|" ++ alt'

486− (_ , _ ) -> "|" ++ tit

487− return $ "{{" ++ source ++ imageDims opts attr ++ txt ++ "}}"

488+ ("", _ ) -> "|" <> alt'

489+ (_ , _ ) -> "|" <> tit

490+ return $ "{{" <> source <> imageDims opts attr <> txt <> "}}"

488491

489492inlineToDokuWiki opts (Note contents) = do

490493 contents' <- blockListToDokuWiki opts contents

491− return $ "((" ++ contents' ++ "))"

494+ return $ "((" <> contents' <> "))"

492495 -- note - may not work for notes with multiple blocks

493496

494−imageDims :: WriterOptions -> Attr -> String

497+imageDims :: WriterOptions -> Attr -> Text

495498imageDims opts attr = go (toPx $ dimension Width attr) (toPx $ dimension Height attr)

496499 where

497500 toPx = fmap (showInPixel opts) . checkPct

498501 checkPct (Just (Percent _)) = Nothing

499502 checkPct maybeDim = maybeDim

500− go (Just w) Nothing = "?" ++ w

501− go (Just w) (Just h) = "?" ++ w ++ "x" ++ h

502− go Nothing (Just h) = "?0x" ++ h

503+ go (Just w) Nothing = "?" <> w

504+ go (Just w) (Just h) = "?" <> w <> "x" <> h

505+ go Nothing (Just h) = "?0x" <> h

503506 go Nothing Nothing = ""

src/Text/Pandoc/Writers/EPUB.hs+80−77

@@ -49,7 +49,7 @@ import Text.Pandoc.Options (EPUBVersion (..), HTMLMathMethod (..),

4949 ObfuscationMethod (NoObfuscation), WrapOption (..),

5050 WriterOptions (..))

5151import Text.Pandoc.Shared (makeSections, normalizeDate, renderTags',

52− safeRead, stringify, trim, uniqueIdent)

52+ safeRead, stringify, trim, uniqueIdent, tshow)

5353import qualified Text.Pandoc.UTF8 as UTF8

5454import Text.Pandoc.UUID (getUUID)

5555import Text.Pandoc.Walk (query, walk, walkM)

@@ -176,10 +176,10 @@ getEPUBMetadata opts meta = do

176176 let localeLang =

177177 case mLang of

178178 Just lang ->

179− map (\c -> if c == '_' then '-' else c) $

180− takeWhile (/='.') lang

179+ TS.map (\c -> if c == '_' then '-' else c) $

180+ TS.takeWhile (/='.') lang

181181 Nothing -> "en-US"

182− return m{ epubLanguage = localeLang }

182+ return m{ epubLanguage = TS.unpack localeLang }

183183 else return m

184184 let fixDate m =

185185 if null (epubDate m)

@@ -194,7 +194,7 @@ getEPUBMetadata opts meta = do

194194 then return m

195195 else do

196196 let authors' = map stringify $ docAuthors meta

197− let toAuthor name = Creator{ creatorText = name

197+ let toAuthor name = Creator{ creatorText = TS.unpack name

198198 , creatorRole = Just "aut"

199199 , creatorFileAs = Nothing }

200200 return $ m{ epubCreator = map toAuthor authors' ++ epubCreator m }

@@ -253,18 +253,18 @@ addMetadataFromXML e@(Element (QName "meta" _ _) attrs _ _) md =

253253addMetadataFromXML _ md = md

254254

255255metaValueToString :: MetaValue -> String

256−metaValueToString (MetaString s) = s

257−metaValueToString (MetaInlines ils) = stringify ils

258−metaValueToString (MetaBlocks bs) = stringify bs

256+metaValueToString (MetaString s) = TS.unpack s

257+metaValueToString (MetaInlines ils) = TS.unpack $ stringify ils

258+metaValueToString (MetaBlocks bs) = TS.unpack $ stringify bs

259259metaValueToString (MetaBool True) = "true"

260260metaValueToString (MetaBool False) = "false"

261261metaValueToString _ = ""

262262

263−metaValueToPaths:: MetaValue -> [FilePath]

263+metaValueToPaths :: MetaValue -> [FilePath]

264264metaValueToPaths (MetaList xs) = map metaValueToString xs

265265metaValueToPaths x = [metaValueToString x]

266266

267−getList :: String -> Meta -> (MetaValue -> a) -> [a]

267+getList :: TS.Text -> Meta -> (MetaValue -> a) -> [a]

268268getList s meta handleMetaValue =

269269 case lookupMeta s meta of

270270 Just (MetaList xs) -> map handleMetaValue xs

@@ -288,7 +288,7 @@ getTitle meta = getList "title" meta handleMetaValue

288288 , titleType = metaValueToString <$> M.lookup "type" m }

289289 handleMetaValue mv = Title (metaValueToString mv) Nothing Nothing

290290

291−getCreator :: String -> Meta -> [Creator]

291+getCreator :: TS.Text -> Meta -> [Creator]

292292getCreator s meta = getList s meta handleMetaValue

293293 where handleMetaValue (MetaMap m) =

294294 Creator{ creatorText = maybe "" metaValueToString $ M.lookup "text" m

@@ -296,7 +296,7 @@ getCreator s meta = getList s meta handleMetaValue

296296 , creatorRole = metaValueToString <$> M.lookup "role" m }

297297 handleMetaValue mv = Creator (metaValueToString mv) Nothing Nothing

298298

299−getDate :: String -> Meta -> [Date]

299+getDate :: TS.Text -> Meta -> [Date]

300300getDate s meta = getList s meta handleMetaValue

301301 where handleMetaValue (MetaMap m) =

302302 Date{ dateText = fromMaybe "" $

@@ -305,7 +305,7 @@ getDate s meta = getList s meta handleMetaValue

305305 handleMetaValue mv = Date { dateText = fromMaybe "" $ normalizeDate' $ metaValueToString mv

306306 , dateEvent = Nothing }

307307

308−simpleList :: String -> Meta -> [String]

308+simpleList :: TS.Text -> Meta -> [String]

309309simpleList s meta =

310310 case lookupMeta s meta of

311311 Just (MetaList xs) -> map metaValueToString xs

@@ -366,11 +366,11 @@ metadataFromMeta opts meta = EPUBMetadata{

366366 _ -> Nothing

367367 ibooksFields = case lookupMeta "ibooks" meta of

368368 Just (MetaMap mp)

369− -> M.toList $ M.map metaValueToString mp

369+ -> M.toList $ M.mapKeys TS.unpack $ M.map metaValueToString mp

370370 _ -> []

371371 calibreFields = case lookupMeta "calibre" meta of

372372 Just (MetaMap mp)

373− -> M.toList $ M.map metaValueToString mp

373+ -> M.toList $ M.mapKeys TS.unpack $ M.map metaValueToString mp

374374 _ -> []

375375

376376-- | Produce an EPUB2 file from a Pandoc document.

@@ -396,9 +396,9 @@ writeEPUB :: PandocMonad m

396396writeEPUB epubVersion opts doc = do

397397 let epubSubdir = writerEpubSubdirectory opts

398398 -- sanity check on epubSubdir

399− unless (all (\c -> isAscii c && isAlphaNum c) epubSubdir) $

399+ unless (TS.all (\c -> isAscii c && isAlphaNum c) epubSubdir) $

400400 throwError $ PandocEpubSubdirectoryError epubSubdir

401− let initState = EPUBState { stMediaPaths = [], stMediaNextId = 0, stEpubSubdir = epubSubdir }

401+ let initState = EPUBState { stMediaPaths = [], stMediaNextId = 0, stEpubSubdir = TS.unpack epubSubdir }

402402 evalStateT (pandocToEPUB epubVersion opts doc) initState

403403

404404pandocToEPUB :: PandocMonad m

@@ -422,7 +422,7 @@ pandocToEPUB version opts doc = do

422422 [] -> case epubTitle metadata of

423423 [] -> "UNTITLED"

424424 (x:_) -> titleText x

425− x -> stringify x

425+ x -> TS.unpack $ stringify x

426426

427427 -- stylesheet

428428 stylesheets <- case epubStylesheets metadata of

@@ -468,13 +468,13 @@ pandocToEPUB version opts doc = do

468468 case imageSize opts' (B.toStrict imgContent) of

469469 Right sz -> return $ sizeInPixels sz

470470 Left err' -> (0, 0) <$ report

471− (CouldNotDetermineImageSize img err')

471+ (CouldNotDetermineImageSize (TS.pack img) err')

472472 cpContent <- lift $ writeHtml

473473 opts'{ writerVariables =

474474 Context (M.fromList [

475475 ("coverpage", toVal' "true"),

476− ("pagetitle", toVal' $

477− escapeStringForXML plainTitle),

476+ ("pagetitle", toVal $

477+ escapeStringForXML $ TS.pack plainTitle),

478478 ("cover-image", toVal' coverImage),

479479 ("cover-image-width", toVal' $

480480 show coverImageWidth),

@@ -494,8 +494,8 @@ pandocToEPUB version opts doc = do

494494 Context (M.fromList [

495495 ("titlepage", toVal' "true"),

496496 ("body-type", toVal' "frontmatter"),

497− ("pagetitle", toVal' $

498− escapeStringForXML plainTitle)])

497+ ("pagetitle", toVal $

498+ escapeStringForXML $ TS.pack plainTitle)])

499499 <> cssvars True <> vars }

500500 (Pandoc meta [])

501501 tpEntry <- mkEntry "text/title_page.xhtml" tpContent

@@ -504,7 +504,7 @@ pandocToEPUB version opts doc = do

504504 let matchingGlob f = do

505505 xs <- lift $ P.glob f

506506 when (null xs) $

507− report $ CouldNotFetchResource f "glob did not match any font files"

507+ report $ CouldNotFetchResource (TS.pack f) "glob did not match any font files"

508508 return xs

509509 let mkFontEntry f = mkEntry ("fonts/" ++ takeFileName f) =<<

510510 lift (P.readFileLazy f)

@@ -551,16 +551,16 @@ pandocToEPUB version opts doc = do

551551

552552 let chapters' = secsToChapters secs

553553

554− let extractLinkURL' :: Int -> Inline -> [(String, String)]

554+ let extractLinkURL' :: Int -> Inline -> [(TS.Text, TS.Text)]

555555 extractLinkURL' num (Span (ident, _, _) _)

556− | not (null ident) = [(ident, showChapter num ++ ('#':ident))]

556+ | not (TS.null ident) = [(ident, TS.pack (showChapter num) <> "#" <> ident)]

557557 extractLinkURL' _ _ = []

558558

559− let extractLinkURL :: Int -> Block -> [(String, String)]

559+ let extractLinkURL :: Int -> Block -> [(TS.Text, TS.Text)]

560560 extractLinkURL num (Div (ident, _, _) _)

561− | not (null ident) = [(ident, showChapter num ++ ('#':ident))]

561+ | not (TS.null ident) = [(ident, TS.pack (showChapter num) <> "#" <> ident)]

562562 extractLinkURL num (Header _ (ident, _, _) _)

563− | not (null ident) = [(ident, showChapter num ++ ('#':ident))]

563+ | not (TS.null ident) = [(ident, TS.pack (showChapter num) <> "#" <> ident)]

564564 extractLinkURL num b = query (extractLinkURL' num) b

565565

566566 let reftable = concat $ zipWith (\(Chapter bs) num ->

@@ -568,10 +568,10 @@ pandocToEPUB version opts doc = do

568568 chapters' [1..]

569569

570570 let fixInternalReferences :: Inline -> Inline

571− fixInternalReferences (Link attr lab ('#':xs, tit)) =

572− case lookup xs reftable of

571+ fixInternalReferences (Link attr lab (src, tit))

572+ | Just ('#', xs) <- TS.uncons src = case lookup xs reftable of

573573 Just ys -> Link attr lab (ys, tit)

574− Nothing -> Link attr lab ('#':xs, tit)

574+ Nothing -> Link attr lab (src, tit)

575575 fixInternalReferences x = x

576576

577577 -- internal reference IDs change when we chunk the file,

@@ -645,14 +645,14 @@ pandocToEPUB version opts doc = do

645645 ("href", makeRelative epubSubdir

646646 $ eRelativePath ent),

647647 ("media-type",

648− fromMaybe "application/octet-stream"

648+ maybe "application/octet-stream" TS.unpack

649649 $ mediaTypeOf $ eRelativePath ent)] $ ()

650650 let fontNode ent = unode "item" !

651651 [("id", toId $ makeRelative epubSubdir

652652 $ eRelativePath ent),

653653 ("href", makeRelative epubSubdir

654654 $ eRelativePath ent),

655− ("media-type", fromMaybe "" $

655+ ("media-type", maybe "" TS.unpack $

656656 getMimeType $ eRelativePath ent)] $ ()

657657

658658 let tocTitle = fromMaybe plainTitle $

@@ -724,7 +724,7 @@ pandocToEPUB version opts doc = do

724724 let tocLevel = writerTOCDepth opts

725725

726726 let navPointNode :: PandocMonad m

727− => (Int -> [Inline] -> String -> [Element] -> Element)

727+ => (Int -> [Inline] -> TS.Text -> [Element] -> Element)

728728 -> Block -> StateT Int m [Element]

729729 navPointNode formatter (Div (ident,_,_)

730730 (Header lvl (_,_,kvs) ils : children)) = do

@@ -734,29 +734,29 @@ pandocToEPUB version opts doc = do

734734 n <- get

735735 modify (+1)

736736 let num = fromMaybe "" $ lookup "number" kvs

737− let tit = if writerNumberSections opts && not (null num)

737+ let tit = if writerNumberSections opts && not (TS.null num)

738738 then Span ("", ["section-header-number"], [])

739739 [Str num] : Space : ils

740740 else ils

741741 src <- case lookup ident reftable of

742742 Just x -> return x

743743 Nothing -> throwError $ PandocSomeError $

744− ident ++ " not found in reftable"

744+ ident <> " not found in reftable"

745745 subs <- concat <$> mapM (navPointNode formatter) children

746746 return [formatter n tit src subs]

747747 navPointNode formatter (Div _ bs) =

748748 concat <$> mapM (navPointNode formatter) bs

749749 navPointNode _ _ = return []

750750

751− let navMapFormatter :: Int -> [Inline] -> String -> [Element] -> Element

751+ let navMapFormatter :: Int -> [Inline] -> TS.Text -> [Element] -> Element

752752 navMapFormatter n tit src subs = unode "navPoint" !

753753 [("id", "navPoint-" ++ show n)] $

754− [ unode "navLabel" $ unode "text" $ stringify tit

755− , unode "content" ! [("src", "text/" ++ src)] $ ()

754+ [ unode "navLabel" $ unode "text" $ TS.unpack $ stringify tit

755+ , unode "content" ! [("src", "text/" <> TS.unpack src)] $ ()

756756 ] ++ subs

757757

758758 let tpNode = unode "navPoint" ! [("id", "navPoint-0")] $

759− [ unode "navLabel" $ unode "text" (stringify $ docTitle' meta)

759+ [ unode "navLabel" $ unode "text" (TS.unpack $ stringify $ docTitle' meta)

760760 , unode "content" ! [("src", "text/title_page.xhtml")]

761761 $ () ]

762762

@@ -784,11 +784,11 @@ pandocToEPUB version opts doc = do

784784 ]

785785 tocEntry <- mkEntry "toc.ncx" tocData

786786

787− let navXhtmlFormatter :: Int -> [Inline] -> String -> [Element] -> Element

787+ let navXhtmlFormatter :: Int -> [Inline] -> TS.Text -> [Element] -> Element

788788 navXhtmlFormatter n tit src subs = unode "li" !

789789 [("id", "toc-li-" ++ show n)] $

790790 (unode "a" !

791− [("href", "text/" ++ src)]

791+ [("href", "text/" <> TS.unpack src)]

792792 $ titElements)

793793 : case subs of

794794 [] -> []

@@ -799,12 +799,12 @@ pandocToEPUB version opts doc = do

799799 opts{ writerTemplate = Nothing

800800 , writerVariables =

801801 Context (M.fromList

802− [("pagetitle", toVal' $

803− escapeStringForXML plainTitle)])

802+ [("pagetitle", toVal $

803+ escapeStringForXML $ TS.pack plainTitle)])

804804 <> writerVariables opts}

805805 (Pandoc nullMeta

806806 [Plain $ walk clean tit])) of

807− Left _ -> TS.pack $ stringify tit

807+ Left _ -> stringify tit

808808 Right x -> x

809809 -- can't have <a> elements inside generated links...

810810 clean (Link _ ils _) = Span ("", [], []) ils

@@ -815,7 +815,7 @@ pandocToEPUB version opts doc = do

815815 tocBlocks <- lift $ evalStateT

816816 (concat <$> mapM (navPointNode navXhtmlFormatter) secs) 1

817817 let navBlocks = [RawBlock (Format "html")

818− $ showElement $ -- prettyprinting introduces bad spaces

818+ $ TS.pack $ showElement $ -- prettyprinting introduces bad spaces

819819 unode navtag ! ([("epub:type","toc") | epub3] ++

820820 [("id","toc")]) $

821821 [ unode "h1" ! [("id","toc-title")] $ tocTitle

@@ -836,7 +836,7 @@ pandocToEPUB version opts doc = do

836836 else []

837837 let landmarks = if null landmarkItems

838838 then []

839− else [RawBlock (Format "html") $ ppElement $

839+ else [RawBlock (Format "html") $ TS.pack $ ppElement $

840840 unode "nav" ! [("epub:type","landmarks")

841841 ,("id","landmarks")

842842 ,("hidden","hidden")] $

@@ -995,49 +995,49 @@ showDateTimeISO8601 :: UTCTime -> String

995995showDateTimeISO8601 = formatTime defaultTimeLocale "%FT%TZ"

996996

997997transformTag :: PandocMonad m

998− => Tag String

999− -> E m (Tag String)

998+ => Tag TS.Text

999+ -> E m (Tag TS.Text)

10001000transformTag tag@(TagOpen name attr)

10011001 | name `elem` ["video", "source", "img", "audio"] &&

10021002 isNothing (lookup "data-external" attr) = do

10031003 let src = fromAttrib "src" tag

10041004 let poster = fromAttrib "poster" tag

1005− newsrc <- modifyMediaRef src

1006− newposter <- modifyMediaRef poster

1005+ newsrc <- modifyMediaRef $ TS.unpack src

1006+ newposter <- modifyMediaRef $ TS.unpack poster

10071007 let attr' = filter (\(x,_) -> x /= "src" && x /= "poster") attr ++

1008− [("src", "../" ++ newsrc) | not (null newsrc)] ++

1009− [("poster", "../" ++ newposter) | not (null newposter)]

1008+ [("src", "../" <> newsrc) | not (TS.null newsrc)] ++

1009+ [("poster", "../" <> newposter) | not (TS.null newposter)]

10101010 return $ TagOpen name attr'

10111011transformTag tag = return tag

10121012

10131013modifyMediaRef :: PandocMonad m

10141014 => FilePath

1015− -> E m FilePath

1015+ -> E m TS.Text

10161016modifyMediaRef "" = return ""

10171017modifyMediaRef oldsrc = do

10181018 media <- gets stMediaPaths

10191019 case lookup oldsrc media of

1020− Just (n,_) -> return n

1020+ Just (n,_) -> return $ TS.pack n

10211021 Nothing -> catchError

1022− (do (img, mbMime) <- P.fetchItem oldsrc

1023− let ext = fromMaybe (takeExtension (takeWhile (/='?') oldsrc))

1024− (('.':) <$> (mbMime >>= extensionFromMimeType))

1022+ (do (img, mbMime) <- P.fetchItem $ TS.pack oldsrc

1023+ let ext = maybe (takeExtension (takeWhile (/='?') oldsrc)) TS.unpack

1024+ (("." <>) <$> (mbMime >>= extensionFromMimeType))

10251025 newName <- getMediaNextNewName ext

10261026 let newPath = "media/" ++ newName

10271027 entry <- mkEntry newPath (B.fromChunks . (:[]) $ img)

10281028 modify $ \st -> st{ stMediaPaths =

10291029 (oldsrc, (newPath, Just entry)):media}

1030− return newPath)

1030+ return $ TS.pack newPath)

10311031 (\e -> do

1032− report $ CouldNotFetchResource oldsrc (show e)

1033− return oldsrc)

1032+ report $ CouldNotFetchResource (TS.pack oldsrc) (tshow e)

1033+ return $ TS.pack oldsrc)

10341034

10351035getMediaNextNewName :: PandocMonad m => String -> E m String

10361036getMediaNextNewName ext = do

10371037 nextId <- gets stMediaNextId

10381038 modify $ \st -> st { stMediaNextId = nextId + 1 }

10391039 let nextName = "file" ++ show nextId ++ ext

1040− (P.fetchItem nextName >> getMediaNextNewName ext) `catchError` const (return nextName)

1040+ (P.fetchItem (TS.pack nextName) >> getMediaNextNewName ext) `catchError` const (return nextName)

10411041

10421042transformBlock :: PandocMonad m

10431043 => Block

@@ -1054,14 +1054,14 @@ transformInline :: PandocMonad m

10541054 -> Inline

10551055 -> E m Inline

10561056transformInline _opts (Image attr lab (src,tit)) = do

1057− newsrc <- modifyMediaRef src

1058− return $ Image attr lab ("../" ++ newsrc, tit)

1057+ newsrc <- modifyMediaRef $ TS.unpack src

1058+ return $ Image attr lab ("../" <> newsrc, tit)

10591059transformInline opts (x@(Math t m))

10601060 | WebTeX url <- writerHTMLMathMethod opts = do

1061− newsrc <- modifyMediaRef (url ++ urlEncode m)

1061+ newsrc <- modifyMediaRef (TS.unpack url <> urlEncode (TS.unpack m))

10621062 let mathclass = if t == DisplayMath then "display" else "inline"

10631063 return $ Span ("",["math",mathclass],[])

1064− [Image nullAttr [x] ("../" ++ newsrc, "")]

1064+ [Image nullAttr [x] ("../" <> newsrc, "")]

10651065transformInline _opts (RawInline fmt raw)

10661066 | fmt == Format "html" = do

10671067 let tags = parseTags raw

@@ -1081,7 +1081,7 @@ ppTopElement = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" ++) . unEntity .

10811081 unEntity ('&':'#':xs) =

10821082 let (ds,ys) = break (==';') xs

10831083 rest = drop 1 ys

1084− in case safeRead ('\'':'\\':ds ++ "'") of

1084+ in case safeRead (TS.pack $ "'\\" <> ds <> "'") of

10851085 Just x -> x : unEntity rest

10861086 Nothing -> '&':'#':unEntity xs

10871087 unEntity (x:xs) = x : unEntity xs

@@ -1090,7 +1090,7 @@ mediaTypeOf :: FilePath -> Maybe MimeType

10901090mediaTypeOf x =

10911091 let mediaPrefixes = ["image", "video", "audio"] in

10921092 case getMimeType x of

1093− Just y | any (`isPrefixOf` y) mediaPrefixes -> Just y

1093+ Just y | any (`TS.isPrefixOf` y) mediaPrefixes -> Just y

10941094 _ -> Nothing

10951095

10961096-- Returns filename for chapter number.

@@ -1102,7 +1102,7 @@ addIdentifiers :: WriterOptions -> [Block] -> [Block]

11021102addIdentifiers opts bs = evalState (mapM go bs) Set.empty

11031103 where go (Header n (ident,classes,kvs) ils) = do

11041104 ids <- get

1105− let ident' = if null ident

1105+ let ident' = if TS.null ident

11061106 then uniqueIdent (writerExtensions opts) ils ids

11071107 else ident

11081108 modify $ Set.insert ident'

@@ -1111,13 +1111,16 @@ addIdentifiers opts bs = evalState (mapM go bs) Set.empty

11111111

11121112-- Variant of normalizeDate that allows partial dates: YYYY, YYYY-MM

11131113normalizeDate' :: String -> Maybe String

1114−normalizeDate' xs =

1115− let xs' = trim xs in

1116− case xs' of

1117− [y1,y2,y3,y4] | all isDigit [y1,y2,y3,y4] -> Just xs' -- YYYY

1118− [y1,y2,y3,y4,'-',m1,m2] | all isDigit [y1,y2,y3,y4,m1,m2] -- YYYY-MM

1119− -> Just xs'

1120− _ -> normalizeDate xs'

1114+normalizeDate' = fmap TS.unpack . go . trim . TS.pack

1115+ where

1116+ go xs

1117+ | TS.length xs == 4 -- YYY

1118+ , TS.all isDigit xs = Just xs

1119+ | (y, s) <- TS.splitAt 4 xs -- YYY-MM

1120+ , Just ('-', m) <- TS.uncons s

1121+ , TS.length m == 2

1122+ , TS.all isDigit y && TS.all isDigit m = Just xs

1123+ | otherwise = normalizeDate xs

11211124

11221125toRelator :: String -> Maybe String

11231126toRelator x

src/Text/Pandoc/Writers/FB2.hs+77−74

@@ -1,5 +1,6 @@

11{-# LANGUAGE NoImplicitPrelude #-}

2−{-# LANGUAGE PatternGuards #-}

2+{-# LANGUAGE PatternGuards #-}

3+{-# LANGUAGE OverloadedStrings #-}

34{- |

45Module : Text.Pandoc.Writers.FB2

@@ -23,11 +24,12 @@ import Control.Monad (zipWithM)

2324import Control.Monad.Except (catchError)

2425import Control.Monad.State.Strict (StateT, evalStateT, get, gets, lift, liftM, modify)

2526import Data.ByteString.Base64 (encode)

26−import qualified Data.ByteString.Char8 as B8

27−import Data.Char (isAscii, isControl, isSpace, toLower)

27+import Data.Char (isAscii, isControl, isSpace)

2828import Data.Either (lefts, rights)

29−import Data.List (intercalate, isPrefixOf, stripPrefix)

29+import Data.List (intercalate)

3030import Data.Text (Text, pack)

31+import qualified Data.Text as T

32+import qualified Data.Text.Encoding as TE

3133import Network.HTTP (urlEncode)

3234import Text.XML.Light

3335import qualified Text.XML.Light as X

@@ -40,15 +42,15 @@ import Text.Pandoc.Definition

4042import Text.Pandoc.Logging

4143import Text.Pandoc.Options (HTMLMathMethod (..), WriterOptions (..), def)

4244import Text.Pandoc.Shared (capitalize, isURI, orderedListMarkers,

43− makeSections)

45+ makeSections, tshow)

4446import Text.Pandoc.Writers.Shared (lookupMetaString)

4547

4648-- | Data to be written at the end of the document:

4749-- (foot)notes, URLs, references, images.

4850data FbRenderState = FbRenderState

49− { footnotes :: [ (Int, String, [Content]) ] -- ^ #, ID, text

50− , imagesToFetch :: [ (String, String) ] -- ^ filename, URL or path

51− , parentListMarker :: String -- ^ list marker of the parent ordered list

51+ { footnotes :: [ (Int, Text, [Content]) ] -- ^ #, ID, text

52+ , imagesToFetch :: [ (Text, Text) ] -- ^ filename, URL or path

53+ , parentListMarker :: Text -- ^ list marker of the parent ordered list

5254 , writerOptions :: WriterOptions

5355 } deriving (Show)

5456

@@ -98,8 +100,8 @@ pandocToFB2 opts (Pandoc meta blocks) = do

98100description :: PandocMonad m => Meta -> FBM m Content

99101description meta' = do

100102 let genre = case lookupMetaString "genre" meta' of

101− "" -> el "genre" "unrecognised"

102− s -> el "genre" s

103+ "" -> el "genre" ("unrecognised" :: String)

104+ s -> el "genre" (T.unpack s)

103105 bt <- booktitle meta'

104106 let as = authors meta'

105107 dd <- docdate meta'

@@ -110,7 +112,7 @@ description meta' = do

110112 Just (MetaInlines [Str s]) -> [el "lang" $ iso639 s]

111113 Just (MetaString s) -> [el "lang" $ iso639 s]

112114 _ -> []

113− where iso639 = takeWhile (/= '-') -- Convert BCP 47 to ISO 639

115+ where iso639 = T.unpack . T.takeWhile (/= '-') -- Convert BCP 47 to ISO 639

114116 let coverimage url = do

115117 let img = Image nullAttr mempty (url, "")

116118 im <- insertImage InlineImage img

@@ -122,7 +124,7 @@ description meta' = do

122124 return $ el "description"

123125 [ el "title-info" (genre :

124126 (as ++ bt ++ annotation ++ dd ++ coverpage ++ lang))

125− , el "document-info" [el "program-used" "pandoc"]

127+ , el "document-info" [el "program-used" ("pandoc" :: String)]

126128 ]

127129

128130booktitle :: PandocMonad m => Meta -> FBM m [Content]

@@ -178,7 +180,7 @@ renderSection lvl (Div (id',"section":_,_) (Header _ _ title : xs)) = do

178180 then return []

179181 else list . el "title" <$> formatTitle title

180182 content <- cMapM (renderSection (lvl + 1)) xs

181− let sectionContent = if null id'

183+ let sectionContent = if T.null id'

182184 then el "section" (title' ++ content)

183185 else el "section" ([uattr "id" id'], title' ++ content)

184186 return [sectionContent]

@@ -213,19 +215,19 @@ renderFootnotes = do

213215

214216-- | Fetch images and encode them for the FictionBook XML.

215217-- Return image data and a list of hrefs of the missing images.

216−fetchImages :: PandocMonad m => [(String,String)] -> m ([Content],[String])

218+fetchImages :: PandocMonad m => [(Text,Text)] -> m ([Content],[Text])

217219fetchImages links = do

218220 imgs <- mapM (uncurry fetchImage) links

219221 return (rights imgs, lefts imgs)

220222

221223-- | Fetch image data from disk or from network and make a <binary> XML section.

222224-- Return either (Left hrefOfMissingImage) or (Right xmlContent).

223−fetchImage :: PandocMonad m => String -> String -> m (Either String Content)

225+fetchImage :: PandocMonad m => Text -> Text -> m (Either Text Content)

224226fetchImage href link = do

225227 mbimg <-

226228 case (isURI link, readDataURI link) of

227229 (True, Just (mime,_,True,base64)) ->

228− let mime' = map toLower mime

230+ let mime' = T.toLower mime

229231 in if mime' == "image/png" || mime' == "image/jpeg"

230232 then return (Just (mime',base64))

231233 else return Nothing

@@ -237,9 +239,9 @@ fetchImage href link = do

237239 report $ CouldNotDetermineMimeType link

238240 return Nothing

239241 Just mime -> return $ Just (mime,

240− B8.unpack $ encode bs))

242+ TE.decodeUtf8 $ encode bs))

241243 (\e ->

242− do report $ CouldNotFetchResource link (show e)

244+ do report $ CouldNotFetchResource link (tshow e)

243245 return Nothing)

244246 case mbimg of

245247 Just (imgtype, imgdata) ->

@@ -247,52 +249,52 @@ fetchImage href link = do

247249 ( [uattr "id" href

248250 , uattr "content-type" imgtype]

249251 , txt imgdata )

250− _ -> return (Left ('#':href))

252+ _ -> return (Left ("#" <> href))

251253

252254

253255-- | Extract mime type and encoded data from the Data URI.

254−readDataURI :: String -- ^ URI

255− -> Maybe (String,String,Bool,String)

256+readDataURI :: Text -- ^ URI

257+ -> Maybe (Text,Text,Bool,Text)

256258 -- ^ Maybe (mime,charset,isBase64,data)

257259readDataURI uri =

258− case stripPrefix "data:" uri of

260+ case T.stripPrefix "data:" uri of

259261 Nothing -> Nothing

260262 Just rest ->

261− let meta = takeWhile (/= ',') rest -- without trailing ','

262− uridata = drop (length meta + 1) rest

263− parts = split (== ';') meta

263+ let meta = T.takeWhile (/= ',') rest -- without trailing ','

264+ uridata = T.drop (T.length meta + 1) rest

265+ parts = T.split (== ';') meta

264266 (mime,cs,enc)=foldr upd ("text/plain","US-ASCII",False) parts

265267 in Just (mime,cs,enc,uridata)

266268

267269 where

268270 upd str m@(mime,cs,enc)

269− | isMimeType str = (str,cs,enc)

270− | Just str' <- stripPrefix "charset=" str = (mime,str',enc)

271− | str == "base64" = (mime,cs,True)

272− | otherwise = m

271+ | isMimeType str = (str,cs,enc)

272+ | Just str' <- T.stripPrefix "charset=" str = (mime,str',enc)

273+ | str == "base64" = (mime,cs,True)

274+ | otherwise = m

273275

274276-- Without parameters like ;charset=...; see RFC 2045, 5.1

275−isMimeType :: String -> Bool

277+isMimeType :: Text -> Bool

276278isMimeType s =

277− case split (=='/') s of

279+ case T.split (=='/') s of

278280 [mtype,msubtype] ->

279− (map toLower mtype `elem` types

280− || "x-" `isPrefixOf` map toLower mtype)

281− && all valid mtype

282− && all valid msubtype

281+ (T.toLower mtype `elem` types

282+ || "x-" `T.isPrefixOf` T.toLower mtype)

283+ && T.all valid mtype

284+ && T.all valid msubtype

283285 _ -> False

284286 where

285287 types = ["text","image","audio","video","application","message","multipart"]

286288 valid c = isAscii c && not (isControl c) && not (isSpace c) &&

287− c `notElem` "()<>@,;:\\\"/[]?="

289+ c `notElem` ("()<>@,;:\\\"/[]?=" :: String)

288290

289−footnoteID :: Int -> String

290−footnoteID i = "n" ++ show i

291+footnoteID :: Int -> Text

292+footnoteID i = "n" <> tshow i

291293

292−mkitem :: PandocMonad m => String -> [Block] -> FBM m [Content]

294+mkitem :: PandocMonad m => Text -> [Block] -> FBM m [Content]

293295mkitem mrk bs = do

294296 pmrk <- gets parentListMarker

295− let nmrk = pmrk ++ mrk ++ " "

297+ let nmrk = pmrk <> mrk <> " "

296298 modify (\s -> s { parentListMarker = nmrk})

297299 item <- cMapM blockToXml $ plainToPara $ indentBlocks nmrk bs

298300 modify (\s -> s { parentListMarker = pmrk }) -- old parent marker

@@ -303,11 +305,12 @@ blockToXml :: PandocMonad m => Block -> FBM m [Content]

303305blockToXml (Plain ss) = cMapM toXml ss -- FIXME: can lead to malformed FB2

304306blockToXml (Para [Math DisplayMath formula]) = insertMath NormalImage formula

305307-- title beginning with fig: indicates that the image is a figure

306−blockToXml (Para [Image atr alt (src,'f':'i':'g':':':tit)]) =

307− insertImage NormalImage (Image atr alt (src,tit))

308+blockToXml (Para [Image atr alt (src,tgt)])

309+ | Just tit <- T.stripPrefix "fig:" tgt

310+ = insertImage NormalImage (Image atr alt (src,tit))

308311blockToXml (Para ss) = list . el "p" <$> cMapM toXml ss

309312blockToXml (CodeBlock _ s) = return . spaceBeforeAfter .

310− map (el "p" . el "code") . lines $ s

313+ map (el "p" . el "code" . T.unpack) . T.lines $ s

311314blockToXml (RawBlock f str) =

312315 if f == Format "fb2"

313316 then return $ XI.parseXML str

@@ -329,7 +332,7 @@ blockToXml (DefinitionList defs) =

329332 cMapM mkdef defs

330333 where

331334 mkdef (term, bss) = do

332− items <- cMapM (cMapM blockToXml . plainToPara . indentBlocks (replicate 4 ' ')) bss

335+ items <- cMapM (cMapM blockToXml . plainToPara . indentBlocks (T.replicate 4 " ")) bss

333336 t <- wrap "strong" term

334337 return (el "p" t : items)

335338blockToXml h@Header{} = do

@@ -376,13 +379,13 @@ unPlain x = x

376379

377380-- Simulate increased indentation level. Will not really work

378381-- for multi-line paragraphs.

379−indentPrefix :: String -> Block -> Block

382+indentPrefix :: Text -> Block -> Block

380383indentPrefix spacer = indentBlock

381384 where

382385 indentBlock (Plain ins) = Plain (Str spacer:ins)

383386 indentBlock (Para ins) = Para (Str spacer:ins)

384387 indentBlock (CodeBlock a s) =

385− let s' = unlines . map (spacer++) . lines $ s

388+ let s' = T.unlines . map (spacer<>) . T.lines $ s

386389 in CodeBlock a s'

387390 indentBlock (BlockQuote bs) = BlockQuote (map indent bs)

388391 indentBlock (Header l attr' ins) = Header l attr' (indentLines ins)

@@ -396,12 +399,12 @@ indent :: Block -> Block

396399indent = indentPrefix spacer

397400 where

398401 -- indentation space

399− spacer :: String

400− spacer = replicate 4 ' '

402+ spacer :: Text

403+ spacer = T.replicate 4 " "

401404

402−indentBlocks :: String -> [Block] -> [Block]

405+indentBlocks :: Text -> [Block] -> [Block]

403406indentBlocks _ [] = []

404−indentBlocks prefix (x:xs) = indentPrefix prefix x : map (indentPrefix $ replicate (length prefix) ' ') xs

407+indentBlocks prefix (x:xs) = indentPrefix prefix x : map (indentPrefix $ T.replicate (T.length prefix) " ") xs

405408

406409-- | Convert a Pandoc's Inline element to FictionBook XML representation.

407410toXml :: PandocMonad m => Inline -> FBM m [Content]

@@ -420,7 +423,7 @@ toXml (Quoted DoubleQuote ss) = do

420423 inner <- cMapM toXml ss

421424 return $ [txt "“"] ++ inner ++ [txt "”"]

422425toXml (Cite _ ss) = cMapM toXml ss -- FIXME: support citation styles

423−toXml (Code _ s) = return [el "code" s]

426+toXml (Code _ s) = return [el "code" $ T.unpack s]

424427toXml Space = return [txt " "]

425428toXml SoftBreak = return [txt "\n"]

426429toXml LineBreak = return [txt "\n"]

@@ -438,40 +441,40 @@ toXml (Note bs) = do

438441 let fn_id = footnoteID n

439442 fn_desc <- cMapM blockToXml bs

440443 modify (\s -> s { footnotes = (n, fn_id, fn_desc) : fns })

441− let fn_ref = txt $ "[" ++ show n ++ "]"

442− return . list $ el "a" ( [ attr ("l","href") ('#':fn_id)

444+ let fn_ref = txt $ "[" <> tshow n <> "]"

445+ return . list $ el "a" ( [ attr ("l","href") ("#" <> fn_id)

443446 , uattr "type" "note" ]

444447 , fn_ref )

445448

446−insertMath :: PandocMonad m => ImageMode -> String -> FBM m [Content]

449+insertMath :: PandocMonad m => ImageMode -> Text -> FBM m [Content]

447450insertMath immode formula = do

448451 htmlMath <- fmap (writerHTMLMathMethod . writerOptions) get

449452 case htmlMath of

450453 WebTeX url -> do

451454 let alt = [Code nullAttr formula]

452− let imgurl = url ++ urlEncode formula

455+ let imgurl = url <> T.pack (urlEncode $ T.unpack formula)

453456 let img = Image nullAttr alt (imgurl, "")

454457 insertImage immode img

455− _ -> return [el "code" formula]

458+ _ -> return [el "code" $ T.unpack formula]

456459

457460insertImage :: PandocMonad m => ImageMode -> Inline -> FBM m [Content]

458461insertImage immode (Image _ alt (url,ttl)) = do

459462 images <- imagesToFetch `liftM` get

460463 let n = 1 + length images

461− let fname = "image" ++ show n

464+ let fname = "image" <> tshow n

462465 modify (\s -> s { imagesToFetch = (fname, url) : images })

463− let ttlattr = case (immode, null ttl) of

466+ let ttlattr = case (immode, T.null ttl) of

464467 (NormalImage, False) -> [ uattr "title" ttl ]

465468 _ -> []

466469 return . list $

467470 el "image" $

468− [ attr ("l","href") ('#':fname)

469− , attr ("l","type") (show immode)

470− , uattr "alt" (cMap plain alt) ]

471+ [ attr ("l","href") ("#" <> fname)

472+ , attr ("l","type") (tshow immode)

473+ , uattr "alt" (T.pack $ cMap plain alt) ]

471474 ++ ttlattr

472475insertImage _ _ = error "unexpected inline instead of image"

473476

474−replaceImagesWithAlt :: [String] -> Content -> Content

477+replaceImagesWithAlt :: [Text] -> Content -> Content

475478replaceImagesWithAlt missingHrefs body =

476479 let cur = XC.fromContent body

477480 cur' = replaceAll cur

@@ -507,8 +510,8 @@ replaceImagesWithAlt missingHrefs body =

507510 (Just alt', Just imtype') ->

508511 if imtype' == show NormalImage

509512 then el "p" alt'

510− else txt alt'

511− (Just alt', Nothing) -> txt alt' -- no type attribute

513+ else txt $ T.pack alt'

514+ (Just alt', Nothing) -> txt $ T.pack alt' -- no type attribute

512515 _ -> n -- don't replace if alt text is not found

513516 replaceNode n = n

514517 --

@@ -529,7 +532,7 @@ list = (:[])

529532

530533-- | Convert an 'Inline' to plaintext.

531534plain :: Inline -> String

532−plain (Str s) = s

535+plain (Str s) = T.unpack s

533536plain (Emph ss) = cMap plain ss

534537plain (Span _ ss) = cMap plain ss

535538plain (Strong ss) = cMap plain ss

@@ -539,13 +542,13 @@ plain (Subscript ss) = cMap plain ss

539542plain (SmallCaps ss) = cMap plain ss

540543plain (Quoted _ ss) = cMap plain ss

541544plain (Cite _ ss) = cMap plain ss -- FIXME

542−plain (Code _ s) = s

545+plain (Code _ s) = T.unpack s

543546plain Space = " "

544547plain SoftBreak = " "

545548plain LineBreak = "\n"

546−plain (Math _ s) = s

549+plain (Math _ s) = T.unpack s

547550plain (RawInline _ _) = ""

548−plain (Link _ text (url,_)) = concat (map plain text ++ [" <", url, ">"])

551+plain (Link _ text (url,_)) = concat (map plain text ++ [" <", T.unpack url, ">"])

549552plain (Image _ alt _) = cMap plain alt

550553plain (Note _) = "" -- FIXME

551554

@@ -563,16 +566,16 @@ spaceBeforeAfter cs =

563566 in [emptyline] ++ cs ++ [emptyline]

564567

565568-- | Create a plain-text XML content.

566−txt :: String -> Content

567−txt s = Text $ CData CDataText s Nothing

569+txt :: Text -> Content

570+txt s = Text $ CData CDataText (T.unpack s) Nothing

568571

569572-- | Create an XML attribute with an unqualified name.

570−uattr :: String -> String -> Text.XML.Light.Attr

571−uattr name = Attr (uname name)

573+uattr :: String -> Text -> Text.XML.Light.Attr

574+uattr name = Attr (uname name) . T.unpack

572575

573576-- | Create an XML attribute with a qualified name from given namespace.

574−attr :: (String, String) -> String -> Text.XML.Light.Attr

575−attr (ns, name) = Attr (qname ns name)

577+attr :: (String, String) -> Text -> Text.XML.Light.Attr

578+attr (ns, name) = Attr (qname ns name) . T.unpack

576579

577580-- | Unqualified name

578581uname :: String -> QName

src/Text/Pandoc/Writers/HTML.hs+138−139