20170c3 · typeable/pandoc

git.iko.soy/typeable/pandoc

Changed type of Shared.uniqueIdent argument from [String] to Set String.

This avoids performance problems in documents with many identically named headers. Closes #2671.

commit20170c328f12fc0214c2c50d14c8f7d03ee05e70 authorJohn MacFarlane <jgm@berkeley.edu> dateFri 22 Jan 2016, 10:16 (11 years ago) parent2a2e3d9

11 files changed+39−31

compare with master

unifiedsplit

src/Text/Pandoc/Parsing.hs+6−6 src/Text/Pandoc/Readers/Docx.hs+4−3 src/Text/Pandoc/Readers/HTML.hs+3−3 src/Text/Pandoc/Readers/MediaWiki.hs+3−2 src/Text/Pandoc/Readers/Odt/ContentReader.hs+2−1 src/Text/Pandoc/Readers/Org.hs+3−2 src/Text/Pandoc/Shared.hs+3−3 src/Text/Pandoc/Writers/Docx.hs+4−3 src/Text/Pandoc/Writers/EPUB.hs+3−2 src/Text/Pandoc/Writers/Markdown.hs+4−3 src/Text/Pandoc/Writers/Texinfo.hs+4−3

src/Text/Pandoc/Parsing.hs+6−6

@@ -915,7 +915,7 @@ data ParserState = ParserState

915915 stateMeta' :: F Meta, -- ^ Document metadata

916916 stateHeaderTable :: [HeaderType], -- ^ Ordered list of header types used

917917 stateHeaders :: M.Map Inlines String, -- ^ List of headers and ids (used for implicit ref links)

918− stateIdentifiers :: [String], -- ^ List of header identifiers used

918+ stateIdentifiers :: Set.Set String, -- ^ Header identifiers used

919919 stateNextExample :: Int, -- ^ Number of next example

920920 stateExamples :: M.Map String Int, -- ^ Map from example labels to numbers

921921 stateHasChapters :: Bool, -- ^ True if \chapter encountered

@@ -973,8 +973,8 @@ instance HasHeaderMap ParserState where

973973 updateHeaderMap f st = st{ stateHeaders = f $ stateHeaders st }

974974

975975class HasIdentifierList st where

976− extractIdentifierList :: st -> [String]

977− updateIdentifierList :: ([String] -> [String]) -> st -> st

976+ extractIdentifierList :: st -> Set.Set String

977+ updateIdentifierList :: (Set.Set String -> Set.Set String) -> st -> st

978978

979979instance HasIdentifierList ParserState where

980980 extractIdentifierList = stateIdentifiers

@@ -1013,7 +1013,7 @@ defaultParserState =

10131013 stateMeta' = return nullMeta,

10141014 stateHeaderTable = [],

10151015 stateHeaders = M.empty,

1016− stateIdentifiers = [],

1016+ stateIdentifiers = Set.empty,

10171017 stateNextExample = 1,

10181018 stateExamples = M.empty,

10191019 stateHasChapters = False,

@@ -1092,8 +1092,8 @@ registerHeader (ident,classes,kvs) header' = do

10921092 let id'' = if Ext_ascii_identifiers `Set.member` exts

10931093 then catMaybes $ map toAsciiChar id'

10941094 else id'

1095− updateState $ updateIdentifierList $

1096− if id' == id'' then (id' :) else ([id', id''] ++)

1095+ updateState $ updateIdentifierList $ Set.insert id'

1096+ updateState $ updateIdentifierList $ Set.insert id''

10971097 updateState $ updateHeaderMap $ insert' header' id'

10981098 return (id'',classes,kvs)

10991099 else do

src/Text/Pandoc/Readers/Docx.hs+4−3

@@ -89,6 +89,7 @@ import Text.TeXMath (writeTeX)

8989import Data.Default (Default)

9090import qualified Data.ByteString.Lazy as B

9191import qualified Data.Map as M

92+import qualified Data.Set as Set

9293import Control.Monad.Reader

9394import Control.Monad.State

9495import Data.Sequence (ViewL(..), viewl)

@@ -350,7 +351,7 @@ parPartToInlines (BookMark _ anchor) =

350351 -- avoid an extra pass.

351352 let newAnchor =

352353 if not inHdrBool && anchor `elem` (M.elems anchorMap)

353− then uniqueIdent [Str anchor] (M.elems anchorMap)

354+ then uniqueIdent [Str anchor] (Set.fromList $ M.elems anchorMap)

354355 else anchor

355356 unless inHdrBool

356357 (modify $ \s -> s { docxAnchorMap = M.insert anchor newAnchor anchorMap})

@@ -393,7 +394,7 @@ makeHeaderAnchor' (Header n (_, classes, kvs) ils)

393394 | (c:cs) <- filter isAnchorSpan ils

394395 , (Span (ident, ["anchor"], _) _) <- c = do

395396 hdrIDMap <- gets docxAnchorMap

396− let newIdent = uniqueIdent ils (M.elems hdrIDMap)

397+ let newIdent = uniqueIdent ils (Set.fromList $ M.elems hdrIDMap)

397398 modify $ \s -> s {docxAnchorMap = M.insert ident newIdent hdrIDMap}

398399 return $ Header n (newIdent, classes, kvs) (ils \\ (c:cs))

399400-- Otherwise we just give it a name, and register that name (associate

@@ -401,7 +402,7 @@ makeHeaderAnchor' (Header n (_, classes, kvs) ils)

401402makeHeaderAnchor' (Header n (_, classes, kvs) ils) =

402403 do

403404 hdrIDMap <- gets docxAnchorMap

404− let newIdent = uniqueIdent ils (M.elems hdrIDMap)

405+ let newIdent = uniqueIdent ils (Set.fromList $ M.elems hdrIDMap)

405406 modify $ \s -> s {docxAnchorMap = M.insert newIdent newIdent hdrIDMap}

406407 return $ Header n (newIdent, classes, kvs) ils

407408makeHeaderAnchor' blk = return blk

src/Text/Pandoc/Readers/HTML.hs+3−3

@@ -68,7 +68,7 @@ import Text.Pandoc.Error

6868import Text.Pandoc.CSS (foldOrElse, pickStyleAttrProps)

6969import Text.Pandoc.Compat.Monoid ((<>))

7070import Text.Parsec.Error

71−

71+import qualified Data.Set as Set

7272

7373-- | Convert HTML-formatted string to 'Pandoc' document.

7474readHtml :: ReaderOptions -- ^ Reader options

@@ -77,7 +77,7 @@ readHtml :: ReaderOptions -- ^ Reader options

7777readHtml opts inp =

7878 mapLeft (ParseFailure . getError) . flip runReader def $

7979 runParserT parseDoc

80− (HTMLState def{ stateOptions = opts } [] Nothing [] M.empty)

80+ (HTMLState def{ stateOptions = opts } [] Nothing Set.empty M.empty)

8181 "source" tags

8282 where tags = stripPrefixes . canonicalizeTags $

8383 parseTagsOptions parseOptions{ optTagPosition = True } inp

@@ -104,7 +104,7 @@ data HTMLState =

104104 { parserState :: ParserState,

105105 noteTable :: [(String, Blocks)],

106106 baseHref :: Maybe String,

107− identifiers :: [String],

107+ identifiers :: Set.Set String,

108108 headerMap :: M.Map Inlines String

109109 }

110110

src/Text/Pandoc/Readers/MediaWiki.hs+3−2

@@ -52,6 +52,7 @@ import Text.HTML.TagSoup

5252import Data.Sequence (viewl, ViewL(..), (<|))

5353import qualified Data.Foldable as F

5454import qualified Data.Map as M

55+import qualified Data.Set as Set

5556import Data.Char (isDigit, isSpace)

5657import Data.Maybe (fromMaybe)

5758import Text.Printf (printf)

@@ -69,7 +70,7 @@ readMediaWiki opts s =

6970 , mwNextLinkNumber = 1

7071 , mwCategoryLinks = []

7172 , mwHeaderMap = M.empty

72− , mwIdentifierList = []

73+ , mwIdentifierList = Set.empty

7374 }

7475 (s ++ "\n")

7576

@@ -78,7 +79,7 @@ data MWState = MWState { mwOptions :: ReaderOptions

7879 , mwNextLinkNumber :: Int

7980 , mwCategoryLinks :: [Inlines]

8081 , mwHeaderMap :: M.Map Inlines String

81− , mwIdentifierList :: [String]

82+ , mwIdentifierList :: Set.Set String

8283 }

8384

8485type MWParser = Parser [Char] MWState

src/Text/Pandoc/Readers/Odt/ContentReader.hs+2−1

@@ -61,6 +61,7 @@ import Text.Pandoc.Readers.Odt.Generic.XMLConverter

6161import Text.Pandoc.Readers.Odt.Generic.Fallible

6262import Text.Pandoc.Readers.Odt.Generic.Utils

6363

64+import qualified Data.Set as Set

6465

6566--------------------------------------------------------------------------------

6667-- State

@@ -221,7 +222,7 @@ getPrettyAnchor = proc (baseIdent, uglyAnchor) -> do

221222getHeaderAnchor :: OdtReaderSafe Inlines Anchor

222223getHeaderAnchor = proc title -> do

223224 state <- getExtraState -< ()

224− let anchor = uniqueIdent (toList title) (usedAnchors state)

225+ let anchor = uniqueIdent (toList title) (Set.fromList $ usedAnchors state)

225226 modifyExtraState (putPrettyAnchor anchor anchor) -<< anchor

226227

227228

src/Text/Pandoc/Readers/Org.hs+3−2

@@ -53,6 +53,7 @@ import Data.Char (isAlphaNum, toLower)

5353import Data.Default

5454import Data.List (intersperse, isPrefixOf, isSuffixOf)

5555import qualified Data.Map as M

56+import qualified Data.Set as Set

5657import Data.Maybe (fromMaybe, isJust)

5758import Network.HTTP (urlEncode)

5859

@@ -144,7 +145,7 @@ data OrgParserState = OrgParserState

144145 , orgStateMeta' :: F Meta

145146 , orgStateNotes' :: OrgNoteTable

146147 , orgStateParserContext :: ParserContext

147− , orgStateIdentifiers :: [String]

148+ , orgStateIdentifiers :: Set.Set String

148149 , orgStateHeaderMap :: M.Map Inlines String

149150 }

150151

@@ -186,7 +187,7 @@ defaultOrgParserState = OrgParserState

186187 , orgStateMeta' = return nullMeta

187188 , orgStateNotes' = []

188189 , orgStateParserContext = NullState

189− , orgStateIdentifiers = []

190+ , orgStateIdentifiers = Set.empty

190191 , orgStateHeaderMap = M.empty

191192 }

192193

src/Text/Pandoc/Shared.hs+3−3

@@ -706,14 +706,14 @@ headerLtEq _ _ = False

706706

707707-- | Generate a unique identifier from a list of inlines.

708708-- Second argument is a list of already used identifiers.

709−uniqueIdent :: [Inline] -> [String] -> String

709+uniqueIdent :: [Inline] -> Set.Set String -> String

710710uniqueIdent title' usedIdents

711711 = let baseIdent = case inlineListToIdentifier title' of

712712 "" -> "section"

713713 x -> x

714714 numIdent n = baseIdent ++ "-" ++ show n

715− in if baseIdent `elem` usedIdents

716− then case find (\x -> numIdent x `notElem` usedIdents) ([1..60000] :: [Int]) of

715+ in if baseIdent `Set.member` usedIdents

716+ then case find (\x -> not $ numIdent x `Set.member` usedIdents) ([1..60000] :: [Int]) of

717717 Just x -> numIdent x

718718 Nothing -> baseIdent -- if we have more than 60,000, allow repeats

719719 else baseIdent

src/Text/Pandoc/Writers/Docx.hs+4−3

@@ -34,6 +34,7 @@ import qualified Data.ByteString as B

3434import qualified Data.ByteString.Lazy as BL

3535import qualified Data.ByteString.Lazy.Char8 as BL8

3636import qualified Data.Map as M

37+import qualified Data.Set as Set

3738import qualified Text.Pandoc.UTF8 as UTF8

3839import Codec.Archive.Zip

3940import Data.Time.Clock.POSIX

@@ -95,7 +96,7 @@ data WriterState = WriterState{

9596 stTextProperties :: [Element]

9697 , stParaProperties :: [Element]

9798 , stFootnotes :: [Element]

98− , stSectionIds :: [String]

99+ , stSectionIds :: Set.Set String

99100 , stExternalLinks :: M.Map String String

100101 , stImages :: M.Map FilePath (String, String, Maybe MimeType, Element, B.ByteString)

101102 , stListLevel :: Int

@@ -117,7 +118,7 @@ defaultWriterState = WriterState{

117118 stTextProperties = []

118119 , stParaProperties = []

119120 , stFootnotes = defaultFootnotes

120− , stSectionIds = []

121+ , stSectionIds = Set.empty

121122 , stExternalLinks = M.empty

122123 , stImages = M.empty

123124 , stListLevel = -1

@@ -742,7 +743,7 @@ blockToOpenXML opts (Header lev (ident,_,_) lst) = do

742743 let bookmarkName = if null ident

743744 then uniqueIdent lst usedIdents

744745 else ident

745− modify $ \s -> s{ stSectionIds = bookmarkName : stSectionIds s }

746+ modify $ \s -> s{ stSectionIds = Set.insert bookmarkName $ stSectionIds s }

746747 id' <- getUniqueId

747748 let bookmarkStart = mknode "w:bookmarkStart" [("w:id", id')

748749 ,("w:name",bookmarkName)] ()

src/Text/Pandoc/Writers/EPUB.hs+3−2

@@ -31,6 +31,7 @@ Conversion of 'Pandoc' documents to EPUB.

3131module Text.Pandoc.Writers.EPUB ( writeEPUB ) where

3232import Data.IORef ( IORef, newIORef, readIORef, modifyIORef )

3333import qualified Data.Map as M

34+import qualified Data.Set as Set

3435import Data.Maybe ( fromMaybe, catMaybes )

3536import Data.List ( isPrefixOf, isInfixOf, intercalate )

3637import System.Environment ( getEnv )

@@ -916,13 +917,13 @@ showChapter = printf "ch%03d.xhtml"

916917

917918-- Add identifiers to any headers without them.

918919addIdentifiers :: [Block] -> [Block]

919−addIdentifiers bs = evalState (mapM go bs) []

920+addIdentifiers bs = evalState (mapM go bs) Set.empty

920921 where go (Header n (ident,classes,kvs) ils) = do

921922 ids <- get

922923 let ident' = if null ident

923924 then uniqueIdent ils ids

924925 else ident

925− put $ ident' : ids

926+ modify $ Set.insert ident'

926927 return $ Header n (ident',classes,kvs) ils

927928 go x = return x

928929

src/Text/Pandoc/Writers/Markdown.hs+4−3

@@ -53,6 +53,7 @@ import Data.Yaml (Value(Object,String,Array,Bool,Number))

5353import qualified Data.HashMap.Strict as H

5454import qualified Data.Vector as V

5555import qualified Data.Text as T

56+import qualified Data.Set as Set

5657

5758type Notes = [[Block]]

5859type Ref = ([Inline], Target, Attr)

@@ -61,11 +62,11 @@ data WriterState = WriterState { stNotes :: Notes

6162 , stRefs :: Refs

6263 , stRefShortcutable :: Bool

6364 , stInList :: Bool

64− , stIds :: [String]

65+ , stIds :: Set.Set String

6566 , stPlain :: Bool }

6667instance Default WriterState

6768 where def = WriterState{ stNotes = [], stRefs = [], stRefShortcutable = True,

68− stInList = False, stIds = [], stPlain = False }

69+ stInList = False, stIds = Set.empty, stPlain = False }

6970

7071-- | Convert Pandoc to Markdown.

7172writeMarkdown :: WriterOptions -> Pandoc -> String

@@ -364,7 +365,7 @@ blockToMarkdown opts (Header level attr inlines) = do

364365 -- so we know whether to print an explicit identifier

365366 ids <- gets stIds

366367 let autoId = uniqueIdent inlines ids

367− modify $ \st -> st{ stIds = autoId : ids }

368+ modify $ \st -> st{ stIds = Set.insert autoId ids }

368369 let attr' = case attr of

369370 ("",[],[]) -> empty

370371 (id',[],[]) | isEnabled Ext_auto_identifiers opts

src/Text/Pandoc/Writers/Texinfo.hs+4−3

@@ -43,13 +43,14 @@ import Text.Pandoc.Pretty

4343import Text.Pandoc.ImageSize

4444import Network.URI ( isURI, unEscapeString )

4545import System.FilePath

46+import qualified Data.Set as Set

4647

4748data WriterState =

4849 WriterState { stStrikeout :: Bool -- document contains strikeout

4950 , stSuperscript :: Bool -- document contains superscript

5051 , stSubscript :: Bool -- document contains subscript

5152 , stEscapeComma :: Bool -- in a context where we need @comma

52− , stIdentifiers :: [String] -- header ids used already

53+ , stIdentifiers :: Set.Set String -- header ids used already

5354 , stOptions :: WriterOptions -- writer options

5455 }

5556

@@ -64,7 +65,7 @@ writeTexinfo options document =

6465 evalState (pandocToTexinfo options $ wrapTop document) $

6566 WriterState { stStrikeout = False, stSuperscript = False,

6667 stEscapeComma = False, stSubscript = False,

67− stIdentifiers = [], stOptions = options}

68+ stIdentifiers = Set.empty, stOptions = options}

6869

6970-- | Add a "Top" node around the document, needed by Texinfo.

7071wrapTop :: Pandoc -> Pandoc

@@ -215,7 +216,7 @@ blockToTexinfo (Header level _ lst) = do

215216 txt <- inlineListToTexinfo lst

216217 idsUsed <- gets stIdentifiers

217218 let id' = uniqueIdent lst idsUsed

218− modify $ \st -> st{ stIdentifiers = id' : idsUsed }

219+ modify $ \st -> st{ stIdentifiers = Set.insert id' idsUsed }

219220 return $ if (level > 0) && (level <= 4)

220221 then blankline <> text "@node " <> node $$

221222 text (seccmd level) <> txt $$