4b4060b · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Simplified parsing of reference keys and notes in markdown and RST

4b4060bfiddlosopher19 years ago

src/Text/Pandoc/Readers/Markdown.hs+30−18

compare with master

unifiedsplit

@@ -140,26 +140,24 @@ titleBlock = try $ do

140parseMarkdown = do

141 -- markdown allows raw HTML

142 updateState (\state -> state { stateParseRaw = True })

143− (title, author, date) <- option ([],[],"") titleBlock

143+ startPos <- getPosition

144 -- go through once just to get list of reference keys

145+ -- docMinusKeys is the raw document with blanks where the keys were...

145− refs <- manyTill (referenceKey <|> (lineClump >>= return . LineClump)) eof

146+ docMinusKeys <- many (referenceKey <|> lineClump) >>= return . concat

146− let keys = map (\(KeyBlock label target) -> (label, target)) $

147+ setInput docMinusKeys

147− filter isKeyBlock refs

148+ setPosition startPos

148− -- strip out keys

149− setInput $ concatMap (\(LineClump ln) -> ln) $ filter isLineClump refs

150− updateState (\state -> state { stateKeys = keys })

151 st <- getState

149 st <- getState

150+ -- go through again for notes unless strict...

152 if stateStrict st

151 if stateStrict st

153 then return ()

152 then return ()

154− else do -- go through for notes (which may contain refs - hence 2nd pass)

153+ else do docMinusNotes <- many (noteBlock <|> lineClump) >>= return . concat

155− refs' <- manyTill (noteBlock <|>

154+ st <- getState

156− (lineClump >>= return . LineClump)) eof

155+ let reversedNotes = stateNotes st

157− let notes = map (\(NoteBlock label blocks) -> (label, blocks)) $

158− filter isNoteBlock refs'

159− updateState (\state -> state { stateNotes = notes })

156+ updateState $ \st -> st { stateNotes = reverse reversedNotes }

160− setInput $ concatMap (\(LineClump ln) -> ln) $

157+ setInput docMinusNotes

161− filter isLineClump refs'

158+ setPosition startPos

162− -- go through again, with note blocks and keys stripped out

159+ -- now parse it for real...

160+ (title, author, date) <- option ([],[],"") titleBlock

163 blocks <- parseBlocks

161 blocks <- parseBlocks

164 return $ Pandoc (Meta title author date) $ filter (/= Null) blocks

162 return $ Pandoc (Meta title author date) $ filter (/= Null) blocks

165

163

@@ -168,6 +166,7 @@ parseMarkdown = do

168--

166--

169

167

170referenceKey = try $ do

168referenceKey = try $ do

169+ startPos <- getPosition

171 nonindentSpaces

170 nonindentSpaces

172 label <- reference

171 label <- reference

173 char ':'

172 char ':'

@@ -177,7 +176,13 @@ referenceKey = try $ do

177 optional (char '>')

176 optional (char '>')

178 tit <- option "" referenceTitle

177 tit <- option "" referenceTitle

179 blanklines

178 blanklines

179+ endPos <- getPosition

180− return $ KeyBlock label (removeTrailingSpace src, tit)

180+ let newkey = (label, (removeTrailingSpace src, tit))

181+ st <- getState

182+ let oldkeys = stateKeys st

183+ updateState $ \st -> st { stateKeys = newkey : oldkeys }

184+ -- return blanks so line count isn't affected

185+ return $ replicate (sourceLine endPos - sourceLine startPos) '\n'

181

186

182referenceTitle = try $ do

187referenceTitle = try $ do

183 skipSpaces

188 skipSpaces

@@ -201,15 +206,22 @@ rawLine = do

201rawLines = many1 rawLine >>= return . concat

206rawLines = many1 rawLine >>= return . concat

202

207

203noteBlock = try $ do

208noteBlock = try $ do

209+ startPos <- getPosition

204 ref <- noteMarker

210 ref <- noteMarker

205 char ':'

211 char ':'

206 optional blankline

212 optional blankline

207 optional indentSpaces

213 optional indentSpaces

208 raw <- sepBy rawLines (try (blankline >> indentSpaces))

214 raw <- sepBy rawLines (try (blankline >> indentSpaces))

209 optional blanklines

215 optional blanklines

216+ endPos <- getPosition

210 -- parse the extracted text, which may contain various block elements:

217 -- parse the extracted text, which may contain various block elements:

211 contents <- parseFromString parseBlocks $ (joinWithSep "\n" raw) ++ "\n\n"

218 contents <- parseFromString parseBlocks $ (joinWithSep "\n" raw) ++ "\n\n"

212− return $ NoteBlock ref contents

219+ let newnote = (ref, contents)

220+ st <- getState

221+ let oldnotes = stateNotes st

222+ updateState $ \st -> st { stateNotes = newnote : oldnotes }

223+ -- return blanks so line count isn't affected

224+ return $ replicate (sourceLine endPos - sourceLine startPos) '\n'

213

225

214--

226--

215-- parsing blocks

227-- parsing blocks

src/Text/Pandoc/Readers/RST.hs+24−15

@@ -83,15 +83,17 @@ titleTransform ((Header 1 head1):rest) = -- title, no subtitle

83titleTransform blocks = (blocks, [])

85parseRST = do

86− -- first pass: get keys

86+ startPos <- getPosition

87+ -- go through once just to get list of reference keys

88+ -- docMinusKeys is the raw document with blanks where the keys were...

87− refs <- manyTill (referenceKey <|> (lineClump >>= return . LineClump)) eof

89+ docMinusKeys <- many (referenceKey <|> lineClump) >>= return . concat

88− let keys = map (\(KeyBlock label target) -> (label, target)) $

90+ setInput docMinusKeys

89− filter isKeyBlock refs

91+ setPosition startPos

90− -- second pass, with keys stripped out

92+ st <- getState

91− let rawlines = map (\(LineClump ln) -> ln) $ filter isLineClump refs

93+ let reversedKeys = stateKeys st

92− setInput $ concat rawlines

93− updateState (\state -> state { stateKeys = keys })

94+ updateState $ \st -> st { stateKeys = reverse reversedKeys }

95+ -- now parse it for real...

94− blocks <- parseBlocks

96+ blocks <- parseBlocks

95 let blocks' = filter (/= Null) blocks

97 let blocks' = filter (/= Null) blocks

96 state <- getState

98 state <- getState

97 let (blocks'', title) = if stateStandalone state

99 let (blocks'', title) = if stateStandalone state

@@ -429,9 +431,16 @@ unknownDirective = try $ do

429-- reference key

431-- reference key

430--

432--

431

433

432−referenceKey =

434+referenceKey = do

435+ startPos <- getPosition

433− choice [imageKey, anonymousKey, regularKeyQuoted, regularKey] >>~

436+ key <- choice [imageKey, anonymousKey, regularKeyQuoted, regularKey]

437+ st <- getState

438+ let oldkeys = stateKeys st

439+ updateState $ \st -> st { stateKeys = key : oldkeys }

434 optional blanklines

440 optional blanklines

441+ endPos <- getPosition

442+ -- return enough blanks to replace key

443+ return $ replicate (sourceLine endPos - sourceLine startPos) '\n'

435

444

436targetURI = do

445targetURI = do

437 skipSpaces

446 skipSpaces

@@ -447,26 +456,26 @@ imageKey = try $ do

447 skipSpaces

456 skipSpaces

448 string "image::"

457 string "image::"

449 src <- targetURI

458 src <- targetURI

450− return $ KeyBlock (normalizeSpaces ref) (removeLeadingTrailingSpace src, "")

459+ return (normalizeSpaces ref, (removeLeadingTrailingSpace src, ""))

451

460

452anonymousKey = try $ do

461anonymousKey = try $ do

453 oneOfStrings [".. __:", "__"]

462 oneOfStrings [".. __:", "__"]

454 src <- targetURI

463 src <- targetURI

455 state <- getState

464 state <- getState

456− return $ KeyBlock [Str "_"] (removeLeadingTrailingSpace src, "")

465+ return ([Str "_"], (removeLeadingTrailingSpace src, ""))

457

466

458regularKeyQuoted = try $ do

467regularKeyQuoted = try $ do

459 string ".. _`"

468 string ".. _`"

460 ref <- manyTill inline (char '`')

469 ref <- manyTill inline (char '`')

461 char ':'

470 char ':'

462 src <- targetURI

471 src <- targetURI

463− return $ KeyBlock (normalizeSpaces ref) (removeLeadingTrailingSpace src, "")

472+ return (normalizeSpaces ref, (removeLeadingTrailingSpace src, ""))

464

473

465regularKey = try $ do

474regularKey = try $ do

466 string ".. _"

475 string ".. _"

467 ref <- manyTill inline (char ':')

476 ref <- manyTill inline (char ':')

468 src <- targetURI

477 src <- targetURI

469− return $ KeyBlock (normalizeSpaces ref) (removeLeadingTrailingSpace src, "")

478+ return (normalizeSpaces ref, (removeLeadingTrailingSpace src, ""))

470

479

471 --

480 --

472 -- inline

481 -- inline

src/Text/Pandoc/Shared.hs−24

@@ -72,10 +72,6 @@ module Text.Pandoc.Shared (

72 testStringWith,

73 ParserState (..),

74 defaultParserState,

75− Reference (..),

76− isNoteBlock,

77− isKeyBlock,

78− isLineClump,

79 HeaderType (..),

75 HeaderType (..),

80 ParserContext (..),

76 ParserContext (..),

81 QuoteContext (..),

77 QuoteContext (..),

@@ -543,26 +539,6 @@ defaultParserState =

543 stateColumns = 80,

539 stateColumns = 80,

544 stateHeaderTable = [] }

540 stateHeaderTable = [] }

545

541

546−-- | References from preliminary parsing.

547−data Reference

548− = KeyBlock [Inline] Target -- ^ Key for reference-style link (label URL title)

549− | NoteBlock String [Block] -- ^ Footnote reference and contents

550− | LineClump String -- ^ Raw clump of lines with blanks at end

551− deriving (Eq, Read, Show)

552−

553−-- | Auxiliary functions used in preliminary parsing.

554−isNoteBlock :: Reference -> Bool

555−isNoteBlock (NoteBlock _ _) = True

556−isNoteBlock _ = False

557−

558−isKeyBlock :: Reference -> Bool

559−isKeyBlock (KeyBlock _ _) = True

560−isKeyBlock _ = False

561−

562−isLineClump :: Reference -> Bool

563−isLineClump (LineClump _) = True

564−isLineClump _ = False

565−

566data HeaderType

542data HeaderType

567 = SingleHeader Char -- ^ Single line of characters underneath

543 = SingleHeader Char -- ^ Single line of characters underneath

568 | DoubleHeader Char -- ^ Lines of characters above and below

544 | DoubleHeader Char -- ^ Lines of characters above and below