git.iko.soy/typeable/pandoc
on master help sign in
Muse reader: refactor list parsing

Now list item contents is parsed as blocks, without resorting to parseFromString. Only the first line of paragraph has to be indented now, just like in Emacs Muse and Text::Amuse. Definition lists are not refactored yet. See also: issue #3865.

commitab85143e8aa94de8927208c7eefa1dbfa97666de authorAlexander Krotov <ilabdsf@gmail.com> dateTue 16 Jan 2018, 13:53 (9 years ago) parentd7f0ecf
2 files changed+117−92
src/Text/Pandoc/Readers/Muse.hs+65−75 test/Tests/Readers/Muse.hs+52−17
src/Text/Pandoc/Readers/Muse.hs+65−75
@@ -42,7 +42,8 @@ module Text.Pandoc.Readers.Muse (readMuse) where
4242import Control.Monad
4343import Control.Monad.Except (throwError)
4444import Data.Char (isLetter)
45−import Data.List (stripPrefix)
45+import Data.List (stripPrefix, intercalate)
46+import Data.List.Split (splitOn)
4647import qualified Data.Map as M
4748import Data.Maybe (fromMaybe)
4849import Data.Text (Text, unpack)
@@ -187,17 +188,19 @@ directive = do
187188-- block parsers
188189--
189190
190−block :: PandocMonad m => MuseParser m (F Blocks)
191−block = do
192− res <- mempty <$ skipMany1 blankline
193− <|> blockElements
194− <|> para
195− skipMany blankline
191+parseBlock :: PandocMonad m => MuseParser m (F Blocks)
192+parseBlock = do
193+ res <- blockElements <|> para
194+ optionMaybe blankline
196195 trace (take 60 $ show $ B.toList $ runF res defaultParserState)
197196 return res
198197
198+block :: PandocMonad m => MuseParser m (F Blocks)
199+block = parseBlock <* skipMany blankline
200+
199201blockElements :: PandocMonad m => MuseParser m (F Blocks)
200−blockElements = choice [ comment
202+blockElements = choice [ mempty <$ blankline
203+ , comment
201204 , separator
202205 , header
203206 , example
@@ -257,15 +260,26 @@ example = try $ do
257260-- in case opening and/or closing tags are on separate lines.
258261chop :: String -> String
259262chop = lchop . rchop
260− where lchop s = case s of
263+
264+lchop :: String -> String
265+lchop s = case s of
261266 '\n':ss -> ss
262267 _ -> s
263− rchop = reverse . lchop . reverse
268+
269+rchop :: String -> String
270+rchop = reverse . lchop . reverse
271+
272+dropSpacePrefix :: [String] -> [String]
273+dropSpacePrefix lns =
274+ map (drop maxIndent) lns
275+ where flns = filter (not . all (== ' ')) lns
276+ maxIndent = if null flns then maximum (map length lns) else length $ takeWhile (== ' ') $ foldl1 commonPrefix flns
264277
265278exampleTag :: PandocMonad m => MuseParser m (F Blocks)
266−exampleTag = do
279+exampleTag = try $ do
280+ many spaceChar
267281 (attr, contents) <- htmlElement "example"
268− return $ return $ B.codeBlockWith attr $ chop contents
282+ return $ return $ B.codeBlockWith attr $ rchop $ intercalate "\n" $ dropSpacePrefix $ splitOn "\n" $ lchop contents
269283
270284literal :: PandocMonad m => MuseParser m (F Blocks)
271285literal = do
@@ -309,7 +323,7 @@ verseLine = do
309323
310324verseLines :: PandocMonad m => MuseParser m (F Blocks)
311325verseLines = do
312− optionMaybe blankline -- Skip blankline after opening tag on separate line
326+ --optionMaybe blankline -- Skip blankline after opening tag on separate line
313327 lns <- many verseLine
314328 lns' <- mapM (parseFromString' (trimInlinesF . mconcat <$> many inline)) lns
315329 return $ B.lineBlock <$> sequence lns'
@@ -317,7 +331,7 @@ verseLines = do
317331verseTag :: PandocMonad m => MuseParser m (F Blocks)
318332verseTag = do
319333 (_, content) <- htmlElement "verse"
320− parseFromString verseLines content
334+ parseFromString verseLines (intercalate "\n" $ dropSpacePrefix $ splitOn "\n" $ lchop content)
321335
322336commentTag :: PandocMonad m => MuseParser m (F Blocks)
323337commentTag = parseHtmlContent "comment" anyChar >> return mempty
@@ -349,7 +363,7 @@ amuseNoteBlock = try $ do
349363 guardEnabled Ext_amuse
350364 pos <- getPosition
351365 ref <- noteMarker <* spaceChar
352− content <- listItemContents $ 3 + length ref
366+ content <- listItemContents
353367 oldnotes <- stateNotes' <$> getState
354368 case M.lookup ref oldnotes of
355369 Just _ -> logMessage $ DuplicateNoteReference ref pos
@@ -399,11 +413,6 @@ lineBlock = try $ do
399413-- lists
400414--
401415
402−listLine :: PandocMonad m => Int -> MuseParser m String
403−listLine markerLength = try $ do
404− indentWith markerLength
405− manyTill anyChar eol
406−
407416withListContext :: PandocMonad m => MuseParser m a -> MuseParser m a
408417withListContext p = do
409418 state <- getState
@@ -413,66 +422,47 @@ withListContext p = do
413422 updateState (\st -> st {stateParserContext = oldContext})
414423 return parsed
415424
416−listContinuation :: PandocMonad m => Int -> MuseParser m [String]
417−listContinuation markerLength = try $ do
418− result <- many1 $ listLine markerLength
419− blank <- option id ((++ [""]) <$ blankline)
420− return $ blank result
421−
422−listStart :: PandocMonad m => MuseParser m Int -> MuseParser m Int
423−listStart marker = try $ do
424− preWhitespace <- length <$> many spaceChar
425− st <- stateParserContext <$> getState
426− getPosition >>= \pos -> guard (st == ListItemState || sourceColumn pos /= 1)
427− markerLength <- marker
428− void spaceChar <|> eol
429− return $ preWhitespace + markerLength + 1
430−
431−dropSpacePrefix :: [String] -> [String]
432−dropSpacePrefix lns =
433− map (drop maxIndent) lns
434− where flns = filter (not . all (== ' ')) lns
435− maxIndent = if null flns then 0 else length $ takeWhile (== ' ') $ foldl1 commonPrefix flns
436−
437−listItemContents :: PandocMonad m => Int -> MuseParser m (F Blocks)
438−listItemContents markerLength = do
439− firstLine <- manyTill anyChar eol
440− restLines <- many $ listLine markerLength
441− blank <- option id ((++ [""]) <$ blankline)
442− let first = firstLine : blank restLines
443− rest <- many $ listContinuation markerLength
444− let allLines = concat (first : rest)
445− parseFromString (withListContext parseBlocks) $ unlines (dropSpacePrefix allLines)
446−
447−listItem :: PandocMonad m => MuseParser m Int -> MuseParser m (F Blocks)
448−listItem start = try $ do
449− markerLength <- start
450− listItemContents markerLength
451−
452−bulletListItems :: PandocMonad m => MuseParser m (F [Blocks])
453−bulletListItems = sequence <$> many1 (listItem bulletListStart)
425+listItemContents :: PandocMonad m => MuseParser m (F Blocks)
426+listItemContents = do
427+ pos <- getPosition
428+ let col = sourceColumn pos - 1
429+ first <- try $ withListContext parseBlock
430+ rest <- many $ try (skipMany blankline >> indentWith col >> withListContext parseBlock)
431+ return $ mconcat (first : rest)
454432
455−bulletListStart :: PandocMonad m => MuseParser m Int
456−bulletListStart = listStart (char '-' >> return 1)
433+listItem :: PandocMonad m => Int -> MuseParser m () -> MuseParser m (F Blocks)
434+listItem n p = try $ do
435+ optionMaybe blankline
436+ count n spaceChar
437+ p
438+ void spaceChar <|> lookAhead eol
439+ listItemContents
457440
458441bulletList :: PandocMonad m => MuseParser m (F Blocks)
459−bulletList = do
460− listItems <- bulletListItems
461− return $ B.bulletList <$> listItems
462−
463−orderedListStart :: PandocMonad m
464− => ListNumberStyle
465− -> ListNumberDelim
466− -> MuseParser m Int
467−orderedListStart style delim = listStart (snd <$> withHorizDisplacement (orderedListMarker style delim))
442+bulletList = try $ do
443+ many spaceChar
444+ pos <- getPosition
445+ let col = sourceColumn pos
446+ guard $ col /= 1
447+ char '-'
448+ void spaceChar <|> lookAhead eol
449+ first <- listItemContents
450+ rest <- many $ listItem (col - 1) (void (char '-'))
451+ return $ B.bulletList <$> sequence (first : rest)
468452
469453orderedList :: PandocMonad m => MuseParser m (F Blocks)
470454orderedList = try $ do
471− p@(_, style, delim) <- lookAhead (many spaceChar *> anyOrderedListMarker <* (eol <|> void spaceChar))
455+ many spaceChar
456+ pos <- getPosition
457+ let col = sourceColumn pos
458+ guard $ col /= 1
459+ p@(_, style, delim) <- anyOrderedListMarker
472460 guard $ style `elem` [Decimal, LowerAlpha, UpperAlpha, LowerRoman, UpperRoman]
473461 guard $ delim == Period
474− items <- sequence <$> many1 (listItem $ orderedListStart style delim)
475− return $ B.orderedListWith p <$> items
462+ void spaceChar <|> lookAhead eol
463+ first <- listItemContents
464+ rest <- many $ listItem (col - 1) (void (orderedListMarker style delim))
465+ return $ B.orderedListWith p <$> sequence (first : rest)
476466
477467definitionListItem :: PandocMonad m => MuseParser m (F (Inlines, [Blocks]))
478468definitionListItem = try $ do
@@ -482,7 +472,7 @@ definitionListItem = try $ do
482472 string "::"
483473 firstLine <- manyTill anyChar eol
484474 restLines <- manyTill anyLine endOfListItemElement
485− let lns = dropWhile (== ' ') firstLine : dropSpacePrefix restLines
475+ let lns = dropWhile (== ' ') firstLine : restLines
486476 lineContent <- parseFromString (withListContext parseBlocks) $ unlines lns
487477 pure $ do lineContent' <- lineContent
488478 term' <- term
@@ -501,8 +491,8 @@ definitionListItems = sequence <$> many1 definitionListItem
501491
502492definitionList :: PandocMonad m => MuseParser m (F Blocks)
503493definitionList = do
504− listItems <- definitionListItems
505− return $ B.definitionList <$> listItems
494+ items <- definitionListItems
495+ return $ B.definitionList <$> items
506496
507497--
508498-- tables
test/Tests/Readers/Muse.hs+52−17
@@ -161,6 +161,8 @@ tests =
161161
162162 , "Verbatim inside code" =: "<code><verbatim>foo</verbatim></code>" =?> para (code "<verbatim>foo</verbatim>")
163163
164+ , "Verbatim tag after text" =: "Foo <verbatim>bar</verbatim>" =?> para "Foo bar"
165+
164166 , testGroup "Links"
165167 [ "Link without description" =:
166168 "[[https://amusewiki.org/]]" =?>
@@ -279,20 +281,12 @@ tests =
279281 , " One two three"
280282 , ""
281283 , "</verse>"
282− , "<verse>Foo bar</verse>"
283− , "<verse>"
284− , "Foo bar</verse>"
285− , "<verse>"
286− , " Foo</verse>"
287284 ] =?>
288285 lineBlock [ ""
289286 , text "Foo bar baz"
290287 , text "\160\160One two three"
291288 , ""
292− ] <>
293− lineBlock [ "Foo bar" ] <>
294− lineBlock [ "Foo bar" ] <>
295− lineBlock [ "\160\160\160Foo" ]
289+ ]
296290 , testGroup "Example"
297291 [ "Braces on separate lines" =:
298292 T.unlines [ "{{{"
@@ -356,6 +350,11 @@ tests =
356350 , " </example>"
357351 ] =?>
358352 bulletList [ codeBlock "foo" ]
353+ , "Empty example inside list" =:
354+ T.unlines [ " - <example>"
355+ , " </example>"
356+ ] =?>
357+ bulletList [ codeBlock "" ]
359358 , "Example inside list with empty lines" =:
360359 T.unlines [ " - <example>"
361360 , " foo"
@@ -537,12 +536,14 @@ tests =
537536 , "[1] First footnote paragraph"
538537 , ""
539538 , " Second footnote paragraph"
539+ , "with continuation"
540+ , ""
540541 , "Not a note"
541542 , "[2] Second footnote"
542543 ] =?>
543544 para (text "Multiparagraph" <>
544545 note (para "First footnote paragraph" <>
545− para "Second footnote paragraph") <>
546+ para "Second footnote paragraph\nwith continuation") <>
546547 text " footnotes" <>
547548 note (para "Second footnote")) <>
548549 para (text "Not a note")
@@ -713,8 +714,48 @@ tests =
713714 , mempty
714715 , para "Item3"
715716 ]
717+ , "Bullet list with last item empty" =:
718+ T.unlines
719+ [ " -"
720+ , ""
721+ , "foo"
722+ ] =?>
723+ bulletList [ mempty ] <>
724+ para "foo"
716725 , testGroup "Nested lists"
717− [ "Nested list" =:
726+ [ "Nested bullet list" =:
727+ T.unlines [ " - Item1"
728+ , " - Item2"
729+ , " - Item3"
730+ , " - Item4"
731+ , " - Item5"
732+ , " - Item6"
733+ ] =?>
734+ bulletList [ para "Item1" <>
735+ bulletList [ para "Item2" <>
736+ bulletList [ para "Item3" ]
737+ , para "Item4" <>
738+ bulletList [ para "Item5" ]
739+ ]
740+ , para "Item6"
741+ ]
742+ , "Nested ordered list" =:
743+ T.unlines [ " 1. Item1"
744+ , " 1. Item2"
745+ , " 1. Item3"
746+ , " 2. Item4"
747+ , " 1. Item5"
748+ , " 2. Item6"
749+ ] =?>
750+ orderedListWith (1, Decimal, Period) [ para "Item1" <>
751+ orderedListWith (1, Decimal, Period) [ para "Item2" <>
752+ orderedListWith (1, Decimal, Period) [ para "Item3" ]
753+ , para "Item4" <>
754+ orderedListWith (1, Decimal, Period) [ para "Item5" ]
755+ ]
756+ , para "Item6"
757+ ]
758+ , "Mixed nested list" =:
718759 T.unlines
719760 [ " - Item1"
720761 , " - Item2"
@@ -736,12 +777,6 @@ tests =
736777 ]
737778 ]
738779 ]
739− , "Incorrectly indented Text::Amuse nested list" =:
740− T.unlines
741− [ " - First item"
742− , " - Not nested item"
743− ] =?>
744− bulletList [ para "First item", para "Not nested item"]
745780 , "Text::Amuse includes only one space in list marker" =:
746781 T.unlines
747782 [ " - First item"