4b9fb7a · typeable/pandoc

git.iko.soy/typeable/pandoc

Combine grid table parsers

The grid table parsers for markdown and rst was combined into one single parser, slightly changing parsing behavior of both parsers: - The markdown parser now compactifies block content cell-wise: pure text blocks in cells are now treated as paragraphs only if the cell contains multiple paragraphs, and as plain blocks otherwise. Before, this was true only for single-column tables. - The rst parser now accepts newlines and multiple blocks in header cells. Closes: #3638

commit4b9fb7a1280f1d923a6bcecbf42a496480020359 authorAlbert Krewinkel <albert@zeitkraut.de> dateWed 10 May 2017, 23:35 (9 years ago) parent7bdf38e

5 files changed+94−143

compare with master

unifiedsplit

src/Text/Pandoc/Parsing.hs+51−18 src/Text/Pandoc/Readers/Markdown.hs+1−83 test/command/3516.md+2−2 test/markdown-reader-more.native+34−34 test/tables-rstsubset.native+6−6

src/Text/Pandoc/Parsing.hs+51−18

@@ -66,6 +66,7 @@ module Text.Pandoc.Parsing ( anyLine,

6666 tableWith,

6767 widthsFromIndices,

6868 gridTableWith,

69+ gridTableWith',

6970 readWith,

7071 readWithM,

7172 testStringWith,

@@ -770,6 +771,20 @@ tableWith :: (Stream s m Char, HasReaderOptions st,

770771 -> ParserT s st m end

771772 -> ParserT s st m (mf Blocks)

772773tableWith headerParser rowParser lineParser footerParser = try $ do

774+ (aligns, widths, heads, rows) <- tableWith' headerParser rowParser

775+ lineParser footerParser

776+ return $ B.table mempty (zip aligns widths) <$> heads <*> rows

777+

778+type TableComponents mf = ([Alignment], [Double], mf [Blocks], mf [[Blocks]])

779+

780+tableWith' :: (Stream s m Char, HasReaderOptions st,

781+ Functor mf, Applicative mf, Monad mf)

782+ => ParserT s st m (mf [Blocks], [Alignment], [Int])

783+ -> ([Int] -> ParserT s st m (mf [Blocks]))

784+ -> ParserT s st m sep

785+ -> ParserT s st m end

786+ -> ParserT s st m (TableComponents mf)

787+tableWith' headerParser rowParser lineParser footerParser = try $ do

773788 (heads, aligns, indices) <- headerParser

774789 lines' <- sequence <$> rowParser indices `sepEndBy1` lineParser

775790 footerParser

@@ -777,7 +792,7 @@ tableWith headerParser rowParser lineParser footerParser = try $ do

777792 let widths = if (indices == [])

778793 then replicate (length aligns) 0.0

779794 else widthsFromIndices numColumns indices

780− return $ B.table mempty (zip aligns widths) <$> heads <*> lines'

795+ return $ (aligns, widths, heads, lines')

781796

782797-- Calculate relative widths of table columns, based on indices

783798widthsFromIndices :: Int -- Number of columns on terminal

@@ -812,24 +827,42 @@ widthsFromIndices numColumns' indices =

812827-- ending with a footer (dashed line followed by blank line).

813828gridTableWith :: (Stream [Char] m Char, HasReaderOptions st,

814829 Functor mf, Applicative mf, Monad mf)

815− => ParserT [Char] st m (mf Blocks) -- ^ Block list parser

830+ => ParserT [Char] st m (mf Blocks) -- ^ Block list parser

816831 -> Bool -- ^ Headerless table

817832 -> ParserT [Char] st m (mf Blocks)

818833gridTableWith blocks headless =

819834 tableWith (gridTableHeader headless blocks) (gridTableRow blocks)

820835 (gridTableSep '-') gridTableFooter

821836

837+gridTableWith' :: (Stream [Char] m Char, HasReaderOptions st,

838+ Functor mf, Applicative mf, Monad mf)

839+ => ParserT [Char] st m (mf Blocks) -- ^ Block list parser

840+ -> Bool -- ^ Headerless table

841+ -> ParserT [Char] st m (TableComponents mf)

842+gridTableWith' blocks headless =

843+ tableWith' (gridTableHeader headless blocks) (gridTableRow blocks)

844+ (gridTableSep '-') gridTableFooter

845+

822846gridTableSplitLine :: [Int] -> String -> [String]

823847gridTableSplitLine indices line = map removeFinalBar $ tail $

824848 splitStringByIndices (init indices) $ trimr line

825849

826−gridPart :: Stream s m Char => Char -> ParserT s st m (Int, Int)

850+gridPart :: Stream s m Char => Char -> ParserT s st m ((Int, Int), Alignment)

827851gridPart ch = do

852+ leftColon <- option False (True <$ char ':')

828853 dashes <- many1 (char ch)

854+ rightColon <- option False (True <$ char ':')

829855 char '+'

830− return (length dashes, length dashes + 1)

831−

832−gridDashedLines :: Stream s m Char => Char -> ParserT s st m [(Int,Int)]

856+ let lengthDashes = length dashes + (if leftColon then 1 else 0) +

857+ (if rightColon then 1 else 0)

858+ let alignment = case (leftColon, rightColon) of

859+ (True, True) -> AlignCenter

860+ (True, False) -> AlignLeft

861+ (False, True) -> AlignRight

862+ (False, False) -> AlignDefault

863+ return ((lengthDashes, lengthDashes + 1), alignment)

864+

865+gridDashedLines :: Stream s m Char => Char -> ParserT s st m [((Int, Int), Alignment)]

833866gridDashedLines ch = try $ char '+' >> many1 (gridPart ch) <* blankline

834867

835868removeFinalBar :: String -> String

@@ -853,18 +886,18 @@ gridTableHeader headless blocks = try $ do

853886 else many1

854887 (notFollowedBy (gridTableSep '=') >> char '|' >>

855888 many1Till anyChar newline)

856− if headless

857− then return ()

858− else gridTableSep '=' >> return ()

859− let lines' = map snd dashes

889+ underDashes <- if headless

890+ then return dashes

891+ else gridDashedLines '='

892+ guard $ length dashes == length underDashes

893+ let lines' = map (snd . fst) underDashes

860894 let indices = scanl (+) 0 lines'

861− let aligns = replicate (length lines') AlignDefault

862− -- RST does not have a notion of alignments

895+ let aligns = map snd underDashes

863896 let rawHeads = if headless

864− then replicate (length dashes) ""

865− else map (intercalate " ") $ transpose

897+ then replicate (length underDashes) ""

898+ else map (unlines . map trim) $ transpose

866899 $ map (gridTableSplitLine indices) rawContent

867− heads <- fmap sequence . mapM (parseFromString blocks) $ map trim rawHeads

900+ heads <- fmap sequence $ mapM (parseFromString blocks . trim) rawHeads

868901 return (heads, aligns, indices)

869902

870903gridTableRawLine :: Stream s m Char => [Int] -> ParserT s st m [String]

@@ -882,6 +915,9 @@ gridTableRow blocks indices = do

882915 colLines <- many1 (gridTableRawLine indices)

883916 let cols = map ((++ "\n") . unlines . removeOneLeadingSpace) $

884917 transpose colLines

918+ compactifyCell bs = case compactify [bs] of

919+ [] -> mempty

920+ x:_ -> x

885921 cells <- sequence <$> mapM (parseFromString blocks) cols

886922 return $ fmap (map compactifyCell) cells

887923

@@ -893,9 +929,6 @@ removeOneLeadingSpace xs =

893929 where startsWithSpace "" = True

894930 startsWithSpace (y:_) = y == ' '

895931

896−compactifyCell :: Blocks -> Blocks

897−compactifyCell bs = head $ compactify [bs]

898−

899932-- | Parse footer for a grid table.

900933gridTableFooter :: Stream s m Char => ParserT s st m [Char]

901934gridTableFooter = blanklines

src/Text/Pandoc/Readers/Markdown.hs+1−83

@@ -1291,89 +1291,7 @@ multilineTableHeader headless = try $ do

12911291-- ending with a footer (dashed line followed by blank line).

12921292gridTable :: PandocMonad m => Bool -- ^ Headerless table

12931293 -> MarkdownParser m ([Alignment], [Double], F [Blocks], F [[Blocks]])

1294−gridTable headless =

1295− tableWith (gridTableHeader headless) gridTableRow

1296− (gridTableSep '-') gridTableFooter

1297−

1298−gridTableSplitLine :: [Int] -> String -> [String]

1299−gridTableSplitLine indices line = map removeFinalBar $ tail $

1300− splitStringByIndices (init indices) $ trimr line

1301−

1302−gridPart :: PandocMonad m => Char -> ParserT [Char] st m ((Int, Int), Alignment)

1303−gridPart ch = do

1304− leftColon <- option False (True <$ char ':')

1305− dashes <- many1 (char ch)

1306− rightColon <- option False (True <$ char ':')

1307− char '+'

1308− let lengthDashes = length dashes + (if leftColon then 1 else 0) +

1309− (if rightColon then 1 else 0)

1310− let alignment = case (leftColon, rightColon) of

1311− (True, True) -> AlignCenter

1312− (True, False) -> AlignLeft

1313− (False, True) -> AlignRight

1314− (False, False) -> AlignDefault

1315− return ((lengthDashes, lengthDashes + 1), alignment)

1316−

1317−gridDashedLines :: PandocMonad m => Char -> ParserT [Char] st m [((Int, Int), Alignment)]

1318−gridDashedLines ch = try $ char '+' >> many1 (gridPart ch) <* blankline

1319−

1320−removeFinalBar :: String -> String

1321−removeFinalBar =

1322− reverse . dropWhile (`elem` " \t") . dropWhile (=='|') . reverse

1323−

1324−-- | Separator between rows of grid table.

1325−gridTableSep :: PandocMonad m => Char -> MarkdownParser m Char

1326−gridTableSep ch = try $ gridDashedLines ch >> return '\n'

1327−

1328−-- | Parse header for a grid table.

1329−gridTableHeader :: PandocMonad m => Bool -- ^ Headerless table

1330− -> MarkdownParser m (F [Blocks], [Alignment], [Int])

1331−gridTableHeader headless = try $ do

1332− optional blanklines

1333− dashes <- gridDashedLines '-'

1334− rawContent <- if headless

1335− then return []

1336− else many1 (try (char '|' >> anyLine))

1337− underDashes <- if headless

1338− then return dashes

1339− else gridDashedLines '='

1340− guard $ length dashes == length underDashes

1341− let lines' = map (snd . fst) underDashes

1342− let indices = scanl (+) 0 lines'

1343− let aligns = map snd underDashes

1344− let rawHeads = if headless

1345− then replicate (length underDashes) ""

1346− else map (unlines . map trim) $ transpose

1347− $ map (gridTableSplitLine indices) rawContent

1348− heads <- fmap sequence $ mapM (parseFromString parseBlocks . trim) rawHeads

1349− return (heads, aligns, indices)

1350−

1351−gridTableRawLine :: PandocMonad m => [Int] -> MarkdownParser m [String]

1352−gridTableRawLine indices = do

1353− char '|'

1354− line <- anyLine

1355− return (gridTableSplitLine indices line)

1356−

1357−-- | Parse row of grid table.

1358−gridTableRow :: PandocMonad m => [Int]

1359− -> MarkdownParser m (F [Blocks])

1360−gridTableRow indices = do

1361− colLines <- many1 (gridTableRawLine indices)

1362− let cols = map ((++ "\n") . unlines . removeOneLeadingSpace) $

1363− transpose colLines

1364− fmap compactify <$> fmap sequence (mapM (parseFromString parseBlocks) cols)

1365−

1366−removeOneLeadingSpace :: [String] -> [String]

1367−removeOneLeadingSpace xs =

1368− if all startsWithSpace xs

1369− then map (drop 1) xs

1370− else xs

1371− where startsWithSpace "" = True

1372− startsWithSpace (y:_) = y == ' '

1373−

1374−-- | Parse footer for a grid table.

1375−gridTableFooter :: PandocMonad m => MarkdownParser m [Char]

1376−gridTableFooter = blanklines

1294+gridTable headless = gridTableWith' parseBlocks headless

13771295

13781296pipeBreak :: PandocMonad m => MarkdownParser m ([Alignment], [Int])

13791297pipeBreak = try $ do

test/command/3516.md+2−2

@@ -27,8 +27,8 @@ on Windows builds.

2727[Table [] [AlignDefault,AlignDefault] [5.555555555555555e-2,5.555555555555555e-2]

2828 [[]

2929 ,[]]

30− [[[Para [Str "1"]]

31− ,[Para [Str "2"]]]

30+ [[[Plain [Str "1"]]

31+ ,[Plain [Str "2"]]]

3232 ,[[]

3333 ,[]]]]

3434```

test/markdown-reader-more.native+34−34

@@ -99,74 +99,74 @@ Pandoc (Meta {unMeta = fromList [("author",MetaList [MetaInlines [Str "Author",S

9999 [[Plain [Str "col",Space,Str "1"]]

100100 ,[Plain [Str "col",Space,Str "2"]]

101101 ,[Plain [Str "col",Space,Str "3"]]]

102− [[[Para [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

103− ,[Para [Str "b",SoftBreak,Str "b",Space,Str "2"]]

104− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

105− ,[[Para [Str "r2",Space,Str "d"]]

106− ,[Para [Str "e"]]

107− ,[Para [Str "f"]]]]

102+ [[[Plain [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

103+ ,[Plain [Str "b",SoftBreak,Str "b",Space,Str "2"]]

104+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

105+ ,[[Plain [Str "r2",Space,Str "d"]]

106+ ,[Plain [Str "e"]]

107+ ,[Plain [Str "f"]]]]

108108,Para [Str "Headless"]

109109,Table [] [AlignDefault,AlignDefault,AlignDefault] [0.2638888888888889,0.16666666666666666,0.18055555555555555]

110110 [[]

111111 ,[]

112112 ,[]]

113− [[[Para [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

114− ,[Para [Str "b",SoftBreak,Str "b",Space,Str "2"]]

115− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

116− ,[[Para [Str "r2",Space,Str "d"]]

117− ,[Para [Str "e"]]

118− ,[Para [Str "f"]]]]

113+ [[[Plain [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

114+ ,[Plain [Str "b",SoftBreak,Str "b",Space,Str "2"]]

115+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

116+ ,[[Plain [Str "r2",Space,Str "d"]]

117+ ,[Plain [Str "e"]]

118+ ,[Plain [Str "f"]]]]

119119,Para [Str "With",Space,Str "alignments"]

120120,Table [] [AlignRight,AlignLeft,AlignCenter] [0.2638888888888889,0.16666666666666666,0.18055555555555555]

121121 [[Plain [Str "col",Space,Str "1"]]

122122 ,[Plain [Str "col",Space,Str "2"]]

123123 ,[Plain [Str "col",Space,Str "3"]]]

124− [[[Para [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

125− ,[Para [Str "b",SoftBreak,Str "b",Space,Str "2"]]

126− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

127− ,[[Para [Str "r2",Space,Str "d"]]

128− ,[Para [Str "e"]]

129− ,[Para [Str "f"]]]]

124+ [[[Plain [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

125+ ,[Plain [Str "b",SoftBreak,Str "b",Space,Str "2"]]

126+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

127+ ,[[Plain [Str "r2",Space,Str "d"]]

128+ ,[Plain [Str "e"]]

129+ ,[Plain [Str "f"]]]]

130130,Para [Str "Headless",Space,Str "with",Space,Str "alignments"]

131131,Table [] [AlignRight,AlignLeft,AlignCenter] [0.2638888888888889,0.16666666666666666,0.18055555555555555]

132132 [[]

133133 ,[]

134134 ,[]]

135− [[[Para [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

136− ,[Para [Str "b",SoftBreak,Str "b",Space,Str "2"]]

137− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

138− ,[[Para [Str "r2",Space,Str "d"]]

139− ,[Para [Str "e"]]

140− ,[Para [Str "f"]]]]

135+ [[[Plain [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

136+ ,[Plain [Str "b",SoftBreak,Str "b",Space,Str "2"]]

137+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

138+ ,[[Plain [Str "r2",Space,Str "d"]]

139+ ,[Plain [Str "e"]]

140+ ,[Plain [Str "f"]]]]

141141,Para [Str "Spaces",Space,Str "at",Space,Str "ends",Space,Str "of",Space,Str "lines"]

142142,Table [] [AlignDefault,AlignDefault,AlignDefault] [0.2638888888888889,0.16666666666666666,0.18055555555555555]

143143 [[]

144144 ,[]

145145 ,[]]

146− [[[Para [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

147− ,[Para [Str "b",SoftBreak,Str "b",Space,Str "2"]]

148− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

149− ,[[Para [Str "r2",Space,Str "d"]]

150− ,[Para [Str "e"]]

151− ,[Para [Str "f"]]]]

146+ [[[Plain [Str "r1",Space,Str "a",SoftBreak,Str "r1",Space,Str "bis"]]

147+ ,[Plain [Str "b",SoftBreak,Str "b",Space,Str "2"]]

148+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2"]]]

149+ ,[[Plain [Str "r2",Space,Str "d"]]

150+ ,[Plain [Str "e"]]

151+ ,[Plain [Str "f"]]]]

152152,Para [Str "Multiple",Space,Str "blocks",Space,Str "in",Space,Str "a",Space,Str "cell"]

153153,Table [] [AlignDefault,AlignDefault,AlignDefault] [0.2638888888888889,0.16666666666666666,0.18055555555555555]

154154 [[]

155155 ,[]

156156 ,[]]

157157 [[[Header 1 ("col-1",[],[]) [Str "col",Space,Str "1"]

158− ,Para [Str "col",Space,Str "1"]]

158+ ,Plain [Str "col",Space,Str "1"]]

159159 ,[Header 1 ("col-2",[],[]) [Str "col",Space,Str "2"]

160− ,Para [Str "col",Space,Str "2"]]

160+ ,Plain [Str "col",Space,Str "2"]]

161161 ,[Header 1 ("col-3",[],[]) [Str "col",Space,Str "3"]

162− ,Para [Str "col",Space,Str "3"]]]

162+ ,Plain [Str "col",Space,Str "3"]]]

163163 ,[[Para [Str "r1",Space,Str "a"]

164164 ,Para [Str "r1",Space,Str "bis"]]

165165 ,[BulletList

166166 [[Plain [Str "b"]]

167167 ,[Plain [Str "b",Space,Str "2"]]

168168 ,[Plain [Str "b",Space,Str "2"]]]]

169− ,[Para [Str "c",SoftBreak,Str "c",Space,Str "2",SoftBreak,Str "c",Space,Str "2"]]]]

169+ ,[Plain [Str "c",SoftBreak,Str "c",Space,Str "2",SoftBreak,Str "c",Space,Str "2"]]]]

170170,Para [Str "Empty",Space,Str "cells"]

171171,Table [] [AlignDefault,AlignDefault] [5.555555555555555e-2,5.555555555555555e-2]

172172 [[]

test/tables-rstsubset.native+6−6

@@ -54,9 +54,9 @@

5454 ,[Plain [Str "1"]]]]

5555,Para [Str "Multiline",Space,Str "table",Space,Str "with",Space,Str "caption:"]

5656,Table [Str "Here\8217s",Space,Str "the",Space,Str "caption.",Space,Str "It",Space,Str "may",Space,Str "span",Space,Str "multiple",Space,Str "lines."] [AlignDefault,AlignDefault,AlignDefault,AlignDefault] [0.1375,0.125,0.15,0.325]

57− [[Plain [Str "Centered",Space,Str "Header"]]

58− ,[Plain [Str "Left",Space,Str "Aligned"]]

59− ,[Plain [Str "Right",Space,Str "Aligned"]]

57+ [[Plain [Str "Centered",SoftBreak,Str "Header"]]

58+ ,[Plain [Str "Left",SoftBreak,Str "Aligned"]]

59+ ,[Plain [Str "Right",SoftBreak,Str "Aligned"]]

6060 ,[Plain [Str "Default",Space,Str "aligned"]]]

6161 [[[Plain [Str "First"]]

6262 ,[Plain [Str "row"]]

@@ -68,9 +68,9 @@

6868 ,[Plain [Str "Here\8217s",Space,Str "another",Space,Str "one.",SoftBreak,Str "Note",Space,Str "the",Space,Str "blank",Space,Str "line",SoftBreak,Str "between",Space,Str "rows."]]]]

6969,Para [Str "Multiline",Space,Str "table",Space,Str "without",Space,Str "caption:"]

7070,Table [] [AlignDefault,AlignDefault,AlignDefault,AlignDefault] [0.1375,0.125,0.15,0.325]

71− [[Plain [Str "Centered",Space,Str "Header"]]

72− ,[Plain [Str "Left",Space,Str "Aligned"]]

73− ,[Plain [Str "Right",Space,Str "Aligned"]]

71+ [[Plain [Str "Centered",SoftBreak,Str "Header"]]

72+ ,[Plain [Str "Left",SoftBreak,Str "Aligned"]]

73+ ,[Plain [Str "Right",SoftBreak,Str "Aligned"]]

7474 ,[Plain [Str "Default",Space,Str "aligned"]]]

7575 [[[Plain [Str "First"]]

7676 ,[Plain [Str "row"]]