f5e26e4 · typeable/pandoc

git.iko.soy/typeable/pandoc

Roff tokenizer: check for first-column before parsing macro.

Also add SourcePos as argument to lexRoff, so we can pass in current source pos when parsing a table cell. Closes #5025.

commitf5e26e4512c1a797fa3a20cbc451f5d56b248870 authorJohn MacFarlane <jgm@berkeley.edu> dateSat 27 Oct 2018, 11:03 (8 years ago) parenta3b351c

2 files changed+10−6

compare with master

unifiedsplit

src/Text/Pandoc/Readers/Man.hs+2−2 src/Text/Pandoc/Readers/Roff.hs+8−4

src/Text/Pandoc/Readers/Man.hs+2−2

@@ -52,7 +52,7 @@ import Text.Pandoc.Shared (crFilter)

5252import Text.Pandoc.Readers.Roff -- TODO explicit imports

5353import Text.Parsec hiding (tokenPrim)

5454import qualified Text.Parsec as Parsec

55−import Text.Parsec.Pos (updatePosString)

55+import Text.Parsec.Pos (updatePosString, initialPos)

5656import qualified Data.Foldable as Foldable

5757

5858data ManState = ManState { readerOptions :: ReaderOptions

@@ -69,7 +69,7 @@ type ManParser m = ParserT [RoffToken] ManState m

6969-- | Read man (troff) from an input string and return a Pandoc document.

7070readMan :: PandocMonad m => ReaderOptions -> T.Text -> m Pandoc

7171readMan opts txt = do

72− tokenz <- lexRoff (crFilter txt)

72+ tokenz <- lexRoff (initialPos "input") (crFilter txt)

7373 let state = def {readerOptions = opts} :: ManState

7474 eitherdoc <- readWithMTokens parseMan state

7575 (Foldable.toList . unRoffTokens $ tokenz)

src/Text/Pandoc/Readers/Roff.hs+8−4

@@ -311,6 +311,7 @@ lexComment = do

311311lexMacro :: PandocMonad m => RoffLexer m RoffTokens

312312lexMacro = do

313313 pos <- getPosition

314+ guard $ sourceColumn pos == 1

314315 char '.' <|> char '\''

315316 skipMany spacetab

316317 macroName <- many (satisfy (not . isSpace))

@@ -369,7 +370,9 @@ lexTableRows = do

369370 return $ zip aligns rows

370371

371372tableCell :: PandocMonad m => RoffLexer m RoffTokens

372−tableCell = (enclosedCell <|> simpleCell) >>= lexRoff . T.pack

373+tableCell = do

374+ pos <- getPosition

375+ (enclosedCell <|> simpleCell) >>= lexRoff pos . T.pack

373376 where

374377 enclosedCell = do

375378 try (string "T{")

@@ -642,9 +645,10 @@ linePartsToString = mconcat . map go

642645 go _ = mempty

643646

644647-- | Tokenize a string as a sequence of groff tokens.

645−lexRoff :: PandocMonad m => T.Text -> m RoffTokens

646−lexRoff txt = do

647− eithertokens <- readWithM (mconcat <$> many manToken) def (T.unpack txt)

648+lexRoff :: PandocMonad m => SourcePos -> T.Text -> m RoffTokens

649+lexRoff pos txt = do

650+ eithertokens <- readWithM (do setPosition pos

651+ mconcat <$> many manToken) def (T.unpack txt)

648652 case eithertokens of

649653 Left e -> throwError e

650654 Right tokenz -> return tokenz