f5e26e4 · typeable/pandoc

git.iko.soy/typeable/pandoc

on master help sign in

Roff tokenizer: check for first-column before parsing macro.

f5e26e4John MacFarlane8 years ago

src/Text/Pandoc/Readers/Man.hs+2−2

compare with master

unifiedsplit

@@ -52,7 +52,7 @@ import Text.Pandoc.Shared (crFilter)

52import Text.Pandoc.Readers.Roff -- TODO explicit imports

53import Text.Parsec hiding (tokenPrim)

54import qualified Text.Parsec as Parsec

55−import Text.Parsec.Pos (updatePosString)

55+import Text.Parsec.Pos (updatePosString, initialPos)

56import qualified Data.Foldable as Foldable

58data ManState = ManState { readerOptions :: ReaderOptions

@@ -69,7 +69,7 @@ type ManParser m = ParserT [RoffToken] ManState m

69-- | Read man (troff) from an input string and return a Pandoc document.

70readMan :: PandocMonad m => ReaderOptions -> T.Text -> m Pandoc

71readMan opts txt = do

72− tokenz <- lexRoff (crFilter txt)

72+ tokenz <- lexRoff (initialPos "input") (crFilter txt)

73 let state = def {readerOptions = opts} :: ManState

74 eitherdoc <- readWithMTokens parseMan state

75 (Foldable.toList . unRoffTokens $ tokenz)

src/Text/Pandoc/Readers/Roff.hs+8−4

@@ -311,6 +311,7 @@ lexComment = do

311lexMacro :: PandocMonad m => RoffLexer m RoffTokens

312lexMacro = do

313 pos <- getPosition

314+ guard $ sourceColumn pos == 1

314 char '.' <|> char '\''

315 char '.' <|> char '\''

315 skipMany spacetab

316 skipMany spacetab

316 macroName <- many (satisfy (not . isSpace))

317 macroName <- many (satisfy (not . isSpace))

@@ -369,7 +370,9 @@ lexTableRows = do

369 return $ zip aligns rows

370 return $ zip aligns rows

370

371

371tableCell :: PandocMonad m => RoffLexer m RoffTokens

372tableCell :: PandocMonad m => RoffLexer m RoffTokens

373+tableCell = do

374+ pos <- getPosition

372−tableCell = (enclosedCell <|> simpleCell) >>= lexRoff . T.pack

375+ (enclosedCell <|> simpleCell) >>= lexRoff pos . T.pack

373 where

376 where

374 enclosedCell = do

377 enclosedCell = do

375 try (string "T{")

378 try (string "T{")

@@ -642,9 +645,10 @@ linePartsToString = mconcat . map go

642 go _ = mempty

645 go _ = mempty

643

646

644-- | Tokenize a string as a sequence of groff tokens.

647-- | Tokenize a string as a sequence of groff tokens.

645−lexRoff :: PandocMonad m => T.Text -> m RoffTokens

648+lexRoff :: PandocMonad m => SourcePos -> T.Text -> m RoffTokens

646−lexRoff txt = do

649+lexRoff pos txt = do

650+ eithertokens <- readWithM (do setPosition pos

647− eithertokens <- readWithM (mconcat <$> many manToken) def (T.unpack txt)

651+ mconcat <$> many manToken) def (T.unpack txt)

648 case eithertokens of

652 case eithertokens of

649 Left e -> throwError e

653 Left e -> throwError e

650 Right tokenz -> return tokenz

654 Right tokenz -> return tokenz