@@ -32,6 +32,8 @@ module Text.Pandoc.Readers.LaTeX.Parsing
3232 , totoks
3333 , toksToString
3434 , satisfyTok
35+ , parseFromToks
36+ , disablingWithRaw
3537 , doMacros
3638 , doMacros'
3739 , setpos
@@ -87,13 +89,15 @@ import Control.Monad.Trans (lift)
8789import Data.Char (chr, isAlphaNum, isDigit, isLetter, ord)
8890import Data.Default
8991import Data.List (intercalate)
92+import qualified Data.IntMap as IntMap
9093import qualified Data.Map as M
9194import qualified Data.Set as Set
9295import Data.Text (Text)
9396import qualified Data.Text as T
9497import Text.Pandoc.Builder
9598import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
96−import Text.Pandoc.Error (PandocError (PandocMacroLoop))
99+import Text.Pandoc.Error
100+ (PandocError (PandocMacroLoop,PandocShouldNeverHappenError))
97101import Text.Pandoc.Logging
98102import Text.Pandoc.Options
99103import Text.Pandoc.Parsing hiding (blankline, many, mathDisplay, mathInline,
@@ -153,6 +157,8 @@ data LaTeXState = LaTeXState{ sOptions :: ReaderOptions
153157 , sToggles :: M.Map Text Bool
154158 , sExpanded :: Bool
155159 , sFileContents :: M.Map Text Text
160+ , sEnableWithRaw :: Bool
161+ , sRawTokens :: IntMap.IntMap [Tok]
156162 }
157163 deriving Show
158164
@@ -179,6 +185,8 @@ defaultLaTeXState = LaTeXState{ sOptions = def
179185 , sToggles = M.empty
180186 , sExpanded = False
181187 , sFileContents = M.empty
188+ , sEnableWithRaw = True
189+ , sRawTokens = IntMap.empty
182190 }
183191
184192instance PandocMonad m => HasQuoteContext LaTeXState m where
@@ -404,11 +412,31 @@ untoken t = untokenAccum t mempty
404412toksToString :: [Tok] -> String
405413toksToString = T.unpack . untokenize
406414
415+parseFromToks :: PandocMonad m => LP m a -> [Tok] -> LP m a
416+parseFromToks parser toks = do
417+ oldInput <- getInput
418+ setInput toks
419+ result <- disablingWithRaw parser
420+ setInput oldInput
421+ return result
422+
423+disablingWithRaw :: PandocMonad m => LP m a -> LP m a
424+disablingWithRaw parser = do
425+ oldEnableWithRaw <- sEnableWithRaw <$> getState
426+ updateState $ \st -> st{ sEnableWithRaw = False }
427+ result <- parser
428+ updateState $ \st -> st{ sEnableWithRaw = oldEnableWithRaw }
429+ return result
430+
407431satisfyTok :: PandocMonad m => (Tok -> Bool) -> LP m Tok
408432satisfyTok f = do
409433 doMacros -- apply macros on remaining input stream
410434 res <- tokenPrim (T.unpack . untoken) updatePos matcher
411− updateState $ \st -> st{ sExpanded = False }
435+ updateState $ \st -> st{ sExpanded = False
436+ , sRawTokens =
437+ if sEnableWithRaw st
438+ then IntMap.map (res:) $ sRawTokens st
439+ else sRawTokens st }
412440 return res
413441 where matcher t | f t = Just t
414442 | otherwise = Nothing
@@ -594,18 +622,22 @@ isCommentTok _ = False
594622anyTok :: PandocMonad m => LP m Tok
595623anyTok = satisfyTok (const True)
596624
625+singleCharTok :: PandocMonad m => LP m Tok
626+singleCharTok =
627+ satisfyTok $ \case
628+ Tok _ Word t -> T.length t == 1
629+ Tok _ Symbol t -> not (T.any (`Set.member` specialChars) t)
630+ _ -> False
631+
597632singleChar :: PandocMonad m => LP m Tok
598−singleChar = try $ do
599− Tok pos toktype t <- satisfyTok (tokTypeIn [Word, Symbol])
600− guard $ not $ toktype == Symbol &&
601− T.any (`Set.member` specialChars) t
602− if T.length t > 1
603− then do
604− let (t1, t2) = (T.take 1 t, T.drop 1 t)
605− inp <- getInput
606− setInput $ Tok (incSourceColumn pos 1) toktype t2 : inp
607− return $ Tok pos toktype t1
608− else return $ Tok pos toktype t
633+singleChar = singleCharTok <|> singleCharFromWord
634+ where
635+ singleCharFromWord = do
636+ Tok pos toktype t <- disablingWithRaw $ satisfyTok isWordTok
637+ let (t1, t2) = (T.take 1 t, T.drop 1 t)
638+ inp <- getInput
639+ setInput $ Tok pos toktype t1 : Tok (incSourceColumn pos 1) toktype t2 : inp
640+ anyTok
609641
610642specialChars :: Set.Set Char
611643specialChars = Set.fromList "#$%&~_^\\{}"
@@ -725,11 +757,23 @@ ignore raw = do
725757
726758withRaw :: PandocMonad m => LP m a -> LP m (a, [Tok])
727759withRaw parser = do
728− inp <- getInput
760+ rawTokensMap <- sRawTokens <$> getState
761+ let key = case IntMap.lookupMax rawTokensMap of
762+ Nothing -> 0
763+ Just (n,_) -> n + 1
764+ -- insert empty list at key
765+ updateState $ \st -> st{ sRawTokens =
766+ IntMap.insert key [] $ sRawTokens st }
729767 result <- parser
730− nxtpos <- option Nothing ((\(Tok pos' _ _) -> Just pos') <$> lookAhead anyTok)
731− let raw = takeWhile (\(Tok pos _ _) -> maybe True
732− (\p -> sourceName p /= sourceName pos || pos < p) nxtpos) inp
768+ mbRevToks <- IntMap.lookup key . sRawTokens <$> getState
769+ raw <- case mbRevToks of
770+ Just revtoks -> do
771+ updateState $ \st -> st{ sRawTokens =
772+ IntMap.delete key $ sRawTokens st}
773+ return $ reverse revtoks
774+ Nothing ->
775+ throwError $ PandocShouldNeverHappenError $
776+ "sRawTokens has nothing at key " <> T.pack (show key)
733777 return (result, raw)
734778
735779keyval :: PandocMonad m => LP m (Text, Text)
@@ -794,7 +838,7 @@ getRawCommand name txt = do
794838 (_, rawargs) <- withRaw $
795839 case name of
796840 "write" -> do
797− void $ satisfyTok isWordTok -- digits
841+ void $ many $ satisfyTok isDigitTok -- digits
798842 void braced
799843 "titleformat" -> do
800844 void braced
@@ -809,6 +853,10 @@ getRawCommand name txt = do
809853 void $ many braced
810854 return $ txt <> untokenize rawargs
811855
856+isDigitTok :: Tok -> Bool
857+isDigitTok (Tok _ Word t) = T.all isDigit t
858+isDigitTok _ = False
859+
812860skipopts :: PandocMonad m => LP m ()
813861skipopts = skipMany (void overlaySpecification <|> void rawopt)
814862