git.iko.soy/typeable/pandoc
on master help sign in
Roff tokenizer: better handling of escapes.
8d55dc1John MacFarlane
src/Text/Pandoc/Readers/Roff.hs+39−10
compare with master
unifiedsplit
@@ -215,20 +215,24 @@ escapeNormal :: PandocMonad m => RoffLexer m [LinePart]
215escapeNormal = do
215escapeNormal = do
216 c <- anyChar
216 c <- anyChar
217 case c of
217 case c of
218+ 'A' -> quoteArg >>= checkDefined
218 'C' -> quoteArg >>= resolveGlyph '\''
219 'C' -> quoteArg >>= resolveGlyph '\''
219 'f' -> escFont
220 'f' -> escFont
220 's' -> escFontSize
221 's' -> escFontSize
221 '*' -> escString
222 '*' -> escString
222 '"' -> mempty <$ skipMany (satisfy (/='\n')) -- line comment
223 '"' -> mempty <$ skipMany (satisfy (/='\n')) -- line comment
223 '#' -> mempty <$ manyTill anyChar newline
224 '#' -> mempty <$ manyTill anyChar newline
224− '%' -> return mempty
225+ '%' -> return mempty -- optional hyphenation
226+ ':' -> return mempty -- zero-width break
225 '{' -> return mempty
227 '{' -> return mempty
226 '}' -> return mempty
228 '}' -> return mempty
227− '&' -> return mempty
229+ '&' -> return mempty -- nonprintable zero-width
230+ ')' -> return mempty -- nonprintable zero-width
231+ '/' -> return mempty -- to fix spacing before roman
232+ ',' -> return mempty -- to fix spacing after roman
228− '\n' -> return mempty
233+ '\n' -> return mempty -- line continuation
229− ':' -> return mempty
230− '0' -> return mempty
231− 'c' -> return mempty
234+ 'c' -> return mempty -- interrupt text processing
235+ 'a' -> return mempty -- "non-interpreted leader character"
232 '-' -> return [RoffStr "-"]
236 '-' -> return [RoffStr "-"]
233 '_' -> return [RoffStr "_"]
237 '_' -> return [RoffStr "_"]
234 ' ' -> return [RoffStr " "]
238 ' ' -> return [RoffStr " "]
@@ -240,13 +244,31 @@ escapeNormal = do
240 return [RoffStr "\\"]
244 return [RoffStr "\\"]
241 't' -> return [RoffStr "\t"]
245 't' -> return [RoffStr "\t"]
242 'e' -> return [RoffStr "\\"]
246 'e' -> return [RoffStr "\\"]
247+ 'E' -> do
248+ mode <- roffMode <$> getState
249+ case mode of
250+ CopyMode -> return mempty
251+ NormalMode -> return [RoffStr "\\"]
243 '`' -> return [RoffStr "`"]
252 '`' -> return [RoffStr "`"]
244− '^' -> return [RoffStr " "]
253+ '^' -> return [RoffStr "\x200A"] -- 1/12 em space
245− '|' -> return [RoffStr " "]
254+ '|' -> return [RoffStr "\x2006"] --1/6 em space
246 '\'' -> return [RoffStr "`"]
255 '\'' -> return [RoffStr "`"]
247 '.' -> return [RoffStr "`"]
256 '.' -> return [RoffStr "`"]
248 '~' -> return [RoffStr "\160"] -- nonbreaking space
257 '~' -> return [RoffStr "\160"] -- nonbreaking space
249− _ -> escUnknown ['\\',c]
258+ '0' -> return [RoffStr "\x2007"] -- digit-width space
259+ _ -> escIgnore c
260+
261+escIgnore :: PandocMonad m => Char -> RoffLexer m [LinePart]
262+escIgnore c = do
263+ pos <- getPosition
264+ nextc <- lookAhead anyChar
265+ arg <- case nextc of
266+ '[' -> (\x -> "[" ++ x ++ "]") <$> escapeArg
267+ '(' -> ('(':) <$> escapeArg
268+ '\'' -> (\x -> "'" ++ x ++ "'") <$> quoteArg
269+ _ -> count 1 anyChar
270+ report $ SkippedContent ('\\':c:arg) pos
271+ return mempty
250
272
251escUnknown :: PandocMonad m => String -> RoffLexer m [LinePart]
273escUnknown :: PandocMonad m => String -> RoffLexer m [LinePart]
252escUnknown s = do
274escUnknown s = do
@@ -291,7 +313,7 @@ quoteArg = char '\'' *> manyTill (noneOf ['\n','\'']) (char '\'')
291escFont :: PandocMonad m => RoffLexer m [LinePart]
313escFont :: PandocMonad m => RoffLexer m [LinePart]
292escFont = do
314escFont = do
293 font <- escapeArg <|> count 1 alphaNum
315 font <- escapeArg <|> count 1 alphaNum
294− font' <- if null font
316+ font' <- if null font || font == "P"
295 then prevFont <$> getState
317 then prevFont <$> getState
296 else return $ foldr processFontLetter defaultFontSpec font
318 else return $ foldr processFontLetter defaultFontSpec font
297 modifyState $ \st -> st{ prevFont = currentFont st
319 modifyState $ \st -> st{ prevFont = currentFont st
@@ -581,6 +603,13 @@ lexArgs = do
581 char '"'
603 char '"'
582 return [RoffStr "\""]
604 return [RoffStr "\""]
583
605
606+checkDefined :: PandocMonad m => String -> RoffLexer m [LinePart]
607+checkDefined name = do
608+ macros <- customMacros <$> getState
609+ case M.lookup name macros of
610+ Just _ -> return [RoffStr "1"]
611+ Nothing -> return [RoffStr "0"]
612+
584escString :: PandocMonad m => RoffLexer m [LinePart]
613escString :: PandocMonad m => RoffLexer m [LinePart]
585escString = try $ do
614escString = try $ do
586 pos <- getPosition
615 pos <- getPosition
test/Tests/Readers/Man.hs+2−2
@@ -63,12 +63,12 @@ tests = [
63 text " ok")
63 text " ok")
64 , "skip" =:
64 , "skip" =:
65 "a\\%\\{\\}\\\n\\:b\\0"
65 "a\\%\\{\\}\\\n\\:b\\0"
66− =?> (para $ str "ab")
66+ =?> (para $ str "ab\8199")
67 , "replace" =:
67 , "replace" =:
68 "\\-\\ \\\\\\[lq]\\[rq]\\[em]\\[en]\\*(lq\\*(rq"
68 "\\-\\ \\\\\\[lq]\\[rq]\\[em]\\[en]\\*(lq\\*(rq"
69 =?> (para $ text "- \\“”—–“”")
69 =?> (para $ text "- \\“”—–“”")
70 , "replace2" =:
70 , "replace2" =:
71− "\\t\\e\\`\\^\\|\\'" =?> (para $ text "\\` `")
71+ "\\t\\e\\`\\^\\|\\'" =?> (para $ text "\\`\8202\8198`")
72 , "comment with \\\"" =:
72 , "comment with \\\"" =:
73 "Foo \\\" bar\n" =?> (para $ text "Foo")
73 "Foo \\\" bar\n" =?> (para $ text "Foo")
74 , "comment with \\#" =:
74 , "comment with \\#" =: