git.iko.soy/typeable/pandoc
on master help sign in
Markdown reader: improved efficiency of abbreviation parsing.

Instead of a separate abbrev parser, we just check for abbreviations each time we parse a string. This gives a huge performance boost with -S. Resolves Issue #141. git-svn-id: https://pandoc.googlecode.com/svn/trunk@1570 788f1e2b-df1e-0410-8736-df70ead52e1b

commitab100f7c5e5174620e8cff8dc965be68a1f3ba5a authorfiddlosopher <fiddlosopher@788f1e2b-df1e-0410-8736-df70ead52e1b> dateThu 30 Apr 2009, 04:39 (17 years ago) parent490c430
3 files changed+28−31
src/Text/Pandoc/Readers/Markdown.hs+26−29 tests/testsuite.native+1−1 tests/writer.native+1−1
src/Text/Pandoc/Readers/Markdown.hs+26−29
@@ -34,7 +34,7 @@ module Text.Pandoc.Readers.Markdown (
3434
3535import Data.List ( transpose, isPrefixOf, isSuffixOf, lookup, sortBy, findIndex, intercalate )
3636import Data.Ord ( comparing )
37−import Data.Char ( isAlphaNum, isAlpha, isLower, isDigit, isUpper )
37+import Data.Char ( isAlphaNum, isUpper )
3838import Data.Maybe
3939import Text.Pandoc.Definition
4040import Text.Pandoc.Shared
@@ -820,8 +820,7 @@ inline :: GenParser Char ParserState Inline
820820inline = choice inlineParsers <?> "inline"
821821
822822inlineParsers :: [GenParser Char ParserState Inline]
823−inlineParsers = [ abbrev
824− , str
823+inlineParsers = [ str
825824 , smartPunctuation
826825 , whitespace
827826 , endline
@@ -944,30 +943,6 @@ subscript = failIfStrict >> enclosed (char '~') (char '~')
944943 (notFollowedBy spaceChar >> inline) >>= -- may not contain Space
945944 return . Subscript
946945
947−abbrev :: GenParser Char ParserState Inline
948−abbrev = failUnlessSmart >>
949− (assumedAbbrev <|> knownAbbrev) >>= return . Str . (++ ".\160")
950−
951−-- an string of letters followed by a period that does not end a sentence
952−-- is assumed to be an abbreviation. It is assumed that sentences don't
953−-- start with lowercase letters or numerals.
954−assumedAbbrev :: GenParser Char ParserState [Char]
955−assumedAbbrev = try $ do
956− result <- many1 $ satisfy isAlpha
957− string ". "
958− lookAhead $ satisfy (\x -> isLower x || isDigit x)
959− return result
960−
961−-- these strings are treated as abbreviations even if they are followed
962−-- by a capital letter (such as a name).
963−knownAbbrev :: GenParser Char ParserState [Char]
964−knownAbbrev = try $ do
965− result <- oneOfStrings [ "Mr", "Mrs", "Ms", "Capt", "Dr", "Prof", "Gen",
966− "Gov", "e.g", "i.e", "Sgt", "St", "vol", "vs",
967− "Sen", "Rep", "Pres", "Hon", "Rev" ]
968− string ". "
969− return result
970−
971946smartPunctuation :: GenParser Char ParserState Inline
972947smartPunctuation = failUnlessSmart >>
973948 choice [ quoted, apostrophe, dash, ellipses ]
@@ -1060,8 +1035,30 @@ nonEndline = satisfy (/='\n')
10601035strChar :: GenParser Char st Char
10611036strChar = noneOf (specialChars ++ " \t\n")
10621037
1063−str :: GenParser Char st Inline
1064−str = many1 strChar >>= return . Str
1038+str :: GenParser Char ParserState Inline
1039+str = do
1040+ result <- many1 strChar
1041+ state <- getState
1042+ if stateSmart state
1043+ then case likelyAbbrev result of
1044+ [] -> return $ Str result
1045+ xs -> choice (map (\x ->
1046+ try (string x >> char ' ' >>
1047+ notFollowedBy spaceChar >>
1048+ return (Str $ result ++ x ++ "\160"))) xs)
1049+ <|> (return $ Str result)
1050+ else return $ Str result
1051+
1052+-- | if the string matches the beginning of an abbreviation (before
1053+-- the first period, return strings that would finish the abbreviation.
1054+likelyAbbrev :: String -> [String]
1055+likelyAbbrev x =
1056+ let abbrevs = [ "Mr.", "Mrs.", "Ms.", "Capt.", "Dr.", "Prof.",
1057+ "Gen.", "Gov.", "e.g.", "i.e.", "Sgt.", "St.",
1058+ "vol.", "vs.", "Sen.", "Rep.", "Pres.", "Hon.",
1059+ "Rev.", "Ph.D.", "M.D.", "M.A." ]
1060+ abbrPairs = map (break (=='.')) abbrevs
1061+ in map snd $ filter (\(y,_) -> y == x) abbrPairs
10651062
10661063-- an endline character that can be treated as a space, not a structural break
10671064endline :: GenParser Char ParserState Inline
tests/testsuite.native+1−1
@@ -165,7 +165,7 @@ Pandoc (Meta [Str "Pandoc",Space,Str "Test",Space,Str "Suite"] ["John MacFarlane
165165 [ [ Plain [Str "Nested",Str "."] ]
166166 ] ] ]
167167, Para [Str "Should",Space,Str "not",Space,Str "be",Space,Str "a",Space,Str "list",Space,Str "item:"]
168−, Para [Str "M",Str ".",Str "A.\160",Str "2007"]
168+, Para [Str "M.A.\160",Str "2007"]
169169, Para [Str "B",Str ".",Space,Str "Williams"]
170170, HorizontalRule
171171, Header 1 [Str "Definition",Space,Str "Lists"]
tests/writer.native+1−1
@@ -165,7 +165,7 @@ Pandoc (Meta [Str "Pandoc",Space,Str "Test",Space,Str "Suite"] ["John MacFarlane
165165 [ [ Plain [Str "Nested",Str "."] ]
166166 ] ] ]
167167, Para [Str "Should",Space,Str "not",Space,Str "be",Space,Str "a",Space,Str "list",Space,Str "item:"]
168−, Para [Str "M",Str ".",Str "A.\160",Str "2007"]
168+, Para [Str "M.A.\160",Str "2007"]
169169, Para [Str "B",Str ".",Space,Str "Williams"]
170170, HorizontalRule
171171, Header 1 [Str "Definition",Space,Str "Lists"]