5debb0d · typeable/pandoc

git.iko.soy/typeable/pandoc

Shared: Provide custom isURI that rejects unknown schemes [isURI]

We also export the set of known `schemes`. The new function replaces the function of the same name from `Network.URI`, as the latter did not check whether a scheme is well-known. E.g. MediaWiki wikis frequently feature pages with names like `User:John`. These links were interpreted as URIs, thus turning internal links into global links. This is prevented by also checking whether the scheme of a URI is frequently used (i.e. is IANA registered or an otherwise well-known scheme). Fixes: #2713 Update set of well-known URIs from IANA list All official IANA schemes (as of 2017-05-22) are included in the set of known schemes. The four non-official schemes doi, isbn, javascript, and pmid are kept.

commit5debb0da0f94d1454d51cacede7c4844f01cc2f5 authorAlbert Krewinkel <albert+github@zeitkraut.de> dateTue 23 May 2017, 09:48 (9 years ago) parent4d1e9b8

16 files changed+81−48

compare with master

unifiedsplit

src/Text/Pandoc/App.hs+2−2

@@ -57,7 +57,7 @@ import Data.Maybe (fromMaybe, isJust, isNothing)

5757import qualified Data.Text as T

5858import Data.Yaml (decode)

5959import qualified Data.Yaml as Yaml

60−import Network.URI (URI (..), isURI, parseURI)

60+import Network.URI (URI (..), parseURI)

6161import Paths_pandoc (getDataDir)

6262import Skylighting (Style, Syntax (..), defaultSyntaxMap, parseTheme)

6363import Skylighting.Parser (missingIncludes, parseSyntaxDefinition,

@@ -80,7 +80,7 @@ import Text.Pandoc.Lua ( runLuaFilter )

8080import Text.Pandoc.PDF (makePDF)

8181import Text.Pandoc.Process (pipeProcess)

8282import Text.Pandoc.SelfContained (makeSelfContained, makeDataURI)

83−import Text.Pandoc.Shared (headerShift, openURL, readDataFile,

83+import Text.Pandoc.Shared (isURI, headerShift, openURL, readDataFile,

8484 readDataFileUTF8, safeRead, tabFilter)

8585import qualified Text.Pandoc.UTF8 as UTF8

8686import Text.Pandoc.XML (toEntities)

src/Text/Pandoc/Parsing.hs+1−26

@@ -465,33 +465,8 @@ emailAddress = try $ toResult <$> mailbox <*> (char '@' *> domain)

465465 sepby1 p sep = (:) <$> p <*> (many (try $ sep >> p))

466466

467467

468−-- Schemes from http://www.iana.org/assignments/uri-schemes.html plus

469−-- the unofficial schemes coap, doi, javascript, isbn, pmid

470−schemes :: [String]

471−schemes = ["coap","doi","javascript","aaa","aaas","about","acap","cap","cid",

472− "crid","data","dav","dict","dns","file","ftp","geo","go","gopher",

473− "h323","http","https","iax","icap","im","imap","info","ipp","iris",

474− "iris.beep","iris.xpc","iris.xpcs","iris.lwz","ldap","mailto","mid",

475− "msrp","msrps","mtqp","mupdate","news","nfs","ni","nih","nntp",

476− "opaquelocktoken","pop","pres","rtsp","service","session","shttp","sieve",

477− "sip","sips","sms","snmp","soap.beep","soap.beeps","tag","tel","telnet",

478− "tftp","thismessage","tn3270","tip","tv","urn","vemmi","ws","wss","xcon",

479− "xcon-userid","xmlrpc.beep","xmlrpc.beeps","xmpp","z39.50r","z39.50s",

480− "adiumxtra","afp","afs","aim","apt","attachment","aw","beshare","bitcoin",

481− "bolo","callto","chrome","chrome-extension","com-eventbrite-attendee",

482− "content", "cvs","dlna-playsingle","dlna-playcontainer","dtn","dvb",

483− "ed2k","facetime","feed","finger","fish","gg","git","gizmoproject",

484− "gtalk","hcp","icon","ipn","irc","irc6","ircs","itms","jar","jms",

485− "keyparc","lastfm","ldaps","magnet","maps","market","message","mms",

486− "ms-help","msnim","mumble","mvn","notes","oid","palm","paparazzi",

487− "platform","proxy","psyc","query","res","resource","rmi","rsync",

488− "rtmp","secondlife","sftp","sgn","skype","smb","soldat","spotify",

489− "ssh","steam","svn","teamspeak","things","udp","unreal","ut2004",

490− "ventrilo","view-source","webcal","wtai","wyciwyg","xfire","xri",

491− "ymsgr", "isbn", "pmid"]

492−

493468uriScheme :: Stream s m Char => ParserT s st m String

494−uriScheme = oneOfStringsCI schemes

469+uriScheme = oneOfStringsCI (Set.toList schemes)

495470

496471-- | Parses a URI. Returns pair of original and URI-escaped version.

497472uri :: Stream [Char] m Char => ParserT [Char] st m (String, String)

src/Text/Pandoc/Readers/Txt2Tags.hs−1

@@ -42,7 +42,6 @@ import Text.Pandoc.Definition

4242import Text.Pandoc.Options

4343import Text.Pandoc.Parsing hiding (macro, space, spaces, uri)

4444import Text.Pandoc.Shared (compactify, compactifyDL, escapeURI)

45−--import Network.URI (isURI) -- Not sure whether to use this function

4645import Control.Monad (guard, void, when)

4746import Control.Monad.Reader (Reader, asks, runReader)

4847import Data.Default

src/Text/Pandoc/SelfContained.hs+2−2

@@ -42,7 +42,7 @@ import qualified Data.ByteString.Char8 as B

4242import qualified Data.ByteString.Lazy as L

4343import Data.Char (isAlphaNum, isAscii, toLower)

4444import Data.List (isPrefixOf)

45−import Network.URI (URI (..), escapeURIString, isURI, parseURI)

45+import Network.URI (URI (..), escapeURIString, parseURI)

4646import System.FilePath (takeDirectory, takeExtension, (</>))

4747import Text.HTML.TagSoup

4848import Text.Pandoc.Class (PandocMonad (..), fetchItem, report)

@@ -50,7 +50,7 @@ import Text.Pandoc.Error

5050import Text.Pandoc.Logging

5151import Text.Pandoc.MIME (MimeType)

5252import Text.Pandoc.Options (WriterOptions (..))

53−import Text.Pandoc.Shared (renderTags', trim)

53+import Text.Pandoc.Shared (isURI, renderTags', trim)

5454import Text.Pandoc.UTF8 (toString)

5555import Text.Parsec (ParsecT, runParserT)

5656import qualified Text.Parsec as P

src/Text/Pandoc/Shared.hs+68−1

@@ -81,6 +81,9 @@ module Text.Pandoc.Shared (

8181 openURL,

8282 collapseFilePath,

8383 filteredFilesFromArchive,

84+ -- * URI handling

85+ schemes,

86+ isURI,

8487 -- * Error handling

8588 mapLeft,

8689 -- * for squashing blocks

@@ -104,7 +107,7 @@ import Data.List ( find, stripPrefix, intercalate )

104107import Data.Maybe (mapMaybe)

105108import Data.Version ( showVersion )

106109import qualified Data.Map as M

107−import Network.URI ( escapeURIString, unEscapeString )

110+import Network.URI ( URI(uriScheme), escapeURIString, unEscapeString, parseURI )

108111import qualified Data.Set as Set

109112import System.Directory

110113import System.FilePath (splitDirectories, isPathSeparator)

@@ -774,6 +777,70 @@ filteredFilesFromArchive zf f =

774777 fileAndBinary :: Archive -> FilePath -> Maybe (FilePath, BL.ByteString)

775778 fileAndBinary a fp = findEntryByPath fp a >>= \e -> Just (fp, fromEntry e)

776779

780+

781+--

782+-- IANA URIs

783+--

784+

785+-- | Schemes from http://www.iana.org/assignments/uri-schemes.html plus

786+-- the unofficial schemes doi, javascript, isbn, pmid.

787+schemes :: Set.Set String

788+schemes = Set.fromList

789+ -- Official IANA schemes

790+ [ "aaa", "aaas", "about", "acap", "acct", "acr", "adiumxtra", "afp", "afs"

791+ , "aim", "appdata", "apt", "attachment", "aw", "barion", "beshare", "bitcoin"

792+ , "blob", "bolo", "browserext", "callto", "cap", "chrome", "chrome-extension"

793+ , "cid", "coap", "coaps", "com-eventbrite-attendee", "content", "crid", "cvs"

794+ , "data", "dav", "dict", "dis", "dlna-playcontainer", "dlna-playsingle"

795+ , "dns", "dntp", "dtn", "dvb", "ed2k", "example", "facetime", "fax", "feed"

796+ , "feedready", "file", "filesystem", "finger", "fish", "ftp", "geo", "gg"

797+ , "git", "gizmoproject", "go", "gopher", "graph", "gtalk", "h323", "ham"

798+ , "hcp", "http", "https", "hxxp", "hxxps", "hydrazone", "iax", "icap", "icon"

799+ , "im", "imap", "info", "iotdisco", "ipn", "ipp", "ipps", "irc", "irc6"

800+ , "ircs", "iris", "iris.beep", "iris.lwz", "iris.xpc", "iris.xpcs"

801+ , "isostore", "itms", "jabber", "jar", "jms", "keyparc", "lastfm", "ldap"

802+ , "ldaps", "lvlt", "magnet", "mailserver", "mailto", "maps", "market"

803+ , "message", "mid", "mms", "modem", "mongodb", "moz", "ms-access"

804+ , "ms-browser-extension", "ms-drive-to", "ms-enrollment", "ms-excel"

805+ , "ms-gamebarservices", "ms-getoffice", "ms-help", "ms-infopath"

806+ , "ms-media-stream-id", "ms-officeapp", "ms-project", "ms-powerpoint"

807+ , "ms-publisher", "ms-search-repair", "ms-secondary-screen-controller"

808+ , "ms-secondary-screen-setup", "ms-settings", "ms-settings-airplanemode"

809+ , "ms-settings-bluetooth", "ms-settings-camera", "ms-settings-cellular"

810+ , "ms-settings-cloudstorage", "ms-settings-connectabledevices"

811+ , "ms-settings-displays-topology", "ms-settings-emailandaccounts"

812+ , "ms-settings-language", "ms-settings-location", "ms-settings-lock"

813+ , "ms-settings-nfctransactions", "ms-settings-notifications"

814+ , "ms-settings-power", "ms-settings-privacy", "ms-settings-proximity"

815+ , "ms-settings-screenrotation", "ms-settings-wifi", "ms-settings-workplace"

816+ , "ms-spd", "ms-sttoverlay", "ms-transit-to", "ms-virtualtouchpad"

817+ , "ms-visio", "ms-walk-to", "ms-whiteboard", "ms-whiteboard-cmd", "ms-word"

818+ , "msnim", "msrp", "msrps", "mtqp", "mumble", "mupdate", "mvn", "news", "nfs"

819+ , "ni", "nih", "nntp", "notes", "ocf", "oid", "onenote", "onenote-cmd"

820+ , "opaquelocktoken", "pack", "palm", "paparazzi", "pkcs11", "platform", "pop"

821+ , "pres", "prospero", "proxy", "pwid", "psyc", "qb", "query", "redis"

822+ , "rediss", "reload", "res", "resource", "rmi", "rsync", "rtmfp", "rtmp"

823+ , "rtsp", "rtsps", "rtspu", "secondlife", "service", "session", "sftp", "sgn"

824+ , "shttp", "sieve", "sip", "sips", "skype", "smb", "sms", "smtp", "snews"

825+ , "snmp", "soap.beep", "soap.beeps", "soldat", "spotify", "ssh", "steam"

826+ , "stun", "stuns", "submit", "svn", "tag", "teamspeak", "tel", "teliaeid"

827+ , "telnet", "tftp", "things", "thismessage", "tip", "tn3270", "tool", "turn"

828+ , "turns", "tv", "udp", "unreal", "urn", "ut2004", "v-event", "vemmi"

829+ , "ventrilo", "videotex", "vnc", "view-source", "wais", "webcal", "wpid"

830+ , "ws", "wss", "wtai", "wyciwyg", "xcon", "xcon-userid", "xfire"

831+ , "xmlrpc.beep", "xmlrpc.beeps", "xmpp", "xri", "ymsgr", "z39.50", "z39.50r"

832+ , "z39.50s"

833+ -- Inofficial schemes

834+ , "doi", "isbn", "javascript", "pmid"

835+ ]

836+

837+-- | Check if the string is a valid URL with a IANA or frequently used but

838+-- unofficial scheme (see @schemes@).

839+isURI :: String -> Bool

840+isURI = maybe False hasKnownScheme . parseURI

841+ where

842+ hasKnownScheme = (`Set.member` schemes) . filter (/= ':') . uriScheme

843+

777844---

778845--- Squash blocks into inlines

779846---

src/Text/Pandoc/Writers/ConTeXt.hs+1−1

@@ -33,7 +33,7 @@ import Control.Monad.State

3333import Data.Char (ord)

3434import Data.List (intercalate, intersperse)

3535import Data.Maybe (catMaybes)

36−import Network.URI (isURI, unEscapeString)

36+import Network.URI (unEscapeString)

3737import Text.Pandoc.Class (PandocMonad, report)

3838import Text.Pandoc.Logging

3939import Text.Pandoc.Definition

src/Text/Pandoc/Writers/DokuWiki.hs+1−2

@@ -44,13 +44,12 @@ import Control.Monad.Reader (ReaderT, ask, local, runReaderT)

4444import Control.Monad.State (StateT, evalStateT, gets, modify)

4545import Data.Default (Default (..))

4646import Data.List (intercalate, intersect, isPrefixOf, transpose)

47−import Network.URI (isURI)

4847import Text.Pandoc.Class (PandocMonad, report)

4948import Text.Pandoc.Logging

5049import Text.Pandoc.Definition

5150import Text.Pandoc.ImageSize

5251import Text.Pandoc.Options (WrapOption (..), WriterOptions (writerTableOfContents, writerTemplate, writerWrapText))

53−import Text.Pandoc.Shared (camelCaseToHyphenated, escapeURI, linesToPara,

52+import Text.Pandoc.Shared (camelCaseToHyphenated, escapeURI, isURI, linesToPara,

5453 removeFormatting, substitute, trimr)

5554import Text.Pandoc.Templates (renderTemplate')

5655import Text.Pandoc.Writers.Shared (defField, metaToJSON)

src/Text/Pandoc/Writers/FB2.hs+1−2

@@ -46,7 +46,6 @@ import Data.Char (isAscii, isControl, isSpace, toLower)

4646import Data.Either (lefts, rights)

4747import Data.List (intercalate, intersperse, isPrefixOf, stripPrefix)

4848import Network.HTTP (urlEncode)

49−import Network.URI (isURI)

5049import Text.XML.Light

5150import qualified Text.XML.Light as X

5251import qualified Text.XML.Light.Cursor as XC

@@ -57,7 +56,7 @@ import Text.Pandoc.Definition

5756import Text.Pandoc.Error

5857import Text.Pandoc.Logging

5958import Text.Pandoc.Options (HTMLMathMethod (..), WriterOptions (..), def)

60−import Text.Pandoc.Shared (capitalize, isHeaderBlock, linesToPara,

59+import Text.Pandoc.Shared (capitalize, isHeaderBlock, isURI, linesToPara,

6160 orderedListMarkers)

6261

6362-- | Data to be written at the end of the document:

src/Text/Pandoc/Writers/Haddock.hs−1

@@ -36,7 +36,6 @@ module Text.Pandoc.Writers.Haddock (writeHaddock) where

3636import Control.Monad.State

3737import Data.Default

3838import Data.List (intersperse, transpose)

39−import Network.URI (isURI)

4039import Text.Pandoc.Class (PandocMonad, report)

4140import Text.Pandoc.Definition

4241import Text.Pandoc.Logging

src/Text/Pandoc/Writers/ICML.hs+1−2

@@ -21,7 +21,6 @@ import Control.Monad.State

2121import Data.List (intersperse, isInfixOf, isPrefixOf, stripPrefix)

2222import qualified Data.Set as Set

2323import Data.Text as Text (breakOnAll, pack)

24−import Network.URI (isURI)

2524import Text.Pandoc.Class (PandocMonad, report)

2625import qualified Text.Pandoc.Class as P

2726import Text.Pandoc.Definition

@@ -29,7 +28,7 @@ import Text.Pandoc.ImageSize

2928import Text.Pandoc.Logging

3029import Text.Pandoc.Options

3130import Text.Pandoc.Pretty

32−import Text.Pandoc.Shared (linesToPara, splitBy)

31+import Text.Pandoc.Shared (isURI, linesToPara, splitBy)

3332import Text.Pandoc.Templates (renderTemplate')

3433import Text.Pandoc.Writers.Math (texMathToInlines)

3534import Text.Pandoc.Writers.Shared

src/Text/Pandoc/Writers/LaTeX.hs+1−1

@@ -43,7 +43,7 @@ import Data.List (foldl', intercalate, intersperse, isInfixOf, nub, nubBy,

4343 stripPrefix, (\\))

4444import Data.Maybe (catMaybes, fromMaybe, isJust)

4545import qualified Data.Text as T

46−import Network.URI (isURI, unEscapeString)

46+import Network.URI (unEscapeString)

4747import Text.Pandoc.Class (PandocMonad, report)

4848import Text.Pandoc.Definition

4949import Text.Pandoc.Highlighting (formatLaTeXBlock, formatLaTeXInline, highlight,

src/Text/Pandoc/Writers/Markdown.hs−1

@@ -49,7 +49,6 @@ import qualified Data.Text as T

4949import qualified Data.Vector as V

5050import Data.Yaml (Value (Array, Bool, Number, Object, String))

5151import Network.HTTP (urlEncode)

52−import Network.URI (isURI)

5352import Text.HTML.TagSoup (Tag (..), isTagText, parseTags)

5453import Text.Pandoc.Class (PandocMonad, report)

5554import Text.Pandoc.Definition

src/Text/Pandoc/Writers/MediaWiki.hs−1

@@ -34,7 +34,6 @@ import Control.Monad.Reader

3434import Control.Monad.State

3535import Data.List (intercalate)

3636import qualified Data.Set as Set

37−import Network.URI (isURI)

3837import Text.Pandoc.Class (PandocMonad, report)

3938import Text.Pandoc.Logging

4039import Text.Pandoc.Definition

src/Text/Pandoc/Writers/RST.hs−1

@@ -35,7 +35,6 @@ import Control.Monad.State

3535import Data.Char (isSpace, toLower)

3636import Data.List (isPrefixOf, stripPrefix)

3737import Data.Maybe (fromMaybe)

38−import Network.URI (isURI)

3938import qualified Text.Pandoc.Builder as B

4039import Text.Pandoc.Class (PandocMonad, report)

4140import Text.Pandoc.Logging

src/Text/Pandoc/Writers/Texinfo.hs+1−1

@@ -37,7 +37,7 @@ import Data.Char (chr, ord)

3737import Data.List (maximumBy, transpose)

3838import Data.Ord (comparing)

3939import qualified Data.Set as Set

40−import Network.URI (isURI, unEscapeString)

40+import Network.URI (unEscapeString)

4141import System.FilePath

4242import Text.Pandoc.Class (PandocMonad, report)

4343import Text.Pandoc.Definition

src/Text/Pandoc/Writers/ZimWiki.hs+2−3

@@ -38,14 +38,13 @@ import Data.Default (Default (..))

3838import Data.List (intercalate, isInfixOf, isPrefixOf, transpose)

3939import qualified Data.Map as Map

4040import Data.Text (breakOnAll, pack)

41−import Network.URI (isURI)

4241import Text.Pandoc.Class (PandocMonad, report)

4342import Text.Pandoc.Logging

4443import Text.Pandoc.Definition

4544import Text.Pandoc.ImageSize

4645import Text.Pandoc.Options (WrapOption (..), WriterOptions (writerTableOfContents, writerTemplate, writerWrapText))

47−import Text.Pandoc.Shared (escapeURI, linesToPara, removeFormatting, substitute,

48− trimr)

46+import Text.Pandoc.Shared (isURI, escapeURI, linesToPara, removeFormatting,

47+ substitute, trimr)

4948import Text.Pandoc.Templates (renderTemplate')

5049import Text.Pandoc.Writers.Shared (defField, metaToJSON)

5150