Merge pull request #338 from projectdiscovery/issue-328-exclude-cdn

[Feature] Adding support to scan only ports 80/443 if the ip is part of CDN range

common/httpx/httpx.go

@@ -113,7 +113,7 @@ func New(options *Options) (*HTTPX, error) {
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
 	httpx.RequestOverride = &options.RequestOverride
-	if options.CdnCheck {
+	if options.CdnCheck || options.ExcludeCdn {
 		httpx.cdn, err = cdncheck.NewWithCache()
 		if err != nil {
 			return nil, fmt.Errorf("could not create cdn check: %s", err)

common/httpx/option.go

@@ -13,6 +13,7 @@ type Options struct {
 	SocksProxy       string
 	Threads          int
 	CdnCheck         bool
+	ExcludeCdn       bool
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
 	// RetryMax is the maximum number of retries
@@ -44,6 +45,7 @@ var DefaultOptions = Options{
 	RetryMax:    5,
 	Unsafe:      false,
 	CdnCheck:    true,
+	ExcludeCdn:  false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,
 	VHostIgnoreContentLength: true,

runner/options.go

@@ -58,6 +58,7 @@ type scanOptions struct {
 	MaxResponseBodySizeToRead int
 	OutputExtractRegex        string
 	extractRegex              *regexp.Regexp
+	ExcludeCDN                bool
 }
 
 func (s *scanOptions) Clone() *scanOptions {
@@ -178,6 +179,7 @@ type Options struct {
 	OutputExtractRegex        string
 	RateLimit                 int
 	Probe                     bool
+	ExcludeCDN                bool
 }
 
 // ParseOptions parses the command line options for application
@@ -252,6 +254,7 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
 	flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second")
 	flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
+	flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
 
 	flag.Parse()
 

runner/runner.go

@@ -84,6 +84,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
 	httpxOptions.CdnCheck = options.OutputCDN
+	httpxOptions.ExcludeCdn = options.ExcludeCDN
 	httpxOptions.RandomAgent = options.RandomAgent
 	httpxOptions.Deny = options.Deny
 	httpxOptions.Allow = options.Allow
@@ -204,6 +205,7 @@ func New(options *Options) (*Runner, error) {
 		scanopts.OutputMethod = true
 	}
 
+	scanopts.ExcludeCDN = options.ExcludeCDN
 	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
@@ -627,6 +629,13 @@ retry:
 	if err != nil {
 		return Result{URL: domain, err: err}
 	}
+
+	// check if the combination host:port should be skipped if belonging to a cdn
+	if r.skipCDNPort(URL.Host, URL.Port) {
+		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
+		return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")}
+	}
+
 	URL.Scheme = protocol
 
 	if !strings.Contains(domain, URL.Port) {
@@ -1101,3 +1110,35 @@ func (r Result) JSON(scanopts *scanOptions) string { //nolint
 
 	return ""
 }
+
+func (r *Runner) skipCDNPort(host string, port string) bool {
+	// if the option is not enabled we don't skip
+	if !r.options.ExcludeCDN {
+		return false
+	}
+	// uses the dealer to pre-resolve the target
+	dnsData, err := r.hp.Dialer.GetDNSData(host)
+	// if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly
+	if err != nil {
+		return false
+	}
+
+	if len(dnsData.A) == 0 {
+		return false
+	}
+
+	// pick the first ip as target
+	hostIP := dnsData.A[0]
+
+	isCdnIP, err := r.hp.CdnCheck(hostIP)
+	if err != nil {
+		return false
+	}
+
+	// If the target is part of the CDN ips range - only ports 80 and 443 are allowed
+	if isCdnIP && port != "80" && port != "443" {
+		return true
+	}
+
+	return false
+}