Merge pull request #1517 from projectdiscovery/introduce_exclude_flag

introduce `-exclude` flag

README.md

@@ -233,8 +233,7 @@ OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
    -nfs, -no-fallback-scheme          probe with protocol scheme specified in input 
    -maxhr, -max-host-error int        max error count per host before skipping remaining path/s (default 30)
-   -ec, -exclude-cdn                  skip full port scans for CDN/WAF (only checks for 80,443)
-   -eph, -exclude-private-hosts       skip any hosts which have a private ip address
+   -e, -exclude string[]              exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
    -retries int                       number of retries
    -timeout int                       timeout in seconds (default 10)
    -delay value                       duration between each http request (eg: 200ms, 1s) (default -1ns)

cmd/functional-test/testcases.txt

@@ -13,7 +13,7 @@ scanme.sh {{binary}} -silent -tls-grab
 scanme.sh {{binary}} -silent -unsafe
 scanme.sh {{binary}} -silent -x all
 scanme.sh {{binary}} -silent -body 'a=b'
-scanme.sh {{binary}} -silent -exclude-cdn
+scanme.sh {{binary}} -silent -exclude cdn
 scanme.sh {{binary}} -silent -ports https:443
 scanme.sh {{binary}} -silent -ztls
 scanme.sh {{binary}} -silent -jarm

common/httpx/option.go

@@ -8,14 +8,13 @@ import (
 
 // Options contains configuration options for the client
 type Options struct {
-	RandomAgent         bool
-	DefaultUserAgent    string
-	HTTPProxy           string
-	SocksProxy          string
-	Threads             int
-	CdnCheck            bool
-	ExcludeCdn          bool
-	ExcludePrivateHosts bool
+	RandomAgent      bool
+	DefaultUserAgent string
+	HTTPProxy        string
+	SocksProxy       string
+	Threads          int
+	CdnCheck         bool
+	ExcludeCdn       bool
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
 	// RetryMax is the maximum number of retries
@@ -49,15 +48,14 @@ type Options struct {
 
 // DefaultOptions contains the default options
 var DefaultOptions = Options{
-	RandomAgent:         true,
-	Threads:             25,
-	Timeout:             30 * time.Second,
-	RetryMax:            5,
-	MaxRedirects:        10,
-	Unsafe:              false,
-	CdnCheck:            true,
-	ExcludeCdn:          false,
-	ExcludePrivateHosts: false,
+	RandomAgent:  true,
+	Threads:      25,
+	Timeout:      30 * time.Second,
+	RetryMax:     5,
+	MaxRedirects: 10,
+	Unsafe:       false,
+	CdnCheck:     true,
+	ExcludeCdn:   false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,
 	VHostIgnoreContentLength: true,

runner/options.go

@@ -78,7 +78,6 @@ type ScanOptions struct {
 	OutputExtractRegex        string
 	extractRegexps            map[string]*regexp.Regexp
 	ExcludeCDN                bool
-	ExcludePrivateHosts       bool
 	HostMaxErrors             int
 	ProbeAllIPS               bool
 	Favicon                   bool
@@ -243,8 +242,7 @@ type Options struct {
 	Probe                     bool
 	Resume                    bool
 	resumeCfg                 *ResumeCfg
-	ExcludeCDN                bool
-	ExcludePrivateHosts       bool
+	Exclude                   goflags.StringSlice
 	HostMaxErrors             int
 	Stream                    bool
 	SkipDedupe                bool
@@ -457,8 +455,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"),
 		flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "),
 		flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
-		flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDN/WAF (only checks for 80,443)"),
-		flagSet.BoolVarP(&options.ExcludePrivateHosts, "exclude-private-hosts", "eph", false, "skip any hosts which have a private ip address"),
+		flagSet.StringSliceVarP(&options.Exclude, "exclude", "e", nil, "exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
 		flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),

runner/runner.go

@@ -34,6 +34,7 @@ import (
 	"github.com/projectdiscovery/httpx/common/hashes/jarm"
 	"github.com/projectdiscovery/httpx/static"
 	"github.com/projectdiscovery/mapcidr/asn"
+	"github.com/projectdiscovery/networkpolicy"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	osutil "github.com/projectdiscovery/utils/os"
 
@@ -77,6 +78,8 @@ type Runner struct {
 	wappalyzer          *wappalyzer.Wappalyze
 	scanopts            ScanOptions
 	hm                  *hybrid.HybridMap
+	excludePorts        map[string]struct{}
+	excludeCdn          bool
 	stats               clistats.StatisticsClient
 	ratelimiter         ratelimit.Limiter
 	HostErrorsCache     gcache.Cache[string, int]
@@ -114,6 +117,28 @@ func New(options *Options) (*Runner, error) {
 		os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
 	}
 
+	runner.excludePorts = make(map[string]struct{})
+	for _, exclude := range options.Exclude {
+		switch {
+		case exclude == "cdn":
+			runner.excludeCdn = true
+		case exclude == "private-ips":
+			options.Deny = append(options.Deny, networkpolicy.DefaultIPv4Denylist...)
+			options.Deny = append(options.Deny, networkpolicy.DefaultIPv4DenylistRanges...)
+			options.Deny = append(options.Deny, networkpolicy.DefaultIPv6Denylist...)
+			options.Deny = append(options.Deny, networkpolicy.DefaultIPv6DenylistRanges...)
+		case iputil.IsCIDR(exclude):
+			options.Deny = append(options.Deny, exclude)
+		case asn.IsASN(exclude):
+			ips := expandASNInputValue(exclude)
+			options.Deny = append(options.Deny, ips...)
+		case iputil.IsPort(exclude):
+			runner.excludePorts[exclude] = struct{}{}
+		default:
+			options.Deny = append(options.Deny, exclude)
+		}
+	}
+
 	httpxOptions := httpx.DefaultOptions
 	// Enables automatically tlsgrab if tlsprobe is requested
 	httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe
@@ -127,8 +152,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.UnsafeURI = options.RequestURI
 	httpxOptions.CdnCheck = options.OutputCDN
-	httpxOptions.ExcludeCdn = options.ExcludeCDN
-	httpxOptions.ExcludePrivateHosts = options.ExcludePrivateHosts
+	httpxOptions.ExcludeCdn = runner.excludeCdn
 	if options.CustomHeaders.Has("User-Agent:") {
 		httpxOptions.RandomAgent = false
 	} else {
@@ -293,8 +317,7 @@ func New(options *Options) (*Runner, error) {
 		scanopts.OutputMethod = true
 	}
 
-	scanopts.ExcludeCDN = options.ExcludeCDN
-	scanopts.ExcludePrivateHosts = options.ExcludePrivateHosts
+	scanopts.ExcludeCDN = runner.excludeCdn
 	scanopts.HostMaxErrors = options.HostMaxErrors
 	scanopts.ProbeAllIPS = options.ProbeAllIPS
 	scanopts.Favicon = options.Favicon
@@ -340,6 +363,24 @@ func New(options *Options) (*Runner, error) {
 	return runner, nil
 }
 
+func expandCIDRInputValue(value string) []string {
+	var ips []string
+	ipsCh, _ := mapcidr.IPAddressesAsStream(value)
+	for ip := range ipsCh {
+		ips = append(ips, ip)
+	}
+	return ips
+}
+
+func expandASNInputValue(value string) []string {
+	var ips []string
+	cidrs, _ := asn.GetCIDRsForASNNum(value)
+	for _, cidr := range cidrs {
+		ips = append(ips, expandCIDRInputValue(cidr.String())...)
+	}
+	return ips
+}
+
 func (r *Runner) prepareInputPaths() {
 	// most likely, the user would provide the most simplified path to an existing file
 	isAbsoluteOrRelativePath := filepath.Clean(r.options.RequestURIs) == r.options.RequestURIs
@@ -1306,15 +1347,10 @@ retry:
 		}
 	}
 
-	if r.skipPrivateHosts(URL.Hostname()) {
-		gologger.Debug().Msgf("Skipping private host %s\n", URL.Host)
-		return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip and will only connect within same local network")}
-	}
-
 	// check if the combination host:port should be skipped if belonging to a cdn
-	if r.skipCDNPort(URL.Host, URL.Port()) {
-		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())
-		return Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")}
+	skip, reason := r.skip(URL, target, origInput)
+	if skip {
+		return reason
 	}
 
 	URL.Scheme = protocol
@@ -2031,6 +2067,20 @@ retry:
 	return result
 }
 
+func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) {
+	if r.skipCDNPort(URL.Host, URL.Port()) {
+		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())
+		return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")}
+	}
+
+	if _, ok := r.excludePorts[URL.Port()]; ok {
+		gologger.Debug().Msgf("Skipping excluded port: %s:%s\n", URL.Hostname(), URL.Port())
+		return true, Result{URL: target.Host, Input: origInput, Err: errors.New("port is in the exclude list")}
+	}
+
+	return false, Result{}
+}
+
 func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) {
 	reader := bytes.NewReader(screenshotBytes)
 	img, _, err := image.Decode(reader)
@@ -2195,7 +2245,7 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint
 
 func (r *Runner) skipCDNPort(host string, port string) bool {
 	// if the option is not enabled we don't skip
-	if !r.options.ExcludeCDN {
+	if !r.scanopts.ExcludeCDN {
 		return false
 	}
 	// uses the dealer to pre-resolve the target
@@ -2218,45 +2268,13 @@ func (r *Runner) skipCDNPort(host string, port string) bool {
 	}
 
 	// If the target is part of the CDN ips range - only ports 80 and 443 are allowed
-	if isCdnIP && port != "80" && port != "443" {
+	if isCdnIP && port != "" && port != "80" && port != "443" {
 		return true
 	}
 
 	return false
 }
 
-func (r *Runner) skipPrivateHosts(host string) bool {
-	// if the option is not enabled we don't skip
-	if !r.options.ExcludePrivateHosts {
-		return false
-	}
-	dnsData, err := r.hp.Dialer.GetDNSData(host)
-
-	// if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly
-	if err != nil {
-		return false
-	}
-	if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 {
-		return false
-	}
-
-	var ipsToCheck []string
-	ipsToCheck = make([]string, 0, len(dnsData.A)+len(dnsData.AAAA))
-	ipsToCheck = append(ipsToCheck, dnsData.A...)
-	ipsToCheck = append(ipsToCheck, dnsData.AAAA...)
-
-	for _, ipAddr := range ipsToCheck {
-		ip := net.ParseIP(ipAddr)
-		if ip == nil {
-			continue //skip any bad ip addresses
-		}
-		if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
-			return true
-		}
-	}
-	return false
-}
-
 // parseURL parses url based on cli option(unsafe)
 func (r *Runner) parseURL(url string) (*urlutil.URL, error) {
 	urlx, err := urlutil.ParseURL(url, r.options.Unsafe)