Adding option to skip remaining paths on unresponsive host

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
 	github.com/corpix/uarand v0.1.1
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect

go.sum

@@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=

runner/options.go

@@ -61,6 +61,7 @@ type scanOptions struct {
 	OutputExtractRegex        string
 	extractRegex              *regexp.Regexp
 	ExcludeCDN                bool
+	SkipRemainingPathsOnError bool
 }
 
 func (s *scanOptions) Clone() *scanOptions {
@@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		OutputExtractRegex:        s.OutputExtractRegex,
 		MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
 		MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
+		SkipRemainingPathsOnError: s.SkipRemainingPathsOnError,
 	}
 }
 
@@ -184,6 +186,7 @@ type Options struct {
 	Resume                    bool
 	resumeCfg                 *ResumeCfg
 	ExcludeCDN                bool
+	SkipRemainingPathsOnError bool
 }
 
 // ParseOptions parses the command line options for application
@@ -260,6 +263,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
 	flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
 	flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
+	flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers")
 
 	flag.Parse()
 

runner/runner.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -19,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bluele/gcache"
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/clistats"
@@ -52,13 +54,14 @@ const (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	options     *Options
-	hp          *httpx.HTTPX
-	wappalyzer  *wappalyzer.Wappalyze
-	scanopts    scanOptions
-	hm          *hybrid.HybridMap
-	stats       clistats.StatisticsClient
-	ratelimiter ratelimit.Limiter
+	options       *Options
+	hp            *httpx.HTTPX
+	wappalyzer    *wappalyzer.Wappalyze
+	scanopts      scanOptions
+	hm            *hybrid.HybridMap
+	stats         clistats.StatisticsClient
+	ratelimiter   ratelimit.Limiter
+	FailedTargets gcache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -207,6 +210,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	scanopts.ExcludeCDN = options.ExcludeCDN
+	scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError
 	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
@@ -228,6 +232,13 @@ func New(options *Options) (*Runner, error) {
 		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
 
+	if options.SkipRemainingPathsOnError {
+		gc := gcache.New(1000).
+			ARC().
+			Build()
+		runner.FailedTargets = gc
+	}
+
 	return runner, nil
 }
 
@@ -378,6 +389,9 @@ func (r *Runner) Close() {
 	// nolint:errcheck // ignore
 	r.hm.Close()
 	r.hp.Dialer.Close()
+	if r.options.SkipRemainingPathsOnError {
+		r.FailedTargets.Purge()
+	}
 }
 
 // RunEnumeration on targets for httpx client
@@ -612,6 +626,12 @@ retry:
 		return Result{URL: domain, err: err}
 	}
 
+	// check if we have to skip the host:port as a result of a previous failure
+	hostPort := net.JoinHostPort(URL.Host, URL.Port)
+	if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) {
+		return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
+	}
+
 	// check if the combination host:port should be skipped if belonging to a cdn
 	if r.skipCDNPort(URL.Host, URL.Port) {
 		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
@@ -724,6 +744,12 @@ retry:
 			retried = true
 			goto retry
 		}
+
+		// mark the host:port as failed to avoid further checks
+		if r.options.SkipRemainingPathsOnError {
+			_ = r.FailedTargets.Set(hostPort, nil)
+		}
+
 		if r.options.Probe {
 			return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
 		} else {