Merge branch 'dev' into 354-bugfix-custom-ports-with-scheme

.goreleaser.yml

@@ -22,7 +22,7 @@ builds:
       goarch: 'arm'
 
   binary: '{{ .ProjectName }}'
-  main: cmd/httpx/main.go
+  main: cmd/httpx/httpx.go
 
 archives:
 - format: zip
@@ -30,4 +30,4 @@ archives:
       darwin: macOS
 
 checksum:
-  algorithm: sha256
+  algorithm: sha256

common/httpx/httpx.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -18,6 +19,7 @@ import (
 	pdhttputil "github.com/projectdiscovery/httputil"
 	"github.com/projectdiscovery/rawhttp"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/stringsutil"
 	"golang.org/x/net/http2"
 )
 
@@ -59,20 +61,30 @@ func New(options *Options) (*HTTPX, error) {
 	}
 
 	if httpx.Options.FollowRedirects {
-		// Follow redirects
+		// Follow redirects up to a maximum number
-		redirectFunc = nil
+		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
+			if len(previousRequests) >= options.MaxRedirects {
+				// https://github.com/golang/go/issues/10069
+				return http.ErrUseLastResponse
+			}
+			return nil
+		}
 	}
 
 	if httpx.Options.FollowHostRedirects {
-		// Only follow redirects on the same host
+		// Only follow redirects on the same host up to a maximum number
-		redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error {
+		redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
 			// Check if we get a redirect to a differen host
 			var newHost = redirectedRequest.URL.Host
-			var oldHost = previousRequest[0].URL.Host
+			var oldHost = previousRequests[0].URL.Host
 			if newHost != oldHost {
 				// Tell the http client to not follow redirect
 				return http.ErrUseLastResponse
 			}
+			if len(previousRequests) >= options.MaxRedirects {
+				// https://github.com/golang/go/issues/10069
+				return http.ErrUseLastResponse
+			}
 			return nil
 		}
 	}
@@ -134,6 +146,13 @@ get_response:
 		return nil, err
 	}
 
+	var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
+	switch {
+	case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
+		shouldIgnoreErrors = true
+		shouldIgnoreBodyErrors = true
+	}
+
 	var resp Response
 
 	resp.Headers = httpresp.Header.Clone()
@@ -148,23 +167,25 @@ get_response:
 			req.Header.Set("Accept-Encoding", "identity")
 			goto get_response
 		}
-		return nil, err
+		if !shouldIgnoreErrors {
+			return nil, err
+		}
 	}
-	resp.Raw = rawResp
+	resp.Raw = string(rawResp)
-	resp.RawHeaders = headers
+	resp.RawHeaders = string(headers)
 
 	var respbody []byte
 	// websockets don't have a readable body
 	if httpresp.StatusCode != http.StatusSwitchingProtocols {
 		var err error
 		respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
-		if err != nil {
+		if err != nil && !shouldIgnoreBodyErrors {
 			return nil, err
 		}
 	}
 
 	closeErr := httpresp.Body.Close()
-	if closeErr != nil {
+	if closeErr != nil && !shouldIgnoreBodyErrors {
 		return nil, closeErr
 	}
 
@@ -175,7 +196,15 @@ get_response:
 		respbodystr = h.htmlPolicy.Sanitize(respbodystr)
 	}
 
-	resp.ContentLength = utf8.RuneCountInString(respbodystr)
+	if contentLength, ok := resp.Headers["Content-Length"]; ok {
+		contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
+		if err != nil {
+			resp.ContentLength = utf8.RuneCountInString(respbodystr)
+		} else {
+			resp.ContentLength = contentLengthInt
+		}
+	}
+
 	resp.Data = respbody
 
 	// fill metrics

common/httpx/option.go

@@ -23,6 +23,7 @@ type Options struct {
 	VHostSimilarityRatio int
 	FollowRedirects      bool
 	FollowHostRedirects  bool
+	MaxRedirects         int
 	Unsafe               bool
 	TLSGrab              bool
 	// VHOSTs options
@@ -39,13 +40,14 @@ type Options struct {
 
 // DefaultOptions contains the default options
 var DefaultOptions = Options{
-	RandomAgent: true,
+	RandomAgent:  true,
-	Threads:     25,
+	Threads:      25,
-	Timeout:     30 * time.Second,
+	Timeout:      30 * time.Second,
-	RetryMax:    5,
+	RetryMax:     5,
-	Unsafe:      false,
+	MaxRedirects: 10,
-	CdnCheck:    true,
+	Unsafe:       false,
-	ExcludeCdn:  false,
+	CdnCheck:     true,
+	ExcludeCdn:   false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,
 	VHostIgnoreContentLength: true,

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/bluele/gcache v0.0.2 // indirect
 	github.com/corpix/uarand v0.1.1
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
@@ -21,7 +22,7 @@ require (
 	github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
-	github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
+	github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
 	github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
 	github.com/projectdiscovery/mapcidr v0.0.8
 	github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e

go.sum

@@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
@@ -153,6 +155,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ
 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
 github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
 github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
+github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
+github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
 github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
 github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=

runner/banner.go

@@ -8,11 +8,11 @@ const banner = `
   / __ \/ __/ __/ __ \|   /
  / / / / /_/ /_/ /_/ /   |
 /_/ /_/\__/\__/ .___/_/|_|
-             /_/              v1.1.1
+             /_/              v1.1.2
 `
 
 // Version is the current version of httpx
-const Version = `v1.1.1`
+const Version = `v1.1.2`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

runner/options.go

@@ -61,6 +61,7 @@ type scanOptions struct {
 	OutputExtractRegex        string
 	extractRegex              *regexp.Regexp
 	ExcludeCDN                bool
+	HostMaxErrors             int
 }
 
 func (s *scanOptions) Clone() *scanOptions {
@@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		OutputExtractRegex:        s.OutputExtractRegex,
 		MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
 		MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
+		HostMaxErrors:             s.HostMaxErrors,
 	}
 }
 
@@ -154,6 +156,7 @@ type Options struct {
 	responseInStdout          bool
 	chainInStdout             bool
 	FollowHostRedirects       bool
+	MaxRedirects              int
 	OutputMethod              bool
 	TLSProbe                  bool
 	CSPProbe                  bool
@@ -184,6 +187,7 @@ type Options struct {
 	Resume                    bool
 	resumeCfg                 *ResumeCfg
 	ExcludeCDN                bool
+	HostMaxErrors             int
 }
 
 // ParseOptions parses the command line options for application
@@ -209,10 +213,11 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
 	flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
 	flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
+	flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host")
 	flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
 	flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
 	flag.StringVar(&options.InputFile, "l", "", "File containing domains")
-	flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
+	flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)")
 	flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
 	flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
 	flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
@@ -260,6 +265,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
 	flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
 	flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
+	flag.IntVar(&options.HostMaxErrors, "max-host-error", -1, "Max error count per host before skipping remaining path/s")
 
 	flag.Parse()
 

runner/runner.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -19,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bluele/gcache"
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/clistats"
@@ -52,13 +54,14 @@ const (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	options     *Options
+	options         *Options
-	hp          *httpx.HTTPX
+	hp              *httpx.HTTPX
-	wappalyzer  *wappalyzer.Wappalyze
+	wappalyzer      *wappalyzer.Wappalyze
-	scanopts    scanOptions
+	scanopts        scanOptions
-	hm          *hybrid.HybridMap
+	hm              *hybrid.HybridMap
-	stats       clistats.StatisticsClient
+	stats           clistats.StatisticsClient
-	ratelimiter ratelimit.Limiter
+	ratelimiter     ratelimit.Limiter
+	HostErrorsCache gcache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -81,6 +84,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.RetryMax = options.Retries
 	httpxOptions.FollowRedirects = options.FollowRedirects
 	httpxOptions.FollowHostRedirects = options.FollowHostRedirects
+	httpxOptions.MaxRedirects = options.MaxRedirects
 	httpxOptions.HTTPProxy = options.HTTPProxy
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
@@ -155,6 +159,10 @@ func New(options *Options) (*Runner, error) {
 	if strings.EqualFold(options.Methods, "all") {
 		scanopts.Methods = pdhttputil.AllHTTPMethods()
 	} else if options.Methods != "" {
+		// if unsafe is specified then converts the methods to uppercase
+		if !options.Unsafe {
+			options.Methods = strings.ToUpper(options.Methods)
+		}
 		scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
 	}
 	if len(scanopts.Methods) == 0 {
@@ -207,6 +215,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	scanopts.ExcludeCDN = options.ExcludeCDN
+	scanopts.HostMaxErrors = options.HostMaxErrors
 	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
@@ -228,6 +237,13 @@ func New(options *Options) (*Runner, error) {
 		runner.ratelimiter = ratelimit.NewUnlimited()
 	}
 
+	if options.HostMaxErrors >= 0 {
+		gc := gcache.New(1000).
+			ARC().
+			Build()
+		runner.HostErrorsCache = gc
+	}
+
 	return runner, nil
 }
 
@@ -378,6 +394,9 @@ func (r *Runner) Close() {
 	// nolint:errcheck // ignore
 	r.hm.Close()
 	r.hp.Dialer.Close()
+	if r.options.HostMaxErrors >= 0 {
+		r.HostErrorsCache.Purge()
+	}
 }
 
 // RunEnumeration on targets for httpx client
@@ -513,7 +532,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 					wg.Add()
 					go func(target, method, protocol string) {
 						defer wg.Done()
-						result := r.analyze(hp, protocol, target, method, scanopts)
+						result := r.analyze(hp, protocol, target, method, t, scanopts)
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
@@ -546,7 +565,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 					go func(port int, method, protocol string) {
 						defer wg.Done()
 						h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
-						result := r.analyze(hp, protocol, h, method, scanopts)
+						result := r.analyze(hp, protocol, h, method, t, scanopts)
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
@@ -596,7 +615,7 @@ func targets(target string) chan string {
 	return results
 }
 
-func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result {
+func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result {
 	origProtocol := protocol
 	if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
 		protocol = httpx.HTTPS
@@ -608,20 +627,29 @@ retry:
 		parts := strings.Split(domain, ",")
 		//nolint:gomnd // not a magic number
 		if len(parts) != 2 {
-			return Result{}
+			return Result{Input: origInput}
 		}
 		domain = parts[0]
 		customHost = parts[1]
 	}
 	URL, err := urlutil.Parse(domain)
 	if err != nil {
-		return Result{URL: domain, err: err}
+		return Result{URL: domain, Input: origInput, err: err}
+	}
+
+	// check if we have to skip the host:port as a result of a previous failure
+	hostPort := net.JoinHostPort(URL.Host, URL.Port)
+	if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
+		numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
+		if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors {
+			return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
+		}
 	}
 
 	// check if the combination host:port should be skipped if belonging to a cdn
 	if r.skipCDNPort(URL.Host, URL.Port) {
 		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
-		return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")}
+		return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")}
 	}
 
 	URL.Scheme = protocol
@@ -635,7 +663,7 @@ retry:
 	}
 	req, err := hp.NewRequest(method, URL.String())
 	if err != nil {
-		return Result{URL: URL.String(), err: err}
+		return Result{URL: URL.String(), Input: origInput, err: err}
 	}
 	if customHost != "" {
 		req.Host = customHost
@@ -665,7 +693,7 @@ retry:
 		var errDump error
 		requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
 		if errDump != nil {
-			return Result{URL: URL.String(), err: errDump}
+			return Result{URL: URL.String(), Input: origInput, err: errDump}
 		}
 	} else {
 		// Create a copy on the fly of the request body
@@ -674,7 +702,7 @@ retry:
 		var errDump error
 		requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
 		if errDump != nil {
-			return Result{URL: URL.String(), err: errDump}
+			return Result{URL: URL.String(), Input: origInput, err: errDump}
 		}
 		// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
 		// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
@@ -730,10 +758,21 @@ retry:
 			retried = true
 			goto retry
 		}
+
+		// mark the host:port as failed to avoid further checks
+		if r.options.HostMaxErrors >= 0 {
+			errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort)
+			if err != nil || errorCount == nil {
+				_ = r.HostErrorsCache.Set(hostPort, 1)
+			} else if errorCount != nil {
+				_ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1)
+			}
+		}
+
 		if r.options.Probe {
-			return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
+			return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
 		} else {
-			return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err}
+			return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err}
 		}
 	}
 
@@ -978,7 +1017,7 @@ retry:
 
 	parsed, err := urlutil.Parse(fullURL)
 	if err != nil {
-		return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")}
+		return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")}
 	}
 
 	finalPort := parsed.Port
@@ -1022,7 +1061,7 @@ retry:
 		HeaderSHA256:     headersSha,
 		raw:              resp.Raw,
 		URL:              fullURL,
-		Input:            domain,
+		Input:            origInput,
 		ContentLength:    resp.ContentLength,
 		ChainStatusCodes: chainStatusCodes,
 		Chain:            chainItems,