From 5fef6f6b5f19c3be71c084d99ab7f97f03acfd5b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 9 Aug 2021 18:55:43 +0530 Subject: [PATCH 01/11] Main update --- .goreleaser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index a10e943..ea6706a 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -22,7 +22,7 @@ builds: goarch: 'arm' binary: '{{ .ProjectName }}' - main: cmd/httpx/main.go + main: cmd/httpx/httpx.go archives: - format: zip @@ -30,4 +30,4 @@ archives: darwin: macOS checksum: - algorithm: sha256 \ No newline at end of file + algorithm: sha256 From d89b3c14fbc8a70eded5c45faacd3f7d15477b90 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 13 Aug 2021 20:01:48 +0200 Subject: [PATCH 02/11] Adding option to skip remaining paths on unresponsive host --- go.mod | 1 + go.sum | 2 ++ runner/options.go | 4 ++++ runner/runner.go | 40 +++++++++++++++++++++++++++++++++------- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 3d163c1..22dbf61 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/bluele/gcache v0.0.2 // indirect github.com/corpix/uarand v0.1.1 github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect diff --git a/go.sum b/go.sum index 418df5c..52ef7f5 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= diff --git a/runner/options.go b/runner/options.go index 9d42c6a..24055dd 100644 --- a/runner/options.go +++ b/runner/options.go @@ -61,6 +61,7 @@ type scanOptions struct { OutputExtractRegex string extractRegex *regexp.Regexp ExcludeCDN bool + SkipRemainingPathsOnError bool } func (s *scanOptions) Clone() *scanOptions { @@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions { OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, + SkipRemainingPathsOnError: s.SkipRemainingPathsOnError, } } @@ -184,6 +186,7 @@ type Options struct { Resume bool resumeCfg *ResumeCfg ExcludeCDN bool + SkipRemainingPathsOnError bool } // ParseOptions parses the command line options for application @@ -260,6 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") + flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers") flag.Parse() diff --git a/runner/runner.go b/runner/runner.go index e0798f4..1d4a3a5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "net/http/httputil" "net/url" @@ -19,6 +20,7 @@ import ( "strings" "time" + "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/projectdiscovery/clistats" @@ -52,13 +54,14 @@ const ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts scanOptions - hm *hybrid.HybridMap - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts scanOptions + hm *hybrid.HybridMap + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + FailedTargets gcache.Cache } // New creates a new client for running enumeration process. @@ -207,6 +210,7 @@ func New(options *Options) (*Runner, error) { } scanopts.ExcludeCDN = options.ExcludeCDN + scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError runner.scanopts = scanopts if options.ShowStatistics { @@ -228,6 +232,13 @@ func New(options *Options) (*Runner, error) { runner.ratelimiter = ratelimit.NewUnlimited() } + if options.SkipRemainingPathsOnError { + gc := gcache.New(1000). + ARC(). + Build() + runner.FailedTargets = gc + } + return runner, nil } @@ -378,6 +389,9 @@ func (r *Runner) Close() { // nolint:errcheck // ignore r.hm.Close() r.hp.Dialer.Close() + if r.options.SkipRemainingPathsOnError { + r.FailedTargets.Purge() + } } // RunEnumeration on targets for httpx client @@ -612,6 +626,12 @@ retry: return Result{URL: domain, err: err} } + // check if we have to skip the host:port as a result of a previous failure + hostPort := net.JoinHostPort(URL.Host, URL.Port) + if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) { + return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + } + // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) @@ -724,6 +744,12 @@ retry: retried = true goto retry } + + // mark the host:port as failed to avoid further checks + if r.options.SkipRemainingPathsOnError { + _ = r.FailedTargets.Set(hostPort, nil) + } + if r.options.Probe { return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { From 800362e1dfcbd95793560ef8d3fb4778eecc1cc5 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 15 Aug 2021 16:04:24 +0200 Subject: [PATCH 03/11] adding host max errors support --- runner/options.go | 8 ++++---- runner/runner.go | 22 +++++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/runner/options.go b/runner/options.go index 24055dd..82d94cc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -61,7 +61,7 @@ type scanOptions struct { OutputExtractRegex string extractRegex *regexp.Regexp ExcludeCDN bool - SkipRemainingPathsOnError bool + HostMaxErrors int } func (s *scanOptions) Clone() *scanOptions { @@ -100,7 +100,7 @@ func (s *scanOptions) Clone() *scanOptions { OutputExtractRegex: s.OutputExtractRegex, MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, - SkipRemainingPathsOnError: s.SkipRemainingPathsOnError, + HostMaxErrors: s.HostMaxErrors, } } @@ -186,7 +186,7 @@ type Options struct { Resume bool resumeCfg *ResumeCfg ExcludeCDN bool - SkipRemainingPathsOnError bool + HostMaxErrors int } // ParseOptions parses the command line options for application @@ -263,7 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers") + flag.IntVar(&options.HostMaxErrors, "host-max-errors", -1, "Allowed error count per host before skipping remaining path/s") flag.Parse() diff --git a/runner/runner.go b/runner/runner.go index 1d4a3a5..44dd9db 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -210,7 +210,7 @@ func New(options *Options) (*Runner, error) { } scanopts.ExcludeCDN = options.ExcludeCDN - scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError + scanopts.HostMaxErrors = options.HostMaxErrors runner.scanopts = scanopts if options.ShowStatistics { @@ -232,7 +232,7 @@ func New(options *Options) (*Runner, error) { runner.ratelimiter = ratelimit.NewUnlimited() } - if options.SkipRemainingPathsOnError { + if options.HostMaxErrors >= 0 { gc := gcache.New(1000). ARC(). Build() @@ -389,7 +389,7 @@ func (r *Runner) Close() { // nolint:errcheck // ignore r.hm.Close() r.hp.Dialer.Close() - if r.options.SkipRemainingPathsOnError { + if r.options.HostMaxErrors >= 0 { r.FailedTargets.Purge() } } @@ -628,8 +628,11 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port) - if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) { - return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) { + numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort) + if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { + return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + } } // check if the combination host:port should be skipped if belonging to a cdn @@ -746,8 +749,13 @@ retry: } // mark the host:port as failed to avoid further checks - if r.options.SkipRemainingPathsOnError { - _ = r.FailedTargets.Set(hostPort, nil) + if r.options.HostMaxErrors >= 0 { + errorCount, err := r.FailedTargets.GetIFPresent(hostPort) + if err != nil || errorCount == nil { + _ = r.FailedTargets.Set(hostPort, 1) + } else if errorCount != nil { + _ = r.FailedTargets.Set(hostPort, errorCount.(int)+1) + } } if r.options.Probe { From 89f77e50d202f4f0d8dfdeacdc83ad204e75fb54 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 14:25:30 +0200 Subject: [PATCH 04/11] Adding support for maximum redirects number --- common/httpx/httpx.go | 20 +++++++++++++++----- common/httpx/option.go | 16 +++++++++------- runner/options.go | 2 ++ runner/runner.go | 1 + 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852..9d21771 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -59,20 +59,30 @@ func New(options *Options) (*HTTPX, error) { } if httpx.Options.FollowRedirects { - // Follow redirects - redirectFunc = nil + // Follow redirects up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } + return nil + } } if httpx.Options.FollowHostRedirects { - // Only follow redirects on the same host - redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error { + // Only follow redirects on the same host up to a maximum number + redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { // Check if we get a redirect to a differen host var newHost = redirectedRequest.URL.Host - var oldHost = previousRequest[0].URL.Host + var oldHost = previousRequests[0].URL.Host if newHost != oldHost { // Tell the http client to not follow redirect return http.ErrUseLastResponse } + if len(previousRequests) >= options.MaxRedirects { + // https://github.com/golang/go/issues/10069 + return http.ErrUseLastResponse + } return nil } } diff --git a/common/httpx/option.go b/common/httpx/option.go index 840a4ac..1106e9c 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -23,6 +23,7 @@ type Options struct { VHostSimilarityRatio int FollowRedirects bool FollowHostRedirects bool + MaxRedirects int Unsafe bool TLSGrab bool // VHOSTs options @@ -39,13 +40,14 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - Unsafe: false, - CdnCheck: true, - ExcludeCdn: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: true, + ExcludeCdn: false, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/runner/options.go b/runner/options.go index 9d42c6a..1b6bcba 100644 --- a/runner/options.go +++ b/runner/options.go @@ -154,6 +154,7 @@ type Options struct { responseInStdout bool chainInStdout bool FollowHostRedirects bool + MaxRedirects int OutputMethod bool TLSProbe bool CSPProbe bool @@ -209,6 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") diff --git a/runner/runner.go b/runner/runner.go index e0798f4..577ba58 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -81,6 +81,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.RetryMax = options.Retries httpxOptions.FollowRedirects = options.FollowRedirects httpxOptions.FollowHostRedirects = options.FollowHostRedirects + httpxOptions.MaxRedirects = options.MaxRedirects httpxOptions.HTTPProxy = options.HTTPProxy httpxOptions.Unsafe = options.Unsafe httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI} From 5931b811fbc8d5681b0800fb535691450b1870ab Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 15:08:04 +0200 Subject: [PATCH 05/11] Fixing inconsistent "input" field behavior --- runner/runner.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index e0798f4..9ed1946 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -513,7 +513,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. wg.Add() go func(target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, method, scanopts) + result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -541,7 +541,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. go func(port int, method, protocol string) { defer wg.Done() h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) - result := r.analyze(hp, protocol, h, method, scanopts) + result := r.analyze(hp, protocol, h, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -590,7 +590,7 @@ func targets(target string) chan string { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS { protocol = httpx.HTTPS @@ -602,20 +602,20 @@ retry: parts := strings.Split(domain, ",") //nolint:gomnd // not a magic number if len(parts) != 2 { - return Result{} + return Result{Input: origInput} } domain = parts[0] customHost = parts[1] } URL, err := urlutil.Parse(domain) if err != nil { - return Result{URL: domain, err: err} + return Result{URL: domain, Input: origInput, err: err} } // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) - return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")} + return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} } URL.Scheme = protocol @@ -629,7 +629,7 @@ retry: } req, err := hp.NewRequest(method, URL.String()) if err != nil { - return Result{URL: URL.String(), err: err} + return Result{URL: URL.String(), Input: origInput, err: err} } if customHost != "" { req.Host = customHost @@ -659,7 +659,7 @@ retry: var errDump error requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) if errDump != nil { - return Result{URL: URL.String(), err: errDump} + return Result{URL: URL.String(), Input: origInput, err: errDump} } } else { // Create a copy on the fly of the request body @@ -668,7 +668,7 @@ retry: var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { - return Result{URL: URL.String(), err: errDump} + return Result{URL: URL.String(), Input: origInput, err: errDump} } // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) @@ -725,9 +725,9 @@ retry: goto retry } if r.options.Probe { - return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { - return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err} + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err} } } @@ -972,7 +972,7 @@ retry: parsed, err := urlutil.Parse(fullURL) if err != nil { - return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")} + return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")} } finalPort := parsed.Port @@ -1016,7 +1016,7 @@ retry: HeaderSHA256: headersSha, raw: resp.Raw, URL: fullURL, - Input: domain, + Input: origInput, ContentLength: resp.ContentLength, ChainStatusCodes: chainStatusCodes, Chain: chainItems, From 38d56d5de94a130aa9b823f45d7ae0e64e4cc2f2 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 16:09:35 +0200 Subject: [PATCH 06/11] Improving case behavior with methods CLI option --- runner/options.go | 2 +- runner/runner.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 9d42c6a..83c23cc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -212,7 +212,7 @@ func ParseOptions() *Options { flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") - flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()") + flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)") flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") flag.BoolVar(&options.Silent, "silent", false, "Silent mode") flag.BoolVar(&options.Version, "version", false, "Show version of httpx") diff --git a/runner/runner.go b/runner/runner.go index e0798f4..25c0e9a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -155,6 +155,10 @@ func New(options *Options) (*Runner, error) { if strings.EqualFold(options.Methods, "all") { scanopts.Methods = pdhttputil.AllHTTPMethods() } else if options.Methods != "" { + // if unsafe is specified then converts the methods to uppercase + if !options.Unsafe { + options.Methods = strings.ToUpper(options.Methods) + } scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...) } if len(scanopts.Methods) == 0 { From d529a41c8d31cfb141d6636bac435fd00d1b13ae Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 16 Aug 2021 21:08:03 +0530 Subject: [PATCH 07/11] Update options.go --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 82d94cc..ac119cb 100644 --- a/runner/options.go +++ b/runner/options.go @@ -263,7 +263,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Probe, "probe", false, "Display probe status") flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.IntVar(&options.HostMaxErrors, "host-max-errors", -1, "Allowed error count per host before skipping remaining path/s") + flag.IntVar(&options.HostMaxErrors, "max-host-error", -1, "Max error count per host before skipping remaining path/s") flag.Parse() From 1e2e2d54e913b61e934d97f78c454364a4e7c5ed Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 19:10:12 +0200 Subject: [PATCH 08/11] Handling edge case with head method and unresponsive server --- common/httpx/httpx.go | 31 +++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 2 ++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852..b387371 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" "time" "unicode/utf8" @@ -18,6 +19,7 @@ import ( pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/stringsutil" "golang.org/x/net/http2" ) @@ -134,6 +136,13 @@ get_response: return nil, err } + var shouldIgnoreErrors, shouldIgnoreBodyErrors bool + switch { + case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"): + shouldIgnoreErrors = true + shouldIgnoreBodyErrors = true + } + var resp Response resp.Headers = httpresp.Header.Clone() @@ -148,23 +157,25 @@ get_response: req.Header.Set("Accept-Encoding", "identity") goto get_response } - return nil, err + if !shouldIgnoreErrors { + return nil, err + } } - resp.Raw = rawResp - resp.RawHeaders = headers + resp.Raw = string(rawResp) + resp.RawHeaders = string(headers) var respbody []byte // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) - if err != nil { + if err != nil && !shouldIgnoreBodyErrors { return nil, err } } closeErr := httpresp.Body.Close() - if closeErr != nil { + if closeErr != nil && !shouldIgnoreBodyErrors { return nil, closeErr } @@ -175,7 +186,15 @@ get_response: respbodystr = h.htmlPolicy.Sanitize(respbodystr) } - resp.ContentLength = utf8.RuneCountInString(respbodystr) + if contentLength, ok := resp.Headers["Content-Length"]; ok { + contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, "")) + if err != nil { + resp.ContentLength = utf8.RuneCountInString(respbodystr) + } else { + resp.ContentLength = contentLengthInt + } + } + resp.Data = respbody // fill metrics diff --git a/go.mod b/go.mod index e5ab7bb..cfd6dde 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa - github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d + github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b github.com/projectdiscovery/mapcidr v0.0.8 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e diff --git a/go.sum b/go.sum index 98b32ac..e585168 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= From 0f4f9f2cac38619994885f4784ff249794362e67 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 16 Aug 2021 23:52:52 +0530 Subject: [PATCH 09/11] minor update --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 1b6bcba..9761513 100644 --- a/runner/options.go +++ b/runner/options.go @@ -210,7 +210,7 @@ func ParseOptions() *Options { flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host") - flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow") + flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") From 6bc6359f3dbdc821a183e214e563f8b74d1d00c9 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 22:23:03 +0200 Subject: [PATCH 10/11] small name refactor --- runner/runner.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index ed83520..b2b4cbb 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -54,14 +54,14 @@ const ( // Runner is a client for running the enumeration process. type Runner struct { - options *Options - hp *httpx.HTTPX - wappalyzer *wappalyzer.Wappalyze - scanopts scanOptions - hm *hybrid.HybridMap - stats clistats.StatisticsClient - ratelimiter ratelimit.Limiter - FailedTargets gcache.Cache + options *Options + hp *httpx.HTTPX + wappalyzer *wappalyzer.Wappalyze + scanopts scanOptions + hm *hybrid.HybridMap + stats clistats.StatisticsClient + ratelimiter ratelimit.Limiter + HostErrorsCache gcache.Cache } // New creates a new client for running enumeration process. @@ -237,7 +237,7 @@ func New(options *Options) (*Runner, error) { gc := gcache.New(1000). ARC(). Build() - runner.FailedTargets = gc + runner.HostErrorsCache = gc } return runner, nil @@ -391,7 +391,7 @@ func (r *Runner) Close() { r.hm.Close() r.hp.Dialer.Close() if r.options.HostMaxErrors >= 0 { - r.FailedTargets.Purge() + r.HostErrorsCache.Purge() } } @@ -629,8 +629,8 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port) - if r.options.HostMaxErrors >= 0 && r.FailedTargets.Has(hostPort) { - numberOfErrors, err := r.FailedTargets.GetIFPresent(hostPort) + if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { + numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} } @@ -751,11 +751,11 @@ retry: // mark the host:port as failed to avoid further checks if r.options.HostMaxErrors >= 0 { - errorCount, err := r.FailedTargets.GetIFPresent(hostPort) + errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort) if err != nil || errorCount == nil { - _ = r.FailedTargets.Set(hostPort, 1) + _ = r.HostErrorsCache.Set(hostPort, 1) } else if errorCount != nil { - _ = r.FailedTargets.Set(hostPort, errorCount.(int)+1) + _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1) } } From b7d1d1794905ea9483c5838f467bae0f0fdcaa05 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 Aug 2021 17:24:23 +0530 Subject: [PATCH 11/11] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 713adc0..2beadea 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.1 + /_/ v1.1.2 ` // Version is the current version of httpx -const Version = `v1.1.1` +const Version = `v1.1.2` // showBanner is used to show the banner to the user func showBanner() {