mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-10-26 18:08:27 +03:00
Adding option to skip remaining paths on unresponsive host
This commit is contained in:
parent
07a14c712c
commit
d89b3c14fb
1
go.mod
1
go.mod
@ -4,6 +4,7 @@ go 1.14
|
||||
|
||||
require (
|
||||
github.com/akrylysov/pogreb v0.10.1 // indirect
|
||||
github.com/bluele/gcache v0.0.2 // indirect
|
||||
github.com/corpix/uarand v0.1.1
|
||||
github.com/dgraph-io/ristretto v0.1.0 // indirect
|
||||
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
|
||||
|
2
go.sum
2
go.sum
@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
|
@ -61,6 +61,7 @@ type scanOptions struct {
|
||||
OutputExtractRegex string
|
||||
extractRegex *regexp.Regexp
|
||||
ExcludeCDN bool
|
||||
SkipRemainingPathsOnError bool
|
||||
}
|
||||
|
||||
func (s *scanOptions) Clone() *scanOptions {
|
||||
@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
OutputExtractRegex: s.OutputExtractRegex,
|
||||
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
||||
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
||||
SkipRemainingPathsOnError: s.SkipRemainingPathsOnError,
|
||||
}
|
||||
}
|
||||
|
||||
@ -184,6 +186,7 @@ type Options struct {
|
||||
Resume bool
|
||||
resumeCfg *ResumeCfg
|
||||
ExcludeCDN bool
|
||||
SkipRemainingPathsOnError bool
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -260,6 +263,7 @@ func ParseOptions() *Options {
|
||||
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
|
||||
flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
|
||||
flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
|
||||
flag.BoolVar(&options.SkipRemainingPathsOnError, "skip-paths-on-error", false, "Skip remaining paths on unresponsive servers")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
@ -19,6 +20,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/clistats"
|
||||
@ -52,13 +54,14 @@ const (
|
||||
|
||||
// Runner is a client for running the enumeration process.
|
||||
type Runner struct {
|
||||
options *Options
|
||||
hp *httpx.HTTPX
|
||||
wappalyzer *wappalyzer.Wappalyze
|
||||
scanopts scanOptions
|
||||
hm *hybrid.HybridMap
|
||||
stats clistats.StatisticsClient
|
||||
ratelimiter ratelimit.Limiter
|
||||
options *Options
|
||||
hp *httpx.HTTPX
|
||||
wappalyzer *wappalyzer.Wappalyze
|
||||
scanopts scanOptions
|
||||
hm *hybrid.HybridMap
|
||||
stats clistats.StatisticsClient
|
||||
ratelimiter ratelimit.Limiter
|
||||
FailedTargets gcache.Cache
|
||||
}
|
||||
|
||||
// New creates a new client for running enumeration process.
|
||||
@ -207,6 +210,7 @@ func New(options *Options) (*Runner, error) {
|
||||
}
|
||||
|
||||
scanopts.ExcludeCDN = options.ExcludeCDN
|
||||
scanopts.SkipRemainingPathsOnError = options.SkipRemainingPathsOnError
|
||||
runner.scanopts = scanopts
|
||||
|
||||
if options.ShowStatistics {
|
||||
@ -228,6 +232,13 @@ func New(options *Options) (*Runner, error) {
|
||||
runner.ratelimiter = ratelimit.NewUnlimited()
|
||||
}
|
||||
|
||||
if options.SkipRemainingPathsOnError {
|
||||
gc := gcache.New(1000).
|
||||
ARC().
|
||||
Build()
|
||||
runner.FailedTargets = gc
|
||||
}
|
||||
|
||||
return runner, nil
|
||||
}
|
||||
|
||||
@ -378,6 +389,9 @@ func (r *Runner) Close() {
|
||||
// nolint:errcheck // ignore
|
||||
r.hm.Close()
|
||||
r.hp.Dialer.Close()
|
||||
if r.options.SkipRemainingPathsOnError {
|
||||
r.FailedTargets.Purge()
|
||||
}
|
||||
}
|
||||
|
||||
// RunEnumeration on targets for httpx client
|
||||
@ -612,6 +626,12 @@ retry:
|
||||
return Result{URL: domain, err: err}
|
||||
}
|
||||
|
||||
// check if we have to skip the host:port as a result of a previous failure
|
||||
hostPort := net.JoinHostPort(URL.Host, URL.Port)
|
||||
if r.options.SkipRemainingPathsOnError && r.FailedTargets.Has(hostPort) {
|
||||
return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
|
||||
}
|
||||
|
||||
// check if the combination host:port should be skipped if belonging to a cdn
|
||||
if r.skipCDNPort(URL.Host, URL.Port) {
|
||||
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
|
||||
@ -724,6 +744,12 @@ retry:
|
||||
retried = true
|
||||
goto retry
|
||||
}
|
||||
|
||||
// mark the host:port as failed to avoid further checks
|
||||
if r.options.SkipRemainingPathsOnError {
|
||||
_ = r.FailedTargets.Set(hostPort, nil)
|
||||
}
|
||||
|
||||
if r.options.Probe {
|
||||
return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
|
||||
} else {
|
||||
|
Loading…
Reference in New Issue
Block a user