mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-11-28 22:01:28 +03:00
Merge branch 'dev' into 354-bugfix-custom-ports-with-scheme
This commit is contained in:
commit
06bb53bbc8
@ -22,7 +22,7 @@ builds:
|
|||||||
goarch: 'arm'
|
goarch: 'arm'
|
||||||
|
|
||||||
binary: '{{ .ProjectName }}'
|
binary: '{{ .ProjectName }}'
|
||||||
main: cmd/httpx/main.go
|
main: cmd/httpx/httpx.go
|
||||||
|
|
||||||
archives:
|
archives:
|
||||||
- format: zip
|
- format: zip
|
||||||
|
@ -7,6 +7,7 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
@ -18,6 +19,7 @@ import (
|
|||||||
pdhttputil "github.com/projectdiscovery/httputil"
|
pdhttputil "github.com/projectdiscovery/httputil"
|
||||||
"github.com/projectdiscovery/rawhttp"
|
"github.com/projectdiscovery/rawhttp"
|
||||||
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
|
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
|
||||||
|
"github.com/projectdiscovery/stringsutil"
|
||||||
"golang.org/x/net/http2"
|
"golang.org/x/net/http2"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -59,20 +61,30 @@ func New(options *Options) (*HTTPX, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if httpx.Options.FollowRedirects {
|
if httpx.Options.FollowRedirects {
|
||||||
// Follow redirects
|
// Follow redirects up to a maximum number
|
||||||
redirectFunc = nil
|
redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
|
||||||
|
if len(previousRequests) >= options.MaxRedirects {
|
||||||
|
// https://github.com/golang/go/issues/10069
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if httpx.Options.FollowHostRedirects {
|
if httpx.Options.FollowHostRedirects {
|
||||||
// Only follow redirects on the same host
|
// Only follow redirects on the same host up to a maximum number
|
||||||
redirectFunc = func(redirectedRequest *http.Request, previousRequest []*http.Request) error {
|
redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error {
|
||||||
// Check if we get a redirect to a differen host
|
// Check if we get a redirect to a differen host
|
||||||
var newHost = redirectedRequest.URL.Host
|
var newHost = redirectedRequest.URL.Host
|
||||||
var oldHost = previousRequest[0].URL.Host
|
var oldHost = previousRequests[0].URL.Host
|
||||||
if newHost != oldHost {
|
if newHost != oldHost {
|
||||||
// Tell the http client to not follow redirect
|
// Tell the http client to not follow redirect
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
}
|
}
|
||||||
|
if len(previousRequests) >= options.MaxRedirects {
|
||||||
|
// https://github.com/golang/go/issues/10069
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -134,6 +146,13 @@ get_response:
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
|
||||||
|
switch {
|
||||||
|
case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
|
||||||
|
shouldIgnoreErrors = true
|
||||||
|
shouldIgnoreBodyErrors = true
|
||||||
|
}
|
||||||
|
|
||||||
var resp Response
|
var resp Response
|
||||||
|
|
||||||
resp.Headers = httpresp.Header.Clone()
|
resp.Headers = httpresp.Header.Clone()
|
||||||
@ -148,23 +167,25 @@ get_response:
|
|||||||
req.Header.Set("Accept-Encoding", "identity")
|
req.Header.Set("Accept-Encoding", "identity")
|
||||||
goto get_response
|
goto get_response
|
||||||
}
|
}
|
||||||
return nil, err
|
if !shouldIgnoreErrors {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
resp.Raw = rawResp
|
resp.Raw = string(rawResp)
|
||||||
resp.RawHeaders = headers
|
resp.RawHeaders = string(headers)
|
||||||
|
|
||||||
var respbody []byte
|
var respbody []byte
|
||||||
// websockets don't have a readable body
|
// websockets don't have a readable body
|
||||||
if httpresp.StatusCode != http.StatusSwitchingProtocols {
|
if httpresp.StatusCode != http.StatusSwitchingProtocols {
|
||||||
var err error
|
var err error
|
||||||
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
|
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
|
||||||
if err != nil {
|
if err != nil && !shouldIgnoreBodyErrors {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
closeErr := httpresp.Body.Close()
|
closeErr := httpresp.Body.Close()
|
||||||
if closeErr != nil {
|
if closeErr != nil && !shouldIgnoreBodyErrors {
|
||||||
return nil, closeErr
|
return nil, closeErr
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -175,7 +196,15 @@ get_response:
|
|||||||
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
|
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.ContentLength = utf8.RuneCountInString(respbodystr)
|
if contentLength, ok := resp.Headers["Content-Length"]; ok {
|
||||||
|
contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
|
||||||
|
if err != nil {
|
||||||
|
resp.ContentLength = utf8.RuneCountInString(respbodystr)
|
||||||
|
} else {
|
||||||
|
resp.ContentLength = contentLengthInt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
resp.Data = respbody
|
resp.Data = respbody
|
||||||
|
|
||||||
// fill metrics
|
// fill metrics
|
||||||
|
@ -23,6 +23,7 @@ type Options struct {
|
|||||||
VHostSimilarityRatio int
|
VHostSimilarityRatio int
|
||||||
FollowRedirects bool
|
FollowRedirects bool
|
||||||
FollowHostRedirects bool
|
FollowHostRedirects bool
|
||||||
|
MaxRedirects int
|
||||||
Unsafe bool
|
Unsafe bool
|
||||||
TLSGrab bool
|
TLSGrab bool
|
||||||
// VHOSTs options
|
// VHOSTs options
|
||||||
@ -39,13 +40,14 @@ type Options struct {
|
|||||||
|
|
||||||
// DefaultOptions contains the default options
|
// DefaultOptions contains the default options
|
||||||
var DefaultOptions = Options{
|
var DefaultOptions = Options{
|
||||||
RandomAgent: true,
|
RandomAgent: true,
|
||||||
Threads: 25,
|
Threads: 25,
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
RetryMax: 5,
|
RetryMax: 5,
|
||||||
Unsafe: false,
|
MaxRedirects: 10,
|
||||||
CdnCheck: true,
|
Unsafe: false,
|
||||||
ExcludeCdn: false,
|
CdnCheck: true,
|
||||||
|
ExcludeCdn: false,
|
||||||
// VHOSTs options
|
// VHOSTs options
|
||||||
VHostIgnoreStatusCode: false,
|
VHostIgnoreStatusCode: false,
|
||||||
VHostIgnoreContentLength: true,
|
VHostIgnoreContentLength: true,
|
||||||
|
3
go.mod
3
go.mod
@ -4,6 +4,7 @@ go 1.14
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/akrylysov/pogreb v0.10.1 // indirect
|
github.com/akrylysov/pogreb v0.10.1 // indirect
|
||||||
|
github.com/bluele/gcache v0.0.2 // indirect
|
||||||
github.com/corpix/uarand v0.1.1
|
github.com/corpix/uarand v0.1.1
|
||||||
github.com/dgraph-io/ristretto v0.1.0 // indirect
|
github.com/dgraph-io/ristretto v0.1.0 // indirect
|
||||||
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
|
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
|
||||||
@ -21,7 +22,7 @@ require (
|
|||||||
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
|
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
|
||||||
github.com/projectdiscovery/gologger v1.1.4
|
github.com/projectdiscovery/gologger v1.1.4
|
||||||
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
|
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
|
||||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
|
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
|
||||||
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
|
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
|
||||||
github.com/projectdiscovery/mapcidr v0.0.8
|
github.com/projectdiscovery/mapcidr v0.0.8
|
||||||
github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e
|
github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e
|
||||||
|
4
go.sum
4
go.sum
@ -15,6 +15,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
|
|||||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||||
|
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||||
|
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||||
@ -153,6 +155,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ
|
|||||||
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
|
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
|
||||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
|
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
|
||||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
|
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
|
||||||
|
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
|
||||||
|
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
|
||||||
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
|
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
|
||||||
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||||
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||||
|
@ -8,11 +8,11 @@ const banner = `
|
|||||||
/ __ \/ __/ __/ __ \| /
|
/ __ \/ __/ __/ __ \| /
|
||||||
/ / / / /_/ /_/ /_/ / |
|
/ / / / /_/ /_/ /_/ / |
|
||||||
/_/ /_/\__/\__/ .___/_/|_|
|
/_/ /_/\__/\__/ .___/_/|_|
|
||||||
/_/ v1.1.1
|
/_/ v1.1.2
|
||||||
`
|
`
|
||||||
|
|
||||||
// Version is the current version of httpx
|
// Version is the current version of httpx
|
||||||
const Version = `v1.1.1`
|
const Version = `v1.1.2`
|
||||||
|
|
||||||
// showBanner is used to show the banner to the user
|
// showBanner is used to show the banner to the user
|
||||||
func showBanner() {
|
func showBanner() {
|
||||||
|
@ -61,6 +61,7 @@ type scanOptions struct {
|
|||||||
OutputExtractRegex string
|
OutputExtractRegex string
|
||||||
extractRegex *regexp.Regexp
|
extractRegex *regexp.Regexp
|
||||||
ExcludeCDN bool
|
ExcludeCDN bool
|
||||||
|
HostMaxErrors int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *scanOptions) Clone() *scanOptions {
|
func (s *scanOptions) Clone() *scanOptions {
|
||||||
@ -99,6 +100,7 @@ func (s *scanOptions) Clone() *scanOptions {
|
|||||||
OutputExtractRegex: s.OutputExtractRegex,
|
OutputExtractRegex: s.OutputExtractRegex,
|
||||||
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
||||||
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
||||||
|
HostMaxErrors: s.HostMaxErrors,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -154,6 +156,7 @@ type Options struct {
|
|||||||
responseInStdout bool
|
responseInStdout bool
|
||||||
chainInStdout bool
|
chainInStdout bool
|
||||||
FollowHostRedirects bool
|
FollowHostRedirects bool
|
||||||
|
MaxRedirects int
|
||||||
OutputMethod bool
|
OutputMethod bool
|
||||||
TLSProbe bool
|
TLSProbe bool
|
||||||
CSPProbe bool
|
CSPProbe bool
|
||||||
@ -184,6 +187,7 @@ type Options struct {
|
|||||||
Resume bool
|
Resume bool
|
||||||
resumeCfg *ResumeCfg
|
resumeCfg *ResumeCfg
|
||||||
ExcludeCDN bool
|
ExcludeCDN bool
|
||||||
|
HostMaxErrors int
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseOptions parses the command line options for application
|
// ParseOptions parses the command line options for application
|
||||||
@ -209,10 +213,11 @@ func ParseOptions() *Options {
|
|||||||
flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
|
flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
|
||||||
flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
|
flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
|
||||||
flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
|
flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
|
||||||
|
flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host")
|
||||||
flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
|
flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
|
||||||
flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
|
flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
|
||||||
flag.StringVar(&options.InputFile, "l", "", "File containing domains")
|
flag.StringVar(&options.InputFile, "l", "", "File containing domains")
|
||||||
flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
|
flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)")
|
||||||
flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
|
flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
|
||||||
flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
|
flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
|
||||||
flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
|
flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
|
||||||
@ -260,6 +265,7 @@ func ParseOptions() *Options {
|
|||||||
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
|
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")
|
||||||
flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
|
flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg")
|
||||||
flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
|
flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)")
|
||||||
|
flag.IntVar(&options.HostMaxErrors, "max-host-error", -1, "Max error count per host before skipping remaining path/s")
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
@ -8,6 +8,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
"net/url"
|
"net/url"
|
||||||
@ -19,6 +20,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/bluele/gcache"
|
||||||
"github.com/logrusorgru/aurora"
|
"github.com/logrusorgru/aurora"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/projectdiscovery/clistats"
|
"github.com/projectdiscovery/clistats"
|
||||||
@ -52,13 +54,14 @@ const (
|
|||||||
|
|
||||||
// Runner is a client for running the enumeration process.
|
// Runner is a client for running the enumeration process.
|
||||||
type Runner struct {
|
type Runner struct {
|
||||||
options *Options
|
options *Options
|
||||||
hp *httpx.HTTPX
|
hp *httpx.HTTPX
|
||||||
wappalyzer *wappalyzer.Wappalyze
|
wappalyzer *wappalyzer.Wappalyze
|
||||||
scanopts scanOptions
|
scanopts scanOptions
|
||||||
hm *hybrid.HybridMap
|
hm *hybrid.HybridMap
|
||||||
stats clistats.StatisticsClient
|
stats clistats.StatisticsClient
|
||||||
ratelimiter ratelimit.Limiter
|
ratelimiter ratelimit.Limiter
|
||||||
|
HostErrorsCache gcache.Cache
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new client for running enumeration process.
|
// New creates a new client for running enumeration process.
|
||||||
@ -81,6 +84,7 @@ func New(options *Options) (*Runner, error) {
|
|||||||
httpxOptions.RetryMax = options.Retries
|
httpxOptions.RetryMax = options.Retries
|
||||||
httpxOptions.FollowRedirects = options.FollowRedirects
|
httpxOptions.FollowRedirects = options.FollowRedirects
|
||||||
httpxOptions.FollowHostRedirects = options.FollowHostRedirects
|
httpxOptions.FollowHostRedirects = options.FollowHostRedirects
|
||||||
|
httpxOptions.MaxRedirects = options.MaxRedirects
|
||||||
httpxOptions.HTTPProxy = options.HTTPProxy
|
httpxOptions.HTTPProxy = options.HTTPProxy
|
||||||
httpxOptions.Unsafe = options.Unsafe
|
httpxOptions.Unsafe = options.Unsafe
|
||||||
httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
|
httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
|
||||||
@ -155,6 +159,10 @@ func New(options *Options) (*Runner, error) {
|
|||||||
if strings.EqualFold(options.Methods, "all") {
|
if strings.EqualFold(options.Methods, "all") {
|
||||||
scanopts.Methods = pdhttputil.AllHTTPMethods()
|
scanopts.Methods = pdhttputil.AllHTTPMethods()
|
||||||
} else if options.Methods != "" {
|
} else if options.Methods != "" {
|
||||||
|
// if unsafe is specified then converts the methods to uppercase
|
||||||
|
if !options.Unsafe {
|
||||||
|
options.Methods = strings.ToUpper(options.Methods)
|
||||||
|
}
|
||||||
scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
|
scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
|
||||||
}
|
}
|
||||||
if len(scanopts.Methods) == 0 {
|
if len(scanopts.Methods) == 0 {
|
||||||
@ -207,6 +215,7 @@ func New(options *Options) (*Runner, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
scanopts.ExcludeCDN = options.ExcludeCDN
|
scanopts.ExcludeCDN = options.ExcludeCDN
|
||||||
|
scanopts.HostMaxErrors = options.HostMaxErrors
|
||||||
runner.scanopts = scanopts
|
runner.scanopts = scanopts
|
||||||
|
|
||||||
if options.ShowStatistics {
|
if options.ShowStatistics {
|
||||||
@ -228,6 +237,13 @@ func New(options *Options) (*Runner, error) {
|
|||||||
runner.ratelimiter = ratelimit.NewUnlimited()
|
runner.ratelimiter = ratelimit.NewUnlimited()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if options.HostMaxErrors >= 0 {
|
||||||
|
gc := gcache.New(1000).
|
||||||
|
ARC().
|
||||||
|
Build()
|
||||||
|
runner.HostErrorsCache = gc
|
||||||
|
}
|
||||||
|
|
||||||
return runner, nil
|
return runner, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -378,6 +394,9 @@ func (r *Runner) Close() {
|
|||||||
// nolint:errcheck // ignore
|
// nolint:errcheck // ignore
|
||||||
r.hm.Close()
|
r.hm.Close()
|
||||||
r.hp.Dialer.Close()
|
r.hp.Dialer.Close()
|
||||||
|
if r.options.HostMaxErrors >= 0 {
|
||||||
|
r.HostErrorsCache.Purge()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunEnumeration on targets for httpx client
|
// RunEnumeration on targets for httpx client
|
||||||
@ -513,7 +532,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
|
|||||||
wg.Add()
|
wg.Add()
|
||||||
go func(target, method, protocol string) {
|
go func(target, method, protocol string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
result := r.analyze(hp, protocol, target, method, scanopts)
|
result := r.analyze(hp, protocol, target, method, t, scanopts)
|
||||||
output <- result
|
output <- result
|
||||||
if scanopts.TLSProbe && result.TLSData != nil {
|
if scanopts.TLSProbe && result.TLSData != nil {
|
||||||
scanopts.TLSProbe = false
|
scanopts.TLSProbe = false
|
||||||
@ -546,7 +565,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
|
|||||||
go func(port int, method, protocol string) {
|
go func(port int, method, protocol string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
|
h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
|
||||||
result := r.analyze(hp, protocol, h, method, scanopts)
|
result := r.analyze(hp, protocol, h, method, t, scanopts)
|
||||||
output <- result
|
output <- result
|
||||||
if scanopts.TLSProbe && result.TLSData != nil {
|
if scanopts.TLSProbe && result.TLSData != nil {
|
||||||
scanopts.TLSProbe = false
|
scanopts.TLSProbe = false
|
||||||
@ -596,7 +615,7 @@ func targets(target string) chan string {
|
|||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result {
|
func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result {
|
||||||
origProtocol := protocol
|
origProtocol := protocol
|
||||||
if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
|
if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
|
||||||
protocol = httpx.HTTPS
|
protocol = httpx.HTTPS
|
||||||
@ -608,20 +627,29 @@ retry:
|
|||||||
parts := strings.Split(domain, ",")
|
parts := strings.Split(domain, ",")
|
||||||
//nolint:gomnd // not a magic number
|
//nolint:gomnd // not a magic number
|
||||||
if len(parts) != 2 {
|
if len(parts) != 2 {
|
||||||
return Result{}
|
return Result{Input: origInput}
|
||||||
}
|
}
|
||||||
domain = parts[0]
|
domain = parts[0]
|
||||||
customHost = parts[1]
|
customHost = parts[1]
|
||||||
}
|
}
|
||||||
URL, err := urlutil.Parse(domain)
|
URL, err := urlutil.Parse(domain)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{URL: domain, err: err}
|
return Result{URL: domain, Input: origInput, err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if we have to skip the host:port as a result of a previous failure
|
||||||
|
hostPort := net.JoinHostPort(URL.Host, URL.Port)
|
||||||
|
if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
|
||||||
|
numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
|
||||||
|
if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors {
|
||||||
|
return Result{URL: domain, err: errors.New("skipping as previously unresponsive")}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if the combination host:port should be skipped if belonging to a cdn
|
// check if the combination host:port should be skipped if belonging to a cdn
|
||||||
if r.skipCDNPort(URL.Host, URL.Port) {
|
if r.skipCDNPort(URL.Host, URL.Port) {
|
||||||
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
|
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port)
|
||||||
return Result{URL: domain, err: errors.New("cdn target only allows ports 80 and 443")}
|
return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")}
|
||||||
}
|
}
|
||||||
|
|
||||||
URL.Scheme = protocol
|
URL.Scheme = protocol
|
||||||
@ -635,7 +663,7 @@ retry:
|
|||||||
}
|
}
|
||||||
req, err := hp.NewRequest(method, URL.String())
|
req, err := hp.NewRequest(method, URL.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{URL: URL.String(), err: err}
|
return Result{URL: URL.String(), Input: origInput, err: err}
|
||||||
}
|
}
|
||||||
if customHost != "" {
|
if customHost != "" {
|
||||||
req.Host = customHost
|
req.Host = customHost
|
||||||
@ -665,7 +693,7 @@ retry:
|
|||||||
var errDump error
|
var errDump error
|
||||||
requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
|
requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
|
||||||
if errDump != nil {
|
if errDump != nil {
|
||||||
return Result{URL: URL.String(), err: errDump}
|
return Result{URL: URL.String(), Input: origInput, err: errDump}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Create a copy on the fly of the request body
|
// Create a copy on the fly of the request body
|
||||||
@ -674,7 +702,7 @@ retry:
|
|||||||
var errDump error
|
var errDump error
|
||||||
requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
|
requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
|
||||||
if errDump != nil {
|
if errDump != nil {
|
||||||
return Result{URL: URL.String(), err: errDump}
|
return Result{URL: URL.String(), Input: origInput, err: errDump}
|
||||||
}
|
}
|
||||||
// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
|
// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
|
||||||
// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
|
// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
|
||||||
@ -730,10 +758,21 @@ retry:
|
|||||||
retried = true
|
retried = true
|
||||||
goto retry
|
goto retry
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mark the host:port as failed to avoid further checks
|
||||||
|
if r.options.HostMaxErrors >= 0 {
|
||||||
|
errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort)
|
||||||
|
if err != nil || errorCount == nil {
|
||||||
|
_ = r.HostErrorsCache.Set(hostPort, 1)
|
||||||
|
} else if errorCount != nil {
|
||||||
|
_ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if r.options.Probe {
|
if r.options.Probe {
|
||||||
return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
|
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
|
||||||
} else {
|
} else {
|
||||||
return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err}
|
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -978,7 +1017,7 @@ retry:
|
|||||||
|
|
||||||
parsed, err := urlutil.Parse(fullURL)
|
parsed, err := urlutil.Parse(fullURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")}
|
return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")}
|
||||||
}
|
}
|
||||||
|
|
||||||
finalPort := parsed.Port
|
finalPort := parsed.Port
|
||||||
@ -1022,7 +1061,7 @@ retry:
|
|||||||
HeaderSHA256: headersSha,
|
HeaderSHA256: headersSha,
|
||||||
raw: resp.Raw,
|
raw: resp.Raw,
|
||||||
URL: fullURL,
|
URL: fullURL,
|
||||||
Input: domain,
|
Input: origInput,
|
||||||
ContentLength: resp.ContentLength,
|
ContentLength: resp.ContentLength,
|
||||||
ChainStatusCodes: chainStatusCodes,
|
ChainStatusCodes: chainStatusCodes,
|
||||||
Chain: chainItems,
|
Chain: chainItems,
|
||||||
|
Loading…
Reference in New Issue
Block a user