From 38d56d5de94a130aa9b823f45d7ae0e64e4cc2f2 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 16:09:35 +0200 Subject: [PATCH 1/2] Improving case behavior with methods CLI option --- runner/options.go | 2 +- runner/runner.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 9d42c6a..83c23cc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -212,7 +212,7 @@ func ParseOptions() *Options { flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") - flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()") + flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)") flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") flag.BoolVar(&options.Silent, "silent", false, "Silent mode") flag.BoolVar(&options.Version, "version", false, "Show version of httpx") diff --git a/runner/runner.go b/runner/runner.go index e0798f4..25c0e9a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -155,6 +155,10 @@ func New(options *Options) (*Runner, error) { if strings.EqualFold(options.Methods, "all") { scanopts.Methods = pdhttputil.AllHTTPMethods() } else if options.Methods != "" { + // if unsafe is specified then converts the methods to uppercase + if !options.Unsafe { + options.Methods = strings.ToUpper(options.Methods) + } scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...) } if len(scanopts.Methods) == 0 { From 1e2e2d54e913b61e934d97f78c454364a4e7c5ed Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 16 Aug 2021 19:10:12 +0200 Subject: [PATCH 2/2] Handling edge case with head method and unresponsive server --- common/httpx/httpx.go | 31 +++++++++++++++++++++++++------ go.mod | 2 +- go.sum | 2 ++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 33b3852..b387371 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "strings" "time" "unicode/utf8" @@ -18,6 +19,7 @@ import ( pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/stringsutil" "golang.org/x/net/http2" ) @@ -134,6 +136,13 @@ get_response: return nil, err } + var shouldIgnoreErrors, shouldIgnoreBodyErrors bool + switch { + case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"): + shouldIgnoreErrors = true + shouldIgnoreBodyErrors = true + } + var resp Response resp.Headers = httpresp.Header.Clone() @@ -148,23 +157,25 @@ get_response: req.Header.Set("Accept-Encoding", "identity") goto get_response } - return nil, err + if !shouldIgnoreErrors { + return nil, err + } } - resp.Raw = rawResp - resp.RawHeaders = headers + resp.Raw = string(rawResp) + resp.RawHeaders = string(headers) var respbody []byte // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) - if err != nil { + if err != nil && !shouldIgnoreBodyErrors { return nil, err } } closeErr := httpresp.Body.Close() - if closeErr != nil { + if closeErr != nil && !shouldIgnoreBodyErrors { return nil, closeErr } @@ -175,7 +186,15 @@ get_response: respbodystr = h.htmlPolicy.Sanitize(respbodystr) } - resp.ContentLength = utf8.RuneCountInString(respbodystr) + if contentLength, ok := resp.Headers["Content-Length"]; ok { + contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, "")) + if err != nil { + resp.ContentLength = utf8.RuneCountInString(respbodystr) + } else { + resp.ContentLength = contentLengthInt + } + } + resp.Data = respbody // fill metrics diff --git a/go.mod b/go.mod index e5ab7bb..cfd6dde 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa - github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d + github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b github.com/projectdiscovery/mapcidr v0.0.8 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e diff --git a/go.sum b/go.sum index 98b32ac..e585168 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0= github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= +github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=