From 0c9d679acc4e097bd87dfb87b4850b15bad83899 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 26 Aug 2020 22:13:13 +0200 Subject: [PATCH 1/2] adding http2 slow probe --- cmd/httpx/httpx.go | 15 ++++++++++++++ common/httpx/http2.go | 44 ++++++++++++++++++++++++++++++++++++++++ common/httpx/httpx.go | 12 +++++++++++ common/httpx/response.go | 2 ++ go.mod | 2 -- go.sum | 6 ------ 6 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 common/httpx/http2.go diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 3198881..e5bbfa4 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -112,6 +112,7 @@ func main() { scanopts.OutputContentType = options.OutputContentType scanopts.RequestBody = options.RequestBody scanopts.Unsafe = options.Unsafe + scanopts.HTTP2Probe = options.HTTP2Probe // Try to create output folder if it doesnt exist if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) { @@ -298,6 +299,7 @@ type scanOptions struct { OutputContentType bool RequestBody string Unsafe bool + HTTP2Probe bool } func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions) Result { @@ -436,6 +438,15 @@ retry: builder.WriteString(" [websocket]") } + var http2 bool + // if requested probes for http2 + if scanopts.HTTP2Probe { + http2 = hp.SupportHTTP2(protocol, scanopts.Method, URL) + if http2 { + builder.WriteString(" [http2]") + } + } + // store responses in directory if scanopts.StoreResponse { domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI) @@ -464,6 +475,7 @@ retry: WebSocket: isWebSocket, TlsData: resp.TlsData, CspData: resp.CspData, + HTTP2: http2, } } @@ -483,6 +495,7 @@ type Result struct { ContentType string `json:"content-type,omitempty"` TlsData *httpx.TlsData `json:"tls,omitempty"` CspData *httpx.CspData `json:"csp,omitempty"` + HTTP2 bool `json:"http2,omitempty"` } // JSON the result @@ -541,6 +554,7 @@ type Options struct { Unsafe bool RequestBody string Debug bool + HTTP2Probe bool } // ParseOptions parses the command line options for application @@ -585,6 +599,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization") flag.StringVar(&options.RequestBody, "body", "", "Request Body") flag.BoolVar(&options.Debug, "debug", false, "Debug mode") + flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") flag.Parse() // Read the inputs and configure the logging diff --git a/common/httpx/http2.go b/common/httpx/http2.go new file mode 100644 index 0000000..9f9e582 --- /dev/null +++ b/common/httpx/http2.go @@ -0,0 +1,44 @@ +package httpx + +import ( + "io" + "io/ioutil" + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +// SupportHTTP2 checks if the target host supports HTTP2 +func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool { + // http => supports HTTP1.1 => HTTP/2 (H2C) + if protocol == "http" { + req, err := retryablehttp.NewRequest(method, URL, nil) + if err != nil { + return false + } + req.Header.Set("Connection", "Upgrade, HTTP2-Settings") + req.Header.Set("Upgrade", "h2c") + req.Header.Set("HTTP2-Settings", "AAMAAABkAARAAAAAAAIAAAAA") + httpresp, err := h.client.Do(req) + if err != nil { + return false + } + io.Copy(ioutil.Discard, httpresp.Body) + httpresp.Body.Close() + + return httpresp.StatusCode == 101 + } + + // attempts a direct http2 connection + req, err := http.NewRequest(method, URL, nil) + if err != nil { + return false + } + httpresp, err := h.client2.Do(req) + if err != nil { + return false + } + io.Copy(ioutil.Discard, httpresp.Body) + httpresp.Body.Close() + return httpresp.Proto == "HTTP/2.0" +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 929e957..cff2e2a 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -14,11 +14,13 @@ import ( "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" + "golang.org/x/net/http2" ) // HTTPX represent an instance of the library client type HTTPX struct { client *retryablehttp.Client + client2 *http.Client Filters []Filter Options *Options htmlPolicy *bluemonday.Policy @@ -86,6 +88,16 @@ func New(options *Options) (*HTTPX, error) { CheckRedirect: redirectFunc, }, retryablehttpOptions) + httpx.client2 = &http.Client{ + Transport: &http2.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + AllowHTTP: true, + }, + Timeout: httpx.Options.Timeout, + } + httpx.htmlPolicy = bluemonday.NewPolicy() httpx.CustomHeaders = httpx.Options.CustomHeaders httpx.RequestOverride = &options.RequestOverride diff --git a/common/httpx/response.go b/common/httpx/response.go index bacf91b..9bda8ce 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -15,6 +15,8 @@ type Response struct { Lines int TlsData *TlsData CspData *CspData + Http2 bool + Pipeline bool } // GetHeader value diff --git a/go.mod b/go.mod index aa16cd4..2768df3 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,6 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.2.1 - golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect golang.org/x/net v0.0.0-20200822124328-c89045814202 - golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect golang.org/x/text v0.3.3 ) diff --git a/go.sum b/go.sum index a7ddb91..a119b3b 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= -github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087 h1:FV+/XrXTWOaiW2hZVt/VWGxGBmOx/P9ChZnIkz7O96o= -github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4= github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0= github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -41,8 +39,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -56,8 +52,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 6dc35b22b5daa39e0c873e5ec3d499e50389b7a5 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 30 Aug 2020 17:47:38 +0200 Subject: [PATCH 2/2] always showing http2 in json structure --- cmd/httpx/httpx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index e5bbfa4..48acfe3 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -495,7 +495,7 @@ type Result struct { ContentType string `json:"content-type,omitempty"` TlsData *httpx.TlsData `json:"tls,omitempty"` CspData *httpx.CspData `json:"csp,omitempty"` - HTTP2 bool `json:"http2,omitempty"` + HTTP2 bool `json:"http2"` } // JSON the result