Merge pull request #79 from projectdiscovery/feature-http2-probe

adding http2 slow probe

cmd/httpx/httpx.go

@@ -112,6 +112,7 @@ func main() {
 	scanopts.OutputContentType = options.OutputContentType
 	scanopts.RequestBody = options.RequestBody
 	scanopts.Unsafe = options.Unsafe
+	scanopts.HTTP2Probe = options.HTTP2Probe
 
 	// Try to create output folder if it doesnt exist
 	if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
@@ -298,6 +299,7 @@ type scanOptions struct {
 	OutputContentType      bool
 	RequestBody            string
 	Unsafe                 bool
+	HTTP2Probe             bool
 }
 
 func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions) Result {
@@ -436,6 +438,15 @@ retry:
 		builder.WriteString(" [websocket]")
 	}
 
+	var http2 bool
+	// if requested probes for http2
+	if scanopts.HTTP2Probe {
+		http2 = hp.SupportHTTP2(protocol, scanopts.Method, URL)
+		if http2 {
+			builder.WriteString(" [http2]")
+		}
+	}
+
 	// store responses in directory
 	if scanopts.StoreResponse {
 		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@@ -464,6 +475,7 @@ retry:
 		WebSocket:     isWebSocket,
 		TlsData:       resp.TlsData,
 		CspData:       resp.CspData,
+		HTTP2:         http2,
 	}
 }
 
@@ -483,6 +495,7 @@ type Result struct {
 	ContentType   string         `json:"content-type,omitempty"`
 	TlsData       *httpx.TlsData `json:"tls,omitempty"`
 	CspData       *httpx.CspData `json:"csp,omitempty"`
+	HTTP2         bool           `json:"http2"`
 }
 
 // JSON the result
@@ -541,6 +554,7 @@ type Options struct {
 	Unsafe                    bool
 	RequestBody               string
 	Debug                     bool
+	HTTP2Probe                bool
 }
 
 // ParseOptions parses the command line options for application
@@ -585,6 +599,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization")
 	flag.StringVar(&options.RequestBody, "body", "", "Request Body")
 	flag.BoolVar(&options.Debug, "debug", false, "Debug mode")
+	flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
 	flag.Parse()
 
 	// Read the inputs and configure the logging

common/httpx/http2.go

@@ -0,0 +1,44 @@
+package httpx
+
+import (
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+)
+
+// SupportHTTP2 checks if the target host supports HTTP2
+func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool {
+	// http => supports HTTP1.1 => HTTP/2 (H2C)
+	if protocol == "http" {
+		req, err := retryablehttp.NewRequest(method, URL, nil)
+		if err != nil {
+			return false
+		}
+		req.Header.Set("Connection", "Upgrade, HTTP2-Settings")
+		req.Header.Set("Upgrade", "h2c")
+		req.Header.Set("HTTP2-Settings", "AAMAAABkAARAAAAAAAIAAAAA")
+		httpresp, err := h.client.Do(req)
+		if err != nil {
+			return false
+		}
+		io.Copy(ioutil.Discard, httpresp.Body)
+		httpresp.Body.Close()
+
+		return httpresp.StatusCode == 101
+	}
+
+	// attempts a direct http2 connection
+	req, err := http.NewRequest(method, URL, nil)
+	if err != nil {
+		return false
+	}
+	httpresp, err := h.client2.Do(req)
+	if err != nil {
+		return false
+	}
+	io.Copy(ioutil.Discard, httpresp.Body)
+	httpresp.Body.Close()
+	return httpresp.Proto == "HTTP/2.0"
+}

common/httpx/httpx.go

@@ -14,11 +14,13 @@ import (
 	"github.com/projectdiscovery/httpx/common/httputilz"
 	"github.com/projectdiscovery/rawhttp"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
+	"golang.org/x/net/http2"
 )
 
 // HTTPX represent an instance of the library client
 type HTTPX struct {
 	client          *retryablehttp.Client
+	client2         *http.Client
 	Filters         []Filter
 	Options         *Options
 	htmlPolicy      *bluemonday.Policy
@@ -86,6 +88,16 @@ func New(options *Options) (*HTTPX, error) {
 		CheckRedirect: redirectFunc,
 	}, retryablehttpOptions)
 
+	httpx.client2 = &http.Client{
+		Transport: &http2.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			AllowHTTP: true,
+		},
+		Timeout: httpx.Options.Timeout,
+	}
+
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
 	httpx.RequestOverride = &options.RequestOverride

common/httpx/response.go

@@ -15,6 +15,8 @@ type Response struct {
 	Lines         int
 	TlsData       *TlsData
 	CspData       *CspData
+	Http2         bool
+	Pipeline      bool
 }
 
 // GetHeader value

go.mod

@@ -14,8 +14,6 @@ require (
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.2.1
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
-	golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
 	golang.org/x/text v0.3.3
 )

go.sum

@@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
 github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
-github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087 h1:FV+/XrXTWOaiW2hZVt/VWGxGBmOx/P9ChZnIkz7O96o=
-github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
 github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0=
 github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
@@ -41,8 +39,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
-golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -56,8 +52,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
-golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=