Merge pull request #79 from projectdiscovery/feature-http2-probe

adding http2 slow probe
This commit is contained in:
bauthard 2020-08-30 21:22:59 +05:30 committed by GitHub
commit a632271728
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 73 additions and 8 deletions

View File

@ -112,6 +112,7 @@ func main() {
scanopts.OutputContentType = options.OutputContentType
scanopts.RequestBody = options.RequestBody
scanopts.Unsafe = options.Unsafe
scanopts.HTTP2Probe = options.HTTP2Probe
// Try to create output folder if it doesnt exist
if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
@ -298,6 +299,7 @@ type scanOptions struct {
OutputContentType bool
RequestBody string
Unsafe bool
HTTP2Probe bool
}
func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions) Result {
@ -436,6 +438,15 @@ retry:
builder.WriteString(" [websocket]")
}
var http2 bool
// if requested probes for http2
if scanopts.HTTP2Probe {
http2 = hp.SupportHTTP2(protocol, scanopts.Method, URL)
if http2 {
builder.WriteString(" [http2]")
}
}
// store responses in directory
if scanopts.StoreResponse {
domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@ -464,6 +475,7 @@ retry:
WebSocket: isWebSocket,
TlsData: resp.TlsData,
CspData: resp.CspData,
HTTP2: http2,
}
}
@ -483,6 +495,7 @@ type Result struct {
ContentType string `json:"content-type,omitempty"`
TlsData *httpx.TlsData `json:"tls,omitempty"`
CspData *httpx.CspData `json:"csp,omitempty"`
HTTP2 bool `json:"http2"`
}
// JSON the result
@ -541,6 +554,7 @@ type Options struct {
Unsafe bool
RequestBody string
Debug bool
HTTP2Probe bool
}
// ParseOptions parses the command line options for application
@ -585,6 +599,7 @@ func ParseOptions() *Options {
flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization")
flag.StringVar(&options.RequestBody, "body", "", "Request Body")
flag.BoolVar(&options.Debug, "debug", false, "Debug mode")
flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
flag.Parse()
// Read the inputs and configure the logging

44
common/httpx/http2.go Normal file
View File

@ -0,0 +1,44 @@
package httpx
import (
"io"
"io/ioutil"
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
// SupportHTTP2 checks if the target host supports HTTP2
func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool {
// http => supports HTTP1.1 => HTTP/2 (H2C)
if protocol == "http" {
req, err := retryablehttp.NewRequest(method, URL, nil)
if err != nil {
return false
}
req.Header.Set("Connection", "Upgrade, HTTP2-Settings")
req.Header.Set("Upgrade", "h2c")
req.Header.Set("HTTP2-Settings", "AAMAAABkAARAAAAAAAIAAAAA")
httpresp, err := h.client.Do(req)
if err != nil {
return false
}
io.Copy(ioutil.Discard, httpresp.Body)
httpresp.Body.Close()
return httpresp.StatusCode == 101
}
// attempts a direct http2 connection
req, err := http.NewRequest(method, URL, nil)
if err != nil {
return false
}
httpresp, err := h.client2.Do(req)
if err != nil {
return false
}
io.Copy(ioutil.Discard, httpresp.Body)
httpresp.Body.Close()
return httpresp.Proto == "HTTP/2.0"
}

View File

@ -14,11 +14,13 @@ import (
"github.com/projectdiscovery/httpx/common/httputilz"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/net/http2"
)
// HTTPX represent an instance of the library client
type HTTPX struct {
client *retryablehttp.Client
client2 *http.Client
Filters []Filter
Options *Options
htmlPolicy *bluemonday.Policy
@ -86,6 +88,16 @@ func New(options *Options) (*HTTPX, error) {
CheckRedirect: redirectFunc,
}, retryablehttpOptions)
httpx.client2 = &http.Client{
Transport: &http2.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
AllowHTTP: true,
},
Timeout: httpx.Options.Timeout,
}
httpx.htmlPolicy = bluemonday.NewPolicy()
httpx.CustomHeaders = httpx.Options.CustomHeaders
httpx.RequestOverride = &options.RequestOverride

View File

@ -15,6 +15,8 @@ type Response struct {
Lines int
TlsData *TlsData
CspData *CspData
Http2 bool
Pipeline bool
}
// GetHeader value

2
go.mod
View File

@ -14,8 +14,6 @@ require (
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.2.1
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
golang.org/x/net v0.0.0-20200822124328-c89045814202
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
golang.org/x/text v0.3.3
)

6
go.sum
View File

@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087 h1:FV+/XrXTWOaiW2hZVt/VWGxGBmOx/P9ChZnIkz7O96o=
github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0=
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
@ -41,8 +39,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -56,8 +52,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=