mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-11-28 22:01:28 +03:00
Merge branch 'master' into feature-pipeline
This commit is contained in:
Mzack9999
GitHub
commit
94c8c2741d
@ -117,6 +117,7 @@ func main() {
|
||||
scanopts.RequestBody = options.RequestBody
|
||||
scanopts.Unsafe = options.Unsafe
|
||||
scanopts.Pipeline = options.Pipeline
|
||||
scanopts.HTTP2Probe = options.HTTP2Probe
|
||||
|
||||
// Try to create output folder if it doesnt exist
|
||||
if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
|
||||
@ -304,6 +305,7 @@ type scanOptions struct {
|
||||
RequestBody string
|
||||
Unsafe bool
|
||||
Pipeline bool
|
||||
HTTP2Probe bool
|
||||
}
|
||||
|
||||
func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions) Result {
|
||||
@ -442,11 +444,21 @@ retry:
|
||||
builder.WriteString(" [websocket]")
|
||||
}
|
||||
|
||||
|
||||
pipeline := false
|
||||
if scanopts.Pipeline {
|
||||
pipeline = hp.SupportPipeline(protocol, scanopts.Method, domain, port)
|
||||
if pipeline {
|
||||
builder.WriteString(" [pipeline]")
|
||||
}
|
||||
}
|
||||
|
||||
var http2 bool
|
||||
// if requested probes for http2
|
||||
if scanopts.HTTP2Probe {
|
||||
http2 = hp.SupportHTTP2(protocol, scanopts.Method, URL)
|
||||
if http2 {
|
||||
builder.WriteString(" [http2]")
|
||||
}
|
||||
}
|
||||
|
||||
@ -479,6 +491,7 @@ retry:
|
||||
TlsData: resp.TlsData,
|
||||
CspData: resp.CspData,
|
||||
Pipeline: pipeline,
|
||||
HTTP2: http2,
|
||||
}
|
||||
}
|
||||
|
||||
@ -499,6 +512,7 @@ type Result struct {
|
||||
TlsData *httpx.TlsData `json:"tls,omitempty"`
|
||||
CspData *httpx.CspData `json:"csp,omitempty"`
|
||||
Pipeline bool `json:"pipeline,omitempty"`
|
||||
HTTP2 bool `json:"http2"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
@ -558,6 +572,7 @@ type Options struct {
|
||||
RequestBody string
|
||||
Debug bool
|
||||
Pipeline bool
|
||||
HTTP2Probe bool
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -603,6 +618,7 @@ func ParseOptions() *Options {
|
||||
flag.StringVar(&options.RequestBody, "body", "", "Request Body")
|
||||
flag.BoolVar(&options.Debug, "debug", false, "Debug mode")
|
||||
flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline")
|
||||
flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
|
||||
flag.Parse()
|
||||
|
||||
// Read the inputs and configure the logging
|
||||
|
||||
2
common/cache/dialer.go
vendored
2
common/cache/dialer.go
vendored
@ -39,7 +39,7 @@ func NewDialer(options Options) (DialerFunc, error) {
|
||||
finalIps = append(finalIps, ip)
|
||||
}
|
||||
}
|
||||
if err != nil || len(finalIps) <= 0 {
|
||||
if err != nil || len(finalIps) == 0 {
|
||||
return nil, &NoAddressFoundError{}
|
||||
} // Dial to the IPs finally.
|
||||
for _, ip := range ips {
|
||||
|
||||
@ -26,7 +26,7 @@ func (c *CustomPorts) Set(value string) error {
|
||||
// splits on comma
|
||||
potentialPorts := strings.Split(value, ",")
|
||||
|
||||
// check if port is a single integer value or needs to be expanded futher
|
||||
// check if port is a single integer value or needs to be expanded further
|
||||
for _, potentialPort := range potentialPorts {
|
||||
potentialRange := strings.Split(strings.TrimSpace(potentialPort), "-")
|
||||
// it's a single port?
|
||||
|
||||
@ -14,10 +14,7 @@ func FileExists(filename string) bool {
|
||||
// FolderExists checks if a folder exists
|
||||
func FolderExists(folderpath string) bool {
|
||||
_, err := os.Stat(folderpath)
|
||||
if os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// HasStdin determines if the user has piped input
|
||||
|
||||
44
common/httpx/http2.go
Normal file
44
common/httpx/http2.go
Normal file
@ -0,0 +1,44 @@
|
||||
package httpx
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
// SupportHTTP2 checks if the target host supports HTTP2
|
||||
func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool {
|
||||
// http => supports HTTP1.1 => HTTP/2 (H2C)
|
||||
if protocol == "http" {
|
||||
req, err := retryablehttp.NewRequest(method, URL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Set("Connection", "Upgrade, HTTP2-Settings")
|
||||
req.Header.Set("Upgrade", "h2c")
|
||||
req.Header.Set("HTTP2-Settings", "AAMAAABkAARAAAAAAAIAAAAA")
|
||||
httpresp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
io.Copy(ioutil.Discard, httpresp.Body)
|
||||
httpresp.Body.Close()
|
||||
|
||||
return httpresp.StatusCode == 101
|
||||
}
|
||||
|
||||
// attempts a direct http2 connection
|
||||
req, err := http.NewRequest(method, URL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
httpresp, err := h.client2.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
io.Copy(ioutil.Discard, httpresp.Body)
|
||||
httpresp.Body.Close()
|
||||
return httpresp.Proto == "HTTP/2.0"
|
||||
}
|
||||
@ -14,11 +14,13 @@ import (
|
||||
"github.com/projectdiscovery/httpx/common/httputilz"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
// HTTPX represent an instance of the library client
|
||||
type HTTPX struct {
|
||||
client *retryablehttp.Client
|
||||
client2 *http.Client
|
||||
Filters []Filter
|
||||
Options *Options
|
||||
htmlPolicy *bluemonday.Policy
|
||||
@ -86,6 +88,16 @@ func New(options *Options) (*HTTPX, error) {
|
||||
CheckRedirect: redirectFunc,
|
||||
}, retryablehttpOptions)
|
||||
|
||||
httpx.client2 = &http.Client{
|
||||
Transport: &http2.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
AllowHTTP: true,
|
||||
},
|
||||
Timeout: httpx.Options.Timeout,
|
||||
}
|
||||
|
||||
httpx.htmlPolicy = bluemonday.NewPolicy()
|
||||
httpx.CustomHeaders = httpx.Options.CustomHeaders
|
||||
httpx.RequestOverride = &options.RequestOverride
|
||||
@ -117,7 +129,7 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp.Raw = string(rawresp)
|
||||
resp.Raw = rawresp
|
||||
|
||||
var respbody []byte
|
||||
// websockets don't have a readable body
|
||||
|
||||
@ -15,6 +15,8 @@ type Response struct {
|
||||
Lines int
|
||||
TlsData *TlsData
|
||||
CspData *CspData
|
||||
Http2 bool
|
||||
Pipeline bool
|
||||
}
|
||||
|
||||
// GetHeader value
|
||||
|
||||
@ -5,11 +5,7 @@ import "net"
|
||||
// IsCidr determines if the given ip is a cidr range
|
||||
func IsCidr(ip string) bool {
|
||||
_, _, err := net.ParseCIDR(ip)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// IsIP determines if the given string is a valid ip
|
||||
|
||||
2
go.mod
2
go.mod
@ -14,8 +14,6 @@ require (
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.2.1
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202
|
||||
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
|
||||
golang.org/x/text v0.3.3
|
||||
)
|
||||
|
||||
6
go.sum
6
go.sum
@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
|
||||
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
|
||||
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
|
||||
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
|
||||
github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087 h1:FV+/XrXTWOaiW2hZVt/VWGxGBmOx/P9ChZnIkz7O96o=
|
||||
github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
|
||||
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0=
|
||||
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
|
||||
@ -41,8 +39,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
@ -56,8 +52,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
|
||||
golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user