projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-12-01 12:13:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #369 from projectdiscovery/348-bugfix-content-length-head

Browse Source 
Improving case behavior with methods CLI option
This commit is contained in:
Mzack9999 2021-08-17 00:29:11 +02:00 committed by GitHub
commit 4ca01fe686
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
common/httpx/httpx.go
View File

@ -7,6 +7,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode/utf8"
@ -18,6 +19,7 @@ import (
pdhttputil "github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/stringsutil"
"golang.org/x/net/http2"
)
@ -144,6 +146,13 @@ get_response:
return nil, err
}
var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
switch {
case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
shouldIgnoreErrors = true
shouldIgnoreBodyErrors = true
}
var resp Response
resp.Headers = httpresp.Header.Clone()
@ -158,23 +167,25 @@ get_response:
req.Header.Set("Accept-Encoding", "identity")
goto get_response
}
return nil, err
if !shouldIgnoreErrors {
return nil, err
}
}
resp.Raw = rawResp
resp.RawHeaders = headers
resp.Raw = string(rawResp)
resp.RawHeaders = string(headers)
var respbody []byte
// websockets don't have a readable body
if httpresp.StatusCode != http.StatusSwitchingProtocols {
var err error
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
if err != nil {
if err != nil && !shouldIgnoreBodyErrors {
return nil, err
}
}
closeErr := httpresp.Body.Close()
if closeErr != nil {
if closeErr != nil && !shouldIgnoreBodyErrors {
return nil, closeErr
}
@ -185,7 +196,15 @@ get_response:
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
}
resp.ContentLength = utf8.RuneCountInString(respbodystr)
if contentLength, ok := resp.Headers["Content-Length"]; ok {
contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
if err != nil {
resp.ContentLength = utf8.RuneCountInString(respbodystr)
} else {
resp.ContentLength = contentLengthInt
}
}
resp.Data = respbody
// fill metrics

2
go.mod
View File

@ -22,7 +22,7 @@ require (
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
github.com/projectdiscovery/mapcidr v0.0.8
github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e

2
go.sum
View File

@ -155,6 +155,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=

2
runner/options.go
View File

@ -217,7 +217,7 @@ func ParseOptions() *Options {
flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
flag.StringVar(&options.InputFile, "l", "", "File containing domains")
flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs (GET, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE)")
flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
flag.BoolVar(&options.Version, "version", false, "Show version of httpx")

4
runner/runner.go
View File

@ -159,6 +159,10 @@ func New(options *Options) (*Runner, error) {
if strings.EqualFold(options.Methods, "all") {
scanopts.Methods = pdhttputil.AllHTTPMethods()
} else if options.Methods != "" {
// if unsafe is specified then converts the methods to uppercase
if !options.Unsafe {
options.Methods = strings.ToUpper(options.Methods)
}
scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
}
if len(scanopts.Methods) == 0 {