mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-09-11 17:46:08 +03:00
Handling edge case with head method and unresponsive server
This commit is contained in:
parent
38d56d5de9
commit
1e2e2d54e9
@ -7,6 +7,7 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
@ -18,6 +19,7 @@ import (
|
||||
pdhttputil "github.com/projectdiscovery/httputil"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/stringsutil"
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
@ -134,6 +136,13 @@ get_response:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
|
||||
switch {
|
||||
case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
|
||||
shouldIgnoreErrors = true
|
||||
shouldIgnoreBodyErrors = true
|
||||
}
|
||||
|
||||
var resp Response
|
||||
|
||||
resp.Headers = httpresp.Header.Clone()
|
||||
@ -148,23 +157,25 @@ get_response:
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
goto get_response
|
||||
}
|
||||
return nil, err
|
||||
if !shouldIgnoreErrors {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
resp.Raw = rawResp
|
||||
resp.RawHeaders = headers
|
||||
resp.Raw = string(rawResp)
|
||||
resp.RawHeaders = string(headers)
|
||||
|
||||
var respbody []byte
|
||||
// websockets don't have a readable body
|
||||
if httpresp.StatusCode != http.StatusSwitchingProtocols {
|
||||
var err error
|
||||
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
|
||||
if err != nil {
|
||||
if err != nil && !shouldIgnoreBodyErrors {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
closeErr := httpresp.Body.Close()
|
||||
if closeErr != nil {
|
||||
if closeErr != nil && !shouldIgnoreBodyErrors {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
@ -175,7 +186,15 @@ get_response:
|
||||
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
|
||||
}
|
||||
|
||||
resp.ContentLength = utf8.RuneCountInString(respbodystr)
|
||||
if contentLength, ok := resp.Headers["Content-Length"]; ok {
|
||||
contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
|
||||
if err != nil {
|
||||
resp.ContentLength = utf8.RuneCountInString(respbodystr)
|
||||
} else {
|
||||
resp.ContentLength = contentLengthInt
|
||||
}
|
||||
}
|
||||
|
||||
resp.Data = respbody
|
||||
|
||||
// fill metrics
|
||||
|
2
go.mod
2
go.mod
@ -21,7 +21,7 @@ require (
|
||||
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
|
||||
github.com/projectdiscovery/gologger v1.1.4
|
||||
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
|
||||
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
|
||||
github.com/projectdiscovery/mapcidr v0.0.8
|
||||
github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e
|
||||
|
2
go.sum
2
go.sum
@ -153,6 +153,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ
|
||||
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
|
||||
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
|
||||
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
|
||||
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||
|
Loading…
Reference in New Issue
Block a user