projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-09-11 17:46:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Handling edge case with head method and unresponsive server

Browse Source
This commit is contained in:
mzack 2021-08-16 19:10:12 +02:00
parent 38d56d5de9
commit 1e2e2d54e9
3 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
common/httpx/httpx.go
View File

@ -7,6 +7,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode/utf8"
@ -18,6 +19,7 @@ import (
pdhttputil "github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/stringsutil"
"golang.org/x/net/http2"
)
@ -134,6 +136,13 @@ get_response:
return nil, err
}
var shouldIgnoreErrors, shouldIgnoreBodyErrors bool
switch {
case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"):
shouldIgnoreErrors = true
shouldIgnoreBodyErrors = true
}
var resp Response
resp.Headers = httpresp.Header.Clone()
@ -148,23 +157,25 @@ get_response:
req.Header.Set("Accept-Encoding", "identity")
goto get_response
}
return nil, err
if !shouldIgnoreErrors {
return nil, err
}
}
resp.Raw = rawResp
resp.RawHeaders = headers
resp.Raw = string(rawResp)
resp.RawHeaders = string(headers)
var respbody []byte
// websockets don't have a readable body
if httpresp.StatusCode != http.StatusSwitchingProtocols {
var err error
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
if err != nil {
if err != nil && !shouldIgnoreBodyErrors {
return nil, err
}
}
closeErr := httpresp.Body.Close()
if closeErr != nil {
if closeErr != nil && !shouldIgnoreBodyErrors {
return nil, closeErr
}
@ -175,7 +186,15 @@ get_response:
respbodystr = h.htmlPolicy.Sanitize(respbodystr)
}
resp.ContentLength = utf8.RuneCountInString(respbodystr)
if contentLength, ok := resp.Headers["Content-Length"]; ok {
contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, ""))
if err != nil {
resp.ContentLength = utf8.RuneCountInString(respbodystr)
} else {
resp.ContentLength = contentLengthInt
}
}
resp.Data = respbody
// fill metrics

2
go.mod
View File

@ -21,7 +21,7 @@ require (
github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b
github.com/projectdiscovery/mapcidr v0.0.8
github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e

2
go.sum
View File

@ -153,6 +153,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZ
github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8=
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo=
github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4=
github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=