Merge branch 'master' into bugfix-full-rawhttp

cmd/httpx/httpx.go

@@ -138,6 +138,7 @@ func main() {
 	scanopts.OutputIP = options.OutputIP
 	scanopts.OutputCName = options.OutputCName
 	scanopts.OutputCDN = options.OutputCDN
+	scanopts.OutputResponseTime = options.OutputResponseTime
 	// output verb if more than one is specified
 	if len(scanopts.Methods) > 1 && !options.Silent {
 		scanopts.OutputMethod = true
@@ -360,6 +361,7 @@ type scanOptions struct {
 	OutputIP               bool
 	OutputCName            bool
 	OutputCDN              bool
+	OutputResponseTime     bool
 }
 
 func analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
@@ -381,6 +383,7 @@ retry:
 
 	hp.SetCustomHeaders(req, hp.CustomHeaders)
 	if scanopts.RequestBody != "" {
+		req.ContentLength = int64(len(scanopts.RequestBody))
 		req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody))
 	}
 
@@ -552,6 +555,10 @@ retry:
 		builder.WriteString(" [cdn]")
 	}
 
+	if scanopts.OutputResponseTime {
+		builder.WriteString(fmt.Sprintf(" [%s]", resp.Duration))
+	}
+
 	// store responses in directory
 	if scanopts.StoreResponse {
 		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@@ -595,6 +602,7 @@ retry:
 		IPs:           ips,
 		CNAMEs:        cnames,
 		CDN:           isCDN,
+		Duration:      resp.Duration,
 	}
 }
 
@@ -622,6 +630,7 @@ type Result struct {
 	Pipeline      bool           `json:"pipeline,omitempty"`
 	HTTP2         bool           `json:"http2"`
 	CDN           bool           `json:"cdn"`
+	Duration      time.Duration  `json:"duration"`
 }
 
 // JSON the result
@@ -692,6 +701,7 @@ type Options struct {
 	Pipeline                  bool
 	HTTP2Probe                bool
 	OutputCDN                 bool
+	OutputResponseTime        bool
 }
 
 // ParseOptions parses the command line options for application
@@ -746,6 +756,7 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match Regex")
 	flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname")
 	flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)")
+	flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Output the response time")
 
 	flag.Parse()
 

common/httpx/httpx.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/microcosm-cc/bluemonday"
@@ -112,13 +113,15 @@ func New(options *Options) (*HTTPX, error) {
 
 // Do http request
 func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
-	httpresp, err := h.getResponse(req)
+	timeStart := time.Now()
 
+	httpresp, err := h.getResponse(req)
 	if err != nil {
 		return nil, err
 	}
 
 	var resp Response
+
 	resp.Headers = httpresp.Header.Clone()
 
 	// httputil.DumpResponse does not handle websockets
@@ -167,6 +170,7 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 	}
 
 	resp.CSPData = h.CSPGrab(httpresp)
+	resp.Duration = time.Since(timeStart)
 
 	return &resp, nil
 }

common/httpx/response.go

@@ -2,6 +2,7 @@ package httpx
 
 import (
 	"strings"
+	"time"
 )
 
 // Response contains the response to a server
@@ -17,6 +18,7 @@ type Response struct {
 	CSPData       *CSPData
 	HTTP2         bool
 	Pipeline      bool
+	Duration      time.Duration
 }
 
 // GetHeader value

go.sum

@@ -53,8 +53,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
-golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA=
+golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=