manual merging dev

common/httpx/httpx.go

@@ -3,6 +3,7 @@ package httpx
 import (
 	"crypto/tls"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -156,7 +157,7 @@ get_response:
 	// websockets don't have a readable body
 	if httpresp.StatusCode != http.StatusSwitchingProtocols {
 		var err error
-		respbody, err = ioutil.ReadAll(httpresp.Body)
+		respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
 		if err != nil {
 			return nil, err
 		}

common/httpx/option.go

@@ -26,13 +26,15 @@ type Options struct {
 	Unsafe               bool
 	TLSGrab              bool
 	// VHOSTs options
-	VHostIgnoreStatusCode    bool
+	VHostIgnoreStatusCode     bool
-	VHostIgnoreContentLength bool
+	VHostIgnoreContentLength  bool
-	VHostIgnoreNumberOfWords bool
+	VHostIgnoreNumberOfWords  bool
-	VHostIgnoreNumberOfLines bool
+	VHostIgnoreNumberOfLines  bool
-	VHostStripHTML           bool
+	VHostStripHTML            bool
-	Allow                    []string
+	Allow                     []string
-	Deny                     []string
+	Deny                      []string
+	MaxResponseBodySizeToSave int64
+	MaxResponseBodySizeToRead int64
 }
 
 // DefaultOptions contains the default options

go.mod

@@ -25,8 +25,8 @@ require (
 	github.com/projectdiscovery/rawhttp v0.0.7
 	github.com/projectdiscovery/retryabledns v1.0.12 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
-	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d // indirect
+	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
-	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
+	github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
 	github.com/projectdiscovery/wappalyzergo v0.0.7
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0

go.sum

@@ -167,8 +167,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
 github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
+github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
-github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
+github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/wappalyzergo v0.0.7 h1:MvlienkiFUbO3nDvlc5mNy1C5XiHzD2EklLDgnG9Zv4=
 github.com/projectdiscovery/wappalyzergo v0.0.7/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=

runner/options.go

@@ -22,78 +22,81 @@ const (
 )
 
 type scanOptions struct {
-	Methods                []string
+	Methods                   []string
-	StoreResponseDirectory string
+	StoreResponseDirectory    string
-	RequestURI             string
+	RequestURI                string
-	RequestBody            string
+	RequestBody               string
-	VHost                  bool
+	VHost                     bool
-	OutputTitle            bool
+	OutputTitle               bool
-	OutputStatusCode       bool
+	OutputStatusCode          bool
-	OutputLocation         bool
+	OutputLocation            bool
-	OutputContentLength    bool
+	OutputContentLength       bool
-	StoreResponse          bool
+	StoreResponse             bool
-	OutputServerHeader     bool
+	OutputServerHeader        bool
-	OutputWebSocket        bool
+	OutputWebSocket           bool
-	OutputWithNoColor      bool
+	OutputWithNoColor         bool
-	OutputMethod           bool
+	OutputMethod              bool
-	ResponseInStdout       bool
+	ResponseInStdout          bool
-	ChainInStdout          bool
+	ChainInStdout             bool
-	TLSProbe               bool
+	TLSProbe                  bool
-	CSPProbe               bool
+	CSPProbe                  bool
-	VHostInput             bool
+	VHostInput                bool
-	OutputContentType      bool
+	OutputContentType         bool
-	Unsafe                 bool
+	Unsafe                    bool
-	Pipeline               bool
+	Pipeline                  bool
-	HTTP2Probe             bool
+	HTTP2Probe                bool
-	OutputIP               bool
+	OutputIP                  bool
-	OutputCName            bool
+	OutputCName               bool
-	OutputCDN              bool
+	OutputCDN                 bool
-	OutputResponseTime     bool
+	OutputResponseTime        bool
-	PreferHTTPS            bool
+	PreferHTTPS               bool
-	NoFallback             bool
+	NoFallback                bool
-	NoFallbackScheme       bool
+	NoFallbackScheme          bool
-	TechDetect             bool
+	TechDetect                bool
-	StoreChain             bool
+	StoreChain                bool
-	MaxResponseBodySize    int
+	MaxResponseBodySizeToSave int
-	OutputExtractRegex     string
+	MaxResponseBodySizeToRead int
-	extractRegex           *regexp.Regexp
+	OutputExtractRegex        string
-	ExcludeCDN             bool
+	extractRegex              *regexp.Regexp
+	ExcludeCDN                bool
 }
 
 func (s *scanOptions) Clone() *scanOptions {
 	return &scanOptions{
-		Methods:                s.Methods,
+		Methods:                   s.Methods,
-		StoreResponseDirectory: s.StoreResponseDirectory,
+		StoreResponseDirectory:    s.StoreResponseDirectory,
-		RequestURI:             s.RequestURI,
+		RequestURI:                s.RequestURI,
-		RequestBody:            s.RequestBody,
+		RequestBody:               s.RequestBody,
-		VHost:                  s.VHost,
+		VHost:                     s.VHost,
-		OutputTitle:            s.OutputTitle,
+		OutputTitle:               s.OutputTitle,
-		OutputStatusCode:       s.OutputStatusCode,
+		OutputStatusCode:          s.OutputStatusCode,
-		OutputLocation:         s.OutputLocation,
+		OutputLocation:            s.OutputLocation,
-		OutputContentLength:    s.OutputContentLength,
+		OutputContentLength:       s.OutputContentLength,
-		StoreResponse:          s.StoreResponse,
+		StoreResponse:             s.StoreResponse,
-		OutputServerHeader:     s.OutputServerHeader,
+		OutputServerHeader:        s.OutputServerHeader,
-		OutputWebSocket:        s.OutputWebSocket,
+		OutputWebSocket:           s.OutputWebSocket,
-		OutputWithNoColor:      s.OutputWithNoColor,
+		OutputWithNoColor:         s.OutputWithNoColor,
-		OutputMethod:           s.OutputMethod,
+		OutputMethod:              s.OutputMethod,
-		ResponseInStdout:       s.ResponseInStdout,
+		ResponseInStdout:          s.ResponseInStdout,
-		ChainInStdout:          s.ChainInStdout,
+		ChainInStdout:             s.ChainInStdout,
-		TLSProbe:               s.TLSProbe,
+		TLSProbe:                  s.TLSProbe,
-		CSPProbe:               s.CSPProbe,
+		CSPProbe:                  s.CSPProbe,
-		OutputContentType:      s.OutputContentType,
+		OutputContentType:         s.OutputContentType,
-		Unsafe:                 s.Unsafe,
+		Unsafe:                    s.Unsafe,
-		Pipeline:               s.Pipeline,
+		Pipeline:                  s.Pipeline,
-		HTTP2Probe:             s.HTTP2Probe,
+		HTTP2Probe:                s.HTTP2Probe,
-		OutputIP:               s.OutputIP,
+		OutputIP:                  s.OutputIP,
-		OutputCName:            s.OutputCName,
+		OutputCName:               s.OutputCName,
-		OutputCDN:              s.OutputCDN,
+		OutputCDN:                 s.OutputCDN,
-		OutputResponseTime:     s.OutputResponseTime,
+		OutputResponseTime:        s.OutputResponseTime,
-		PreferHTTPS:            s.PreferHTTPS,
+		PreferHTTPS:               s.PreferHTTPS,
-		NoFallback:             s.NoFallback,
+		NoFallback:                s.NoFallback,
-		NoFallbackScheme:       s.NoFallbackScheme,
+		NoFallbackScheme:          s.NoFallbackScheme,
-		TechDetect:             s.TechDetect,
+		TechDetect:                s.TechDetect,
-		StoreChain:             s.StoreChain,
+		StoreChain:                s.StoreChain,
-		OutputExtractRegex:     s.OutputExtractRegex,
+		OutputExtractRegex:        s.OutputExtractRegex,
+		MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
+		MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
 	}
 }
 
@@ -171,7 +174,8 @@ type Options struct {
 	StoreChain                bool
 	Deny                      customlist.CustomList
 	Allow                     customlist.CustomList
-	MaxResponseBodySize       int
+	MaxResponseBodySizeToSave int
+	MaxResponseBodySizeToRead int
 	OutputExtractRegex        string
 	RateLimit                 int
 	Probe                     bool
@@ -182,6 +186,7 @@ type Options struct {
 func ParseOptions() *Options {
 	options := &Options{}
 
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
 	flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing")
 	flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection")
 	flag.IntVar(&options.Threads, "threads", 50, "Number of threads")
@@ -244,7 +249,8 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')")
 	flag.Var(&options.Allow, "allow", "Allowlist ip/cidr")
 	flag.Var(&options.Deny, "deny", "Denylist ip/cidr")
-	flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size")
+	flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)")
+	flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)")
 	flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
 	flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second")
 	flag.BoolVar(&options.Probe, "probe", false, "Display probe status")

runner/runner.go

@@ -14,14 +14,15 @@ import (
 	"os"
 	"path"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/logrusorgru/aurora"
 	"github.com/pkg/errors"
-
 	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/stringsutil"
 	"github.com/projectdiscovery/urlutil"
 
 	// automatic fd max increase if running as root
@@ -40,6 +41,7 @@ import (
 	"github.com/projectdiscovery/rawhttp"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
 	"github.com/remeh/sizedwaitgroup"
+	"go.uber.org/ratelimit"
 )
 
 const (
@@ -48,12 +50,13 @@ const (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	options    *Options
+	options     *Options
-	hp         *httpx.HTTPX
+	hp          *httpx.HTTPX
-	wappalyzer *wappalyzer.Wappalyze
+	wappalyzer  *wappalyzer.Wappalyze
-	scanopts   scanOptions
+	scanopts    scanOptions
-	hm         *hybrid.HybridMap
+	hm          *hybrid.HybridMap
-	stats      clistats.StatisticsClient
+	stats       clistats.StatisticsClient
+	ratelimiter ratelimit.Limiter
 }
 
 // New creates a new client for running enumeration process.
@@ -84,6 +87,12 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.RandomAgent = options.RandomAgent
 	httpxOptions.Deny = options.Deny
 	httpxOptions.Allow = options.Allow
+	httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave)
+	httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead)
+	// adjust response size saved according to the max one read by the server
+	if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead {
+		httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead
+	}
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)
@@ -182,7 +191,8 @@ func New(options *Options) (*Runner, error) {
 	scanopts.NoFallbackScheme = options.NoFallbackScheme
 	scanopts.TechDetect = options.TechDetect
 	scanopts.StoreChain = options.StoreChain
-	scanopts.MaxResponseBodySize = options.MaxResponseBodySize
+	scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
+	scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
 	if options.OutputExtractRegex != "" {
 		if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
 			return nil, err
@@ -210,6 +220,12 @@ func New(options *Options) (*Runner, error) {
 	}
 	runner.hm = hm
 
+	if options.RateLimit > 0 {
+		runner.ratelimiter = ratelimit.New(options.RateLimit)
+	} else {
+		runner.ratelimiter = ratelimit.NewUnlimited()
+	}
+
 	return runner, nil
 }
 
@@ -256,7 +272,7 @@ func (r *Runner) prepareInput() {
 		}
 		numTargets += numTargetsStdin
 	}
-	// reqLength = numTargets
+
 	if r.options.ShowStatistics {
 		numPorts := len(customport.Ports)
 		if numPorts == 0 {
@@ -280,6 +296,9 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	for scanner.Scan() {
 		target := strings.TrimSpace(scanner.Text())
 		// Used just to get the exact number of targets
+		if target == "" {
+			continue
+		}
 		if _, ok := r.hm.Get(target); ok {
 			continue
 		}
@@ -340,7 +359,6 @@ func (r *Runner) Close() {
 
 // RunEnumeration on targets for httpx client
 func (r *Runner) RunEnumeration() {
-	//fmt.Println("result : == >> 45646 ")
 	// Try to create output folder if it doesnt exist
 	if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) {
 		if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil {
@@ -368,7 +386,7 @@ func (r *Runner) RunEnumeration() {
 		}
 		for resp := range output {
 			if resp.err != nil {
-				gologger.Debug().Msgf("Failure '%s': %s\n", resp.URL, resp.err)
+				gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err)
 			}
 			if resp.str == "" {
 				continue
@@ -404,7 +422,6 @@ func (r *Runner) RunEnumeration() {
 			if r.options.JSONOutput {
 				row = resp.JSON(&r.scanopts)
 			}
-
 			gologger.Silent().Msgf("%s\n", row)
 			if f != nil {
 				//nolint:errcheck // this method needs a small refactor to reduce complexity
@@ -456,7 +473,6 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 	if scanopts.NoFallback {
 		protocols = []string{httpx.HTTPS, httpx.HTTP}
 	}
-
 	for target := range targets(stringz.TrimProtocol(t, scanopts.NoFallback || scanopts.NoFallbackScheme)) {
 		// if no custom ports specified then test the default ones
 		if len(customport.Ports) == 0 {
@@ -615,9 +631,46 @@ retry:
 			req.Body = nil
 		}
 	}
-
+	r.ratelimiter.Take()
 	resp, err := hp.Do(req)
+
+	fullURL := req.URL.String()
+
+	builder := &strings.Builder{}
+
+	// if the full url doesn't end with the custom path we pick the original input value
+	if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
+		parsedURL, _ := urlutil.Parse(fullURL)
+		parsedURL.RequestURI = scanopts.RequestURI
+		fullURL = parsedURL.String()
+	}
+	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
+
+	if r.options.Probe {
+		builder.WriteString(" [")
+
+		outputStatus := "SUCCESS"
+		if err != nil {
+			outputStatus = "FAILED"
+		}
+
+		if !scanopts.OutputWithNoColor && err != nil {
+			builder.WriteString(aurora.Red(outputStatus).String())
+		} else if !scanopts.OutputWithNoColor && err == nil {
+			builder.WriteString(aurora.Green(outputStatus).String())
+		} else {
+			builder.WriteString(outputStatus)
+		}
+
+		builder.WriteRune(']')
+	}
+
 	if err != nil {
+		errString := ""
+		errString = err.Error()
+		splitErr := strings.Split(errString, ":")
+		errString = strings.TrimSpace(splitErr[len(splitErr)-1])
+
 		if !retried && origProtocol == httpx.HTTPorHTTPS {
 			if protocol == httpx.HTTPS {
 				protocol = httpx.HTTP
@@ -627,19 +680,13 @@ retry:
 			retried = true
 			goto retry
 		}
-		return Result{URL: URL.String(), err: err}
+		if r.options.Probe {
+			return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
+		} else {
+			return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err}
+		}
 	}
 
-	var fullURL string
-
-	if resp.StatusCode >= 0 {
-		fullURL = req.URL.String()
-	}
-
-	builder := &strings.Builder{}
-
-	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
-	// portLst = append(portLst, fullURL)
 	if scanopts.OutputStatusCode {
 		builder.WriteString(" [")
 		for i, chainItem := range resp.Chain {
@@ -733,6 +780,7 @@ retry:
 	// check for virtual host
 	isvhost := false
 	if scanopts.VHost {
+		r.ratelimiter.Take()
 		isvhost, _ = hp.IsVirtualHost(req)
 		if isvhost {
 			builder.WriteString(" [vhost]")
@@ -748,6 +796,7 @@ retry:
 	pipeline := false
 	if scanopts.Pipeline {
 		port, _ := strconv.Atoi(URL.Port)
+		r.ratelimiter.Take()
 		pipeline = hp.SupportPipeline(protocol, method, URL.Host, port)
 		if pipeline {
 			builder.WriteString(" [pipeline]")
@@ -757,6 +806,7 @@ retry:
 	var http2 bool
 	// if requested probes for http2
 	if scanopts.HTTP2Probe {
+		r.ratelimiter.Take()
 		http2 = hp.SupportHTTP2(protocol, method, URL.String())
 		if http2 {
 			builder.WriteString(" [http2]")
@@ -802,6 +852,7 @@ retry:
 		}
 
 		if len(technologies) > 0 {
+			sort.Strings(technologies)
 			technologies := strings.Join(technologies, ",")
 
 			builder.WriteString(" [")
@@ -852,8 +903,8 @@ retry:
 		// store response
 		responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
 		respRaw := resp.Raw
-		if len(respRaw) > scanopts.MaxResponseBodySize {
+		if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
-			respRaw = respRaw[:scanopts.MaxResponseBodySize]
+			respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
 		}
 		writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644)
 		if writeErr != nil {
@@ -903,6 +954,7 @@ retry:
 	if scanopts.ChainInStdout && resp.HasChain() {
 		chainItems = append(chainItems, resp.GetChainAsSlice()...)
 	}
+
 	return Result{
 		Timestamp:        time.Now(),
 		Request:          request,
@@ -914,6 +966,7 @@ retry:
 		HeaderSHA256:     headersSha,
 		raw:              resp.Raw,
 		URL:              fullURL,
+		Input:            domain,
 		ContentLength:    resp.ContentLength,
 		ChainStatusCodes: chainStatusCodes,
 		Chain:            chainItems,
@@ -938,8 +991,6 @@ retry:
 		ResponseTime:     resp.Duration.String(),
 		Technologies:     technologies,
 		FinalURL:         finalURL,
-		// UniqueUrl:        uniqueUrlStrngs,
-		// PortLst:          portLst,
 	}
 }
 
@@ -957,10 +1008,12 @@ type Result struct {
 	CNAMEs           []string  `json:"cnames,omitempty"`
 	raw              string
 	URL              string `json:"url,omitempty"`
+	Input            string `json:"input,omitempty"`
 	Location         string `json:"location,omitempty"`
 	Title            string `json:"title,omitempty"`
 	str              string
 	err              error
+	Error            string            `json:"error,omitempty"`
 	WebServer        string            `json:"webserver,omitempty"`
 	ResponseBody     string            `json:"response-body,omitempty"`
 	ContentType      string            `json:"content-type,omitempty"`
@@ -980,12 +1033,13 @@ type Result struct {
 	Technologies     []string          `json:"technologies,omitempty"`
 	Chain            []httpx.ChainItem `json:"chain,omitempty"`
 	FinalURL         string            `json:"final-url,omitempty"`
+	Failed           bool              `json:"failed"`
 }
 
 // JSON the result
 func (r Result) JSON(scanopts *scanOptions) string { //nolint
-	if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySize {
+	if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
-		r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySize]
+		r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
 	}
 
 	if js, err := json.Marshal(r); err == nil {