Merge pull request #1761 from projectdiscovery/support_multiple_values

Support multiple values

README.md

@@ -95,27 +95,28 @@ INPUT:
    -u, -target string[]  input target host(s) to probe
 
 PROBES:
-   -sc, -status-code     display response status-code
+   -sc, -status-code      display response status-code
-   -cl, -content-length  display response content-length
+   -cl, -content-length   display response content-length
-   -ct, -content-type    display response content-type
+   -ct, -content-type     display response content-type
-   -location             display response redirect location
+   -location              display response redirect location
-   -favicon              display mmh3 hash for '/favicon.ico' file
+   -favicon               display mmh3 hash for '/favicon.ico' file
-   -hash string          display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
+   -hash string           display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
-   -jarm                 display jarm fingerprint hash
+   -jarm                  display jarm fingerprint hash
-   -rt, -response-time   display response time
+   -rt, -response-time    display response time
-   -lc, -line-count      display response body line count
+   -lc, -line-count       display response body line count
-   -wc, -word-count      display response body word count
+   -wc, -word-count       display response body word count
-   -title                display page title
+   -title                 display page title
-   -bp, -body-preview    display first N characters of response body (default 100)
+   -bp, -body-preview     display first N characters of response body (default 100)
-   -server, -web-server  display server name
+   -server, -web-server   display server name
-   -td, -tech-detect     display technology in use based on wappalyzer dataset
+   -td, -tech-detect      display technology in use based on wappalyzer dataset
-   -method               display http request method
+   -method                display http request method
-   -websocket            display server using websocket
+   -websocket             display server using websocket
-   -ip                   display host ip
+   -ip                    display host ip
-   -cname                display host cname
+   -cname                 display host cname
-   -asn                  display host asn information
+   -extract-fqdn, -efqdn  get domain and subdomains from response body and header in jsonl/csv output
-   -cdn                  display cdn/waf in use (default true)
+   -asn                   display host asn information
-   -probe                display probe status
+   -cdn                   display cdn/waf in use (default true)
+   -probe                 display probe status
 
 HEADLESS:
    -ss, -screenshot                 enable saving screenshot of the page using headless browser
@@ -131,15 +132,15 @@ MATCHERS:
    -mlc, -match-line-count string     match response body with specified line count (-mlc 423,532)
    -mwc, -match-word-count string     match response body with specified word count (-mwc 43,55)
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
-   -ms, -match-string string          match response with specified string (-ms admin)
+   -ms, -match-string string[]        match response with specified string (-ms admin)
-   -mr, -match-regex string           match response with specified regex (-mr admin)
+   -mr, -match-regex string[]         match response with specified regex (-mr admin)
    -mcdn, -match-cdn string[]         match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
 EXTRACTOR:
    -er, -extract-regex string[]   display response content with matched regex
-   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (mail,url,ipv4)
+   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (url,ipv4,mail)
 
 FILTERS:
    -fc, -filter-code string            filter response with specified status code (-fc 403,401)
@@ -148,8 +149,8 @@ FILTERS:
    -flc, -filter-line-count string     filter response body with specified line count (-flc 423,532)
    -fwc, -filter-word-count string     filter response body with specified word count (-fwc 423,532)
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-ffc 1494302000)
-   -fs, -filter-string string          filter response with specified string (-fs admin)
+   -fs, -filter-string string[]        filter response with specified string (-fs admin)
-   -fe, -filter-regex string           filter response with specified regex (-fe admin)
+   -fe, -filter-regex string[]         filter response with specified regex (-fe admin)
    -fcdn, -filter-cdn string[]         filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
@@ -193,31 +194,32 @@ OUTPUT:
    -pr, -protocol string               protocol to use (unknown, http11)
 
 CONFIGURATIONS:
-   -config string                path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
+   -config string                   path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
-   -auth                         configure projectdiscovery cloud (pdcp) api key (default true)
+   -auth                            configure projectdiscovery cloud (pdcp) api key (default true)
-   -r, -resolvers string[]       list of custom resolver (file or comma separated)
+   -r, -resolvers string[]          list of custom resolver (file or comma separated)
-   -allow string[]               allowed list of IP/CIDR's to process (file or comma separated)
+   -allow string[]                  allowed list of IP/CIDR's to process (file or comma separated)
-   -deny string[]                denied list of IP/CIDR's to process (file or comma separated)
+   -deny string[]                   denied list of IP/CIDR's to process (file or comma separated)
-   -sni, -sni-name string        custom TLS SNI name
+   -sni, -sni-name string           custom TLS SNI name
-   -random-agent                 enable Random User-Agent to use (default true)
+   -random-agent                    enable Random User-Agent to use (default true)
-   -H, -header string[]          custom http headers to send with request
+   -H, -header string[]             custom http headers to send with request
-   -http-proxy, -proxy string    http proxy to use (eg http://127.0.0.1:8080)
+   -http-proxy, -proxy string       http proxy to use (eg http://127.0.0.1:8080)
-   -unsafe                       send raw requests skipping golang normalization
+   -unsafe                          send raw requests skipping golang normalization
-   -resume                       resume scan using resume.cfg
+   -resume                          resume scan using resume.cfg
-   -fr, -follow-redirects        follow http redirects
+   -fr, -follow-redirects           follow http redirects
-   -maxr, -max-redirects int     max number of redirects to follow per host (default 10)
+   -maxr, -max-redirects int        max number of redirects to follow per host (default 10)
-   -fhr, -follow-host-redirects  follow redirects on the same host
+   -fhr, -follow-host-redirects     follow redirects on the same host
-   -rhsts, -respect-hsts         respect HSTS response headers for redirect requests
+   -rhsts, -respect-hsts            respect HSTS response headers for redirect requests
-   -vhost-input                  get a list of vhosts as input
+   -vhost-input                     get a list of vhosts as input
-   -x string                     request methods to probe, use 'all' to probe all HTTP methods
+   -x string                        request methods to probe, use 'all' to probe all HTTP methods
-   -body string                  post body to include in http request
+   -body string                     post body to include in http request
-   -s, -stream                   stream mode - start elaborating input targets without sorting
+   -s, -stream                      stream mode - start elaborating input targets without sorting
-   -sd, -skip-dedupe             disable dedupe input items (only used with stream mode)
+   -sd, -skip-dedupe                disable dedupe input items (only used with stream mode)
-   -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https://host:443
+   -ldp, -leave-default-ports       leave default http/https ports in host header (eg. http://host:80 - https://host:443
-   -ztls                         use ztls library with autofallback to standard one for tls13
+   -ztls                            use ztls library with autofallback to standard one for tls13
-   -no-decode                    avoid decoding body
+   -no-decode                       avoid decoding body
-   -tlsi, -tls-impersonate       enable experimental client hello (ja3) tls randomization
+   -tlsi, -tls-impersonate          enable experimental client hello (ja3) tls randomization
-   -no-stdin                     Disable Stdin processing
+   -no-stdin                        Disable Stdin processing
+   -hae, -http-api-endpoint string  experimental http api endpoint
 
 DEBUG:
    -health-check, -hc        run diagnostic check up

runner/options.go

@@ -180,16 +180,16 @@ type Options struct {
 	InputRawRequest           string
 	rawRequest                string
 	RequestBody               string
-	OutputFilterString        string
+	OutputFilterString        goflags.StringSlice
-	OutputMatchString         string
+	OutputMatchString         goflags.StringSlice
-	OutputFilterRegex         string
+	OutputFilterRegex         goflags.StringSlice
-	OutputMatchRegex          string
+	OutputMatchRegex          goflags.StringSlice
 	Retries                   int
 	Threads                   int
 	Timeout                   int
 	Delay                     time.Duration
-	filterRegex               *regexp.Regexp
+	filterRegexes             []*regexp.Regexp
-	matchRegex                *regexp.Regexp
+	matchRegexes              []*regexp.Regexp
 	VHost                     bool
 	VHostInput                bool
 	Smuggling                 bool
@@ -359,8 +359,8 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
 		flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
 		flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
-		flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"),
+		flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions),
-		flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"),
+		flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
 		flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
 		flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
@@ -378,8 +378,8 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
 		flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
 		flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
-		flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"),
+		flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions),
-		flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"),
+		flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
 		flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
 		flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
@@ -610,16 +610,22 @@ func (options *Options) ValidateOptions() error {
 	if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
 		return errors.Wrap(err, "Invalid value for filter content length option")
 	}
-	if options.OutputFilterRegex != "" {
+	for _, filterRegexStr := range options.OutputFilterRegex {
-		if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil {
+		filterRegex, err := regexp.Compile(filterRegexStr)
+		if err != nil {
 			return errors.Wrap(err, "Invalid value for regex filter option")
 		}
+		options.filterRegexes = append(options.filterRegexes, filterRegex)
 	}
-	if options.OutputMatchRegex != "" {
+
-		if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil {
+	for _, matchRegexStr := range options.OutputMatchRegex {
+		matchRegex, err := regexp.Compile(matchRegexStr)
+		if err != nil {
 			return errors.Wrap(err, "Invalid value for match regex option")
 		}
+		options.matchRegexes = append(options.matchRegexes, matchRegex)
 	}
+
 	if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
 		return errors.Wrap(err, "Invalid value for match lines count option")
 	}

runner/runner.go

@@ -846,10 +846,19 @@ func (r *Runner) RunEnumeration() {
 			if len(r.options.filterWordsCount) > 0 && sliceutil.Contains(r.options.filterWordsCount, resp.Words) {
 				continue
 			}
-			if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) {
+			if r.options.filterRegexes != nil {
-				continue
+				shouldContinue := false
+				for _, filterRegex := range r.options.filterRegexes {
+					if filterRegex.MatchString(resp.Raw) {
+						shouldContinue = true
+						break
+					}
+				}
+				if shouldContinue {
+					continue
+				}
 			}
-			if r.options.OutputFilterString != "" && stringsutil.ContainsAnyI(resp.Raw, r.options.OutputFilterString) {
+			if len(r.options.OutputFilterString) > 0 && stringsutil.EqualFoldAny(resp.Raw, r.options.OutputFilterString...) {
 				continue
 			}
 			if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
@@ -861,10 +870,19 @@ func (r *Runner) RunEnumeration() {
 			if len(r.options.matchContentLength) > 0 && !sliceutil.Contains(r.options.matchContentLength, resp.ContentLength) {
 				continue
 			}
-			if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) {
+			if r.options.matchRegexes != nil {
-				continue
+				shouldContinue := false
+				for _, matchRegex := range r.options.matchRegexes {
+					if !matchRegex.MatchString(resp.Raw) {
+						shouldContinue = true
+						break
+					}
+				}
+				if shouldContinue {
+					continue
+				}
 			}
-			if r.options.OutputMatchString != "" && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString) {
+			if len(r.options.OutputMatchString) > 0 && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString...) {
 				continue
 			}
 			if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {