mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-09-11 09:35:28 +03:00
Merge pull request #1761 from projectdiscovery/support_multiple_values
Support multiple values
This commit is contained in:
commit
e432123e95
104
README.md
104
README.md
@ -95,27 +95,28 @@ INPUT:
|
|||||||
-u, -target string[] input target host(s) to probe
|
-u, -target string[] input target host(s) to probe
|
||||||
|
|
||||||
PROBES:
|
PROBES:
|
||||||
-sc, -status-code display response status-code
|
-sc, -status-code display response status-code
|
||||||
-cl, -content-length display response content-length
|
-cl, -content-length display response content-length
|
||||||
-ct, -content-type display response content-type
|
-ct, -content-type display response content-type
|
||||||
-location display response redirect location
|
-location display response redirect location
|
||||||
-favicon display mmh3 hash for '/favicon.ico' file
|
-favicon display mmh3 hash for '/favicon.ico' file
|
||||||
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
|
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
|
||||||
-jarm display jarm fingerprint hash
|
-jarm display jarm fingerprint hash
|
||||||
-rt, -response-time display response time
|
-rt, -response-time display response time
|
||||||
-lc, -line-count display response body line count
|
-lc, -line-count display response body line count
|
||||||
-wc, -word-count display response body word count
|
-wc, -word-count display response body word count
|
||||||
-title display page title
|
-title display page title
|
||||||
-bp, -body-preview display first N characters of response body (default 100)
|
-bp, -body-preview display first N characters of response body (default 100)
|
||||||
-server, -web-server display server name
|
-server, -web-server display server name
|
||||||
-td, -tech-detect display technology in use based on wappalyzer dataset
|
-td, -tech-detect display technology in use based on wappalyzer dataset
|
||||||
-method display http request method
|
-method display http request method
|
||||||
-websocket display server using websocket
|
-websocket display server using websocket
|
||||||
-ip display host ip
|
-ip display host ip
|
||||||
-cname display host cname
|
-cname display host cname
|
||||||
-asn display host asn information
|
-extract-fqdn, -efqdn get domain and subdomains from response body and header in jsonl/csv output
|
||||||
-cdn display cdn/waf in use (default true)
|
-asn display host asn information
|
||||||
-probe display probe status
|
-cdn display cdn/waf in use (default true)
|
||||||
|
-probe display probe status
|
||||||
|
|
||||||
HEADLESS:
|
HEADLESS:
|
||||||
-ss, -screenshot enable saving screenshot of the page using headless browser
|
-ss, -screenshot enable saving screenshot of the page using headless browser
|
||||||
@ -131,15 +132,15 @@ MATCHERS:
|
|||||||
-mlc, -match-line-count string match response body with specified line count (-mlc 423,532)
|
-mlc, -match-line-count string match response body with specified line count (-mlc 423,532)
|
||||||
-mwc, -match-word-count string match response body with specified word count (-mwc 43,55)
|
-mwc, -match-word-count string match response body with specified word count (-mwc 43,55)
|
||||||
-mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000)
|
-mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000)
|
||||||
-ms, -match-string string match response with specified string (-ms admin)
|
-ms, -match-string string[] match response with specified string (-ms admin)
|
||||||
-mr, -match-regex string match response with specified regex (-mr admin)
|
-mr, -match-regex string[] match response with specified regex (-mr admin)
|
||||||
-mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
|
-mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
|
||||||
-mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1')
|
-mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1')
|
||||||
-mdc, -match-condition string match response with dsl expression condition
|
-mdc, -match-condition string match response with dsl expression condition
|
||||||
|
|
||||||
EXTRACTOR:
|
EXTRACTOR:
|
||||||
-er, -extract-regex string[] display response content with matched regex
|
-er, -extract-regex string[] display response content with matched regex
|
||||||
-ep, -extract-preset string[] display response content matched by a pre-defined regex (mail,url,ipv4)
|
-ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail)
|
||||||
|
|
||||||
FILTERS:
|
FILTERS:
|
||||||
-fc, -filter-code string filter response with specified status code (-fc 403,401)
|
-fc, -filter-code string filter response with specified status code (-fc 403,401)
|
||||||
@ -148,8 +149,8 @@ FILTERS:
|
|||||||
-flc, -filter-line-count string filter response body with specified line count (-flc 423,532)
|
-flc, -filter-line-count string filter response body with specified line count (-flc 423,532)
|
||||||
-fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532)
|
-fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532)
|
||||||
-ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000)
|
-ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000)
|
||||||
-fs, -filter-string string filter response with specified string (-fs admin)
|
-fs, -filter-string string[] filter response with specified string (-fs admin)
|
||||||
-fe, -filter-regex string filter response with specified regex (-fe admin)
|
-fe, -filter-regex string[] filter response with specified regex (-fe admin)
|
||||||
-fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
|
-fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
|
||||||
-frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1')
|
-frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1')
|
||||||
-fdc, -filter-condition string filter response with dsl expression condition
|
-fdc, -filter-condition string filter response with dsl expression condition
|
||||||
@ -193,31 +194,32 @@ OUTPUT:
|
|||||||
-pr, -protocol string protocol to use (unknown, http11)
|
-pr, -protocol string protocol to use (unknown, http11)
|
||||||
|
|
||||||
CONFIGURATIONS:
|
CONFIGURATIONS:
|
||||||
-config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
|
-config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
|
||||||
-auth configure projectdiscovery cloud (pdcp) api key (default true)
|
-auth configure projectdiscovery cloud (pdcp) api key (default true)
|
||||||
-r, -resolvers string[] list of custom resolver (file or comma separated)
|
-r, -resolvers string[] list of custom resolver (file or comma separated)
|
||||||
-allow string[] allowed list of IP/CIDR's to process (file or comma separated)
|
-allow string[] allowed list of IP/CIDR's to process (file or comma separated)
|
||||||
-deny string[] denied list of IP/CIDR's to process (file or comma separated)
|
-deny string[] denied list of IP/CIDR's to process (file or comma separated)
|
||||||
-sni, -sni-name string custom TLS SNI name
|
-sni, -sni-name string custom TLS SNI name
|
||||||
-random-agent enable Random User-Agent to use (default true)
|
-random-agent enable Random User-Agent to use (default true)
|
||||||
-H, -header string[] custom http headers to send with request
|
-H, -header string[] custom http headers to send with request
|
||||||
-http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080)
|
-http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080)
|
||||||
-unsafe send raw requests skipping golang normalization
|
-unsafe send raw requests skipping golang normalization
|
||||||
-resume resume scan using resume.cfg
|
-resume resume scan using resume.cfg
|
||||||
-fr, -follow-redirects follow http redirects
|
-fr, -follow-redirects follow http redirects
|
||||||
-maxr, -max-redirects int max number of redirects to follow per host (default 10)
|
-maxr, -max-redirects int max number of redirects to follow per host (default 10)
|
||||||
-fhr, -follow-host-redirects follow redirects on the same host
|
-fhr, -follow-host-redirects follow redirects on the same host
|
||||||
-rhsts, -respect-hsts respect HSTS response headers for redirect requests
|
-rhsts, -respect-hsts respect HSTS response headers for redirect requests
|
||||||
-vhost-input get a list of vhosts as input
|
-vhost-input get a list of vhosts as input
|
||||||
-x string request methods to probe, use 'all' to probe all HTTP methods
|
-x string request methods to probe, use 'all' to probe all HTTP methods
|
||||||
-body string post body to include in http request
|
-body string post body to include in http request
|
||||||
-s, -stream stream mode - start elaborating input targets without sorting
|
-s, -stream stream mode - start elaborating input targets without sorting
|
||||||
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
|
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
|
||||||
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
|
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
|
||||||
-ztls use ztls library with autofallback to standard one for tls13
|
-ztls use ztls library with autofallback to standard one for tls13
|
||||||
-no-decode avoid decoding body
|
-no-decode avoid decoding body
|
||||||
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
|
||||||
-no-stdin Disable Stdin processing
|
-no-stdin Disable Stdin processing
|
||||||
|
-hae, -http-api-endpoint string experimental http api endpoint
|
||||||
|
|
||||||
DEBUG:
|
DEBUG:
|
||||||
-health-check, -hc run diagnostic check up
|
-health-check, -hc run diagnostic check up
|
||||||
|
@ -180,16 +180,16 @@ type Options struct {
|
|||||||
InputRawRequest string
|
InputRawRequest string
|
||||||
rawRequest string
|
rawRequest string
|
||||||
RequestBody string
|
RequestBody string
|
||||||
OutputFilterString string
|
OutputFilterString goflags.StringSlice
|
||||||
OutputMatchString string
|
OutputMatchString goflags.StringSlice
|
||||||
OutputFilterRegex string
|
OutputFilterRegex goflags.StringSlice
|
||||||
OutputMatchRegex string
|
OutputMatchRegex goflags.StringSlice
|
||||||
Retries int
|
Retries int
|
||||||
Threads int
|
Threads int
|
||||||
Timeout int
|
Timeout int
|
||||||
Delay time.Duration
|
Delay time.Duration
|
||||||
filterRegex *regexp.Regexp
|
filterRegexes []*regexp.Regexp
|
||||||
matchRegex *regexp.Regexp
|
matchRegexes []*regexp.Regexp
|
||||||
VHost bool
|
VHost bool
|
||||||
VHostInput bool
|
VHostInput bool
|
||||||
Smuggling bool
|
Smuggling bool
|
||||||
@ -359,8 +359,8 @@ func ParseOptions() *Options {
|
|||||||
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
|
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
|
||||||
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
|
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
|
||||||
flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
|
flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"),
|
flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"),
|
flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
|
flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
|
flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
|
||||||
flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
|
flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
|
||||||
@ -378,8 +378,8 @@ func ParseOptions() *Options {
|
|||||||
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
|
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
|
||||||
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
|
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
|
||||||
flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
|
flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"),
|
flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"),
|
flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
|
flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
|
||||||
flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
|
flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
|
||||||
flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
|
flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
|
||||||
@ -610,16 +610,22 @@ func (options *Options) ValidateOptions() error {
|
|||||||
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
|
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
|
||||||
return errors.Wrap(err, "Invalid value for filter content length option")
|
return errors.Wrap(err, "Invalid value for filter content length option")
|
||||||
}
|
}
|
||||||
if options.OutputFilterRegex != "" {
|
for _, filterRegexStr := range options.OutputFilterRegex {
|
||||||
if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil {
|
filterRegex, err := regexp.Compile(filterRegexStr)
|
||||||
|
if err != nil {
|
||||||
return errors.Wrap(err, "Invalid value for regex filter option")
|
return errors.Wrap(err, "Invalid value for regex filter option")
|
||||||
}
|
}
|
||||||
|
options.filterRegexes = append(options.filterRegexes, filterRegex)
|
||||||
}
|
}
|
||||||
if options.OutputMatchRegex != "" {
|
|
||||||
if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil {
|
for _, matchRegexStr := range options.OutputMatchRegex {
|
||||||
|
matchRegex, err := regexp.Compile(matchRegexStr)
|
||||||
|
if err != nil {
|
||||||
return errors.Wrap(err, "Invalid value for match regex option")
|
return errors.Wrap(err, "Invalid value for match regex option")
|
||||||
}
|
}
|
||||||
|
options.matchRegexes = append(options.matchRegexes, matchRegex)
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
|
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
|
||||||
return errors.Wrap(err, "Invalid value for match lines count option")
|
return errors.Wrap(err, "Invalid value for match lines count option")
|
||||||
}
|
}
|
||||||
|
@ -846,10 +846,19 @@ func (r *Runner) RunEnumeration() {
|
|||||||
if len(r.options.filterWordsCount) > 0 && sliceutil.Contains(r.options.filterWordsCount, resp.Words) {
|
if len(r.options.filterWordsCount) > 0 && sliceutil.Contains(r.options.filterWordsCount, resp.Words) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) {
|
if r.options.filterRegexes != nil {
|
||||||
continue
|
shouldContinue := false
|
||||||
|
for _, filterRegex := range r.options.filterRegexes {
|
||||||
|
if filterRegex.MatchString(resp.Raw) {
|
||||||
|
shouldContinue = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shouldContinue {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if r.options.OutputFilterString != "" && stringsutil.ContainsAnyI(resp.Raw, r.options.OutputFilterString) {
|
if len(r.options.OutputFilterString) > 0 && stringsutil.EqualFoldAny(resp.Raw, r.options.OutputFilterString...) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
|
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
|
||||||
@ -861,10 +870,19 @@ func (r *Runner) RunEnumeration() {
|
|||||||
if len(r.options.matchContentLength) > 0 && !sliceutil.Contains(r.options.matchContentLength, resp.ContentLength) {
|
if len(r.options.matchContentLength) > 0 && !sliceutil.Contains(r.options.matchContentLength, resp.ContentLength) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) {
|
if r.options.matchRegexes != nil {
|
||||||
continue
|
shouldContinue := false
|
||||||
|
for _, matchRegex := range r.options.matchRegexes {
|
||||||
|
if !matchRegex.MatchString(resp.Raw) {
|
||||||
|
shouldContinue = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if shouldContinue {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if r.options.OutputMatchString != "" && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString) {
|
if len(r.options.OutputMatchString) > 0 && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString...) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
|
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
|
||||||
|
Loading…
Reference in New Issue
Block a user