From 608d6d40d773ca91771ae413f600d475290f7be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 11 Jun 2024 12:04:54 +0300 Subject: [PATCH 1/3] support multiple values --- runner/options.go | 40 ++++++++++++++++++++++++---------------- runner/runner.go | 30 ++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/runner/options.go b/runner/options.go index 2321c78..970ae3c 100644 --- a/runner/options.go +++ b/runner/options.go @@ -179,16 +179,16 @@ type Options struct { InputRawRequest string rawRequest string RequestBody string - OutputFilterString string - OutputMatchString string - OutputFilterRegex string - OutputMatchRegex string + OutputFilterString goflags.StringSlice + OutputMatchString goflags.StringSlice + OutputFilterRegex goflags.StringSlice + OutputMatchRegex goflags.StringSlice Retries int Threads int Timeout int Delay time.Duration - filterRegex *regexp.Regexp - matchRegex *regexp.Regexp + filterRegexes []*regexp.Regexp + matchRegexes []*regexp.Regexp VHost bool VHostInput bool Smuggling bool @@ -357,8 +357,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"), flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"), flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions), - flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), - flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), + flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"), @@ -376,8 +376,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"), flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions), - flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), - flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), + flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"), @@ -607,14 +607,22 @@ func (options *Options) ValidateOptions() error { if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil { return errors.Wrap(err, "Invalid value for filter content length option") } - if options.OutputFilterRegex != "" { - if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil { - return errors.Wrap(err, "Invalid value for regex filter option") + if len(options.OutputFilterRegex) > 0 { + for _, filterRegexStr := range options.OutputFilterRegex { + filterRegex, err := regexp.Compile(filterRegexStr) + if err != nil { + return errors.Wrap(err, "Invalid value for regex filter option") + } + options.filterRegexes = append(options.filterRegexes, filterRegex) } } - if options.OutputMatchRegex != "" { - if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil { - return errors.Wrap(err, "Invalid value for match regex option") + if len(options.OutputMatchRegex) > 0 { + for _, matchRegexStr := range options.OutputMatchRegex { + matchRegex, err := regexp.Compile(matchRegexStr) + if err != nil { + return errors.Wrap(err, "Invalid value for match regex option") + } + options.matchRegexes = append(options.matchRegexes, matchRegex) } } if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil { diff --git a/runner/runner.go b/runner/runner.go index 9b0915b..2d17853 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -833,10 +833,19 @@ func (r *Runner) RunEnumeration() { if len(r.options.filterWordsCount) > 0 && sliceutil.Contains(r.options.filterWordsCount, resp.Words) { continue } - if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) { - continue + if r.options.filterRegexes != nil { + shouldContinue := false + for _, filterRegex := range r.options.filterRegexes { + if filterRegex.MatchString(resp.Raw) { + shouldContinue = true + break + } + } + if shouldContinue { + continue + } } - if r.options.OutputFilterString != "" && stringsutil.ContainsAnyI(resp.Raw, r.options.OutputFilterString) { + if len(r.options.OutputFilterString) > 0 && stringsutil.EqualFoldAny(resp.Raw, r.options.OutputFilterString...) { continue } if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) { @@ -848,10 +857,19 @@ func (r *Runner) RunEnumeration() { if len(r.options.matchContentLength) > 0 && !sliceutil.Contains(r.options.matchContentLength, resp.ContentLength) { continue } - if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) { - continue + if r.options.matchRegexes != nil { + shouldContinue := false + for _, matchRegex := range r.options.matchRegexes { + if !matchRegex.MatchString(resp.Raw) { + shouldContinue = true + break + } + } + if shouldContinue { + continue + } } - if r.options.OutputMatchString != "" && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString) { + if len(r.options.OutputMatchString) > 0 && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString...) { continue } if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) { From 812d7d892f089d5506bfb6534a9a94a61627a6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 11 Jun 2024 12:07:28 +0300 Subject: [PATCH 2/3] update README --- README.md | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 03e006c..a0bd597 100644 --- a/README.md +++ b/README.md @@ -95,27 +95,28 @@ INPUT: -u, -target string[] input target host(s) to probe PROBES: - -sc, -status-code display response status-code - -cl, -content-length display response content-length - -ct, -content-type display response content-type - -location display response redirect location - -favicon display mmh3 hash for '/favicon.ico' file - -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) - -jarm display jarm fingerprint hash - -rt, -response-time display response time - -lc, -line-count display response body line count - -wc, -word-count display response body word count - -title display page title - -bp, -body-preview display first N characters of response body (default 100) - -server, -web-server display server name - -td, -tech-detect display technology in use based on wappalyzer dataset - -method display http request method - -websocket display server using websocket - -ip display host ip - -cname display host cname - -asn display host asn information - -cdn display cdn/waf in use (default true) - -probe display probe status + -sc, -status-code display response status-code + -cl, -content-length display response content-length + -ct, -content-type display response content-type + -location display response redirect location + -favicon display mmh3 hash for '/favicon.ico' file + -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) + -jarm display jarm fingerprint hash + -rt, -response-time display response time + -lc, -line-count display response body line count + -wc, -word-count display response body word count + -title display page title + -bp, -body-preview display first N characters of response body (default 100) + -server, -web-server display server name + -td, -tech-detect display technology in use based on wappalyzer dataset + -method display http request method + -websocket display server using websocket + -ip display host ip + -cname display host cname + -extract-fqdn, -efqdn get domain and subdomains from response body and header in jsonl/csv output + -asn display host asn information + -cdn display cdn/waf in use (default true) + -probe display probe status HEADLESS: -ss, -screenshot enable saving screenshot of the page using headless browser @@ -131,15 +132,15 @@ MATCHERS: -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (-ms admin) - -mr, -match-regex string match response with specified regex (-mr admin) + -ms, -match-string string[] match response with specified string (-ms admin) + -mr, -match-regex string[] match response with specified regex (-mr admin) -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition EXTRACTOR: -er, -extract-regex string[] display response content with matched regex - -ep, -extract-preset string[] display response content matched by a pre-defined regex (mail,url,ipv4) + -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) @@ -148,8 +149,8 @@ FILTERS: -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) - -fs, -filter-string string filter response with specified string (-fs admin) - -fe, -filter-regex string filter response with specified regex (-fe admin) + -fs, -filter-string string[] filter response with specified string (-fs admin) + -fe, -filter-regex string[] filter response with specified regex (-fe admin) -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition From 6811b18e853ff489118026ee6dcf50e4111d84a3 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 12 Jun 2024 15:36:16 +0200 Subject: [PATCH 3/3] removing redundant code + update readme --- README.md | 51 ++++++++++++++++++++++++----------------------- runner/options.go | 26 +++++++++++------------- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index a0bd597..d9b0a3f 100644 --- a/README.md +++ b/README.md @@ -194,31 +194,32 @@ OUTPUT: -pr, -protocol string protocol to use (unknown, http11) CONFIGURATIONS: - -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) - -auth configure projectdiscovery cloud (pdcp) api key (default true) - -r, -resolvers string[] list of custom resolver (file or comma separated) - -allow string[] allowed list of IP/CIDR's to process (file or comma separated) - -deny string[] denied list of IP/CIDR's to process (file or comma separated) - -sni, -sni-name string custom TLS SNI name - -random-agent enable Random User-Agent to use (default true) - -H, -header string[] custom http headers to send with request - -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) - -unsafe send raw requests skipping golang normalization - -resume resume scan using resume.cfg - -fr, -follow-redirects follow http redirects - -maxr, -max-redirects int max number of redirects to follow per host (default 10) - -fhr, -follow-host-redirects follow redirects on the same host - -rhsts, -respect-hsts respect HSTS response headers for redirect requests - -vhost-input get a list of vhosts as input - -x string request methods to probe, use 'all' to probe all HTTP methods - -body string post body to include in http request - -s, -stream stream mode - start elaborating input targets without sorting - -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 - -ztls use ztls library with autofallback to standard one for tls13 - -no-decode avoid decoding body - -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization - -no-stdin Disable Stdin processing + -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) + -auth configure projectdiscovery cloud (pdcp) api key (default true) + -r, -resolvers string[] list of custom resolver (file or comma separated) + -allow string[] allowed list of IP/CIDR's to process (file or comma separated) + -deny string[] denied list of IP/CIDR's to process (file or comma separated) + -sni, -sni-name string custom TLS SNI name + -random-agent enable Random User-Agent to use (default true) + -H, -header string[] custom http headers to send with request + -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) + -unsafe send raw requests skipping golang normalization + -resume resume scan using resume.cfg + -fr, -follow-redirects follow http redirects + -maxr, -max-redirects int max number of redirects to follow per host (default 10) + -fhr, -follow-host-redirects follow redirects on the same host + -rhsts, -respect-hsts respect HSTS response headers for redirect requests + -vhost-input get a list of vhosts as input + -x string request methods to probe, use 'all' to probe all HTTP methods + -body string post body to include in http request + -s, -stream stream mode - start elaborating input targets without sorting + -sd, -skip-dedupe disable dedupe input items (only used with stream mode) + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 + -ztls use ztls library with autofallback to standard one for tls13 + -no-decode avoid decoding body + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization + -no-stdin Disable Stdin processing + -hae, -http-api-endpoint string experimental http api endpoint DEBUG: -health-check, -hc run diagnostic check up diff --git a/runner/options.go b/runner/options.go index 970ae3c..681602c 100644 --- a/runner/options.go +++ b/runner/options.go @@ -607,24 +607,22 @@ func (options *Options) ValidateOptions() error { if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil { return errors.Wrap(err, "Invalid value for filter content length option") } - if len(options.OutputFilterRegex) > 0 { - for _, filterRegexStr := range options.OutputFilterRegex { - filterRegex, err := regexp.Compile(filterRegexStr) - if err != nil { - return errors.Wrap(err, "Invalid value for regex filter option") - } - options.filterRegexes = append(options.filterRegexes, filterRegex) + for _, filterRegexStr := range options.OutputFilterRegex { + filterRegex, err := regexp.Compile(filterRegexStr) + if err != nil { + return errors.Wrap(err, "Invalid value for regex filter option") } + options.filterRegexes = append(options.filterRegexes, filterRegex) } - if len(options.OutputMatchRegex) > 0 { - for _, matchRegexStr := range options.OutputMatchRegex { - matchRegex, err := regexp.Compile(matchRegexStr) - if err != nil { - return errors.Wrap(err, "Invalid value for match regex option") - } - options.matchRegexes = append(options.matchRegexes, matchRegex) + + for _, matchRegexStr := range options.OutputMatchRegex { + matchRegex, err := regexp.Compile(matchRegexStr) + if err != nil { + return errors.Wrap(err, "Invalid value for match regex option") } + options.matchRegexes = append(options.matchRegexes, matchRegex) } + if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil { return errors.Wrap(err, "Invalid value for match lines count option") }