mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-10-27 02:22:13 +03:00
Merge branch 'dev' into issue-464-default-ports
This commit is contained in:
mzack
commit
a152dcd42d
47
README.md
47
README.md
@ -43,18 +43,19 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi
|
||||
|
||||
### Supported probes:-
|
||||
|
||||
| Probes | Default check | Probes | Default check |
|
||||
|--------------------|-----------------|--------------------|-----------------|
|
||||
| URL | true | IP | true |
|
||||
| Title | true | CNAME | true |
|
||||
| Status Code | true | Raw HTTP | false |
|
||||
| Content Length | true | HTTP2 | false |
|
||||
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
|
||||
| CSP Header | true | Virtual host | false |
|
||||
| Location Header | true | CDN | false |
|
||||
| Web Server | true | Path | false |
|
||||
| Web Socket | true | Ports | false |
|
||||
| Response Time | true | Request method | false |
|
||||
| Probes | Default check | Probes | Default check |
|
||||
| --------------- | ------------- | ----------------- | ------------- |
|
||||
| URL | true | IP | true |
|
||||
| Title | true | CNAME | true |
|
||||
| Status Code | true | Raw HTTP | false |
|
||||
| Content Length | true | HTTP2 | false |
|
||||
| Line Count | true | Word Count | true |
|
||||
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
|
||||
| CSP Header | true | Virtual host | false |
|
||||
| Location Header | true | CDN | false |
|
||||
| Web Server | true | Paths | false |
|
||||
| Web Socket | true | Ports | false |
|
||||
| Response Time | true | Request Method | false |
|
||||
|
||||
|
||||
# Installation Instructions
|
||||
@ -87,6 +88,8 @@ PROBES:
|
||||
-sc, -status-code Display Status Code
|
||||
-td, -tech-detect Display wappalyzer based technology detection
|
||||
-cl, -content-length Display Content-Length
|
||||
-lc, -line-count Display Response body line count
|
||||
-wc, -word-count Display Response body word count
|
||||
-server, -web-server Display Server header
|
||||
-ct, -content-type Display Content-Type header
|
||||
-rt, -response-time Display the response time
|
||||
@ -105,12 +108,16 @@ MATCHERS:
|
||||
-ms, -match-string string Match response with given string
|
||||
-mr, -match-regex string Match response with specific regex
|
||||
-er, -extract-regex string Display response content with matched regex
|
||||
-mlc, -match-line-count string Match Response body line count
|
||||
-mwc, -match-word-count string Match Response body word count
|
||||
|
||||
FILTERS:
|
||||
-fc, -filter-code string Filter response with given status code (-fc 403,401)
|
||||
-fl, -filter-length string Filter response with given content length (-fl 23,33)
|
||||
-fs, -filter-string string Filter response with specific string
|
||||
-fe, -filter-regex string Filter response with specific regex
|
||||
-flc, -filter-line-count string Filter Response body line count
|
||||
-fwc, -filter-word-count string Filter Response body word count
|
||||
|
||||
RATE-LIMIT:
|
||||
-t, -threads int Number of threads (default 50)
|
||||
@ -128,14 +135,14 @@ MISCELLANEOUS:
|
||||
-paths string File or comma separated paths to request (deprecated)
|
||||
|
||||
OUTPUT:
|
||||
-o, -output string File to write output
|
||||
-sr, -store-response Store HTTP responses
|
||||
-srd, -store-response-dir string Custom directory to store HTTP responses (default "output")
|
||||
-json Output in JSONL(ines) format
|
||||
-irr, -include-response Include HTTP request/response in JSON output (-json only)
|
||||
-include-chain Include redirect HTTP Chain in JSON output (-json only)
|
||||
-store-chain Include HTTP redirect chain in responses (-sr only)
|
||||
-csv Output in CSV format
|
||||
-o, -output string file to write output
|
||||
-sr, -store-response store http response to output directory
|
||||
-srd, -store-response-dir string store http response to custom directory (default "output")
|
||||
-csv store output in CSV format
|
||||
-json store output in JSONL(ines) format
|
||||
-irr, -include-response include http request/response in JSON output (-json only)
|
||||
-include-chain include redirect http chain in JSON output (-json only)
|
||||
-store-chain include http redirect chain in responses (-sr only)
|
||||
|
||||
CONFIGURATIONS:
|
||||
-r, -resolvers string[] List of custom resolvers (file or comma separated)
|
||||
|
||||
4
go.mod
4
go.mod
@ -10,7 +10,7 @@ require (
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/microcosm-cc/bluemonday v1.0.16
|
||||
github.com/microcosm-cc/bluemonday v1.0.17
|
||||
github.com/miekg/dns v1.1.43 // indirect
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/cdncheck v0.0.2
|
||||
@ -32,7 +32,7 @@ require (
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.22
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.23
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.3.0
|
||||
github.com/smartystreets/assertions v1.0.0 // indirect
|
||||
|
||||
8
go.sum
8
go.sum
@ -70,8 +70,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
|
||||
github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.17 h1:Z1a//hgsQ4yjC+8zEkV8IWySkXnsxmdSY642CTFQb5Y=
|
||||
github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.22 h1:nPEl0eUfYvAUH9xEZPF2kuboh92DZJQz1G6pbLS7XdU=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.22/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.23 h1:ciuuPOPvkDjRXIzxTXtIp/nOhpE/l2tne33FpXHaMJA=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.23/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
|
||||
@ -67,6 +67,8 @@ type scanOptions struct {
|
||||
HostMaxErrors int
|
||||
ProbeAllIPS bool
|
||||
LeaveDefaultPorts bool
|
||||
OutputLinesCount bool
|
||||
OutputWordsCount bool
|
||||
}
|
||||
|
||||
func (s *scanOptions) Clone() *scanOptions {
|
||||
@ -107,6 +109,8 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
||||
HostMaxErrors: s.HostMaxErrors,
|
||||
LeaveDefaultPorts: s.LeaveDefaultPorts,
|
||||
OutputLinesCount: s.OutputLinesCount,
|
||||
OutputWordsCount: s.OutputWordsCount,
|
||||
}
|
||||
}
|
||||
|
||||
@ -202,6 +206,16 @@ type Options struct {
|
||||
ProbeAllIPS bool
|
||||
Resolvers goflags.NormalizedStringSlice
|
||||
LeaveDefaultPorts bool
|
||||
OutputLinesCount bool
|
||||
OutputMatchLinesCount string
|
||||
matchLinesCount []int
|
||||
OutputFilterLinesCount string
|
||||
filterLinesCount []int
|
||||
OutputWordsCount bool
|
||||
OutputMatchWordsCount string
|
||||
matchWordsCount []int
|
||||
OutputFilterWordsCount string
|
||||
filterWordsCount []int
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -222,6 +236,8 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"),
|
||||
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"),
|
||||
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"),
|
||||
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "Display Response body line count"),
|
||||
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "Display Response body word count"),
|
||||
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"),
|
||||
flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"),
|
||||
flagSet.BoolVar(&options.Location, "location", false, "Display Location header"),
|
||||
@ -231,6 +247,7 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
|
||||
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
|
||||
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "matchers", "Matchers",
|
||||
@ -238,7 +255,8 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
|
||||
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
|
||||
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "filters", "Filters",
|
||||
@ -246,6 +264,8 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"),
|
||||
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"),
|
||||
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "rate-limit", "Rate-Limit",
|
||||
@ -266,14 +286,14 @@ func ParseOptions() *Options {
|
||||
)
|
||||
|
||||
createGroup(flagSet, "output", "Output",
|
||||
flagSet.StringVarP(&options.Output, "output", "o", "", "File to write output"),
|
||||
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"),
|
||||
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "Custom directory to store HTTP responses"),
|
||||
flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"),
|
||||
flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "Include HTTP request/response in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"),
|
||||
flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"),
|
||||
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
|
||||
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
|
||||
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "store http response to custom directory"),
|
||||
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in CSV format"),
|
||||
flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"),
|
||||
flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "configs", "Configurations",
|
||||
@ -377,6 +397,18 @@ func (options *Options) validateOptions() {
|
||||
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
|
||||
}
|
||||
}
|
||||
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
|
||||
}
|
||||
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
|
||||
}
|
||||
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
|
||||
}
|
||||
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
|
||||
}
|
||||
|
||||
var resolvers []string
|
||||
for _, resolver := range options.Resolvers {
|
||||
@ -395,7 +427,11 @@ func (options *Options) validateOptions() {
|
||||
options.Resolvers = resolvers
|
||||
if len(options.Resolvers) > 0 {
|
||||
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
|
||||
}
|
||||
|
||||
if options.StoreResponseDir != "" && !options.StoreResponse {
|
||||
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -228,6 +228,8 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.HostMaxErrors = options.HostMaxErrors
|
||||
scanopts.ProbeAllIPS = options.ProbeAllIPS
|
||||
scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
|
||||
scanopts.OutputLinesCount = options.OutputLinesCount
|
||||
scanopts.OutputWordsCount = options.OutputWordsCount
|
||||
runner.scanopts = scanopts
|
||||
|
||||
if options.ShowStatistics {
|
||||
@ -553,6 +555,12 @@ func (r *Runner) RunEnumeration() {
|
||||
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
|
||||
continue
|
||||
}
|
||||
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
|
||||
continue
|
||||
}
|
||||
@ -571,6 +579,12 @@ func (r *Runner) RunEnumeration() {
|
||||
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) {
|
||||
continue
|
||||
}
|
||||
|
||||
row := resp.str
|
||||
if r.options.JSONOutput {
|
||||
@ -1178,6 +1192,26 @@ retry:
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
if scanopts.OutputLinesCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(resp.Lines).String())
|
||||
} else {
|
||||
builder.WriteString(fmt.Sprint(resp.Lines))
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
if scanopts.OutputWordsCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(resp.Words).String())
|
||||
} else {
|
||||
builder.WriteString(fmt.Sprint(resp.Words))
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
// store responses or chain in directory
|
||||
if scanopts.StoreResponse || scanopts.StoreChain {
|
||||
domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
|
||||
@ -1281,6 +1315,8 @@ retry:
|
||||
ResponseTime: resp.Duration.String(),
|
||||
Technologies: technologies,
|
||||
FinalURL: finalURL,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
}
|
||||
}
|
||||
|
||||
@ -1332,6 +1368,8 @@ type Result struct {
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
||||
Loading…
Reference in New Issue
Block a user