mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-10-26 18:08:27 +03:00
Adding support for lines/words matchers/filters
This commit is contained in:
parent
72ce90ec4e
commit
47bbacf877
@ -66,6 +66,8 @@ type scanOptions struct {
|
||||
ExcludeCDN bool
|
||||
HostMaxErrors int
|
||||
ProbeAllIPS bool
|
||||
OutputLinesCount bool
|
||||
OutputWordsCount bool
|
||||
}
|
||||
|
||||
func (s *scanOptions) Clone() *scanOptions {
|
||||
@ -105,6 +107,8 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
||||
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
||||
HostMaxErrors: s.HostMaxErrors,
|
||||
OutputLinesCount: s.OutputLinesCount,
|
||||
OutputWordsCount: s.OutputWordsCount,
|
||||
}
|
||||
}
|
||||
|
||||
@ -199,6 +203,16 @@ type Options struct {
|
||||
SkipDedupe bool
|
||||
ProbeAllIPS bool
|
||||
Resolvers goflags.NormalizedStringSlice
|
||||
OutputLinesCount bool
|
||||
OutputMatchLinesCount string
|
||||
matchLinesCount []int
|
||||
OutputFilterLinesCount string
|
||||
filterLinesCount []int
|
||||
OutputWordsCount bool
|
||||
OutputMatchWordsCount string
|
||||
matchWordsCount []int
|
||||
OutputFilterWordsCount string
|
||||
filterWordsCount []int
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -228,6 +242,9 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
|
||||
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
|
||||
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, " Display Response body line count"),
|
||||
flagSet.BoolVarP(&options.OutputWordsCount, "words-count", "wc", false, " Display Response body words count"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "matchers", "Matchers",
|
||||
@ -235,7 +252,8 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
|
||||
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
|
||||
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-words-count", "mwc", "", "Match Response body words count"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "filters", "Filters",
|
||||
@ -243,6 +261,8 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"),
|
||||
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"),
|
||||
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-words-count", "fwc", "", "Filter Response body words count"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "rate-limit", "Rate-Limit",
|
||||
@ -373,6 +393,18 @@ func (options *Options) validateOptions() {
|
||||
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
|
||||
}
|
||||
}
|
||||
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
|
||||
}
|
||||
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
|
||||
}
|
||||
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
|
||||
}
|
||||
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
|
||||
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
|
||||
}
|
||||
|
||||
var resolvers []string
|
||||
for _, resolver := range options.Resolvers {
|
||||
|
@ -227,6 +227,8 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.ExcludeCDN = options.ExcludeCDN
|
||||
scanopts.HostMaxErrors = options.HostMaxErrors
|
||||
scanopts.ProbeAllIPS = options.ProbeAllIPS
|
||||
scanopts.OutputLinesCount = options.OutputLinesCount
|
||||
scanopts.OutputWordsCount = options.OutputWordsCount
|
||||
runner.scanopts = scanopts
|
||||
|
||||
if options.ShowStatistics {
|
||||
@ -552,6 +554,12 @@ func (r *Runner) RunEnumeration() {
|
||||
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
|
||||
continue
|
||||
}
|
||||
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
|
||||
continue
|
||||
}
|
||||
@ -570,6 +578,12 @@ func (r *Runner) RunEnumeration() {
|
||||
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) {
|
||||
continue
|
||||
}
|
||||
|
||||
row := resp.str
|
||||
if r.options.JSONOutput {
|
||||
@ -1168,6 +1182,26 @@ retry:
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
if scanopts.OutputLinesCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(resp.Lines).String())
|
||||
} else {
|
||||
builder.WriteString(fmt.Sprint(resp.Lines))
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
if scanopts.OutputWordsCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(resp.Words).String())
|
||||
} else {
|
||||
builder.WriteString(fmt.Sprint(resp.Words))
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
// store responses or chain in directory
|
||||
if scanopts.StoreResponse || scanopts.StoreChain {
|
||||
domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
|
||||
@ -1271,6 +1305,8 @@ retry:
|
||||
ResponseTime: resp.Duration.String(),
|
||||
Technologies: technologies,
|
||||
FinalURL: finalURL,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
}
|
||||
}
|
||||
|
||||
@ -1322,6 +1358,8 @@ type Result struct {
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
Loading…
Reference in New Issue
Block a user