projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-10-26 18:08:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adding support for lines/words matchers/filters

Browse Source
This commit is contained in:
mzack 2021-12-31 09:26:39 +01:00
parent 72ce90ec4e
commit 47bbacf877
2 changed files with 71 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
runner/options.go
View File

@ -66,6 +66,8 @@ type scanOptions struct {
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
OutputLinesCount bool
OutputWordsCount bool
}
func (s *scanOptions) Clone() *scanOptions {
@ -105,6 +107,8 @@ func (s *scanOptions) Clone() *scanOptions {
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
}
}
@ -199,6 +203,16 @@ type Options struct {
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.NormalizedStringSlice
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
OutputFilterLinesCount string
filterLinesCount []int
OutputWordsCount bool
OutputMatchWordsCount string
matchWordsCount []int
OutputFilterWordsCount string
filterWordsCount []int
}
// ParseOptions parses the command line options for application
@ -228,6 +242,9 @@ func ParseOptions() *Options {
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, " Display Response body line count"),
flagSet.BoolVarP(&options.OutputWordsCount, "words-count", "wc", false, " Display Response body words count"),
)
createGroup(flagSet, "matchers", "Matchers",
@ -235,7 +252,8 @@ func ParseOptions() *Options {
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-words-count", "mwc", "", "Match Response body words count"),
)
createGroup(flagSet, "filters", "Filters",
@ -243,6 +261,8 @@ func ParseOptions() *Options {
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"),
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"),
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-words-count", "fwc", "", "Filter Response body words count"),
)
createGroup(flagSet, "rate-limit", "Rate-Limit",
@ -373,6 +393,18 @@ func (options *Options) validateOptions() {
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
}
}
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
}
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
}
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
}
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
}
var resolvers []string
for _, resolver := range options.Resolvers {

38
runner/runner.go
View File

@ -227,6 +227,8 @@ func New(options *Options) (*Runner, error) {
scanopts.ExcludeCDN = options.ExcludeCDN
scanopts.HostMaxErrors = options.HostMaxErrors
scanopts.ProbeAllIPS = options.ProbeAllIPS
scanopts.OutputLinesCount = options.OutputLinesCount
scanopts.OutputWordsCount = options.OutputWordsCount
runner.scanopts = scanopts
if options.ShowStatistics {
@ -552,6 +554,12 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
continue
}
if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) {
continue
}
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
continue
}
@ -570,6 +578,12 @@ func (r *Runner) RunEnumeration() {
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
continue
}
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
continue
}
if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) {
continue
}
row := resp.str
if r.options.JSONOutput {
@ -1168,6 +1182,26 @@ retry:
builder.WriteRune(']')
}
if scanopts.OutputLinesCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Lines).String())
} else {
builder.WriteString(fmt.Sprint(resp.Lines))
}
builder.WriteRune(']')
}
if scanopts.OutputWordsCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Words).String())
} else {
builder.WriteString(fmt.Sprint(resp.Words))
}
builder.WriteRune(']')
}
// store responses or chain in directory
if scanopts.StoreResponse || scanopts.StoreChain {
domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
@ -1271,6 +1305,8 @@ retry:
ResponseTime: resp.Duration.String(),
Technologies: technologies,
FinalURL: finalURL,
Lines: resp.Lines,
Words: resp.Words,
}
}
@ -1322,6 +1358,8 @@ type Result struct {
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
Failed bool `json:"failed" csv:"failed"`
Lines int `json:"lines" csv:"lines"`
Words int `json:"words" csv:"words"`
}
// JSON the result