diff --git a/README.md b/README.md index 988af0b..d8d103b 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ MATCHERS: -er, -extract-regex string Display response content with matched regex -mlc, -match-line-count string Match Response body line count -mwc, -match-word-count string Match Response body word count + -mfc, -match-favicon string[] Match response with specific favicon FILTERS: -fc, -filter-code string Filter response with given status code (-fc 403,401) @@ -118,12 +119,14 @@ FILTERS: -fe, -filter-regex string Filter response with specific regex -flc, -filter-line-count string Filter Response body line count -fwc, -filter-word-count string Filter Response body word count + -ffc, -filter-favicon string[] Filter response with specific favicon RATE-LIMIT: -t, -threads int Number of threads (default 50) -rl, -rate-limit int Maximum requests to send per second (default 150) MISCELLANEOUS: + -favicon Probes for favicon ("favicon.ico" as path) and display phythonic hash -tls-grab Perform TLS(SSL) data grabbing -tls-probe Send HTTP probes on the extracted TLS domains -csp-probe Send HTTP probes on the extracted CSP domains @@ -328,8 +331,8 @@ https://support.hackerone.com - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable. - For printing both HTTP/HTTPS results, `no-fallback` flag can be used. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` -- `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. -- Unique flags should be used for specific use cases instead of running them as default with other flags. +- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. +- Unique flags should be used for specific use cases instead of running them as default with other probes. - When using `json` flag, all the information (default probes) included in the JSON output. - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**) - Invalid custom resolvers/files are ignored. diff --git a/common/slice/slice.go b/common/slice/slice.go index 0102490..2426d84 100644 --- a/common/slice/slice.go +++ b/common/slice/slice.go @@ -10,6 +10,16 @@ func IntSliceContains(sl []int, v int) bool { return false } +// UIntSliceContains check if a slice contains the specified uint value +func UInt32SliceContains(sl []uint32, v uint32) bool { + for _, vv := range sl { + if vv == v { + return true + } + } + return false +} + // ToSlice creates a slice with all string keys from a map func ToSlice(m map[string]struct{}) (s []string) { for k := range m { diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 34eb790..6c73201 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -1,11 +1,14 @@ package stringz import ( + "bytes" + "encoding/base64" "net/url" "strconv" "strings" "github.com/projectdiscovery/urlutil" + "github.com/spaolacci/murmur3" ) // TrimProtocol removes the HTTP scheme from an URI @@ -39,6 +42,24 @@ func StringToSliceInt(s string) ([]int, error) { return r, nil } +// StringToSliceUInt converts string to slice of ints +func StringToSliceUInt32(s string) ([]uint32, error) { + var r []uint32 + if s == "" { + return r, nil + } + for _, v := range strings.Split(s, ",") { + vTrim := strings.TrimSpace(v) + if i, err := strconv.ParseUint(vTrim, 10, 64); err == nil { + r = append(r, uint32(i)) + } else { + return r, err + } + } + + return r, nil +} + // SplitByCharAndTrimSpace splits string by a character and remove spaces func SplitByCharAndTrimSpace(s, splitchar string) (result []string) { for _, token := range strings.Split(s, splitchar) { @@ -84,3 +105,25 @@ func GetInvalidURI(rawURL string) (bool, string) { return false, "" } + +func FaviconHash(data []byte) int32 { + stdBase64 := base64.StdEncoding.EncodeToString(data) + stdBase64 = InsertInto(stdBase64, 76, '\n') + hasher := murmur3.New32WithSeed(0) + hasher.Write([]byte(stdBase64)) + return int32(hasher.Sum32()) +} + +func InsertInto(s string, interval int, sep rune) string { + var buffer bytes.Buffer + before := interval - 1 + last := len(s) - 1 + for i, char := range s { + buffer.WriteRune(char) + if i%interval == before && i != last { + buffer.WriteRune(sep) + } + } + buffer.WriteRune(sep) + return buffer.String() +} diff --git a/go.mod b/go.mod index 71fb315..7672f9b 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,8 @@ require ( golang.org/x/text v0.3.7 ) +require github.com/spaolacci/murmur3 v1.1.0 + require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 6b311c7..1a83f40 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,8 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/runner/options.go b/runner/options.go index 39c44bc..ca2f90e 100644 --- a/runner/options.go +++ b/runner/options.go @@ -67,6 +67,7 @@ type scanOptions struct { ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool + Favicon bool LeaveDefaultPorts bool OutputLinesCount bool OutputWordsCount bool @@ -109,6 +110,7 @@ func (s *scanOptions) Clone() *scanOptions { MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, HostMaxErrors: s.HostMaxErrors, + Favicon: s.Favicon, LeaveDefaultPorts: s.LeaveDefaultPorts, OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, @@ -206,6 +208,9 @@ type Options struct { SkipDedupe bool ProbeAllIPS bool Resolvers goflags.NormalizedStringSlice + Favicon bool + OutputFilterFavicon goflags.NormalizedStringSlice + OutputMatchFavicon goflags.NormalizedStringSlice LeaveDefaultPorts bool OutputLinesCount bool OutputMatchLinesCount string @@ -248,7 +253,6 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), ) createGroup(flagSet, "matchers", "Matchers", @@ -256,8 +260,10 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"), flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "Match response with specific favicon"), ) createGroup(flagSet, "filters", "Filters", @@ -267,6 +273,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"), + flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "Filter response with specific favicon"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -275,6 +282,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Misc", "Miscellaneous", + flagSet.BoolVar(&options.Favicon, "favicon", false, "Probes for favicon (\"favicon.ico\" as path) and display phythonic hash"), flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), @@ -438,6 +446,11 @@ func (options *Options) validateOptions() { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } + + if options.Favicon { + gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n") + options.RequestURIs = "/favicon.ico" + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index c3aee24..c8d8ee1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) { scanopts.ExcludeCDN = options.ExcludeCDN scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS + scanopts.Favicon = options.Favicon scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts scanopts.OutputLinesCount = options.OutputLinesCount scanopts.OutputWordsCount = options.OutputWordsCount @@ -567,6 +568,9 @@ func (r *Runner) RunEnumeration() { if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) { continue } + if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) { + continue + } if len(r.options.matchStatusCode) > 0 && !slice.IntSliceContains(r.options.matchStatusCode, resp.StatusCode) { continue } @@ -579,6 +583,9 @@ func (r *Runner) RunEnumeration() { if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) { continue } + if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) { + continue + } if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) { continue } @@ -1192,6 +1199,18 @@ retry: builder.WriteRune(']') } + var faviconMMH3 string + if scanopts.Favicon { + faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(resp.Data)) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(faviconMMH3).String()) + } else { + builder.WriteString(faviconMMH3) + } + builder.WriteRune(']') + } + if scanopts.OutputLinesCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1315,6 +1334,7 @@ retry: ResponseTime: resp.Duration.String(), Technologies: technologies, FinalURL: finalURL, + FavIconMMH3: faviconMMH3, Lines: resp.Lines, Words: resp.Words, } @@ -1368,6 +1388,7 @@ type Result struct { Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` FinalURL string `json:"final-url,omitempty" csv:"final-url"` Failed bool `json:"failed" csv:"failed"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` }