Add support for favicon hash (#476)

* Add support for favicon hash

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

README.md

@@ -110,6 +110,7 @@ MATCHERS:
    -er, -extract-regex string  Display response content with matched regex
    -mlc, -match-line-count string  Match Response body line count
    -mwc, -match-word-count string  Match Response body word count
+   -mfc, -match-favicon string[]   Match response with specific favicon
 
 FILTERS:
    -fc, -filter-code string    Filter response with given status code (-fc 403,401)
@@ -118,12 +119,14 @@ FILTERS:
    -fe, -filter-regex string   Filter response with specific regex
    -flc, -filter-line-count string  Filter Response body line count
    -fwc, -filter-word-count string  Filter Response body word count
+   -ffc, -filter-favicon string[]   Filter response with specific favicon
 
 RATE-LIMIT:
    -t, -threads int      Number of threads (default 50)
    -rl, -rate-limit int  Maximum requests to send per second (default 150)
 
 MISCELLANEOUS:
+   -favicon             Probes for favicon ("favicon.ico" as path) and display phythonic hash
    -tls-grab            Perform TLS(SSL) data grabbing
    -tls-probe           Send HTTP probes on the extracted TLS domains
    -csp-probe           Send HTTP probes on the extracted CSP domains
@@ -328,8 +331,8 @@ https://support.hackerone.com
 - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable.
 - For printing both HTTP/HTTPS results, `no-fallback` flag can be used.
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
-- `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
-- Unique flags should be used for specific use cases instead of running them as default with other flags.
+- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
+- Unique flags should be used for specific use cases instead of running them as default with other probes.
 - When using `json` flag, all the information (default probes) included in the JSON output.
 - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port`  (eg **udp:127.0.0.1:53**)
 - Invalid custom resolvers/files are ignored.

common/slice/slice.go

@@ -10,6 +10,16 @@ func IntSliceContains(sl []int, v int) bool {
 	return false
 }
 
+// UIntSliceContains check if a slice contains the specified uint value
+func UInt32SliceContains(sl []uint32, v uint32) bool {
+	for _, vv := range sl {
+		if vv == v {
+			return true
+		}
+	}
+	return false
+}
+
 // ToSlice creates a slice with all string keys from a map
 func ToSlice(m map[string]struct{}) (s []string) {
 	for k := range m {

common/stringz/stringz.go

@@ -1,11 +1,14 @@
 package stringz
 
 import (
+	"bytes"
+	"encoding/base64"
 	"net/url"
 	"strconv"
 	"strings"
 
 	"github.com/projectdiscovery/urlutil"
+	"github.com/spaolacci/murmur3"
 )
 
 // TrimProtocol removes the HTTP scheme from an URI
@@ -39,6 +42,24 @@ func StringToSliceInt(s string) ([]int, error) {
 	return r, nil
 }
 
+// StringToSliceUInt converts string to slice of ints
+func StringToSliceUInt32(s string) ([]uint32, error) {
+	var r []uint32
+	if s == "" {
+		return r, nil
+	}
+	for _, v := range strings.Split(s, ",") {
+		vTrim := strings.TrimSpace(v)
+		if i, err := strconv.ParseUint(vTrim, 10, 64); err == nil {
+			r = append(r, uint32(i))
+		} else {
+			return r, err
+		}
+	}
+
+	return r, nil
+}
+
 // SplitByCharAndTrimSpace splits string by a character and remove spaces
 func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
 	for _, token := range strings.Split(s, splitchar) {
@@ -84,3 +105,25 @@ func GetInvalidURI(rawURL string) (bool, string) {
 
 	return false, ""
 }
+
+func FaviconHash(data []byte) int32 {
+	stdBase64 := base64.StdEncoding.EncodeToString(data)
+	stdBase64 = InsertInto(stdBase64, 76, '\n')
+	hasher := murmur3.New32WithSeed(0)
+	hasher.Write([]byte(stdBase64))
+	return int32(hasher.Sum32())
+}
+
+func InsertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}

go.mod

@@ -44,6 +44,8 @@ require (
 	golang.org/x/text v0.3.7
 )
 
+require github.com/spaolacci/murmur3 v1.1.0
+
 require (
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect

go.sum

@@ -170,6 +170,8 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

runner/options.go

@@ -67,6 +67,7 @@ type scanOptions struct {
 	ExcludeCDN                bool
 	HostMaxErrors             int
 	ProbeAllIPS               bool
+	Favicon                   bool
 	LeaveDefaultPorts         bool
 	OutputLinesCount          bool
 	OutputWordsCount          bool
@@ -109,6 +110,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
 		MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
 		HostMaxErrors:             s.HostMaxErrors,
+		Favicon:                   s.Favicon,
 		LeaveDefaultPorts:         s.LeaveDefaultPorts,
 		OutputLinesCount:          s.OutputLinesCount,
 		OutputWordsCount:          s.OutputWordsCount,
@@ -206,6 +208,9 @@ type Options struct {
 	SkipDedupe                bool
 	ProbeAllIPS               bool
 	Resolvers                 goflags.NormalizedStringSlice
+	Favicon                   bool
+	OutputFilterFavicon       goflags.NormalizedStringSlice
+	OutputMatchFavicon        goflags.NormalizedStringSlice
 	LeaveDefaultPorts         bool
 	OutputLinesCount          bool
 	OutputMatchLinesCount     string
@@ -248,7 +253,6 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
 		flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
 		flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
-		flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
 	)
 
 	createGroup(flagSet, "matchers", "Matchers",
@@ -256,8 +260,10 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
 		flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
 		flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
+		flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
 		flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
 		flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"),
+		flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "Match response with specific favicon"),
 	)
 
 	createGroup(flagSet, "filters", "Filters",
@@ -267,6 +273,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
 		flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
 		flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"),
+		flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "Filter response with specific favicon"),
 	)
 
 	createGroup(flagSet, "rate-limit", "Rate-Limit",
@@ -275,6 +282,7 @@ func ParseOptions() *Options {
 	)
 
 	createGroup(flagSet, "Misc", "Miscellaneous",
+		flagSet.BoolVar(&options.Favicon, "favicon", false, "Probes for favicon (\"favicon.ico\" as path) and display phythonic hash"),
 		flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"),
 		flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"),
 		flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"),
@@ -438,6 +446,11 @@ func (options *Options) validateOptions() {
 		gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
 		options.StoreResponse = true
 	}
+
+	if options.Favicon {
+		gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n")
+		options.RequestURIs = "/favicon.ico"
+	}
 }
 
 // configureOutput configures the output on the screen

runner/runner.go

@@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) {
 	scanopts.ExcludeCDN = options.ExcludeCDN
 	scanopts.HostMaxErrors = options.HostMaxErrors
 	scanopts.ProbeAllIPS = options.ProbeAllIPS
+	scanopts.Favicon = options.Favicon
 	scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
 	scanopts.OutputLinesCount = options.OutputLinesCount
 	scanopts.OutputWordsCount = options.OutputWordsCount
@@ -567,6 +568,9 @@ func (r *Runner) RunEnumeration() {
 			if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) {
 				continue
 			}
+			if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
+				continue
+			}
 			if len(r.options.matchStatusCode) > 0 && !slice.IntSliceContains(r.options.matchStatusCode, resp.StatusCode) {
 				continue
 			}
@@ -579,6 +583,9 @@ func (r *Runner) RunEnumeration() {
 			if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
 				continue
 			}
+			if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
+				continue
+			}
 			if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
 				continue
 			}
@@ -1192,6 +1199,18 @@ retry:
 		builder.WriteRune(']')
 	}
 
+	var faviconMMH3 string
+	if scanopts.Favicon {
+		faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(resp.Data))
+		builder.WriteString(" [")
+		if !scanopts.OutputWithNoColor {
+			builder.WriteString(aurora.Magenta(faviconMMH3).String())
+		} else {
+			builder.WriteString(faviconMMH3)
+		}
+		builder.WriteRune(']')
+	}
+
 	if scanopts.OutputLinesCount {
 		builder.WriteString(" [")
 		if !scanopts.OutputWithNoColor {
@@ -1315,6 +1334,7 @@ retry:
 		ResponseTime:     resp.Duration.String(),
 		Technologies:     technologies,
 		FinalURL:         finalURL,
+		FavIconMMH3:      faviconMMH3,
 		Lines:            resp.Lines,
 		Words:            resp.Words,
 	}
@@ -1368,6 +1388,7 @@ type Result struct {
 	Chain            []httpx.ChainItem   `json:"chain,omitempty" csv:"chain"`
 	FinalURL         string              `json:"final-url,omitempty" csv:"final-url"`
 	Failed           bool                `json:"failed" csv:"failed"`
+	FavIconMMH3      string              `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
 	Lines            int                 `json:"lines" csv:"lines"`
 	Words            int                 `json:"words" csv:"words"`
 }