mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 05:20:29 +03:00
Add support for favicon hash (#476)
* Add support for favicon hash Co-authored-by: sandeep <sandeep@projectdiscovery.io>
This commit is contained in:
parent
47915603a5
commit
bee964315c
@ -110,6 +110,7 @@ MATCHERS:
|
||||
-er, -extract-regex string Display response content with matched regex
|
||||
-mlc, -match-line-count string Match Response body line count
|
||||
-mwc, -match-word-count string Match Response body word count
|
||||
-mfc, -match-favicon string[] Match response with specific favicon
|
||||
|
||||
FILTERS:
|
||||
-fc, -filter-code string Filter response with given status code (-fc 403,401)
|
||||
@ -118,12 +119,14 @@ FILTERS:
|
||||
-fe, -filter-regex string Filter response with specific regex
|
||||
-flc, -filter-line-count string Filter Response body line count
|
||||
-fwc, -filter-word-count string Filter Response body word count
|
||||
-ffc, -filter-favicon string[] Filter response with specific favicon
|
||||
|
||||
RATE-LIMIT:
|
||||
-t, -threads int Number of threads (default 50)
|
||||
-rl, -rate-limit int Maximum requests to send per second (default 150)
|
||||
|
||||
MISCELLANEOUS:
|
||||
-favicon Probes for favicon ("favicon.ico" as path) and display phythonic hash
|
||||
-tls-grab Perform TLS(SSL) data grabbing
|
||||
-tls-probe Send HTTP probes on the extracted TLS domains
|
||||
-csp-probe Send HTTP probes on the extracted CSP domains
|
||||
@ -328,8 +331,8 @@ https://support.hackerone.com
|
||||
- As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable.
|
||||
- For printing both HTTP/HTTPS results, `no-fallback` flag can be used.
|
||||
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
|
||||
- `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
|
||||
- Unique flags should be used for specific use cases instead of running them as default with other flags.
|
||||
- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
|
||||
- Unique flags should be used for specific use cases instead of running them as default with other probes.
|
||||
- When using `json` flag, all the information (default probes) included in the JSON output.
|
||||
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**)
|
||||
- Invalid custom resolvers/files are ignored.
|
||||
|
@ -10,6 +10,16 @@ func IntSliceContains(sl []int, v int) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// UIntSliceContains check if a slice contains the specified uint value
|
||||
func UInt32SliceContains(sl []uint32, v uint32) bool {
|
||||
for _, vv := range sl {
|
||||
if vv == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ToSlice creates a slice with all string keys from a map
|
||||
func ToSlice(m map[string]struct{}) (s []string) {
|
||||
for k := range m {
|
||||
|
@ -1,11 +1,14 @@
|
||||
package stringz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/urlutil"
|
||||
"github.com/spaolacci/murmur3"
|
||||
)
|
||||
|
||||
// TrimProtocol removes the HTTP scheme from an URI
|
||||
@ -39,6 +42,24 @@ func StringToSliceInt(s string) ([]int, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// StringToSliceUInt converts string to slice of ints
|
||||
func StringToSliceUInt32(s string) ([]uint32, error) {
|
||||
var r []uint32
|
||||
if s == "" {
|
||||
return r, nil
|
||||
}
|
||||
for _, v := range strings.Split(s, ",") {
|
||||
vTrim := strings.TrimSpace(v)
|
||||
if i, err := strconv.ParseUint(vTrim, 10, 64); err == nil {
|
||||
r = append(r, uint32(i))
|
||||
} else {
|
||||
return r, err
|
||||
}
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// SplitByCharAndTrimSpace splits string by a character and remove spaces
|
||||
func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
|
||||
for _, token := range strings.Split(s, splitchar) {
|
||||
@ -84,3 +105,25 @@ func GetInvalidURI(rawURL string) (bool, string) {
|
||||
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func FaviconHash(data []byte) int32 {
|
||||
stdBase64 := base64.StdEncoding.EncodeToString(data)
|
||||
stdBase64 = InsertInto(stdBase64, 76, '\n')
|
||||
hasher := murmur3.New32WithSeed(0)
|
||||
hasher.Write([]byte(stdBase64))
|
||||
return int32(hasher.Sum32())
|
||||
}
|
||||
|
||||
func InsertInto(s string, interval int, sep rune) string {
|
||||
var buffer bytes.Buffer
|
||||
before := interval - 1
|
||||
last := len(s) - 1
|
||||
for i, char := range s {
|
||||
buffer.WriteRune(char)
|
||||
if i%interval == before && i != last {
|
||||
buffer.WriteRune(sep)
|
||||
}
|
||||
}
|
||||
buffer.WriteRune(sep)
|
||||
return buffer.String()
|
||||
}
|
||||
|
2
go.mod
2
go.mod
@ -44,6 +44,8 @@ require (
|
||||
golang.org/x/text v0.3.7
|
||||
)
|
||||
|
||||
require github.com/spaolacci/murmur3 v1.1.0
|
||||
|
||||
require (
|
||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
|
2
go.sum
2
go.sum
@ -170,6 +170,8 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
|
@ -67,6 +67,7 @@ type scanOptions struct {
|
||||
ExcludeCDN bool
|
||||
HostMaxErrors int
|
||||
ProbeAllIPS bool
|
||||
Favicon bool
|
||||
LeaveDefaultPorts bool
|
||||
OutputLinesCount bool
|
||||
OutputWordsCount bool
|
||||
@ -109,6 +110,7 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
||||
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
||||
HostMaxErrors: s.HostMaxErrors,
|
||||
Favicon: s.Favicon,
|
||||
LeaveDefaultPorts: s.LeaveDefaultPorts,
|
||||
OutputLinesCount: s.OutputLinesCount,
|
||||
OutputWordsCount: s.OutputWordsCount,
|
||||
@ -206,6 +208,9 @@ type Options struct {
|
||||
SkipDedupe bool
|
||||
ProbeAllIPS bool
|
||||
Resolvers goflags.NormalizedStringSlice
|
||||
Favicon bool
|
||||
OutputFilterFavicon goflags.NormalizedStringSlice
|
||||
OutputMatchFavicon goflags.NormalizedStringSlice
|
||||
LeaveDefaultPorts bool
|
||||
OutputLinesCount bool
|
||||
OutputMatchLinesCount string
|
||||
@ -248,7 +253,6 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
|
||||
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
|
||||
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "matchers", "Matchers",
|
||||
@ -256,8 +260,10 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
|
||||
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
|
||||
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
|
||||
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"),
|
||||
flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "Match response with specific favicon"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "filters", "Filters",
|
||||
@ -267,6 +273,7 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
|
||||
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
|
||||
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"),
|
||||
flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "Filter response with specific favicon"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "rate-limit", "Rate-Limit",
|
||||
@ -275,6 +282,7 @@ func ParseOptions() *Options {
|
||||
)
|
||||
|
||||
createGroup(flagSet, "Misc", "Miscellaneous",
|
||||
flagSet.BoolVar(&options.Favicon, "favicon", false, "Probes for favicon (\"favicon.ico\" as path) and display phythonic hash"),
|
||||
flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"),
|
||||
flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"),
|
||||
flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"),
|
||||
@ -438,6 +446,11 @@ func (options *Options) validateOptions() {
|
||||
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
|
||||
if options.Favicon {
|
||||
gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n")
|
||||
options.RequestURIs = "/favicon.ico"
|
||||
}
|
||||
}
|
||||
|
||||
// configureOutput configures the output on the screen
|
||||
|
@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.ExcludeCDN = options.ExcludeCDN
|
||||
scanopts.HostMaxErrors = options.HostMaxErrors
|
||||
scanopts.ProbeAllIPS = options.ProbeAllIPS
|
||||
scanopts.Favicon = options.Favicon
|
||||
scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
|
||||
scanopts.OutputLinesCount = options.OutputLinesCount
|
||||
scanopts.OutputWordsCount = options.OutputWordsCount
|
||||
@ -567,6 +568,9 @@ func (r *Runner) RunEnumeration() {
|
||||
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchStatusCode) > 0 && !slice.IntSliceContains(r.options.matchStatusCode, resp.StatusCode) {
|
||||
continue
|
||||
}
|
||||
@ -579,6 +583,9 @@ func (r *Runner) RunEnumeration() {
|
||||
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
|
||||
continue
|
||||
}
|
||||
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
|
||||
continue
|
||||
}
|
||||
@ -1192,6 +1199,18 @@ retry:
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
var faviconMMH3 string
|
||||
if scanopts.Favicon {
|
||||
faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(resp.Data))
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(faviconMMH3).String())
|
||||
} else {
|
||||
builder.WriteString(faviconMMH3)
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
if scanopts.OutputLinesCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
@ -1315,6 +1334,7 @@ retry:
|
||||
ResponseTime: resp.Duration.String(),
|
||||
Technologies: technologies,
|
||||
FinalURL: finalURL,
|
||||
FavIconMMH3: faviconMMH3,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
}
|
||||
@ -1368,6 +1388,7 @@ type Result struct {
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user