diff --git a/common/hashes/doc.go b/common/hashes/doc.go new file mode 100644 index 0000000..0067bdd --- /dev/null +++ b/common/hashes/doc.go @@ -0,0 +1 @@ +package hashes diff --git a/common/hashes/hashes.go b/common/hashes/hashes.go new file mode 100644 index 0000000..a152877 --- /dev/null +++ b/common/hashes/hashes.go @@ -0,0 +1,64 @@ +package hashes + +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "fmt" + "github.com/mfonda/simhash" + "github.com/spaolacci/murmur3" +) + +func stdBase64(braw []byte) []byte { + bckd := base64.StdEncoding.EncodeToString(braw) + var buffer bytes.Buffer + for i := 0; i < len(bckd); i++ { + ch := bckd[i] + buffer.WriteByte(ch) + if (i+1)%76 == 0 { + buffer.WriteByte('\n') + } + } + buffer.WriteByte('\n') + return buffer.Bytes() +} + +func Mmh3(data []byte) string { + var h32 = murmur3.New32WithSeed(0) + h32.Write(stdBase64(data)) + return fmt.Sprintf("%d", h32.Sum32()) +} + +func Md5(data []byte) string { + hash := md5.Sum(data) + return hex.EncodeToString(hash[:]) +} + +func Sha1(data []byte) string { + hash := sha1.Sum(data) + return hex.EncodeToString(hash[:]) +} + +func Sha256(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + +func Sha224(data []byte) string { + hash := sha256.Sum224(data) + return hex.EncodeToString(hash[:]) +} + +func Sha512(data []byte) string { + hash := sha512.Sum512(data) + return hex.EncodeToString(hash[:]) +} + +func Simhash(data []byte) string { + hash := simhash.Simhash(simhash.NewWordFeatureSet(data)) + return fmt.Sprintf("%d", hash) +} diff --git a/common/slice/slice.go b/common/slice/slice.go index 2426d84..ce837b9 100644 --- a/common/slice/slice.go +++ b/common/slice/slice.go @@ -20,6 +20,16 @@ func UInt32SliceContains(sl []uint32, v uint32) bool { return false } +// StringSliceContains check if a slice contains the specified int value +func StringSliceContains(sl []string, v string) bool { + for _, vv := range sl { + if vv == v { + return true + } + } + return false +} + // ToSlice creates a slice with all string keys from a map func ToSlice(m map[string]struct{}) (s []string) { for k := range m { diff --git a/go.mod b/go.mod index 96a22a5..6b92331 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,8 @@ require ( require github.com/spaolacci/murmur3 v1.1.0 +require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 + require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index bfb74c7..5479a83 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= diff --git a/runner/options.go b/runner/options.go index 911ae7c..5e3d01b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,6 +1,7 @@ package runner import ( + "github.com/projectdiscovery/httpx/common/slice" "math" "os" "regexp" @@ -71,6 +72,7 @@ type scanOptions struct { LeaveDefaultPorts bool OutputLinesCount bool OutputWordsCount bool + Hashes string } func (s *scanOptions) Clone() *scanOptions { @@ -114,6 +116,7 @@ func (s *scanOptions) Clone() *scanOptions { LeaveDefaultPorts: s.LeaveDefaultPorts, OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, + Hashes: s.Hashes, } } @@ -224,6 +227,7 @@ type Options struct { matchWordsCount []int OutputFilterWordsCount string filterWordsCount []int + Hashes string } // ParseOptions parses the command line options for application @@ -295,6 +299,7 @@ func ParseOptions() *Options { flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), + flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"), ) createGroup(flagSet, "output", "Output", @@ -460,6 +465,14 @@ func (options *Options) validateOptions() { gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n") options.RequestURIs = "/favicon.ico" } + + if options.Hashes != "" { + for _, hashType := range strings.Split(options.Hashes, ",") { + if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { + gologger.Error().Msgf("Unsupported hash type: %s\n", hashType) + } + } + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index 8484ef1..0f28da8 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -30,6 +30,7 @@ import ( "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" + "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" @@ -228,6 +229,7 @@ func New(options *Options) (*Runner, error) { scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts scanopts.OutputLinesCount = options.OutputLinesCount scanopts.OutputWordsCount = options.OutputWordsCount + scanopts.Hashes = options.Hashes runner.scanopts = scanopts if options.ShowStatistics { @@ -1211,6 +1213,38 @@ retry: builder.WriteRune(']') } + var hashesMap = map[string]string{} + if scanopts.Hashes != "" { + hs := strings.Split(scanopts.Hashes, ",") + for _, hashType := range hs { + var hash string + switch strings.ToLower(hashType) { + case "md5": + hash = hashes.Md5(resp.Data) + case "mmh3": + hash = hashes.Mmh3(resp.Data) + case "sha1": + hash = hashes.Sha1(resp.Data) + case "sha256": + hash = hashes.Sha256(resp.Data) + case "sha512": + hash = hashes.Sha512(resp.Data) + case "simhash": + hash = hashes.Simhash(resp.Data) + } + if hash != "" { + hashesMap[hashType] = hash + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(hash).String()) + } else { + builder.WriteString(hash) + } + builder.WriteRune(']') + } + } + } + if scanopts.OutputLinesCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1336,6 +1370,7 @@ retry: Technologies: technologies, FinalURL: finalURL, FavIconMMH3: faviconMMH3, + Hashes: hashesMap, Lines: resp.Lines, Words: resp.Words, } @@ -1391,6 +1426,7 @@ type Result struct { FinalURL string `json:"final-url,omitempty" csv:"final-url"` Failed bool `json:"failed" csv:"failed"` FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` }