Add hash flag, to support multi body hash type. (#517)

* Add hashes flag, to support multi body hash type. issue:489 and issue:488

* go mod update

* Add hash type validate for -hash flag

Co-authored-by: sandeep <sandeep@projectdiscovery.io>
Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>

common/hashes/doc.go

@@ -0,0 +1 @@
+package hashes

common/hashes/hashes.go

@@ -0,0 +1,64 @@
+package hashes
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"github.com/mfonda/simhash"
+	"github.com/spaolacci/murmur3"
+)
+
+func stdBase64(braw []byte) []byte {
+	bckd := base64.StdEncoding.EncodeToString(braw)
+	var buffer bytes.Buffer
+	for i := 0; i < len(bckd); i++ {
+		ch := bckd[i]
+		buffer.WriteByte(ch)
+		if (i+1)%76 == 0 {
+			buffer.WriteByte('\n')
+		}
+	}
+	buffer.WriteByte('\n')
+	return buffer.Bytes()
+}
+
+func Mmh3(data []byte) string {
+	var h32 = murmur3.New32WithSeed(0)
+	h32.Write(stdBase64(data))
+	return fmt.Sprintf("%d", h32.Sum32())
+}
+
+func Md5(data []byte) string {
+	hash := md5.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func Sha1(data []byte) string {
+	hash := sha1.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func Sha256(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func Sha224(data []byte) string {
+	hash := sha256.Sum224(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func Sha512(data []byte) string {
+	hash := sha512.Sum512(data)
+	return hex.EncodeToString(hash[:])
+}
+
+func Simhash(data []byte) string {
+	hash := simhash.Simhash(simhash.NewWordFeatureSet(data))
+	return fmt.Sprintf("%d", hash)
+}

common/slice/slice.go

@@ -20,6 +20,16 @@ func UInt32SliceContains(sl []uint32, v uint32) bool {
 	return false
 }
 
+// StringSliceContains check if a slice contains the specified int value
+func StringSliceContains(sl []string, v string) bool {
+	for _, vv := range sl {
+		if vv == v {
+			return true
+		}
+	}
+	return false
+}
+
 // ToSlice creates a slice with all string keys from a map
 func ToSlice(m map[string]struct{}) (s []string) {
 	for k := range m {

go.mod

@@ -46,6 +46,8 @@ require (
 
 require github.com/spaolacci/murmur3 v1.1.0
 
+require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
+
 require (
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect

go.sum

@@ -71,6 +71,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY=
+github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8=
 github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
 github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=

runner/options.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"github.com/projectdiscovery/httpx/common/slice"
 	"math"
 	"os"
 	"regexp"
@@ -71,6 +72,7 @@ type scanOptions struct {
 	LeaveDefaultPorts         bool
 	OutputLinesCount          bool
 	OutputWordsCount          bool
+	Hashes                    string
 }
 
 func (s *scanOptions) Clone() *scanOptions {
@@ -114,6 +116,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		LeaveDefaultPorts:         s.LeaveDefaultPorts,
 		OutputLinesCount:          s.OutputLinesCount,
 		OutputWordsCount:          s.OutputWordsCount,
+		Hashes:                    s.Hashes,
 	}
 }
 
@@ -224,6 +227,7 @@ type Options struct {
 	matchWordsCount           []int
 	OutputFilterWordsCount    string
 	filterWordsCount          []int
+	Hashes                    string
 }
 
 // ParseOptions parses the command line options for application
@@ -295,6 +299,7 @@ func ParseOptions() *Options {
 		flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"),
 		flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"),
 		flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"),
+		flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"),
 	)
 
 	createGroup(flagSet, "output", "Output",
@@ -460,6 +465,14 @@ func (options *Options) validateOptions() {
 		gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n")
 		options.RequestURIs = "/favicon.ico"
 	}
+
+	if options.Hashes != "" {
+		for _, hashType := range strings.Split(options.Hashes, ",") {
+			if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
+				gologger.Error().Msgf("Unsupported hash type: %s\n", hashType)
+			}
+		}
+	}
 }
 
 // configureOutput configures the output on the screen

runner/runner.go

@@ -30,6 +30,7 @@ import (
 	"github.com/projectdiscovery/clistats"
 	"github.com/projectdiscovery/cryptoutil"
 	"github.com/projectdiscovery/goconfig"
+	"github.com/projectdiscovery/httpx/common/hashes"
 	"github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/stringsutil"
 	"github.com/projectdiscovery/urlutil"
@@ -228,6 +229,7 @@ func New(options *Options) (*Runner, error) {
 	scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
 	scanopts.OutputLinesCount = options.OutputLinesCount
 	scanopts.OutputWordsCount = options.OutputWordsCount
+	scanopts.Hashes = options.Hashes
 	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
@@ -1211,6 +1213,38 @@ retry:
 		builder.WriteRune(']')
 	}
 
+	var hashesMap = map[string]string{}
+	if scanopts.Hashes != "" {
+		hs := strings.Split(scanopts.Hashes, ",")
+		for _, hashType := range hs {
+			var hash string
+			switch strings.ToLower(hashType) {
+			case "md5":
+				hash = hashes.Md5(resp.Data)
+			case "mmh3":
+				hash = hashes.Mmh3(resp.Data)
+			case "sha1":
+				hash = hashes.Sha1(resp.Data)
+			case "sha256":
+				hash = hashes.Sha256(resp.Data)
+			case "sha512":
+				hash = hashes.Sha512(resp.Data)
+			case "simhash":
+				hash = hashes.Simhash(resp.Data)
+			}
+			if hash != "" {
+				hashesMap[hashType] = hash
+				builder.WriteString(" [")
+				if !scanopts.OutputWithNoColor {
+					builder.WriteString(aurora.Magenta(hash).String())
+				} else {
+					builder.WriteString(hash)
+				}
+				builder.WriteRune(']')
+			}
+		}
+	}
+
 	if scanopts.OutputLinesCount {
 		builder.WriteString(" [")
 		if !scanopts.OutputWithNoColor {
@@ -1336,6 +1370,7 @@ retry:
 		Technologies:     technologies,
 		FinalURL:         finalURL,
 		FavIconMMH3:      faviconMMH3,
+		Hashes:           hashesMap,
 		Lines:            resp.Lines,
 		Words:            resp.Words,
 	}
@@ -1391,6 +1426,7 @@ type Result struct {
 	FinalURL         string              `json:"final-url,omitempty" csv:"final-url"`
 	Failed           bool                `json:"failed" csv:"failed"`
 	FavIconMMH3      string              `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
+	Hashes           map[string]string   `json:"hashes,omitempty" csv:"hashes"`
 	Lines            int                 `json:"lines" csv:"lines"`
 	Words            int                 `json:"words" csv:"words"`
 }