hash improvement (#528)

* hash improvement

* mmh3 conversion to int32

* misc update

Co-authored-by: sandeep <sandeep@projectdiscovery.io>

README.md

@@ -139,6 +139,7 @@ MISCELLANEOUS:
    -p, -ports string[]  Port to scan (nmap syntax: eg 1,2-10,11)
    -path string         File or comma separated paths to request
    -paths string        File or comma separated paths to request (deprecated)
+   -hash  string        Display response body hash (supported: md5,mmh3,sha256,sim)
 
 OUTPUT:
    -o, -output string                file to write output

common/hashes/hashes.go

@@ -30,7 +30,7 @@ func stdBase64(braw []byte) []byte {
 func Mmh3(data []byte) string {
 	var h32 = murmur3.New32WithSeed(0)
 	h32.Write(stdBase64(data))
-	return fmt.Sprintf("%d", h32.Sum32())
+	return fmt.Sprintf("%d", int32(h32.Sum32()))
 }
 
 func Md5(data []byte) string {

runner/options.go

@@ -299,7 +299,7 @@ func ParseOptions() *Options {
 		flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"),
 		flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"),
 		flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"),
-		flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"),
+		flagSet.StringVar(&options.Hashes, "hash", "", "Display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"),
 	)
 
 	createGroup(flagSet, "output", "Output",

runner/runner.go

@@ -4,9 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"encoding/csv"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -1212,39 +1210,54 @@ retry:
 		}
 		builder.WriteRune(']')
 	}
-
+	// adding default hashing for json output format
+	if r.options.JSONOutput && len(scanopts.Hashes) == 0 {
+		scanopts.Hashes = "md5,mmh3,sha256,simhash"
+	}
 	var hashesMap = map[string]string{}
 	if scanopts.Hashes != "" {
 		hs := strings.Split(scanopts.Hashes, ",")
-		for _, hashType := range hs {
-			var hash string
-			switch strings.ToLower(hashType) {
+		builder.WriteString(" [")
+		for index, hashType := range hs {
+			var (
+				hashHeader, hashBody string
+			)
+			hashType = strings.ToLower(hashType)
+			switch hashType {
 			case "md5":
-				hash = hashes.Md5(resp.Data)
+				hashBody = hashes.Md5(resp.Data)
+				hashHeader = hashes.Md5([]byte(resp.RawHeaders))
 			case "mmh3":
-				hash = hashes.Mmh3(resp.Data)
+				hashBody = hashes.Mmh3(resp.Data)
+				hashHeader = hashes.Mmh3([]byte(resp.RawHeaders))
 			case "sha1":
-				hash = hashes.Sha1(resp.Data)
+				hashBody = hashes.Sha1(resp.Data)
+				hashHeader = hashes.Sha1([]byte(resp.RawHeaders))
 			case "sha256":
-				hash = hashes.Sha256(resp.Data)
+				hashBody = hashes.Sha256(resp.Data)
+				hashHeader = hashes.Sha256([]byte(resp.RawHeaders))
 			case "sha512":
-				hash = hashes.Sha512(resp.Data)
+				hashBody = hashes.Sha512(resp.Data)
+				hashHeader = hashes.Sha512([]byte(resp.RawHeaders))
 			case "simhash":
-				hash = hashes.Simhash(resp.Data)
+				hashBody = hashes.Simhash(resp.Data)
+				hashHeader = hashes.Simhash([]byte(resp.RawHeaders))
 			}
-			if hash != "" {
-				hashesMap[hashType] = hash
-				builder.WriteString(" [")
+			if hashBody != "" {
+				hashesMap[fmt.Sprintf("body-%s", hashType)] = hashBody
+				hashesMap[fmt.Sprintf("header-%s", hashType)] = hashHeader
 				if !scanopts.OutputWithNoColor {
-					builder.WriteString(aurora.Magenta(hash).String())
+					builder.WriteString(aurora.Magenta(hashBody).String())
 				} else {
-					builder.WriteString(hash)
+					builder.WriteString(hashBody)
+				}
+				if index != len(hs)-1 {
+					builder.WriteString(",")
 				}
-				builder.WriteRune(']')
 			}
 		}
+		builder.WriteRune(']')
 	}
-
 	if scanopts.OutputLinesCount {
 		builder.WriteString(" [")
 		if !scanopts.OutputWithNoColor {
@@ -1314,15 +1327,6 @@ retry:
 	if finalPath == "" {
 		finalPath = "/"
 	}
-
-	hasher := sha256.New()
-	_, _ = hasher.Write(resp.Data)
-	bodySha := hex.EncodeToString(hasher.Sum(nil))
-	hasher.Reset()
-
-	_, _ = hasher.Write([]byte(resp.RawHeaders))
-	headersSha := hex.EncodeToString(hasher.Sum(nil))
-
 	var chainStatusCodes []int
 	if resp.HasChain() {
 		chainStatusCodes = append(chainStatusCodes, resp.GetChainStatusCodes()...)
@@ -1339,8 +1343,6 @@ retry:
 		Scheme:           parsed.Scheme,
 		Port:             finalPort,
 		Path:             finalPath,
-		BodySHA256:       bodySha,
-		HeaderSHA256:     headersSha,
 		raw:              resp.Raw,
 		URL:              fullURL,
 		Input:            origInput,
@@ -1392,8 +1394,6 @@ type Result struct {
 	Scheme           string    `json:"scheme,omitempty" csv:"scheme"`
 	Port             string    `json:"port,omitempty" csv:"port"`
 	Path             string    `json:"path,omitempty" csv:"path"`
-	BodySHA256       string    `json:"body-sha256,omitempty" csv:"body-sha256"`
-	HeaderSHA256     string    `json:"header-sha256,omitempty" csv:"header-sha256"`
 	A                []string  `json:"a,omitempty" csv:"a"`
 	CNAMEs           []string  `json:"cnames,omitempty" csv:"cnames"`
 	raw              string