mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 13:30:22 +03:00
Add hash flag, to support multi body hash type. (#517)
* Add hashes flag, to support multi body hash type. issue:489 and issue:488 * go mod update * Add hash type validate for -hash flag Co-authored-by: sandeep <sandeep@projectdiscovery.io> Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>
This commit is contained in:
parent
8865959f8e
commit
e2b0c1683b
1
common/hashes/doc.go
Normal file
1
common/hashes/doc.go
Normal file
@ -0,0 +1 @@
|
||||
package hashes
|
64
common/hashes/hashes.go
Normal file
64
common/hashes/hashes.go
Normal file
@ -0,0 +1,64 @@
|
||||
package hashes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"github.com/mfonda/simhash"
|
||||
"github.com/spaolacci/murmur3"
|
||||
)
|
||||
|
||||
func stdBase64(braw []byte) []byte {
|
||||
bckd := base64.StdEncoding.EncodeToString(braw)
|
||||
var buffer bytes.Buffer
|
||||
for i := 0; i < len(bckd); i++ {
|
||||
ch := bckd[i]
|
||||
buffer.WriteByte(ch)
|
||||
if (i+1)%76 == 0 {
|
||||
buffer.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
buffer.WriteByte('\n')
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func Mmh3(data []byte) string {
|
||||
var h32 = murmur3.New32WithSeed(0)
|
||||
h32.Write(stdBase64(data))
|
||||
return fmt.Sprintf("%d", h32.Sum32())
|
||||
}
|
||||
|
||||
func Md5(data []byte) string {
|
||||
hash := md5.Sum(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func Sha1(data []byte) string {
|
||||
hash := sha1.Sum(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func Sha256(data []byte) string {
|
||||
hash := sha256.Sum256(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func Sha224(data []byte) string {
|
||||
hash := sha256.Sum224(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func Sha512(data []byte) string {
|
||||
hash := sha512.Sum512(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
func Simhash(data []byte) string {
|
||||
hash := simhash.Simhash(simhash.NewWordFeatureSet(data))
|
||||
return fmt.Sprintf("%d", hash)
|
||||
}
|
@ -20,6 +20,16 @@ func UInt32SliceContains(sl []uint32, v uint32) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// StringSliceContains check if a slice contains the specified int value
|
||||
func StringSliceContains(sl []string, v string) bool {
|
||||
for _, vv := range sl {
|
||||
if vv == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ToSlice creates a slice with all string keys from a map
|
||||
func ToSlice(m map[string]struct{}) (s []string) {
|
||||
for k := range m {
|
||||
|
2
go.mod
2
go.mod
@ -46,6 +46,8 @@ require (
|
||||
|
||||
require github.com/spaolacci/murmur3 v1.1.0
|
||||
|
||||
require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
|
||||
|
||||
require (
|
||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
|
2
go.sum
2
go.sum
@ -71,6 +71,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY=
|
||||
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8=
|
||||
github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
|
||||
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
|
@ -1,6 +1,7 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"github.com/projectdiscovery/httpx/common/slice"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
@ -71,6 +72,7 @@ type scanOptions struct {
|
||||
LeaveDefaultPorts bool
|
||||
OutputLinesCount bool
|
||||
OutputWordsCount bool
|
||||
Hashes string
|
||||
}
|
||||
|
||||
func (s *scanOptions) Clone() *scanOptions {
|
||||
@ -114,6 +116,7 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
LeaveDefaultPorts: s.LeaveDefaultPorts,
|
||||
OutputLinesCount: s.OutputLinesCount,
|
||||
OutputWordsCount: s.OutputWordsCount,
|
||||
Hashes: s.Hashes,
|
||||
}
|
||||
}
|
||||
|
||||
@ -224,6 +227,7 @@ type Options struct {
|
||||
matchWordsCount []int
|
||||
OutputFilterWordsCount string
|
||||
filterWordsCount []int
|
||||
Hashes string
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -295,6 +299,7 @@ func ParseOptions() *Options {
|
||||
flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"),
|
||||
flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"),
|
||||
flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"),
|
||||
flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "output", "Output",
|
||||
@ -460,6 +465,14 @@ func (options *Options) validateOptions() {
|
||||
gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n")
|
||||
options.RequestURIs = "/favicon.ico"
|
||||
}
|
||||
|
||||
if options.Hashes != "" {
|
||||
for _, hashType := range strings.Split(options.Hashes, ",") {
|
||||
if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
|
||||
gologger.Error().Msgf("Unsupported hash type: %s\n", hashType)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// configureOutput configures the output on the screen
|
||||
|
@ -30,6 +30,7 @@ import (
|
||||
"github.com/projectdiscovery/clistats"
|
||||
"github.com/projectdiscovery/cryptoutil"
|
||||
"github.com/projectdiscovery/goconfig"
|
||||
"github.com/projectdiscovery/httpx/common/hashes"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/stringsutil"
|
||||
"github.com/projectdiscovery/urlutil"
|
||||
@ -228,6 +229,7 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
|
||||
scanopts.OutputLinesCount = options.OutputLinesCount
|
||||
scanopts.OutputWordsCount = options.OutputWordsCount
|
||||
scanopts.Hashes = options.Hashes
|
||||
runner.scanopts = scanopts
|
||||
|
||||
if options.ShowStatistics {
|
||||
@ -1211,6 +1213,38 @@ retry:
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
var hashesMap = map[string]string{}
|
||||
if scanopts.Hashes != "" {
|
||||
hs := strings.Split(scanopts.Hashes, ",")
|
||||
for _, hashType := range hs {
|
||||
var hash string
|
||||
switch strings.ToLower(hashType) {
|
||||
case "md5":
|
||||
hash = hashes.Md5(resp.Data)
|
||||
case "mmh3":
|
||||
hash = hashes.Mmh3(resp.Data)
|
||||
case "sha1":
|
||||
hash = hashes.Sha1(resp.Data)
|
||||
case "sha256":
|
||||
hash = hashes.Sha256(resp.Data)
|
||||
case "sha512":
|
||||
hash = hashes.Sha512(resp.Data)
|
||||
case "simhash":
|
||||
hash = hashes.Simhash(resp.Data)
|
||||
}
|
||||
if hash != "" {
|
||||
hashesMap[hashType] = hash
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(hash).String())
|
||||
} else {
|
||||
builder.WriteString(hash)
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if scanopts.OutputLinesCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
@ -1336,6 +1370,7 @@ retry:
|
||||
Technologies: technologies,
|
||||
FinalURL: finalURL,
|
||||
FavIconMMH3: faviconMMH3,
|
||||
Hashes: hashesMap,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
}
|
||||
@ -1391,6 +1426,7 @@ type Result struct {
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
|
||||
Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user