mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-12-01 04:08:53 +03:00
Issue 549 jarm algorithm (#567)
* jarm hash * lint error fix * remove default jarm value if jarm not calculated * jarm cli boolean flag * removed unused hash type * readme update * example update Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
This commit is contained in:
Sami
GitHub
parent
df5f69fd18
commit
ccf0c9ddc5
29
README.md
29
README.md
@ -58,6 +58,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi
|
||||
| Favicon Hash | false | Probe Status | false |
|
||||
| Body Hash | true | Header Hash | true |
|
||||
| Redirect chain | false | URL Scheme | true |
|
||||
| JARM Hash | false | | |
|
||||
|
||||
|
||||
# Installation Instructions
|
||||
@ -93,6 +94,7 @@ PROBES:
|
||||
-location display response redirect location
|
||||
-favicon display mmh3 hash for '/favicon.ico' file
|
||||
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
|
||||
-jarm display jarm fingerprint hash
|
||||
-rt, -response-time display response time
|
||||
-lc, -line-count display response body line count
|
||||
-wc, -word-count display response body word count
|
||||
@ -334,6 +336,33 @@ https://mta-sts.hackerone.com/favicon.ico [-1700323260]
|
||||
https://www.hackerone.com/favicon.ico [778073381]
|
||||
```
|
||||
|
||||
### [JARM Fingerprint](https://github.com/salesforce/jarm)
|
||||
|
||||
|
||||
```console
|
||||
subfinder -d hackerone.com -silent | httpx -jarm
|
||||
__ __ __ _ __
|
||||
/ /_ / /_/ /_____ | |/ /
|
||||
/ __ \/ __/ __/ __ \| /
|
||||
/ / / / /_/ /_/ /_/ / |
|
||||
/_/ /_/\__/\__/ .___/_/|_|
|
||||
/_/ v1.2.1
|
||||
|
||||
projectdiscovery.io
|
||||
|
||||
Use with caution. You are responsible for your actions.
|
||||
Developers assume no liability and are not responsible for any misuse or damage.
|
||||
https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
|
||||
https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
|
||||
https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
|
||||
https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
|
||||
https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7]
|
||||
https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
|
||||
https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
|
||||
https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5]
|
||||
```
|
||||
|
||||
|
||||
### Path Probe
|
||||
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
"github.com/mfonda/simhash"
|
||||
"github.com/spaolacci/murmur3"
|
||||
)
|
||||
|
||||
93
common/hashes/jarmhash.go
Normal file
93
common/hashes/jarmhash.go
Normal file
@ -0,0 +1,93 @@
|
||||
package hashes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/RumbleDiscovery/jarm-go"
|
||||
"github.com/projectdiscovery/httpx/common/regexhelper"
|
||||
"golang.org/x/net/proxy"
|
||||
)
|
||||
|
||||
const defaultPort int = 443
|
||||
|
||||
var DefualtBackoff = func(r, m int) time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
type target struct {
|
||||
Host string
|
||||
Port int
|
||||
Retries int
|
||||
Backoff func(r, m int) time.Duration
|
||||
}
|
||||
|
||||
// fingerprint probes a single host/port
|
||||
func fingerprint(t target, duration int) string {
|
||||
timeout := time.Duration(duration) * time.Second
|
||||
results := []string{}
|
||||
for _, probe := range jarm.GetProbes(t.Host, t.Port) {
|
||||
dialer := proxy.FromEnvironmentUsing(&net.Dialer{Timeout: timeout})
|
||||
addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port))
|
||||
c := net.Conn(nil)
|
||||
n := 0
|
||||
for c == nil && n <= t.Retries {
|
||||
// Ignoring error since error message was already being dropped.
|
||||
// Also, if theres an error, c == nil.
|
||||
if c, _ = dialer.Dial("tcp", addr); c != nil || t.Retries == 0 {
|
||||
break
|
||||
}
|
||||
bo := t.Backoff
|
||||
if bo == nil {
|
||||
bo = DefualtBackoff
|
||||
}
|
||||
time.Sleep(bo(n, t.Retries))
|
||||
n++
|
||||
}
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
data := jarm.BuildProbe(probe)
|
||||
_ = c.SetWriteDeadline(time.Now().Add(timeout))
|
||||
_, err := c.Write(data)
|
||||
if err != nil {
|
||||
results = append(results, "")
|
||||
c.Close()
|
||||
continue
|
||||
}
|
||||
_ = c.SetReadDeadline(time.Now().Add(timeout))
|
||||
buff := make([]byte, 1484)
|
||||
_, _ = c.Read(buff)
|
||||
c.Close()
|
||||
ans, err := jarm.ParseServerHello(buff, probe)
|
||||
if err != nil {
|
||||
results = append(results, "")
|
||||
continue
|
||||
}
|
||||
results = append(results, ans)
|
||||
}
|
||||
return jarm.RawHashToFuzzyHash(strings.Join(results, ","))
|
||||
}
|
||||
func Jarm(host string, duration int) string {
|
||||
t := target{}
|
||||
if u, err := url.Parse(host); err == nil {
|
||||
if u.Scheme == "http" {
|
||||
return ""
|
||||
}
|
||||
t.Host = u.Hostname()
|
||||
port, _ := strconv.Atoi(u.Port())
|
||||
t.Port = port
|
||||
}
|
||||
if t.Port == 0 {
|
||||
t.Port = defaultPort
|
||||
}
|
||||
hash := fingerprint(t, duration)
|
||||
if regexhelper.JarmHashRegex.MatchString(hash) {
|
||||
return ""
|
||||
}
|
||||
return hash
|
||||
}
|
||||
7
common/regexhelper/regex.go
Normal file
7
common/regexhelper/regex.go
Normal file
@ -0,0 +1,7 @@
|
||||
package regexhelper
|
||||
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
JarmHashRegex = regexp.MustCompile("(?m)0{62}")
|
||||
)
|
||||
1
go.mod
1
go.mod
@ -51,6 +51,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect
|
||||
github.com/ammario/ipisp/v2 v2.0.0 // indirect
|
||||
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@ -3,6 +3,9 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
|
||||
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4=
|
||||
github.com/RumbleDiscovery/jarm-go v0.0.6/go.mod h1:dXV7z5vBXQI0cNaHXwzGtq2PJ2LgM3XgcFiX32FU3bg=
|
||||
github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
|
||||
github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
|
||||
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
|
||||
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
|
||||
@ -32,6 +35,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
@ -85,6 +89,7 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p
|
||||
github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
|
||||
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
|
||||
github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
|
||||
@ -185,6 +190,7 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po
|
||||
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
@ -228,6 +234,7 @@ go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6m
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc=
|
||||
@ -243,6 +250,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@ -256,6 +264,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcx
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -270,6 +279,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
||||
@ -1,12 +1,11 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"github.com/projectdiscovery/httpx/common/slice"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/httpx/common/slice"
|
||||
"github.com/projectdiscovery/fileutil"
|
||||
"github.com/projectdiscovery/goconfig"
|
||||
"github.com/projectdiscovery/goflags"
|
||||
@ -228,6 +227,7 @@ type Options struct {
|
||||
OutputFilterWordsCount string
|
||||
filterWordsCount []int
|
||||
Hashes string
|
||||
Jarm bool
|
||||
Asn bool
|
||||
}
|
||||
|
||||
@ -250,6 +250,7 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"),
|
||||
flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"),
|
||||
flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"),
|
||||
flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"),
|
||||
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"),
|
||||
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"),
|
||||
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"),
|
||||
|
||||
@ -1291,7 +1291,17 @@ retry:
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
jarmhash := ""
|
||||
if r.options.Jarm {
|
||||
jarmhash = hashes.Jarm(fullURL,r.options.Timeout)
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(jarmhash).String())
|
||||
} else {
|
||||
builder.WriteString(fmt.Sprint(jarmhash))
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
if scanopts.OutputWordsCount {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
@ -1397,6 +1407,7 @@ retry:
|
||||
FinalURL: finalURL,
|
||||
FavIconMMH3: faviconMMH3,
|
||||
Hashes: hashesMap,
|
||||
Jarm: jarmhash,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
ASN: asnResponse,
|
||||
@ -1466,6 +1477,7 @@ type Result struct {
|
||||
ASN interface{} `json:"asn,omitempty" csv:"asn"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
Jarm string `json:"jarm,omitempty" csv:"jarm"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
||||
Loading…
Reference in New Issue
Block a user