mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-09-11 17:46:08 +03:00
Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change"
This reverts commit4634851eec
, reversing changes made tob7800f906c
.
This commit is contained in:
parent
4634851eec
commit
eec0fa69e5
47
common/hashes/jarm/connpool.go
Normal file
47
common/hashes/jarm/connpool.go
Normal file
@ -0,0 +1,47 @@
|
||||
package jarm
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/multierr"
|
||||
)
|
||||
|
||||
type inFlightConns struct {
|
||||
sync.RWMutex
|
||||
inflightConns map[net.Conn]struct{}
|
||||
}
|
||||
|
||||
func newInFlightConns() (*inFlightConns, error) {
|
||||
return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Add(conn net.Conn) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
i.inflightConns[conn] = struct{}{}
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Remove(conn net.Conn) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
delete(i.inflightConns, conn)
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Close() error {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
var errs []error
|
||||
|
||||
for conn := range i.inflightConns {
|
||||
if err := conn.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
delete(i.inflightConns, conn)
|
||||
}
|
||||
|
||||
return multierr.Combine(errs...)
|
||||
}
|
@ -13,10 +13,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
)
|
||||
|
||||
const (
|
||||
poolCount = 3
|
||||
defaultPort = 443
|
||||
)
|
||||
const defaultPort int = 443
|
||||
|
||||
type target struct {
|
||||
Host string
|
||||
@ -24,15 +21,11 @@ type target struct {
|
||||
}
|
||||
|
||||
// fingerprint probes a single host/port
|
||||
func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
|
||||
func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string {
|
||||
results := []string{}
|
||||
addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port))
|
||||
timeout := time.Duration(duration) * time.Second
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(duration*poolCount) * time.Second))
|
||||
defer cancel()
|
||||
|
||||
pool, err := newOneTimePool(ctx, addr, poolCount)
|
||||
// using connection pool as we need multiple probes
|
||||
pool, err := newOneTimePool(context.Background(), addr, 3)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@ -42,18 +35,19 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
|
||||
go pool.Run() //nolint
|
||||
|
||||
for _, probe := range jarm.GetProbes(t.Host, t.Port) {
|
||||
conn, err := pool.Acquire(ctx)
|
||||
conn, err := pool.Acquire(context.Background())
|
||||
if err != nil {
|
||||
return ""
|
||||
continue
|
||||
}
|
||||
if conn == nil {
|
||||
return ""
|
||||
continue
|
||||
}
|
||||
_ = conn.SetWriteDeadline(time.Now().Add(timeout))
|
||||
_, err = conn.Write(jarm.BuildProbe(probe))
|
||||
if err != nil {
|
||||
results = append(results, "")
|
||||
_ = conn.Close()
|
||||
return ""
|
||||
continue
|
||||
}
|
||||
_ = conn.SetReadDeadline(time.Now().Add(timeout))
|
||||
buff := make([]byte, 1484)
|
||||
@ -61,7 +55,8 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
|
||||
_ = conn.Close()
|
||||
ans, err := jarm.ParseServerHello(buff, probe)
|
||||
if err != nil {
|
||||
return ""
|
||||
results = append(results, "")
|
||||
continue
|
||||
}
|
||||
results = append(results, ans)
|
||||
}
|
||||
@ -81,5 +76,6 @@ func Jarm(dialer *fastdialer.Dialer, host string, duration int) string {
|
||||
if t.Port == 0 {
|
||||
t.Port = defaultPort
|
||||
}
|
||||
return fingerprint(dialer, t, duration)
|
||||
timeout := time.Duration(duration) * time.Second
|
||||
return fingerprint(dialer, t, timeout)
|
||||
}
|
||||
|
@ -3,10 +3,8 @@ package jarm
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"go.uber.org/multierr"
|
||||
)
|
||||
|
||||
// oneTimePool is a pool designed to create continous bare connections that are for one time only usage
|
||||
@ -77,40 +75,3 @@ func (p *oneTimePool) Close() error {
|
||||
p.cancel()
|
||||
return p.InFlightConns.Close()
|
||||
}
|
||||
|
||||
type inFlightConns struct {
|
||||
sync.Mutex
|
||||
inflightConns map[net.Conn]struct{}
|
||||
}
|
||||
|
||||
func newInFlightConns() (*inFlightConns, error) {
|
||||
return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Add(conn net.Conn) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
i.inflightConns[conn] = struct{}{}
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Remove(conn net.Conn) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
delete(i.inflightConns, conn)
|
||||
}
|
||||
|
||||
func (i *inFlightConns) Close() error {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
|
||||
var errs []error
|
||||
for conn := range i.inflightConns {
|
||||
if err := conn.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
delete(i.inflightConns, conn)
|
||||
}
|
||||
return multierr.Combine(errs...)
|
||||
}
|
||||
|
@ -4,8 +4,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/cryptoutil"
|
||||
"github.com/projectdiscovery/httputil"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
)
|
||||
|
||||
// Response contains the response to a server
|
||||
@ -18,7 +18,7 @@ type Response struct {
|
||||
RawHeaders string
|
||||
Words int
|
||||
Lines int
|
||||
TLSData *clients.Response
|
||||
TLSData *cryptoutil.TLSData
|
||||
CSPData *CSPData
|
||||
HTTP2 bool
|
||||
Pipeline bool
|
||||
|
@ -1,74 +1,15 @@
|
||||
package httpx
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
"github.com/projectdiscovery/cryptoutil"
|
||||
)
|
||||
|
||||
// versionToTLSVersionString converts tls version to version string
|
||||
var versionToTLSVersionString = map[uint16]string{
|
||||
tls.VersionTLS10: "tls10",
|
||||
tls.VersionTLS11: "tls11",
|
||||
tls.VersionTLS12: "tls12",
|
||||
tls.VersionTLS13: "tls13",
|
||||
}
|
||||
|
||||
// TLSGrab fills the TLSData
|
||||
func (h *HTTPX) TLSGrab(r *http.Response) *clients.Response {
|
||||
if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
|
||||
return nil
|
||||
func (h *HTTPX) TLSGrab(r *http.Response) *cryptoutil.TLSData {
|
||||
if r.TLS != nil {
|
||||
return cryptoutil.TLSGrab(r.TLS)
|
||||
}
|
||||
host := r.Request.URL.Host
|
||||
hostname, port, _ := net.SplitHostPort(host)
|
||||
if hostname == "" {
|
||||
hostname = host
|
||||
}
|
||||
if port == "" {
|
||||
port = "443"
|
||||
}
|
||||
|
||||
tlsVersion := versionToTLSVersionString[r.TLS.Version]
|
||||
tlsCipher := tls.CipherSuiteName(r.TLS.CipherSuite)
|
||||
|
||||
leafCertificate := r.TLS.PeerCertificates[0]
|
||||
response := &clients.Response{
|
||||
Host: hostname,
|
||||
ProbeStatus: true,
|
||||
Port: port,
|
||||
Version: tlsVersion,
|
||||
Cipher: tlsCipher,
|
||||
TLSConnection: "ctls",
|
||||
CertificateResponse: convertCertificateToResponse(hostname, leafCertificate),
|
||||
ServerName: r.TLS.ServerName,
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse {
|
||||
response := &clients.CertificateResponse{
|
||||
SubjectAN: cert.DNSNames,
|
||||
Emails: cert.EmailAddresses,
|
||||
NotBefore: cert.NotBefore,
|
||||
NotAfter: cert.NotAfter,
|
||||
Expired: clients.IsExpired(cert.NotAfter),
|
||||
SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId),
|
||||
MisMatched: clients.IsMisMatchedCert(hostname, append(cert.DNSNames, cert.Subject.CommonName)),
|
||||
WildCardCert: clients.IsWildCardCert(append(cert.DNSNames, cert.Subject.CommonName)),
|
||||
IssuerCN: cert.Issuer.CommonName,
|
||||
IssuerOrg: cert.Issuer.Organization,
|
||||
SubjectCN: cert.Subject.CommonName,
|
||||
SubjectOrg: cert.Subject.Organization,
|
||||
FingerprintHash: clients.CertificateResponseFingerprintHash{
|
||||
MD5: clients.MD5Fingerprint(cert.Raw),
|
||||
SHA1: clients.SHA1Fingerprint(cert.Raw),
|
||||
SHA256: clients.SHA256Fingerprint(cert.Raw),
|
||||
},
|
||||
}
|
||||
response.IssuerDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawIssuer, cert.Issuer.String())
|
||||
response.SubjectDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawSubject, cert.Subject.String())
|
||||
return response
|
||||
return nil
|
||||
}
|
||||
|
@ -27,7 +27,6 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/projectdiscovery/httpx/common/customextract"
|
||||
"github.com/projectdiscovery/httpx/common/hashes/jarm"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
|
||||
"github.com/ammario/ipisp/v2"
|
||||
"github.com/bluele/gcache"
|
||||
@ -35,6 +34,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/projectdiscovery/clistats"
|
||||
"github.com/projectdiscovery/cryptoutil"
|
||||
"github.com/projectdiscovery/goconfig"
|
||||
"github.com/projectdiscovery/httpx/common/hashes"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
@ -796,14 +796,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
|
||||
output <- result
|
||||
if scanopts.TLSProbe && result.TLSData != nil {
|
||||
scanopts.TLSProbe = false
|
||||
for _, tt := range result.TLSData.SubjectAN {
|
||||
for _, tt := range result.TLSData.DNSNames {
|
||||
if !r.testAndSet(tt) {
|
||||
continue
|
||||
}
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
if r.testAndSet(result.TLSData.SubjectCN) {
|
||||
r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.CommonName {
|
||||
if !r.testAndSet(tt) {
|
||||
continue
|
||||
}
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
}
|
||||
if scanopts.CSPProbe && result.CSPData != nil {
|
||||
@ -835,14 +838,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
|
||||
output <- result
|
||||
if scanopts.TLSProbe && result.TLSData != nil {
|
||||
scanopts.TLSProbe = false
|
||||
for _, tt := range result.TLSData.SubjectAN {
|
||||
for _, tt := range result.TLSData.DNSNames {
|
||||
if !r.testAndSet(tt) {
|
||||
continue
|
||||
}
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
if r.testAndSet(result.TLSData.SubjectCN) {
|
||||
r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.CommonName {
|
||||
if !r.testAndSet(tt) {
|
||||
continue
|
||||
}
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
}
|
||||
}(port, target, method, wantedProtocol)
|
||||
@ -1567,12 +1573,12 @@ type Result struct {
|
||||
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
|
||||
ASN interface{} `json:"asn,omitempty" csv:"asn"`
|
||||
err error
|
||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
||||
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
||||
Hashes map[string]string `json:"hash,omitempty" csv:"hash"`
|
||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
|
||||
Port string `json:"port,omitempty" csv:"port"`
|
||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
||||
TLSData *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"`
|
||||
Hashes map[string]string `json:"hash,omitempty" csv:"hash"`
|
||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
|
||||
Port string `json:"port,omitempty" csv:"port"`
|
||||
raw string
|
||||
URL string `json:"url,omitempty" csv:"url"`
|
||||
Input string `json:"input,omitempty" csv:"input"`
|
||||
|
Loading…
Reference in New Issue
Block a user