Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change"

This reverts commit 4634851eec, reversing
changes made to b7800f906c.

common/hashes/jarm/connpool.go

@@ -0,0 +1,47 @@
+package jarm
+
+import (
+	"net"
+	"sync"
+
+	"go.uber.org/multierr"
+)
+
+type inFlightConns struct {
+	sync.RWMutex
+	inflightConns map[net.Conn]struct{}
+}
+
+func newInFlightConns() (*inFlightConns, error) {
+	return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
+}
+
+func (i *inFlightConns) Add(conn net.Conn) {
+	i.Lock()
+	defer i.Unlock()
+
+	i.inflightConns[conn] = struct{}{}
+}
+
+func (i *inFlightConns) Remove(conn net.Conn) {
+	i.Lock()
+	defer i.Unlock()
+
+	delete(i.inflightConns, conn)
+}
+
+func (i *inFlightConns) Close() error {
+	i.Lock()
+	defer i.Unlock()
+
+	var errs []error
+
+	for conn := range i.inflightConns {
+		if err := conn.Close(); err != nil {
+			errs = append(errs, err)
+		}
+		delete(i.inflightConns, conn)
+	}
+
+	return multierr.Combine(errs...)
+}

common/hashes/jarm/jarmhash.go

@@ -13,10 +13,7 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 )
 
-const (
-	poolCount   = 3
-	defaultPort = 443
-)
+const defaultPort int = 443
 
 type target struct {
 	Host string
@@ -24,15 +21,11 @@ type target struct {
 }
 
 // fingerprint probes a single host/port
-func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
+func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string {
 	results := []string{}
 	addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port))
-	timeout := time.Duration(duration) * time.Second
-
-	ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(duration*poolCount) * time.Second))
-	defer cancel()
-
-	pool, err := newOneTimePool(ctx, addr, poolCount)
+	// using connection pool as we need multiple probes
+	pool, err := newOneTimePool(context.Background(), addr, 3)
 	if err != nil {
 		return ""
 	}
@@ -42,18 +35,19 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
 	go pool.Run()      //nolint
 
 	for _, probe := range jarm.GetProbes(t.Host, t.Port) {
-		conn, err := pool.Acquire(ctx)
+		conn, err := pool.Acquire(context.Background())
 		if err != nil {
-			return ""
+			continue
 		}
 		if conn == nil {
-			return ""
+			continue
 		}
 		_ = conn.SetWriteDeadline(time.Now().Add(timeout))
 		_, err = conn.Write(jarm.BuildProbe(probe))
 		if err != nil {
+			results = append(results, "")
 			_ = conn.Close()
-			return ""
+			continue
 		}
 		_ = conn.SetReadDeadline(time.Now().Add(timeout))
 		buff := make([]byte, 1484)
@@ -61,7 +55,8 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
 		_ = conn.Close()
 		ans, err := jarm.ParseServerHello(buff, probe)
 		if err != nil {
-			return ""
+			results = append(results, "")
+			continue
 		}
 		results = append(results, ans)
 	}
@@ -81,5 +76,6 @@ func Jarm(dialer *fastdialer.Dialer, host string, duration int) string {
 	if t.Port == 0 {
 		t.Port = defaultPort
 	}
-	return fingerprint(dialer, t, duration)
+	timeout := time.Duration(duration) * time.Second
+	return fingerprint(dialer, t, timeout)
 }

common/hashes/jarm/onetimepool.go

@@ -3,10 +3,8 @@ package jarm
 import (
 	"context"
 	"net"
-	"sync"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
-	"go.uber.org/multierr"
 )
 
 // oneTimePool is a pool designed to create continous bare connections that are for one time only usage
@@ -77,40 +75,3 @@ func (p *oneTimePool) Close() error {
 	p.cancel()
 	return p.InFlightConns.Close()
 }
-
-type inFlightConns struct {
-	sync.Mutex
-	inflightConns map[net.Conn]struct{}
-}
-
-func newInFlightConns() (*inFlightConns, error) {
-	return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
-}
-
-func (i *inFlightConns) Add(conn net.Conn) {
-	i.Lock()
-	defer i.Unlock()
-
-	i.inflightConns[conn] = struct{}{}
-}
-
-func (i *inFlightConns) Remove(conn net.Conn) {
-	i.Lock()
-	defer i.Unlock()
-
-	delete(i.inflightConns, conn)
-}
-
-func (i *inFlightConns) Close() error {
-	i.Lock()
-	defer i.Unlock()
-
-	var errs []error
-	for conn := range i.inflightConns {
-		if err := conn.Close(); err != nil {
-			errs = append(errs, err)
-		}
-		delete(i.inflightConns, conn)
-	}
-	return multierr.Combine(errs...)
-}

common/httpx/response.go

@@ -4,8 +4,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/projectdiscovery/cryptoutil"
 	"github.com/projectdiscovery/httputil"
-	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
 )
 
 // Response contains the response to a server
@@ -18,7 +18,7 @@ type Response struct {
 	RawHeaders    string
 	Words         int
 	Lines         int
-	TLSData       *clients.Response
+	TLSData       *cryptoutil.TLSData
 	CSPData       *CSPData
 	HTTP2         bool
 	Pipeline      bool

common/httpx/tls.go

@@ -1,74 +1,15 @@
 package httpx
 
 import (
-	"crypto/tls"
-	"crypto/x509"
-	"net"
 	"net/http"
 
-	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
+	"github.com/projectdiscovery/cryptoutil"
 )
 
-// versionToTLSVersionString converts tls version to version string
-var versionToTLSVersionString = map[uint16]string{
-	tls.VersionTLS10: "tls10",
-	tls.VersionTLS11: "tls11",
-	tls.VersionTLS12: "tls12",
-	tls.VersionTLS13: "tls13",
-}
-
 // TLSGrab fills the TLSData
-func (h *HTTPX) TLSGrab(r *http.Response) *clients.Response {
-	if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
-		return nil
+func (h *HTTPX) TLSGrab(r *http.Response) *cryptoutil.TLSData {
+	if r.TLS != nil {
+		return cryptoutil.TLSGrab(r.TLS)
 	}
-	host := r.Request.URL.Host
-	hostname, port, _ := net.SplitHostPort(host)
-	if hostname == "" {
-		hostname = host
-	}
-	if port == "" {
-		port = "443"
-	}
-
-	tlsVersion := versionToTLSVersionString[r.TLS.Version]
-	tlsCipher := tls.CipherSuiteName(r.TLS.CipherSuite)
-
-	leafCertificate := r.TLS.PeerCertificates[0]
-	response := &clients.Response{
-		Host:                hostname,
-		ProbeStatus:         true,
-		Port:                port,
-		Version:             tlsVersion,
-		Cipher:              tlsCipher,
-		TLSConnection:       "ctls",
-		CertificateResponse: convertCertificateToResponse(hostname, leafCertificate),
-		ServerName:          r.TLS.ServerName,
-	}
-	return response
-}
-
-func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse {
-	response := &clients.CertificateResponse{
-		SubjectAN:    cert.DNSNames,
-		Emails:       cert.EmailAddresses,
-		NotBefore:    cert.NotBefore,
-		NotAfter:     cert.NotAfter,
-		Expired:      clients.IsExpired(cert.NotAfter),
-		SelfSigned:   clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId),
-		MisMatched:   clients.IsMisMatchedCert(hostname, append(cert.DNSNames, cert.Subject.CommonName)),
-		WildCardCert: clients.IsWildCardCert(append(cert.DNSNames, cert.Subject.CommonName)),
-		IssuerCN:     cert.Issuer.CommonName,
-		IssuerOrg:    cert.Issuer.Organization,
-		SubjectCN:    cert.Subject.CommonName,
-		SubjectOrg:   cert.Subject.Organization,
-		FingerprintHash: clients.CertificateResponseFingerprintHash{
-			MD5:    clients.MD5Fingerprint(cert.Raw),
-			SHA1:   clients.SHA1Fingerprint(cert.Raw),
-			SHA256: clients.SHA256Fingerprint(cert.Raw),
-		},
-	}
-	response.IssuerDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawIssuer, cert.Issuer.String())
-	response.SubjectDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawSubject, cert.Subject.String())
-	return response
+	return nil
 }

runner/runner.go

@@ -27,7 +27,6 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/httpx/common/customextract"
 	"github.com/projectdiscovery/httpx/common/hashes/jarm"
-	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
 
 	"github.com/ammario/ipisp/v2"
 	"github.com/bluele/gcache"
@@ -35,6 +34,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/cryptoutil"
 	"github.com/projectdiscovery/goconfig"
 	"github.com/projectdiscovery/httpx/common/hashes"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -796,14 +796,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
-							for _, tt := range result.TLSData.SubjectAN {
+							for _, tt := range result.TLSData.DNSNames {
 								if !r.testAndSet(tt) {
 									continue
 								}
 								r.process(tt, wg, hp, protocol, scanopts, output)
 							}
-							if r.testAndSet(result.TLSData.SubjectCN) {
-								r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
+							for _, tt := range result.TLSData.CommonName {
+								if !r.testAndSet(tt) {
+									continue
+								}
+								r.process(tt, wg, hp, protocol, scanopts, output)
 							}
 						}
 						if scanopts.CSPProbe && result.CSPData != nil {
@@ -835,14 +838,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
-							for _, tt := range result.TLSData.SubjectAN {
+							for _, tt := range result.TLSData.DNSNames {
 								if !r.testAndSet(tt) {
 									continue
 								}
 								r.process(tt, wg, hp, protocol, scanopts, output)
 							}
-							if r.testAndSet(result.TLSData.SubjectCN) {
-								r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
+							for _, tt := range result.TLSData.CommonName {
+								if !r.testAndSet(tt) {
+									continue
+								}
+								r.process(tt, wg, hp, protocol, scanopts, output)
 							}
 						}
 					}(port, target, method, wantedProtocol)
@@ -1567,12 +1573,12 @@ type Result struct {
 	Timestamp        time.Time   `json:"timestamp,omitempty" csv:"timestamp"`
 	ASN              interface{} `json:"asn,omitempty" csv:"asn"`
 	err              error
-	CSPData          *httpx.CSPData    `json:"csp,omitempty" csv:"csp"`
-	TLSData          *clients.Response `json:"tls,omitempty" csv:"tls"`
-	Hashes           map[string]string `json:"hash,omitempty" csv:"hash"`
-	ExtractRegex     []string          `json:"extract_regex,omitempty" csv:"extract_regex"`
-	CDNName          string            `json:"cdn_name,omitempty" csv:"cdn_name"`
-	Port             string            `json:"port,omitempty" csv:"port"`
+	CSPData          *httpx.CSPData      `json:"csp,omitempty" csv:"csp"`
+	TLSData          *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"`
+	Hashes           map[string]string   `json:"hash,omitempty" csv:"hash"`
+	ExtractRegex     []string            `json:"extract_regex,omitempty" csv:"extract_regex"`
+	CDNName          string              `json:"cdn_name,omitempty" csv:"cdn_name"`
+	Port             string              `json:"port,omitempty" csv:"port"`
 	raw              string
 	URL              string `json:"url,omitempty" csv:"url"`
 	Input            string `json:"input,omitempty" csv:"input"`