remove wildcard in domain

common/httpx/csp.go

@@ -64,7 +64,7 @@ func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
 	// we extracts only potential domains
 	for _, t := range tokens {
 		if isPotentialDomain(t) {
-			if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
+			if dn, err := publicsuffix.Parse(extractDomain(removeWildcards(t))); err == nil {
 				domains[dn.SLD+"."+dn.TLD] = struct{}{}
 				if dn.TRD != "" {
 					fqdns[dn.String()] = struct{}{}
@@ -89,3 +89,13 @@ func extractDomain(str string) string {
 	}
 	return parsedURL.Host
 }
+
+func removeWildcards(domain string) string {
+	parts := []string{}
+	for _, part := range strings.Split(domain, ".") {
+		if part != "*" {
+			parts = append(parts, part)
+		}
+	}
+	return strings.Join(parts, ".")
+}

common/httpx/httpx.go

@@ -311,7 +311,9 @@ get_response:
 		}
 	}
 
-	resp.CSPData = h.CSPGrab(&resp)
+	if h.Options.ExtractFqdn {
+		resp.CSPData = h.CSPGrab(&resp)
+	}
 
 	// build the redirect flow by reverse cycling the response<-request chain
 	if !h.Options.Unsafe {

common/httpx/option.go

@@ -17,6 +17,7 @@ type Options struct {
 	Threads          int
 	CdnCheck         string
 	ExcludeCdn       bool
+	ExtractFqdn      bool
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
 	// RetryMax is the maximum number of retries

common/httpx/response.go

@@ -42,7 +42,6 @@ func (r *Response) GetHeader(name string) string {
 	if ok {
 		return strings.Join(v, " ")
 	}
-
 	return ""
 }
 

runner/runner.go

@@ -157,6 +157,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.UnsafeURI = options.RequestURI
 	httpxOptions.CdnCheck = options.OutputCDN
 	httpxOptions.ExcludeCdn = runner.excludeCdn
+	httpxOptions.ExtractFqdn = options.ExtractFqdn
 	if options.CustomHeaders.Has("User-Agent:") {
 		httpxOptions.RandomAgent = false
 	} else {
@@ -874,11 +875,7 @@ func (r *Runner) RunEnumeration() {
 			if r.options.OnResult != nil {
 				r.options.OnResult(resp)
 			}
-			// Set body domains and fqdns
+
-			if r.options.ExtractFqdn && resp.CSPData != nil {
-				resp.BodyDomains = resp.CSPData.Domains
-				resp.BodyFqdns = resp.CSPData.Fqdns
-			}
 			// store responses or chain in directory
 			URL, _ := urlutil.Parse(resp.URL)
 			domainFile := resp.Method + ":" + URL.EscapedString()

runner/types.go

@@ -36,8 +36,6 @@ type Result struct {
 	ASN                *AsnResponse           `json:"asn,omitempty" csv:"asn"`
 	Err                error                  `json:"-" csv:"-"`
 	CSPData            *httpx.CSPData         `json:"csp,omitempty" csv:"csp"`
-	BodyFqdns          []string               `json:"body_fqdn,omitempty" csv:"body_fqdn"`
-	BodyDomains        []string               `json:"body_domains,omitempty" csv:"body_domains"`
 	TLSData            *clients.Response      `json:"tls,omitempty" csv:"tls"`
 	Hashes             map[string]interface{} `json:"hash,omitempty" csv:"hash"`
 	ExtractRegex       []string               `json:"extract_regex,omitempty" csv:"extract_regex"`