diff --git a/common/httpx/csp.go b/common/httpx/csp.go index 7fbde4b..1992796 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -64,7 +64,7 @@ func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) { // we extracts only potential domains for _, t := range tokens { if isPotentialDomain(t) { - if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil { + if dn, err := publicsuffix.Parse(extractDomain(removeWildcards(t))); err == nil { domains[dn.SLD+"."+dn.TLD] = struct{}{} if dn.TRD != "" { fqdns[dn.String()] = struct{}{} @@ -89,3 +89,13 @@ func extractDomain(str string) string { } return parsedURL.Host } + +func removeWildcards(domain string) string { + parts := []string{} + for _, part := range strings.Split(domain, ".") { + if part != "*" { + parts = append(parts, part) + } + } + return strings.Join(parts, ".") +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 8720d20..ac279d7 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -311,7 +311,9 @@ get_response: } } - resp.CSPData = h.CSPGrab(&resp) + if h.Options.ExtractFqdn { + resp.CSPData = h.CSPGrab(&resp) + } // build the redirect flow by reverse cycling the response<-request chain if !h.Options.Unsafe { diff --git a/common/httpx/option.go b/common/httpx/option.go index 7cb29c3..5f99cd6 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -17,6 +17,7 @@ type Options struct { Threads int CdnCheck string ExcludeCdn bool + ExtractFqdn bool // Timeout is the maximum time to wait for the request Timeout time.Duration // RetryMax is the maximum number of retries diff --git a/common/httpx/response.go b/common/httpx/response.go index 1b1ab47..5b766a3 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -42,7 +42,6 @@ func (r *Response) GetHeader(name string) string { if ok { return strings.Join(v, " ") } - return "" } diff --git a/runner/runner.go b/runner/runner.go index 2e61f64..9b0915b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -157,6 +157,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN httpxOptions.ExcludeCdn = runner.excludeCdn + httpxOptions.ExtractFqdn = options.ExtractFqdn if options.CustomHeaders.Has("User-Agent:") { httpxOptions.RandomAgent = false } else { @@ -874,11 +875,7 @@ func (r *Runner) RunEnumeration() { if r.options.OnResult != nil { r.options.OnResult(resp) } - // Set body domains and fqdns - if r.options.ExtractFqdn && resp.CSPData != nil { - resp.BodyDomains = resp.CSPData.Domains - resp.BodyFqdns = resp.CSPData.Fqdns - } + // store responses or chain in directory URL, _ := urlutil.Parse(resp.URL) domainFile := resp.Method + ":" + URL.EscapedString() diff --git a/runner/types.go b/runner/types.go index f06a431..1928081 100644 --- a/runner/types.go +++ b/runner/types.go @@ -36,8 +36,6 @@ type Result struct { ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` Err error `json:"-" csv:"-"` CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - BodyFqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn"` - BodyDomains []string `json:"body_domains,omitempty" csv:"body_domains"` TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`