mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-16 13:00:24 +03:00
remove wildcard in domain
This commit is contained in:
parent
c5ca655207
commit
de9375e643
@ -64,7 +64,7 @@ func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
|
||||
// we extracts only potential domains
|
||||
for _, t := range tokens {
|
||||
if isPotentialDomain(t) {
|
||||
if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
|
||||
if dn, err := publicsuffix.Parse(extractDomain(removeWildcards(t))); err == nil {
|
||||
domains[dn.SLD+"."+dn.TLD] = struct{}{}
|
||||
if dn.TRD != "" {
|
||||
fqdns[dn.String()] = struct{}{}
|
||||
@ -89,3 +89,13 @@ func extractDomain(str string) string {
|
||||
}
|
||||
return parsedURL.Host
|
||||
}
|
||||
|
||||
func removeWildcards(domain string) string {
|
||||
parts := []string{}
|
||||
for _, part := range strings.Split(domain, ".") {
|
||||
if part != "*" {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
@ -311,7 +311,9 @@ get_response:
|
||||
}
|
||||
}
|
||||
|
||||
resp.CSPData = h.CSPGrab(&resp)
|
||||
if h.Options.ExtractFqdn {
|
||||
resp.CSPData = h.CSPGrab(&resp)
|
||||
}
|
||||
|
||||
// build the redirect flow by reverse cycling the response<-request chain
|
||||
if !h.Options.Unsafe {
|
||||
|
@ -17,6 +17,7 @@ type Options struct {
|
||||
Threads int
|
||||
CdnCheck string
|
||||
ExcludeCdn bool
|
||||
ExtractFqdn bool
|
||||
// Timeout is the maximum time to wait for the request
|
||||
Timeout time.Duration
|
||||
// RetryMax is the maximum number of retries
|
||||
|
@ -42,7 +42,6 @@ func (r *Response) GetHeader(name string) string {
|
||||
if ok {
|
||||
return strings.Join(v, " ")
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
|
@ -157,6 +157,7 @@ func New(options *Options) (*Runner, error) {
|
||||
httpxOptions.UnsafeURI = options.RequestURI
|
||||
httpxOptions.CdnCheck = options.OutputCDN
|
||||
httpxOptions.ExcludeCdn = runner.excludeCdn
|
||||
httpxOptions.ExtractFqdn = options.ExtractFqdn
|
||||
if options.CustomHeaders.Has("User-Agent:") {
|
||||
httpxOptions.RandomAgent = false
|
||||
} else {
|
||||
@ -874,11 +875,7 @@ func (r *Runner) RunEnumeration() {
|
||||
if r.options.OnResult != nil {
|
||||
r.options.OnResult(resp)
|
||||
}
|
||||
// Set body domains and fqdns
|
||||
if r.options.ExtractFqdn && resp.CSPData != nil {
|
||||
resp.BodyDomains = resp.CSPData.Domains
|
||||
resp.BodyFqdns = resp.CSPData.Fqdns
|
||||
}
|
||||
|
||||
// store responses or chain in directory
|
||||
URL, _ := urlutil.Parse(resp.URL)
|
||||
domainFile := resp.Method + ":" + URL.EscapedString()
|
||||
|
@ -36,8 +36,6 @@ type Result struct {
|
||||
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
|
||||
Err error `json:"-" csv:"-"`
|
||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
||||
BodyFqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn"`
|
||||
BodyDomains []string `json:"body_domains,omitempty" csv:"body_domains"`
|
||||
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
||||
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
|
||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||
|
Loading…
Reference in New Issue
Block a user