mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-09-11 09:35:28 +03:00
remove wildcard in domain
This commit is contained in:
parent
c5ca655207
commit
de9375e643
@ -64,7 +64,7 @@ func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
|
|||||||
// we extracts only potential domains
|
// we extracts only potential domains
|
||||||
for _, t := range tokens {
|
for _, t := range tokens {
|
||||||
if isPotentialDomain(t) {
|
if isPotentialDomain(t) {
|
||||||
if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
|
if dn, err := publicsuffix.Parse(extractDomain(removeWildcards(t))); err == nil {
|
||||||
domains[dn.SLD+"."+dn.TLD] = struct{}{}
|
domains[dn.SLD+"."+dn.TLD] = struct{}{}
|
||||||
if dn.TRD != "" {
|
if dn.TRD != "" {
|
||||||
fqdns[dn.String()] = struct{}{}
|
fqdns[dn.String()] = struct{}{}
|
||||||
@ -89,3 +89,13 @@ func extractDomain(str string) string {
|
|||||||
}
|
}
|
||||||
return parsedURL.Host
|
return parsedURL.Host
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func removeWildcards(domain string) string {
|
||||||
|
parts := []string{}
|
||||||
|
for _, part := range strings.Split(domain, ".") {
|
||||||
|
if part != "*" {
|
||||||
|
parts = append(parts, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ".")
|
||||||
|
}
|
||||||
|
@ -311,7 +311,9 @@ get_response:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.CSPData = h.CSPGrab(&resp)
|
if h.Options.ExtractFqdn {
|
||||||
|
resp.CSPData = h.CSPGrab(&resp)
|
||||||
|
}
|
||||||
|
|
||||||
// build the redirect flow by reverse cycling the response<-request chain
|
// build the redirect flow by reverse cycling the response<-request chain
|
||||||
if !h.Options.Unsafe {
|
if !h.Options.Unsafe {
|
||||||
|
@ -17,6 +17,7 @@ type Options struct {
|
|||||||
Threads int
|
Threads int
|
||||||
CdnCheck string
|
CdnCheck string
|
||||||
ExcludeCdn bool
|
ExcludeCdn bool
|
||||||
|
ExtractFqdn bool
|
||||||
// Timeout is the maximum time to wait for the request
|
// Timeout is the maximum time to wait for the request
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
// RetryMax is the maximum number of retries
|
// RetryMax is the maximum number of retries
|
||||||
|
@ -42,7 +42,6 @@ func (r *Response) GetHeader(name string) string {
|
|||||||
if ok {
|
if ok {
|
||||||
return strings.Join(v, " ")
|
return strings.Join(v, " ")
|
||||||
}
|
}
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -157,6 +157,7 @@ func New(options *Options) (*Runner, error) {
|
|||||||
httpxOptions.UnsafeURI = options.RequestURI
|
httpxOptions.UnsafeURI = options.RequestURI
|
||||||
httpxOptions.CdnCheck = options.OutputCDN
|
httpxOptions.CdnCheck = options.OutputCDN
|
||||||
httpxOptions.ExcludeCdn = runner.excludeCdn
|
httpxOptions.ExcludeCdn = runner.excludeCdn
|
||||||
|
httpxOptions.ExtractFqdn = options.ExtractFqdn
|
||||||
if options.CustomHeaders.Has("User-Agent:") {
|
if options.CustomHeaders.Has("User-Agent:") {
|
||||||
httpxOptions.RandomAgent = false
|
httpxOptions.RandomAgent = false
|
||||||
} else {
|
} else {
|
||||||
@ -874,11 +875,7 @@ func (r *Runner) RunEnumeration() {
|
|||||||
if r.options.OnResult != nil {
|
if r.options.OnResult != nil {
|
||||||
r.options.OnResult(resp)
|
r.options.OnResult(resp)
|
||||||
}
|
}
|
||||||
// Set body domains and fqdns
|
|
||||||
if r.options.ExtractFqdn && resp.CSPData != nil {
|
|
||||||
resp.BodyDomains = resp.CSPData.Domains
|
|
||||||
resp.BodyFqdns = resp.CSPData.Fqdns
|
|
||||||
}
|
|
||||||
// store responses or chain in directory
|
// store responses or chain in directory
|
||||||
URL, _ := urlutil.Parse(resp.URL)
|
URL, _ := urlutil.Parse(resp.URL)
|
||||||
domainFile := resp.Method + ":" + URL.EscapedString()
|
domainFile := resp.Method + ":" + URL.EscapedString()
|
||||||
|
@ -36,8 +36,6 @@ type Result struct {
|
|||||||
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
|
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
|
||||||
Err error `json:"-" csv:"-"`
|
Err error `json:"-" csv:"-"`
|
||||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
||||||
BodyFqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn"`
|
|
||||||
BodyDomains []string `json:"body_domains,omitempty" csv:"body_domains"`
|
|
||||||
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
||||||
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
|
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
|
||||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||||
|
Loading…
Reference in New Issue
Block a user