projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-08-16 13:00:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

remove wildcard in domain

Browse Source
This commit is contained in:
Ramana Reddy 2024-05-30 15:03:15 +05:30
parent c5ca655207
commit de9375e643
6 changed files with 17 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
common/httpx/csp.go
View File

@ -64,7 +64,7 @@ func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
// we extracts only potential domains
for _, t := range tokens {
if isPotentialDomain(t) {
if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
if dn, err := publicsuffix.Parse(extractDomain(removeWildcards(t))); err == nil {
domains[dn.SLD+"."+dn.TLD] = struct{}{}
if dn.TRD != "" {
fqdns[dn.String()] = struct{}{}
@ -89,3 +89,13 @@ func extractDomain(str string) string {
}
return parsedURL.Host
}
func removeWildcards(domain string) string {
parts := []string{}
for _, part := range strings.Split(domain, ".") {
if part != "*" {
parts = append(parts, part)
}
}
return strings.Join(parts, ".")
}

4
common/httpx/httpx.go
View File

@ -311,7 +311,9 @@ get_response:
}
}
resp.CSPData = h.CSPGrab(&resp)
if h.Options.ExtractFqdn {
resp.CSPData = h.CSPGrab(&resp)
}
// build the redirect flow by reverse cycling the response<-request chain
if !h.Options.Unsafe {

1
common/httpx/option.go
View File

@ -17,6 +17,7 @@ type Options struct {
Threads int
CdnCheck string
ExcludeCdn bool
ExtractFqdn bool
// Timeout is the maximum time to wait for the request
Timeout time.Duration
// RetryMax is the maximum number of retries

1
common/httpx/response.go
View File

@ -42,7 +42,6 @@ func (r *Response) GetHeader(name string) string {
if ok {
return strings.Join(v, " ")
}
return ""
}

7
runner/runner.go
View File

@ -157,6 +157,7 @@ func New(options *Options) (*Runner, error) {
httpxOptions.UnsafeURI = options.RequestURI
httpxOptions.CdnCheck = options.OutputCDN
httpxOptions.ExcludeCdn = runner.excludeCdn
httpxOptions.ExtractFqdn = options.ExtractFqdn
if options.CustomHeaders.Has("User-Agent:") {
httpxOptions.RandomAgent = false
} else {
@ -874,11 +875,7 @@ func (r *Runner) RunEnumeration() {
if r.options.OnResult != nil {
r.options.OnResult(resp)
}
// Set body domains and fqdns
if r.options.ExtractFqdn && resp.CSPData != nil {
resp.BodyDomains = resp.CSPData.Domains
resp.BodyFqdns = resp.CSPData.Fqdns
}
// store responses or chain in directory
URL, _ := urlutil.Parse(resp.URL)
domainFile := resp.Method + ":" + URL.EscapedString()

2
runner/types.go
View File

@ -36,8 +36,6 @@ type Result struct {
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
Err error `json:"-" csv:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
BodyFqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn"`
BodyDomains []string `json:"body_domains,omitempty" csv:"body_domains"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`