Merge pull request #1769 from projectdiscovery/use-tech-networkpolicy-singletons

feat: use networkpolicy + wappalyzer singletons from library
This commit is contained in:
Ice3man 2024-06-17 21:55:13 +05:30 committed by GitHub
commit 355814014a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 82 additions and 54 deletions

View File

@ -55,7 +55,7 @@ func New(options *Options) (*HTTPX, error) {
if options.NetworkPolicy != nil {
httpx.NetworkPolicy = options.NetworkPolicy
fastdialerOpts.WithNetworkPolicyOptions = options.NetworkPolicy.Options
fastdialerOpts.NetworkPolicy = options.NetworkPolicy
}
fastdialerOpts.WithDialerHistory = true
fastdialerOpts.WithZTLS = options.ZTLS

View File

@ -51,14 +51,15 @@ type Options struct {
// DefaultOptions contains the default options
var DefaultOptions = Options{
RandomAgent: true,
Threads: 25,
Timeout: 30 * time.Second,
RetryMax: 5,
MaxRedirects: 10,
Unsafe: false,
CdnCheck: "true",
ExcludeCdn: false,
RandomAgent: true,
Threads: 25,
Timeout: 30 * time.Second,
RetryMax: 5,
MaxRedirects: 10,
Unsafe: false,
CdnCheck: "true",
ExcludeCdn: false,
MaxResponseBodySizeToRead: 1024 * 1024 * 10,
// VHOSTs options
VHostIgnoreStatusCode: false,
VHostIgnoreContentLength: true,

12
go.mod
View File

@ -24,20 +24,20 @@ require (
github.com/projectdiscovery/cdncheck v1.0.9
github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/dsl v0.1.2
github.com/projectdiscovery/fastdialer v0.1.3
github.com/projectdiscovery/fastdialer v0.1.5
github.com/projectdiscovery/fdmax v0.0.4
github.com/projectdiscovery/goconfig v0.0.1
github.com/projectdiscovery/goflags v0.1.54
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/hmap v0.0.46
github.com/projectdiscovery/hmap v0.0.47
github.com/projectdiscovery/mapcidr v1.1.34
github.com/projectdiscovery/networkpolicy v0.0.8
github.com/projectdiscovery/networkpolicy v0.0.9
github.com/projectdiscovery/ratelimit v0.0.44
github.com/projectdiscovery/rawhttp v0.1.52
github.com/projectdiscovery/retryablehttp-go v1.0.64
github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/useragent v0.0.54
github.com/projectdiscovery/utils v0.1.1
github.com/projectdiscovery/utils v0.1.4
github.com/projectdiscovery/wappalyzergo v0.1.4
github.com/rs/xid v1.5.0
github.com/spaolacci/murmur3 v1.1.0
@ -65,6 +65,7 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/charmbracelet/glamour v0.6.0 // indirect
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/cloudflare/cfssl v1.6.4 // indirect
@ -76,6 +77,7 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/gaissmai/bart v0.9.5 // indirect
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang/protobuf v1.5.3 // indirect
@ -112,7 +114,7 @@ require (
github.com/projectdiscovery/freeport v0.0.5 // indirect
github.com/projectdiscovery/gostruct v0.0.2 // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
github.com/projectdiscovery/retryabledns v1.0.62 // indirect
github.com/projectdiscovery/retryabledns v1.0.64 // indirect
github.com/quic-go/quic-go v0.42.0 // indirect
github.com/refraction-networking/utls v1.5.4 // indirect
github.com/rivo/uniseg v0.4.4 // indirect

26
go.sum
View File

@ -31,8 +31,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
@ -67,6 +67,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI=
@ -226,8 +228,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI=
github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao=
github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@ -240,20 +242,20 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk=
github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g=
github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE=
github.com/projectdiscovery/rawhttp v0.1.52 h1:NaDtkpCfrCiEMJ7b3NEVt0tJGY7S58jWwafy7UiCog8=
github.com/projectdiscovery/rawhttp v0.1.52/go.mod h1:oXzQ8oToaZij1zC0PB8ai5meCeySH34DuVc0UW+c6Wg=
github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA=
github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA=
github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k=
github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg=
github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@ -262,8 +264,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4=
github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o=
github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=

View File

@ -24,12 +24,14 @@ import (
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/env"
fileutil "github.com/projectdiscovery/utils/file"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
updateutils "github.com/projectdiscovery/utils/update"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
const (
@ -306,6 +308,10 @@ type Options struct {
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
HeadlessOptionalArguments goflags.StringSlice
Protocol string
// Optional pre-created objects to reduce allocations
Wappalyzer *wappalyzer.Wappalyze
Networkpolicy *networkpolicy.NetworkPolicy
}
// ParseOptions parses the command line options for application

View File

@ -107,7 +107,9 @@ func New(options *Options) (*Runner, error) {
options: options,
}
var err error
if options.TechDetect || options.JSONOutput || options.CSVOutput {
if options.Wappalyzer != nil {
runner.wappalyzer = options.Wappalyzer
} else if options.TechDetect || options.JSONOutput || options.CSVOutput {
runner.wappalyzer, err = wappalyzer.New()
}
if err != nil {
@ -119,34 +121,19 @@ func New(options *Options) (*Runner, error) {
os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
}
var npOptions networkpolicy.Options
for _, exclude := range options.Exclude {
switch {
case exclude == "cdn":
//implement cdn check in netoworkpolicy pkg??
runner.excludeCdn = true
continue
case exclude == "private-ips":
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
case iputil.IsCIDR(exclude):
npOptions.DenyList = append(npOptions.DenyList, exclude)
case asn.IsASN(exclude):
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
ips := expandASNInputValue(exclude)
npOptions.DenyList = append(npOptions.DenyList, ips...)
case iputil.IsPort(exclude):
port, _ := strconv.Atoi(exclude)
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
default:
npOptions.DenyList = append(npOptions.DenyList, exclude)
}
}
httpxOptions := httpx.DefaultOptions
httpxOptions.NetworkPolicy, _ = networkpolicy.New(npOptions)
var np *networkpolicy.NetworkPolicy
if options.Networkpolicy != nil {
np = options.Networkpolicy
} else {
np, err = runner.createNetworkpolicyInstance(options)
}
if err != nil {
return nil, err
}
httpxOptions.NetworkPolicy = np
// Enables automatically tlsgrab if tlsprobe is requested
httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe
httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second
@ -381,6 +368,36 @@ func New(options *Options) (*Runner, error) {
return runner, nil
}
func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpolicy.NetworkPolicy, error) {
var npOptions networkpolicy.Options
for _, exclude := range options.Exclude {
switch {
case exclude == "cdn":
//implement cdn check in netoworkpolicy pkg??
runner.excludeCdn = true
continue
case exclude == "private-ips":
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
case iputil.IsCIDR(exclude):
npOptions.DenyList = append(npOptions.DenyList, exclude)
case asn.IsASN(exclude):
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
ips := expandASNInputValue(exclude)
npOptions.DenyList = append(npOptions.DenyList, ips...)
case iputil.IsPort(exclude):
port, _ := strconv.Atoi(exclude)
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
default:
npOptions.DenyList = append(npOptions.DenyList, exclude)
}
}
np, err := networkpolicy.New(npOptions)
return np, err
}
func expandCIDRInputValue(value string) []string {
var ips []string
ipsCh, _ := mapcidr.IPAddressesAsStream(value)