From ca3e696834739ffa75fa0f64097f60151a5bdfcd Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 17 Jun 2024 18:11:01 +0530 Subject: [PATCH 1/2] feat: use networkpolicy + wappalyzer singletons from library --- common/httpx/httpx.go | 2 +- go.mod | 12 ++++--- go.sum | 26 ++++++++------- runner/options.go | 6 ++++ runner/runner.go | 73 ++++++++++++++++++++++++++----------------- 5 files changed, 73 insertions(+), 46 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index ac279d7..193b098 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -55,7 +55,7 @@ func New(options *Options) (*HTTPX, error) { if options.NetworkPolicy != nil { httpx.NetworkPolicy = options.NetworkPolicy - fastdialerOpts.WithNetworkPolicyOptions = options.NetworkPolicy.Options + fastdialerOpts.NetworkPolicy = options.NetworkPolicy } fastdialerOpts.WithDialerHistory = true fastdialerOpts.WithZTLS = options.ZTLS diff --git a/go.mod b/go.mod index e96dece..6d53856 100644 --- a/go.mod +++ b/go.mod @@ -24,20 +24,20 @@ require ( github.com/projectdiscovery/cdncheck v1.0.9 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/dsl v0.1.2 - github.com/projectdiscovery/fastdialer v0.1.3 + github.com/projectdiscovery/fastdialer v0.1.5 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.54 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.46 + github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/networkpolicy v0.0.8 + github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.44 github.com/projectdiscovery/rawhttp v0.1.52 github.com/projectdiscovery/retryablehttp-go v1.0.64 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.54 - github.com/projectdiscovery/utils v0.1.1 + github.com/projectdiscovery/utils v0.1.4 github.com/projectdiscovery/wappalyzergo v0.1.4 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 @@ -65,6 +65,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect @@ -76,6 +77,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/gaissmai/bart v0.9.5 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -112,7 +114,7 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.62 // indirect + github.com/projectdiscovery/retryabledns v1.0.64 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index f707634..d4b6d1e 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= -github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -67,6 +67,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= +github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI= @@ -226,8 +228,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw= github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA= -github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI= -github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao= +github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= +github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -240,20 +242,20 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk= -github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8= +github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk= +github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= -github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= -github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= +github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= +github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g= github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE= github.com/projectdiscovery/rawhttp v0.1.52 h1:NaDtkpCfrCiEMJ7b3NEVt0tJGY7S58jWwafy7UiCog8= github.com/projectdiscovery/rawhttp v0.1.52/go.mod h1:oXzQ8oToaZij1zC0PB8ai5meCeySH34DuVc0UW+c6Wg= -github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA= -github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA= +github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= +github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8= github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -262,8 +264,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc= github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM= -github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4= -github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o= +github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI= +github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM= github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4= github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= diff --git a/runner/options.go b/runner/options.go index 85fb95f..b4d18b7 100644 --- a/runner/options.go +++ b/runner/options.go @@ -24,12 +24,14 @@ import ( fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/stringz" + "github.com/projectdiscovery/networkpolicy" "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" updateutils "github.com/projectdiscovery/utils/update" + wappalyzer "github.com/projectdiscovery/wappalyzergo" ) const ( @@ -306,6 +308,10 @@ type Options struct { // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice Protocol string + + // Optional pre-created objects to reduce allocations + Wappalyzer *wappalyzer.Wappalyze + Networkpolicy *networkpolicy.NetworkPolicy } // ParseOptions parses the command line options for application diff --git a/runner/runner.go b/runner/runner.go index deaded4..fb5d7a0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -107,7 +107,9 @@ func New(options *Options) (*Runner, error) { options: options, } var err error - if options.TechDetect || options.JSONOutput || options.CSVOutput { + if options.Wappalyzer != nil { + runner.wappalyzer = options.Wappalyzer + } else if options.TechDetect || options.JSONOutput || options.CSVOutput { runner.wappalyzer, err = wappalyzer.New() } if err != nil { @@ -119,34 +121,19 @@ func New(options *Options) (*Runner, error) { os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) } - var npOptions networkpolicy.Options - for _, exclude := range options.Exclude { - switch { - case exclude == "cdn": - //implement cdn check in netoworkpolicy pkg?? - runner.excludeCdn = true - continue - case exclude == "private-ips": - npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...) - npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...) - npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...) - npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...) - case iputil.IsCIDR(exclude): - npOptions.DenyList = append(npOptions.DenyList, exclude) - case asn.IsASN(exclude): - // update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged - ips := expandASNInputValue(exclude) - npOptions.DenyList = append(npOptions.DenyList, ips...) - case iputil.IsPort(exclude): - port, _ := strconv.Atoi(exclude) - npOptions.DenyPortList = append(npOptions.DenyPortList, port) - default: - npOptions.DenyList = append(npOptions.DenyList, exclude) - } - } - httpxOptions := httpx.DefaultOptions - httpxOptions.NetworkPolicy, _ = networkpolicy.New(npOptions) + + var np *networkpolicy.NetworkPolicy + if options.Networkpolicy != nil { + np = options.Networkpolicy + } else { + np, err = runner.createNetworkpolicyInstance(options) + } + if err != nil { + return nil, err + } + httpxOptions.NetworkPolicy = np + // Enables automatically tlsgrab if tlsprobe is requested httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second @@ -381,6 +368,36 @@ func New(options *Options) (*Runner, error) { return runner, nil } +func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpolicy.NetworkPolicy, error) { + var npOptions networkpolicy.Options + for _, exclude := range options.Exclude { + switch { + case exclude == "cdn": + //implement cdn check in netoworkpolicy pkg?? + runner.excludeCdn = true + continue + case exclude == "private-ips": + npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...) + npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...) + npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...) + npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...) + case iputil.IsCIDR(exclude): + npOptions.DenyList = append(npOptions.DenyList, exclude) + case asn.IsASN(exclude): + // update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged + ips := expandASNInputValue(exclude) + npOptions.DenyList = append(npOptions.DenyList, ips...) + case iputil.IsPort(exclude): + port, _ := strconv.Atoi(exclude) + npOptions.DenyPortList = append(npOptions.DenyPortList, port) + default: + npOptions.DenyList = append(npOptions.DenyList, exclude) + } + } + np, err := networkpolicy.New(npOptions) + return np, err +} + func expandCIDRInputValue(value string) []string { var ips []string ipsCh, _ := mapcidr.IPAddressesAsStream(value) From 80d5532b15408b5a8a2bb316dbcce1eb977d6c75 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 17 Jun 2024 19:58:12 +0530 Subject: [PATCH 2/2] misc fixing tests --- common/httpx/option.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/common/httpx/option.go b/common/httpx/option.go index 5f99cd6..559af4a 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -51,14 +51,15 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - MaxRedirects: 10, - Unsafe: false, - CdnCheck: "true", - ExcludeCdn: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: "true", + ExcludeCdn: false, + MaxResponseBodySizeToRead: 1024 * 1024 * 10, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true,