mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-16 21:10:23 +03:00
Merge pull request #1769 from projectdiscovery/use-tech-networkpolicy-singletons
feat: use networkpolicy + wappalyzer singletons from library
This commit is contained in:
commit
355814014a
@ -55,7 +55,7 @@ func New(options *Options) (*HTTPX, error) {
|
|||||||
|
|
||||||
if options.NetworkPolicy != nil {
|
if options.NetworkPolicy != nil {
|
||||||
httpx.NetworkPolicy = options.NetworkPolicy
|
httpx.NetworkPolicy = options.NetworkPolicy
|
||||||
fastdialerOpts.WithNetworkPolicyOptions = options.NetworkPolicy.Options
|
fastdialerOpts.NetworkPolicy = options.NetworkPolicy
|
||||||
}
|
}
|
||||||
fastdialerOpts.WithDialerHistory = true
|
fastdialerOpts.WithDialerHistory = true
|
||||||
fastdialerOpts.WithZTLS = options.ZTLS
|
fastdialerOpts.WithZTLS = options.ZTLS
|
||||||
|
@ -51,14 +51,15 @@ type Options struct {
|
|||||||
|
|
||||||
// DefaultOptions contains the default options
|
// DefaultOptions contains the default options
|
||||||
var DefaultOptions = Options{
|
var DefaultOptions = Options{
|
||||||
RandomAgent: true,
|
RandomAgent: true,
|
||||||
Threads: 25,
|
Threads: 25,
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
RetryMax: 5,
|
RetryMax: 5,
|
||||||
MaxRedirects: 10,
|
MaxRedirects: 10,
|
||||||
Unsafe: false,
|
Unsafe: false,
|
||||||
CdnCheck: "true",
|
CdnCheck: "true",
|
||||||
ExcludeCdn: false,
|
ExcludeCdn: false,
|
||||||
|
MaxResponseBodySizeToRead: 1024 * 1024 * 10,
|
||||||
// VHOSTs options
|
// VHOSTs options
|
||||||
VHostIgnoreStatusCode: false,
|
VHostIgnoreStatusCode: false,
|
||||||
VHostIgnoreContentLength: true,
|
VHostIgnoreContentLength: true,
|
||||||
|
12
go.mod
12
go.mod
@ -24,20 +24,20 @@ require (
|
|||||||
github.com/projectdiscovery/cdncheck v1.0.9
|
github.com/projectdiscovery/cdncheck v1.0.9
|
||||||
github.com/projectdiscovery/clistats v0.0.20
|
github.com/projectdiscovery/clistats v0.0.20
|
||||||
github.com/projectdiscovery/dsl v0.1.2
|
github.com/projectdiscovery/dsl v0.1.2
|
||||||
github.com/projectdiscovery/fastdialer v0.1.3
|
github.com/projectdiscovery/fastdialer v0.1.5
|
||||||
github.com/projectdiscovery/fdmax v0.0.4
|
github.com/projectdiscovery/fdmax v0.0.4
|
||||||
github.com/projectdiscovery/goconfig v0.0.1
|
github.com/projectdiscovery/goconfig v0.0.1
|
||||||
github.com/projectdiscovery/goflags v0.1.54
|
github.com/projectdiscovery/goflags v0.1.54
|
||||||
github.com/projectdiscovery/gologger v1.1.12
|
github.com/projectdiscovery/gologger v1.1.12
|
||||||
github.com/projectdiscovery/hmap v0.0.46
|
github.com/projectdiscovery/hmap v0.0.47
|
||||||
github.com/projectdiscovery/mapcidr v1.1.34
|
github.com/projectdiscovery/mapcidr v1.1.34
|
||||||
github.com/projectdiscovery/networkpolicy v0.0.8
|
github.com/projectdiscovery/networkpolicy v0.0.9
|
||||||
github.com/projectdiscovery/ratelimit v0.0.44
|
github.com/projectdiscovery/ratelimit v0.0.44
|
||||||
github.com/projectdiscovery/rawhttp v0.1.52
|
github.com/projectdiscovery/rawhttp v0.1.52
|
||||||
github.com/projectdiscovery/retryablehttp-go v1.0.64
|
github.com/projectdiscovery/retryablehttp-go v1.0.64
|
||||||
github.com/projectdiscovery/tlsx v1.1.6
|
github.com/projectdiscovery/tlsx v1.1.6
|
||||||
github.com/projectdiscovery/useragent v0.0.54
|
github.com/projectdiscovery/useragent v0.0.54
|
||||||
github.com/projectdiscovery/utils v0.1.1
|
github.com/projectdiscovery/utils v0.1.4
|
||||||
github.com/projectdiscovery/wappalyzergo v0.1.4
|
github.com/projectdiscovery/wappalyzergo v0.1.4
|
||||||
github.com/rs/xid v1.5.0
|
github.com/rs/xid v1.5.0
|
||||||
github.com/spaolacci/murmur3 v1.1.0
|
github.com/spaolacci/murmur3 v1.1.0
|
||||||
@ -65,6 +65,7 @@ require (
|
|||||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||||
github.com/aymerick/douceur v0.2.0 // indirect
|
github.com/aymerick/douceur v0.2.0 // indirect
|
||||||
|
github.com/bits-and-blooms/bitset v1.13.0 // indirect
|
||||||
github.com/charmbracelet/glamour v0.6.0 // indirect
|
github.com/charmbracelet/glamour v0.6.0 // indirect
|
||||||
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
|
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
|
||||||
github.com/cloudflare/cfssl v1.6.4 // indirect
|
github.com/cloudflare/cfssl v1.6.4 // indirect
|
||||||
@ -76,6 +77,7 @@ require (
|
|||||||
github.com/docker/go-units v0.5.0 // indirect
|
github.com/docker/go-units v0.5.0 // indirect
|
||||||
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
|
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
|
||||||
github.com/fatih/color v1.15.0 // indirect
|
github.com/fatih/color v1.15.0 // indirect
|
||||||
|
github.com/gaissmai/bart v0.9.5 // indirect
|
||||||
github.com/gaukas/godicttls v0.0.4 // indirect
|
github.com/gaukas/godicttls v0.0.4 // indirect
|
||||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||||
github.com/golang/protobuf v1.5.3 // indirect
|
github.com/golang/protobuf v1.5.3 // indirect
|
||||||
@ -112,7 +114,7 @@ require (
|
|||||||
github.com/projectdiscovery/freeport v0.0.5 // indirect
|
github.com/projectdiscovery/freeport v0.0.5 // indirect
|
||||||
github.com/projectdiscovery/gostruct v0.0.2 // indirect
|
github.com/projectdiscovery/gostruct v0.0.2 // indirect
|
||||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||||
github.com/projectdiscovery/retryabledns v1.0.62 // indirect
|
github.com/projectdiscovery/retryabledns v1.0.64 // indirect
|
||||||
github.com/quic-go/quic-go v0.42.0 // indirect
|
github.com/quic-go/quic-go v0.42.0 // indirect
|
||||||
github.com/refraction-networking/utls v1.5.4 // indirect
|
github.com/refraction-networking/utls v1.5.4 // indirect
|
||||||
github.com/rivo/uniseg v0.4.4 // indirect
|
github.com/rivo/uniseg v0.4.4 // indirect
|
||||||
|
26
go.sum
26
go.sum
@ -31,8 +31,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
|
|||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||||
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
|
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
|
||||||
github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||||
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
|
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
|
||||||
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
|
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
|
||||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||||
@ -67,6 +67,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
|
|||||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||||
|
github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
|
||||||
|
github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
|
||||||
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
|
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
|
||||||
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
|
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
|
||||||
github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI=
|
github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI=
|
||||||
@ -226,8 +228,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
|
|||||||
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
|
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
|
||||||
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
|
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
|
||||||
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
|
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
|
||||||
github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI=
|
github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
|
||||||
github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao=
|
github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
|
||||||
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
|
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
|
||||||
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
|
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
|
||||||
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
|
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
|
||||||
@ -240,20 +242,20 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ
|
|||||||
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
|
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
|
||||||
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
|
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
|
||||||
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
||||||
github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk=
|
github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
|
||||||
github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
|
github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
|
||||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
||||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
||||||
github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
|
github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
|
||||||
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
|
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
|
||||||
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
|
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
|
||||||
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
|
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
|
||||||
github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g=
|
github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g=
|
||||||
github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE=
|
github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE=
|
||||||
github.com/projectdiscovery/rawhttp v0.1.52 h1:NaDtkpCfrCiEMJ7b3NEVt0tJGY7S58jWwafy7UiCog8=
|
github.com/projectdiscovery/rawhttp v0.1.52 h1:NaDtkpCfrCiEMJ7b3NEVt0tJGY7S58jWwafy7UiCog8=
|
||||||
github.com/projectdiscovery/rawhttp v0.1.52/go.mod h1:oXzQ8oToaZij1zC0PB8ai5meCeySH34DuVc0UW+c6Wg=
|
github.com/projectdiscovery/rawhttp v0.1.52/go.mod h1:oXzQ8oToaZij1zC0PB8ai5meCeySH34DuVc0UW+c6Wg=
|
||||||
github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA=
|
github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k=
|
||||||
github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA=
|
github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg=
|
||||||
github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
|
github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
|
||||||
github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
|
github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
|
||||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||||
@ -262,8 +264,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
|
|||||||
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
|
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
|
||||||
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
|
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
|
||||||
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
|
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
|
||||||
github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4=
|
github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
|
||||||
github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o=
|
github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
|
||||||
github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
|
github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
|
||||||
github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
|
github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
|
||||||
github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
|
github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
|
||||||
|
@ -24,12 +24,14 @@ import (
|
|||||||
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
|
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
|
||||||
"github.com/projectdiscovery/httpx/common/httpx"
|
"github.com/projectdiscovery/httpx/common/httpx"
|
||||||
"github.com/projectdiscovery/httpx/common/stringz"
|
"github.com/projectdiscovery/httpx/common/stringz"
|
||||||
|
"github.com/projectdiscovery/networkpolicy"
|
||||||
"github.com/projectdiscovery/utils/auth/pdcp"
|
"github.com/projectdiscovery/utils/auth/pdcp"
|
||||||
"github.com/projectdiscovery/utils/env"
|
"github.com/projectdiscovery/utils/env"
|
||||||
fileutil "github.com/projectdiscovery/utils/file"
|
fileutil "github.com/projectdiscovery/utils/file"
|
||||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||||
updateutils "github.com/projectdiscovery/utils/update"
|
updateutils "github.com/projectdiscovery/utils/update"
|
||||||
|
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -306,6 +308,10 @@ type Options struct {
|
|||||||
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
|
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
|
||||||
HeadlessOptionalArguments goflags.StringSlice
|
HeadlessOptionalArguments goflags.StringSlice
|
||||||
Protocol string
|
Protocol string
|
||||||
|
|
||||||
|
// Optional pre-created objects to reduce allocations
|
||||||
|
Wappalyzer *wappalyzer.Wappalyze
|
||||||
|
Networkpolicy *networkpolicy.NetworkPolicy
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseOptions parses the command line options for application
|
// ParseOptions parses the command line options for application
|
||||||
|
@ -107,7 +107,9 @@ func New(options *Options) (*Runner, error) {
|
|||||||
options: options,
|
options: options,
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
if options.TechDetect || options.JSONOutput || options.CSVOutput {
|
if options.Wappalyzer != nil {
|
||||||
|
runner.wappalyzer = options.Wappalyzer
|
||||||
|
} else if options.TechDetect || options.JSONOutput || options.CSVOutput {
|
||||||
runner.wappalyzer, err = wappalyzer.New()
|
runner.wappalyzer, err = wappalyzer.New()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -119,34 +121,19 @@ func New(options *Options) (*Runner, error) {
|
|||||||
os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
|
os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
|
||||||
}
|
}
|
||||||
|
|
||||||
var npOptions networkpolicy.Options
|
|
||||||
for _, exclude := range options.Exclude {
|
|
||||||
switch {
|
|
||||||
case exclude == "cdn":
|
|
||||||
//implement cdn check in netoworkpolicy pkg??
|
|
||||||
runner.excludeCdn = true
|
|
||||||
continue
|
|
||||||
case exclude == "private-ips":
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
|
|
||||||
case iputil.IsCIDR(exclude):
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
|
||||||
case asn.IsASN(exclude):
|
|
||||||
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
|
|
||||||
ips := expandASNInputValue(exclude)
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, ips...)
|
|
||||||
case iputil.IsPort(exclude):
|
|
||||||
port, _ := strconv.Atoi(exclude)
|
|
||||||
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
|
|
||||||
default:
|
|
||||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
httpxOptions := httpx.DefaultOptions
|
httpxOptions := httpx.DefaultOptions
|
||||||
httpxOptions.NetworkPolicy, _ = networkpolicy.New(npOptions)
|
|
||||||
|
var np *networkpolicy.NetworkPolicy
|
||||||
|
if options.Networkpolicy != nil {
|
||||||
|
np = options.Networkpolicy
|
||||||
|
} else {
|
||||||
|
np, err = runner.createNetworkpolicyInstance(options)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
httpxOptions.NetworkPolicy = np
|
||||||
|
|
||||||
// Enables automatically tlsgrab if tlsprobe is requested
|
// Enables automatically tlsgrab if tlsprobe is requested
|
||||||
httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe
|
httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe
|
||||||
httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second
|
httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second
|
||||||
@ -381,6 +368,36 @@ func New(options *Options) (*Runner, error) {
|
|||||||
return runner, nil
|
return runner, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpolicy.NetworkPolicy, error) {
|
||||||
|
var npOptions networkpolicy.Options
|
||||||
|
for _, exclude := range options.Exclude {
|
||||||
|
switch {
|
||||||
|
case exclude == "cdn":
|
||||||
|
//implement cdn check in netoworkpolicy pkg??
|
||||||
|
runner.excludeCdn = true
|
||||||
|
continue
|
||||||
|
case exclude == "private-ips":
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
|
||||||
|
case iputil.IsCIDR(exclude):
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||||
|
case asn.IsASN(exclude):
|
||||||
|
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
|
||||||
|
ips := expandASNInputValue(exclude)
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, ips...)
|
||||||
|
case iputil.IsPort(exclude):
|
||||||
|
port, _ := strconv.Atoi(exclude)
|
||||||
|
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
|
||||||
|
default:
|
||||||
|
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
np, err := networkpolicy.New(npOptions)
|
||||||
|
return np, err
|
||||||
|
}
|
||||||
|
|
||||||
func expandCIDRInputValue(value string) []string {
|
func expandCIDRInputValue(value string) []string {
|
||||||
var ips []string
|
var ips []string
|
||||||
ipsCh, _ := mapcidr.IPAddressesAsStream(value)
|
ipsCh, _ := mapcidr.IPAddressesAsStream(value)
|
||||||
|
Loading…
Reference in New Issue
Block a user