mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-16 21:10:23 +03:00
Merge pull request #1769 from projectdiscovery/use-tech-networkpolicy-singletons
feat: use networkpolicy + wappalyzer singletons from library
This commit is contained in:
commit
355814014a
@ -55,7 +55,7 @@ func New(options *Options) (*HTTPX, error) {
|
||||
|
||||
if options.NetworkPolicy != nil {
|
||||
httpx.NetworkPolicy = options.NetworkPolicy
|
||||
fastdialerOpts.WithNetworkPolicyOptions = options.NetworkPolicy.Options
|
||||
fastdialerOpts.NetworkPolicy = options.NetworkPolicy
|
||||
}
|
||||
fastdialerOpts.WithDialerHistory = true
|
||||
fastdialerOpts.WithZTLS = options.ZTLS
|
||||
|
@ -59,6 +59,7 @@ var DefaultOptions = Options{
|
||||
Unsafe: false,
|
||||
CdnCheck: "true",
|
||||
ExcludeCdn: false,
|
||||
MaxResponseBodySizeToRead: 1024 * 1024 * 10,
|
||||
// VHOSTs options
|
||||
VHostIgnoreStatusCode: false,
|
||||
VHostIgnoreContentLength: true,
|
||||
|
12
go.mod
12
go.mod
@ -24,20 +24,20 @@ require (
|
||||
github.com/projectdiscovery/cdncheck v1.0.9
|
||||
github.com/projectdiscovery/clistats v0.0.20
|
||||
github.com/projectdiscovery/dsl v0.1.2
|
||||
github.com/projectdiscovery/fastdialer v0.1.3
|
||||
github.com/projectdiscovery/fastdialer v0.1.5
|
||||
github.com/projectdiscovery/fdmax v0.0.4
|
||||
github.com/projectdiscovery/goconfig v0.0.1
|
||||
github.com/projectdiscovery/goflags v0.1.54
|
||||
github.com/projectdiscovery/gologger v1.1.12
|
||||
github.com/projectdiscovery/hmap v0.0.46
|
||||
github.com/projectdiscovery/hmap v0.0.47
|
||||
github.com/projectdiscovery/mapcidr v1.1.34
|
||||
github.com/projectdiscovery/networkpolicy v0.0.8
|
||||
github.com/projectdiscovery/networkpolicy v0.0.9
|
||||
github.com/projectdiscovery/ratelimit v0.0.44
|
||||
github.com/projectdiscovery/rawhttp v0.1.52
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.64
|
||||
github.com/projectdiscovery/tlsx v1.1.6
|
||||
github.com/projectdiscovery/useragent v0.0.54
|
||||
github.com/projectdiscovery/utils v0.1.1
|
||||
github.com/projectdiscovery/utils v0.1.4
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.4
|
||||
github.com/rs/xid v1.5.0
|
||||
github.com/spaolacci/murmur3 v1.1.0
|
||||
@ -65,6 +65,7 @@ require (
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.13.0 // indirect
|
||||
github.com/charmbracelet/glamour v0.6.0 // indirect
|
||||
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
|
||||
github.com/cloudflare/cfssl v1.6.4 // indirect
|
||||
@ -76,6 +77,7 @@ require (
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
|
||||
github.com/fatih/color v1.15.0 // indirect
|
||||
github.com/gaissmai/bart v0.9.5 // indirect
|
||||
github.com/gaukas/godicttls v0.0.4 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
@ -112,7 +114,7 @@ require (
|
||||
github.com/projectdiscovery/freeport v0.0.5 // indirect
|
||||
github.com/projectdiscovery/gostruct v0.0.2 // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||
github.com/projectdiscovery/retryabledns v1.0.62 // indirect
|
||||
github.com/projectdiscovery/retryabledns v1.0.64 // indirect
|
||||
github.com/quic-go/quic-go v0.42.0 // indirect
|
||||
github.com/refraction-networking/utls v1.5.4 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
|
26
go.sum
26
go.sum
@ -31,8 +31,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
|
||||
github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
|
||||
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
|
||||
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
|
||||
github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
|
||||
@ -67,6 +67,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
|
||||
github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
|
||||
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
|
||||
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
|
||||
github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI=
|
||||
@ -226,8 +228,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
|
||||
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
|
||||
github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
|
||||
github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
|
||||
github.com/projectdiscovery/fastdialer v0.1.3 h1:DuXMFFDnhHP2pXWYZXqZHS76vSvkMeRYD4NifI/DlXI=
|
||||
github.com/projectdiscovery/fastdialer v0.1.3/go.mod h1:V5fkUburMLylbtBqbei5aAXTt1foDMeX517iM3uPkao=
|
||||
github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
|
||||
github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
|
||||
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
|
||||
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
|
||||
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
|
||||
@ -240,20 +242,20 @@ github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJ
|
||||
github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
|
||||
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
|
||||
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
||||
github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk=
|
||||
github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
|
||||
github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
|
||||
github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
||||
github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
|
||||
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
|
||||
github.com/projectdiscovery/ratelimit v0.0.44 h1:zkoTiDbBSJLSDLX73gTvrCzCDY9CdfrYKaTzolE7y0g=
|
||||
github.com/projectdiscovery/ratelimit v0.0.44/go.mod h1:hm7Mf0Dn1cuId0FVTdRlM2j2XPSYMBXaxcSEom6kdoE=
|
||||
github.com/projectdiscovery/rawhttp v0.1.52 h1:NaDtkpCfrCiEMJ7b3NEVt0tJGY7S58jWwafy7UiCog8=
|
||||
github.com/projectdiscovery/rawhttp v0.1.52/go.mod h1:oXzQ8oToaZij1zC0PB8ai5meCeySH34DuVc0UW+c6Wg=
|
||||
github.com/projectdiscovery/retryabledns v1.0.62 h1:nroTEgMMfppwjwtqT5WqyfMRH8edBPsJ0tosH+5riPA=
|
||||
github.com/projectdiscovery/retryabledns v1.0.62/go.mod h1:K761GKlfiveUrN+byy7ManMpO3oHOIhTe3wVBWXxTSA=
|
||||
github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k=
|
||||
github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
@ -262,8 +264,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
|
||||
github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
|
||||
github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
|
||||
github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
|
||||
github.com/projectdiscovery/utils v0.1.1 h1:iQ/DyrClxbIbKMUCXfXLA1lFkqprrUM9Ti/nMU5dQj4=
|
||||
github.com/projectdiscovery/utils v0.1.1/go.mod h1:EPuSvVIvp61nXJD5EO65vaCv82OuhO+wfZpWAWA0q3o=
|
||||
github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
|
||||
github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
|
||||
github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
|
||||
|
@ -24,12 +24,14 @@ import (
|
||||
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/httpx/common/stringz"
|
||||
"github.com/projectdiscovery/networkpolicy"
|
||||
"github.com/projectdiscovery/utils/auth/pdcp"
|
||||
"github.com/projectdiscovery/utils/env"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
updateutils "github.com/projectdiscovery/utils/update"
|
||||
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -306,6 +308,10 @@ type Options struct {
|
||||
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
|
||||
HeadlessOptionalArguments goflags.StringSlice
|
||||
Protocol string
|
||||
|
||||
// Optional pre-created objects to reduce allocations
|
||||
Wappalyzer *wappalyzer.Wappalyze
|
||||
Networkpolicy *networkpolicy.NetworkPolicy
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
|
@ -107,7 +107,9 @@ func New(options *Options) (*Runner, error) {
|
||||
options: options,
|
||||
}
|
||||
var err error
|
||||
if options.TechDetect || options.JSONOutput || options.CSVOutput {
|
||||
if options.Wappalyzer != nil {
|
||||
runner.wappalyzer = options.Wappalyzer
|
||||
} else if options.TechDetect || options.JSONOutput || options.CSVOutput {
|
||||
runner.wappalyzer, err = wappalyzer.New()
|
||||
}
|
||||
if err != nil {
|
||||
@ -119,34 +121,19 @@ func New(options *Options) (*Runner, error) {
|
||||
os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
|
||||
}
|
||||
|
||||
var npOptions networkpolicy.Options
|
||||
for _, exclude := range options.Exclude {
|
||||
switch {
|
||||
case exclude == "cdn":
|
||||
//implement cdn check in netoworkpolicy pkg??
|
||||
runner.excludeCdn = true
|
||||
continue
|
||||
case exclude == "private-ips":
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
|
||||
case iputil.IsCIDR(exclude):
|
||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||
case asn.IsASN(exclude):
|
||||
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
|
||||
ips := expandASNInputValue(exclude)
|
||||
npOptions.DenyList = append(npOptions.DenyList, ips...)
|
||||
case iputil.IsPort(exclude):
|
||||
port, _ := strconv.Atoi(exclude)
|
||||
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
|
||||
default:
|
||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||
}
|
||||
}
|
||||
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
httpxOptions.NetworkPolicy, _ = networkpolicy.New(npOptions)
|
||||
|
||||
var np *networkpolicy.NetworkPolicy
|
||||
if options.Networkpolicy != nil {
|
||||
np = options.Networkpolicy
|
||||
} else {
|
||||
np, err = runner.createNetworkpolicyInstance(options)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpxOptions.NetworkPolicy = np
|
||||
|
||||
// Enables automatically tlsgrab if tlsprobe is requested
|
||||
httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe
|
||||
httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second
|
||||
@ -381,6 +368,36 @@ func New(options *Options) (*Runner, error) {
|
||||
return runner, nil
|
||||
}
|
||||
|
||||
func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpolicy.NetworkPolicy, error) {
|
||||
var npOptions networkpolicy.Options
|
||||
for _, exclude := range options.Exclude {
|
||||
switch {
|
||||
case exclude == "cdn":
|
||||
//implement cdn check in netoworkpolicy pkg??
|
||||
runner.excludeCdn = true
|
||||
continue
|
||||
case exclude == "private-ips":
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4Denylist...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv4DenylistRanges...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6Denylist...)
|
||||
npOptions.DenyList = append(npOptions.DenyList, networkpolicy.DefaultIPv6DenylistRanges...)
|
||||
case iputil.IsCIDR(exclude):
|
||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||
case asn.IsASN(exclude):
|
||||
// update this to use networkpolicy pkg once https://github.com/projectdiscovery/networkpolicy/pull/55 is merged
|
||||
ips := expandASNInputValue(exclude)
|
||||
npOptions.DenyList = append(npOptions.DenyList, ips...)
|
||||
case iputil.IsPort(exclude):
|
||||
port, _ := strconv.Atoi(exclude)
|
||||
npOptions.DenyPortList = append(npOptions.DenyPortList, port)
|
||||
default:
|
||||
npOptions.DenyList = append(npOptions.DenyList, exclude)
|
||||
}
|
||||
}
|
||||
np, err := networkpolicy.New(npOptions)
|
||||
return np, err
|
||||
}
|
||||
|
||||
func expandCIDRInputValue(value string) []string {
|
||||
var ips []string
|
||||
ipsCh, _ := mapcidr.IPAddressesAsStream(value)
|
||||
|
Loading…
Reference in New Issue
Block a user