From 659c1a33e44f4db3125d34919ad6f95b8a6d2bc5 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 6 Nov 2020 01:48:16 +0100 Subject: [PATCH] adding fastdialer --- common/cache/cache.go | 100 ------------------------------- common/cache/dialer.go | 81 ------------------------- common/cache/doc.go | 2 - common/httpx/httpx.go | 8 ++- common/resolve/client.go | 121 -------------------------------------- common/resolve/doc.go | 3 - common/resolve/utils.go | 31 ---------- go.mod | 4 ++ go.sum | 18 ++++++ internal/runner/runner.go | 7 +-- 10 files changed, 30 insertions(+), 345 deletions(-) delete mode 100644 common/cache/cache.go delete mode 100644 common/cache/dialer.go delete mode 100644 common/cache/doc.go delete mode 100644 common/resolve/client.go delete mode 100644 common/resolve/doc.go delete mode 100644 common/resolve/utils.go diff --git a/common/cache/cache.go b/common/cache/cache.go deleted file mode 100644 index a3a7ffb..0000000 --- a/common/cache/cache.go +++ /dev/null @@ -1,100 +0,0 @@ -package cache - -import ( - "net" - - "github.com/coocood/freecache" - dns "github.com/projectdiscovery/httpx/common/resolve" -) - -const megaByteBytes = 1048576 - -// Cache is a structure for caching DNS lookups -type Cache struct { - dnsClient Resolver - cache *freecache.Cache - defaultExpirationTime int -} - -// Resolver interface -type Resolver interface { - Resolve(string) (dns.Result, error) -} - -// Options of the cache -type Options struct { - BaseResolvers []string - CacheSize int - ExpirationTime int - MaxRetries int -} - -// DefaultOptions of the cache -var DefaultOptions = Options{ - BaseResolvers: DefaultResolvers, - CacheSize: 10, - ExpirationTime: 5 * 60, - MaxRetries: 5, -} - -// DefaultResolvers trusted -var DefaultResolvers = []string{ - "1.1.1.1:53", - "1.0.0.1:53", - "8.8.8.8:53", - "8.8.4.4:53", -} - -// New creates a new caching dns resolver -func New(options Options) (*Cache, error) { - dnsClient, err := dns.New(options.BaseResolvers, options.MaxRetries) - if err != nil { - return nil, err - } - cache := freecache.NewCache(options.CacheSize * megaByteBytes) - return &Cache{dnsClient: dnsClient, cache: cache, defaultExpirationTime: options.ExpirationTime}, nil -} - -// Lookup a hostname -func (c *Cache) Lookup(hostname string) (*dns.Result, error) { - if ip := net.ParseIP(hostname); ip != nil { - if ip.To4() != nil { - return &dns.Result{IP4s: []string{hostname}}, nil - } - if ip.To16() != nil { - return &dns.Result{IP6s: []string{hostname}}, nil - } - } - hostnameBytes := []byte(hostname) - value, err := c.cache.Get(hostnameBytes) - if err != nil { - // continue only if the failure is caused by cache-miss - if err != freecache.ErrNotFound { - return nil, err - } - result, resolveErr := c.dnsClient.Resolve(hostname) - if resolveErr != nil { - return nil, resolveErr - } - if result.TTL == 0 { - result.TTL = c.defaultExpirationTime - } - b, _ := result.Marshal() - - err = c.cache.Set(hostnameBytes, b, result.TTL) - if err != nil { - return nil, err - } - - return &result, nil - } - - var result dns.Result - - err = result.Unmarshal(value) - if err != nil { - return nil, err - } - - return &result, nil -} diff --git a/common/cache/dialer.go b/common/cache/dialer.go deleted file mode 100644 index 1747f82..0000000 --- a/common/cache/dialer.go +++ /dev/null @@ -1,81 +0,0 @@ -package cache - -import ( - "context" - "net" - "strings" - "time" - - "github.com/coocood/freecache" - dns "github.com/projectdiscovery/httpx/common/resolve" -) - -var ( - dialerHistory *freecache.Cache - cache *Cache -) - -// NoAddressFoundError occurs when no addresses are found for the host -type NoAddressFoundError struct{} - -func (m *NoAddressFoundError) Error() string { - return "no address found for host" -} - -// DialerFunc with signature matching of go net/dial -type DialerFunc func(context.Context, string, string) (net.Conn, error) - -// NewDialer gets a new Dialer instance using a resolver cache -func NewDialer(options Options) (DialerFunc, error) { - if cache == nil { - var err error - cache, err = New(options) - if err != nil { - return nil, err - } - dialerHistory = freecache.NewCache(options.CacheSize * megaByteBytes) - } - - dialer := &net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 10 * time.Second, - DualStack: true, - } - return func(ctx context.Context, network, address string) (conn net.Conn, err error) { - separator := strings.LastIndex(address, ":") - - // we need to filter out empty records - hostname := address[:separator] - dnsResult, err := cache.Lookup(hostname) - if err != nil || len(dnsResult.IP4s)+len(dnsResult.IP6s) == 0 { - return nil, &NoAddressFoundError{} - } - - // Dial to the IPs finally. - for _, ip := range append(dnsResult.IP4s, dnsResult.IP6s...) { - conn, err = dialer.DialContext(ctx, network, ip+address[separator:]) - if err == nil { - setErr := dialerHistory.Set([]byte(hostname), []byte(ip), 0) - if setErr != nil { - return nil, err - } - break - } - } - return - }, nil -} - -// GetDialedIP returns the ip dialed by the HTTP client -func GetDialedIP(hostname string) string { - v, err := dialerHistory.Get([]byte(hostname)) - if err != nil { - return "" - } - return string(v) -} - -// GetDNSData cached by the resolver -func GetDNSData(hostname string) (*dns.Result, error) { - return cache.Lookup(hostname) -} diff --git a/common/cache/doc.go b/common/cache/doc.go deleted file mode 100644 index a87d440..0000000 --- a/common/cache/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package cache contains the logic for caching dns resolver -package cache diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 229a64d..a6ce9a6 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -12,7 +12,7 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" - "github.com/projectdiscovery/httpx/common/cache" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" @@ -29,15 +29,17 @@ type HTTPX struct { CustomHeaders map[string]string RequestOverride *RequestOverride cdn *cdncheck.Client + Dialer *fastdialer.Dialer } // New httpx instance func New(options *Options) (*HTTPX, error) { httpx := &HTTPX{} - dialer, err := cache.NewDialer(cache.DefaultOptions) + dialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) } + httpx.Dialer = dialer httpx.Options = options @@ -68,7 +70,7 @@ func New(options *Options) (*HTTPX, error) { } transport := &http.Transport{ - DialContext: dialer, + DialContext: httpx.Dialer.Dial, MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, diff --git a/common/resolve/client.go b/common/resolve/client.go deleted file mode 100644 index f97cd88..0000000 --- a/common/resolve/client.go +++ /dev/null @@ -1,121 +0,0 @@ -package resolve - -import ( - "bytes" - "encoding/gob" - "errors" - "math/rand" - "strings" - "time" - - "github.com/miekg/dns" -) - -const defaultPort = "53" - -// Client dns instance -type Client struct { - resolvers []string - maxRetries int -} - -// Result containing ip and time to live -type Result struct { - IP4s []string - IP6s []string - CNAMEs []string - TTL int -} - -// Marshal structure to bytes -func (r *Result) Marshal() ([]byte, error) { - var b bytes.Buffer - enc := gob.NewEncoder(&b) - err := enc.Encode(r) - if err != nil { - return nil, err - } - - return b.Bytes(), nil -} - -// Unmarshal structure -func (r *Result) Unmarshal(b []byte) error { - dec := gob.NewDecoder(bytes.NewBuffer(b)) - err := dec.Decode(&r) - if err != nil { - return err - } - return nil -} - -// New creates a new dns client -func New(baseResolvers []string, maxRetries int) (*Client, error) { - rand.Seed(time.Now().UnixNano()) - client := Client{maxRetries: maxRetries} - // fails on non unix systems so we just don't care - resolvers, _ := ReadResolveConfig("/etc/resolv.conf") - client.resolvers = append(client.resolvers, resolvers...) - client.resolvers = append(client.resolvers, baseResolvers...) - return &client, nil -} - -// Resolve ips for a record -func (c *Client) Resolve(host string) (result Result, err error) { - // retrieve ipv4 addresses - err = c.query(host, dns.TypeA, &result) - if err != nil { - return - } - - // retrieve ipv6 addresses - err = c.query(host, dns.TypeAAAA, &result) - return -} - -func (c *Client) query(host string, queryType uint16, result *Result) error { - msg := new(dns.Msg) - msg.Id = dns.Id() - msg.RecursionDesired = true - msg.Question = make([]dns.Question, 1) - msg.Question[0] = dns.Question{ - Name: dns.Fqdn(host), - Qtype: queryType, - Qclass: dns.ClassINET, - } - resolver := c.resolvers[rand.Intn(len(c.resolvers))] - for i := 0; i < c.maxRetries; i++ { - answer, err := dns.Exchange(msg, resolver) - if err != nil { - continue - } - if answer != nil && answer.Rcode != dns.RcodeSuccess { - return errors.New(dns.RcodeToString[answer.Rcode]) - } - - for _, record := range answer.Answer { - switch t := record.(type) { - case *dns.A: - ip := t.A.String() - if ip != "" { - result.IP4s = append(result.IP4s, t.A.String()) - result.TTL = int(t.Header().Ttl) - } - case *dns.AAAA: - ip := t.AAAA.String() - if ip != "" { - result.IP6s = append(result.IP6s, t.AAAA.String()) - result.TTL = int(t.Header().Ttl) - } - case *dns.CNAME: - if queryType == dns.TypeA && t.Target != "" { - result.CNAMEs = append(result.CNAMEs, strings.TrimSuffix(t.Target, ".")) - } - } - } - - break - } - - return nil -} diff --git a/common/resolve/doc.go b/common/resolve/doc.go deleted file mode 100644 index e14f359..0000000 --- a/common/resolve/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package resolve is used to handle resolving records -// It also handles wildcard subdomains and rotating resolvers. -package resolve diff --git a/common/resolve/utils.go b/common/resolve/utils.go deleted file mode 100644 index 6b75491..0000000 --- a/common/resolve/utils.go +++ /dev/null @@ -1,31 +0,0 @@ -package resolve - -import ( - "net" - - "github.com/miekg/dns" -) - -// ReadResolveConfig retrieve resolvers from os -func ReadResolveConfig(configFile string) ([]string, error) { - var servers []string - - conf, err := dns.ClientConfigFromFile(configFile) - if err != nil { - return servers, err - } - - for _, nameserver := range conf.Servers { - if nameserver[0] == '[' && nameserver[len(nameserver)-1] == ']' { - nameserver = nameserver[1 : len(nameserver)-1] - } - if ip := net.ParseIP(nameserver); ip != nil { - nameserver = net.JoinHostPort(nameserver, defaultPort) - } else { - nameserver = dns.Fqdn(nameserver) + ":" + defaultPort - } - servers = append(servers, nameserver) - } - - return servers, nil -} diff --git a/go.mod b/go.mod index 0be13f6..c4087f7 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/projectdiscovery/httpx go 1.14 +replace github.com/projectdiscovery/fastdialer => /home/projectdiscovery/go/src/github.com/projectdiscovery/fastdialer + require ( github.com/coocood/freecache v1.1.1 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf @@ -9,8 +11,10 @@ require ( github.com/microcosm-cc/bluemonday v1.0.4 github.com/miekg/dns v1.1.33 github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 + github.com/projectdiscovery/fastdialer v0.0.1 github.com/projectdiscovery/fdmax v0.0.2 github.com/projectdiscovery/gologger v1.0.1 + github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb // indirect github.com/projectdiscovery/mapcidr v0.0.4 github.com/projectdiscovery/rawhttp v0.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 diff --git a/go.sum b/go.sum index 33914d8..fe174a6 100644 --- a/go.sum +++ b/go.sum @@ -8,10 +8,15 @@ github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpR github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -21,6 +26,9 @@ github.com/miekg/dns v1.1.32 h1:MDaYYzWOYscpvDOEgPMT1c1mebCZmIdxZI/J161OdJU= github.com/miekg/dns v1.1.32/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI= github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 h1:YCM8XrQXo9llkohuXw3CBoz2S/T3kVMnb1pM04X23t8= github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw= @@ -28,6 +36,8 @@ github.com/projectdiscovery/fdmax v0.0.2 h1:d0HqNC4kbrMWT669u9W7ksFS7UBvnW0zmgY6 github.com/projectdiscovery/fdmax v0.0.2/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= +github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb h1:y+O2ZCGICrGcSevxu2nydZtjMiwDHPf2cMgwi8PCFgQ= +github.com/projectdiscovery/hmap v0.0.0-20201026185329-db41b5717bcb/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto= @@ -44,6 +54,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/yl2chen/cidranger v1.0.0 h1:9tdo0orHQJvXsX6mf+1Goou/R4kq21AfpbYeTcpXs2Q= github.com/yl2chen/cidranger v1.0.0/go.mod h1:L7Msw4X7EQK7zMVjOtv7o8xMyjv1rJcNlYlMgGwP7ko= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -51,6 +63,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -59,7 +72,9 @@ golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Gh golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -72,4 +87,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1583ddb..50c68ae 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -15,7 +15,6 @@ import ( "github.com/logrusorgru/aurora" _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/httpx/common/cache" customport "github.com/projectdiscovery/httpx/common/customports" "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/httputilz" @@ -156,7 +155,7 @@ func New(options *Options) (*Runner, error) { } func (runner *Runner) Close() { - // not implemented + runner.hp.Dialer.Close() } func (runner *Runner) RunEnumeration() { @@ -526,7 +525,7 @@ retry: } } - ip := cache.GetDialedIP(domain) + ip := hp.Dialer.GetDialedIP(domain) if scanopts.OutputIP { builder.WriteString(fmt.Sprintf(" [%s]", ip)) } @@ -535,7 +534,7 @@ retry: ips []string cnames []string ) - dnsData, err := cache.GetDNSData(domain) + dnsData, err := hp.Dialer.GetDNSData(domain) if dnsData != nil && err == nil { ips = append(ips, dnsData.IP4s...) ips = append(ips, dnsData.IP6s...)