Merge branch 'master' into feature-ips-cnames

cmd/httpx/httpx.go

@@ -122,7 +122,7 @@ func main() {
 	scanopts.OutputMethod = options.OutputMethod
 	scanopts.OutputIP = options.OutputIP
 	scanopts.OutputCName = options.OutputCName
-
+	scanopts.OutputCDN = options.OutputCDN
 	// output verb if more than one is specified
 	if len(scanopts.Methods) > 1 && !options.Silent {
 		scanopts.OutputMethod = true
@@ -323,6 +323,7 @@ type scanOptions struct {
 	HTTP2Probe             bool
 	OutputIP               bool
 	OutputCName            bool
+	OutputCDN              bool
 }
 
 func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, method string, scanopts *scanOptions) Result {
@@ -510,6 +511,11 @@ retry:
 		builder.WriteString(fmt.Sprintf(" [%s]", cnames[0]))
   }
 	
+  isCDN := hp.CdnCheck(ip)
+	if scanopts.OutputCDN && isCDN {
+		builder.WriteString(" [cdn]")
+	}
+
 	// store responses in directory
 	if scanopts.StoreResponse {
 		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@@ -544,6 +550,7 @@ retry:
 		IP:            ip,
 		IPs:           ips,
 		CNAMEs:        cnames,
+		CDN:           isCDN,
 	}
 }
 
@@ -569,6 +576,7 @@ type Result struct {
 	IP            string         `json:"ip"`
 	IPs           []string       `json:"ips"`
 	CNAMEs        []string       `json:"cnames,omitempty"`
+	CDN           bool           `json:"cdn"`
 }
 
 // JSON the result
@@ -632,6 +640,7 @@ type Options struct {
 	Debug                     bool
 	Pipeline                  bool
 	HTTP2Probe                bool
+	OutputCDN                 bool
 }
 
 // ParseOptions parses the command line options for application
@@ -681,6 +690,8 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
 	flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip")
 	flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname")
+	flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)")
+  
 	flag.Parse()
 
 	// Read the inputs and configure the logging

common/httpx/cdn.go

@@ -0,0 +1,10 @@
+package httpx
+
+import "net"
+
+// CdnCheck verifies if the given ip is part of Cdn ranges
+func (h *HTTPX) CdnCheck(ip string) bool {
+	ok, err := h.cdn.Check(net.ParseIP((ip)))
+
+	return ok && err == nil
+}

common/httpx/httpx.go

@@ -10,6 +10,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/microcosm-cc/bluemonday"
+	"github.com/projectdiscovery/cdncheck"
 	"github.com/projectdiscovery/httpx/common/cache"
 	"github.com/projectdiscovery/httpx/common/httputilz"
 	"github.com/projectdiscovery/rawhttp"
@@ -26,6 +27,7 @@ type HTTPX struct {
 	htmlPolicy      *bluemonday.Policy
 	CustomHeaders   map[string]string
 	RequestOverride *RequestOverride
+	cdn             *cdncheck.Client
 }
 
 // New httpx instance
@@ -101,6 +103,10 @@ func New(options *Options) (*HTTPX, error) {
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
 	httpx.RequestOverride = &options.RequestOverride
+	httpx.cdn, err = cdncheck.New()
+	if err != nil {
+		return nil, fmt.Errorf("Could not create cdn check: %s", err)
+	}
 
 	return httpx, nil
 }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.4
 	github.com/miekg/dns v1.1.31
+	github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26
 	github.com/projectdiscovery/fdmax v0.0.1
 	github.com/projectdiscovery/gologger v1.0.1
 	github.com/projectdiscovery/mapcidr v0.0.4

go.sum

@@ -20,6 +20,8 @@ github.com/microcosm-cc/bluemonday v1.0.4/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvza
 github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26 h1:8LwryON7jJ8xKzghSTY6KxLMBgZM/Vqc9LlWMqf3pLA=
+github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw=
 github.com/projectdiscovery/fdmax v0.0.1 h1:EDpan+CgIAAYJ1K1zpTii2SouIDPHQwpGQsYc2UGUj0=
 github.com/projectdiscovery/fdmax v0.0.1/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs=
 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
@@ -36,7 +38,10 @@ github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/yl2chen/cidranger v1.0.0 h1:9tdo0orHQJvXsX6mf+1Goou/R4kq21AfpbYeTcpXs2Q=
+github.com/yl2chen/cidranger v1.0.0/go.mod h1:L7Msw4X7EQK7zMVjOtv7o8xMyjv1rJcNlYlMgGwP7ko=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=