Merge branch 'master' into local_redirect

2024-09-11 17:46:08 +03:00 · 2020-06-08 11:41:48 +00:00 · 2020-06-08 11:41:48 +00:00 · b9d73af0d2
commit b9d73af0d2
parent d1b88ea776 6c68fc5d89
10 changed files with 228 additions and 67 deletions
--- a/.github/workflows/dockerhub-push.yml
+++ b/.github/workflows/dockerhub-push.yml
@ -0,0 +1,17 @@
+# dockerhub-push pushes docker build to dockerhub automatically
+# on the creation of a new release
+name: Publish to Dockerhub on creation of a new release
+on: 
+  release:
+    types: [published]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: Publish to Dockerhub Registry
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: projectdiscovery/httpx
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
--- a/10
+++ b/10
@ -0,0 +1,10 @@
+FROM golang:1.14-alpine AS build-env
+
+RUN apk add --no-cache --upgrade git openssh-client ca-certificates
+RUN go get -u github.com/golang/dep/cmd/dep
+WORKDIR /go/src/app
+
+# Install
+RUN go get -u github.com/projectdiscovery/httpx/cmd/httpx
+
+ENTRYPOINT ["httpx"]
--- a/README.md
+++ b/README.md
@ -30,7 +30,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi
 - Fast And fully configurable flags to probe mutiple elements.
 - Supports vhost, urls, ports, title, content-length, status-code, response-body probbing.
 - Smart auto fallback from https to http as default. 
- - Supports hosts and URLs as input.
+ - Supports hosts, URLs and CIDR as input.
 - Handles edge cases doing retries, backoffs etc for handling WAFs.

 # Usage
@ -48,6 +48,7 @@ This will display help for the tool. Here are all the switches it supports.
 | -follow-host-redirects  | Follow URL redirects only when staying on the same host (default false)  | httpx -follow-host-redirects                                                                                                                         |
 | -http-proxy        | URL of the proxy server                               | httpx -http-proxy hxxp://proxy-host:80             |
 | -l                 | File containing host/urls to process                  | httpx -l hosts.txt                                |
+| -l                 | File containing CIDR to process                       | httpx -l cidr.txt                                |
 | -no-color          | Disable colors in the output.                         | httpx -no-color                                    |
 | -o                 | File to save output result (optional)                 | httpx -o output.txt                                |
 | -json              | Prints all the probes in JSON format (default false)  | httpx -json                                        |
@ -57,6 +58,7 @@ This will display help for the tool. Here are all the switches it supports.
 | -title             | Prints title of page if available                      | httpx -title                                       |
 | -content-length    | Prints content length in the output                   | httpx -content-length                              |
 | -status-code       | Prints status code in the output                    | httpx -status-code                                 |
+| -web-server       | Prints running web sever if available                 | httpx -web-server                               |
 | -store-response    | Store response as domain.txt                          | httpx -store-response                              |
 | -store-response-dir| Directory to store response (default current path)    | httpx -store-response-dir output                   | 
 | -retries           | Number of retries                                     | httpx -retries                                     |
@ -174,6 +176,42 @@ https://api.hackerone.com
 https://support.hackerone.com
 ```

+### Running httpx with CIDR input   
+
+```bash
+> echo 173.0.84.0/24 | httpx 
+
+    __    __  __       _  __
+   / /_  / /_/ /_____ | |/ /
+  / __ \/ __/ __/ __ \|   / 
+ / / / / /_/ /_/ /_/ /   |  
+/_/ /_/\__/\__/ .___/_/|_|  
+             /_/            
+
+		projectdiscovery.io
+
+[WRN] Use with caution. You are responsible for your actions
+[WRN] Developers assume no liability and are not responsible for any misuse or damage.
+https://173.0.84.29
+https://173.0.84.43
+https://173.0.84.31
+https://173.0.84.44
+https://173.0.84.12
+https://173.0.84.4
+https://173.0.84.36
+https://173.0.84.45
+https://173.0.84.14
+https://173.0.84.25
+https://173.0.84.46
+https://173.0.84.24
+https://173.0.84.32
+https://173.0.84.9
+https://173.0.84.13
+https://173.0.84.6
+https://173.0.84.16
+https://173.0.84.34
+```
+

 ### Using httpX with subfinder/chaos and any other similar tool.

@ -221,14 +259,14 @@ https://resources.hackerone.com [301] [0] []
 [WRN] Use with caution. You are responsible for your actions
 [WRN] Developers assume no liability and are not responsible for any misuse or damage.

-{"url":"https://mta-sts.forwarding.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
-{"url":"https://mta-sts.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
-{"url":"https://docs.hackerone.com","content-length":65444,"status-code":200,"title":"","error":null,"vhost":false}
-{"url":"https://mta-sts.managed.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
-{"url":"https://support.hackerone.com","content-length":489,"status-code":301,"title":"","error":null,"vhost":false}
-{"url":"https://resources.hackerone.com","content-length":0,"status-code":301,"title":"","error":null,"vhost":false}
-{"url":"https://api.hackerone.com","content-length":7791,"status-code":200,"title":"","error":null,"vhost":false}
-{"url":"https://www.hackerone.com","content-length":54166,"status-code":200,"title":"","error":null,"vhost":false}
+{"url":"https://mta-sts.managed.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
+{"url":"https://mta-sts.forwarding.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
+{"url":"https://mta-sts.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
+{"url":"https://docs.hackerone.com","content-length":65781,"status-code":200,"title":"HackerOne Platform Documentation","vhost":false,"webserver":"GitHub.com"}
+{"url":"https://api.hackerone.com","content-length":7791,"status-code":200,"title":"HackerOne API","vhost":false,"webserver":"cloudflare"}
+{"url":"https://support.hackerone.com","content-length":98,"status-code":301,"title":"","vhost":false,"webserver":"cloudflare"}
+{"url":"https://resources.hackerone.com","content-length":0,"status-code":301,"title":"","vhost":false,"webserver":""}
+{"url":"https://www.hackerone.com","content-length":54136,"status-code":200,"title":"Bug Bounty - Hacker Powered Security Testing | HackerOne","vhost":false,"webserver":"cloudflare"}

 ```

--- a/cmd/httpx/httpx.go
+++ b/cmd/httpx/httpx.go
@ -14,6 +14,7 @@ import (
 	customport "github.com/projectdiscovery/httpx/common/customports"
 	"github.com/projectdiscovery/httpx/common/fileutil"
 	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/httpx/common/iputil"
 	"github.com/projectdiscovery/httpx/common/stringz"
 	"github.com/remeh/sizedwaitgroup"
 )
@ -54,6 +55,7 @@ func main() {
 	scanopts.StoreResponse = options.StoreResponse
 	scanopts.StoreResponseDirectory = options.StoreResponseDir
 	scanopts.Method = options.Method
+	scanopts.OutputServerHeader = options.OutputServerHeader

 	// Try to create output folder if it doesnt exist
 	if options.StoreResponse && options.StoreResponseDir != "" && options.StoreResponseDir != "." {
@ -79,7 +81,7 @@ func main() {
 			defer f.Close()
 		}
 		for r := range output {
-			if r.Error != nil {
+			if r.err != nil {
 				continue
 			}
 			row := r.str
@ -112,8 +114,7 @@ func main() {
 	}

 	for sc.Scan() {
-		target := stringz.TrimProtocol(sc.Text())
-
+		for target := range targets(stringz.TrimProtocol(sc.Text())) {
 			// if no custom ports specified then test the default ones
 			if len(customport.Ports) == 0 {
 				wg.Add()
@ -137,6 +138,7 @@ func main() {
 				}(port)
 			}
 		}
+	}

 	wg.Wait()

@ -145,6 +147,29 @@ func main() {
 	wgoutput.Wait()
 }

+// returns all the targets within a cidr range or the single target
+func targets(target string) chan string {
+	results := make(chan string)
+	go func() {
+		defer close(results)
+
+		// test if the target is a cidr
+		if iputil.IsCidr(target) {
+			cidrIps, err := iputil.Ips(target)
+			if err != nil {
+				return
+			}
+			for _, ip := range cidrIps {
+				results <- ip
+			}
+		} else {
+			results <- target
+		}
+
+	}()
+	return results
+}
+
 type scanOptions struct {
 	Method                 string
 	VHost                  bool
@ -153,6 +178,7 @@ type scanOptions struct {
 	OutputContentLength    bool
 	StoreResponse          bool
 	StoreResponseDirectory string
+	OutputServerHeader     bool
 }

 func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions, output chan Result) {
@ -165,7 +191,7 @@ retry:

 	req, err := hp.NewRequest(scanopts.Method, URL)
 	if err != nil {
-		output <- Result{URL: URL, Error: err}
+		output <- Result{URL: URL, err: err}
 		return
 	}

@ -173,7 +199,7 @@ retry:

 	resp, err := hp.Do(req)
 	if err != nil {
-		output <- Result{URL: URL, Error: err}
+		output <- Result{URL: URL, err: err}
 		if !retried {
 			if protocol == "https" {
 				protocol = "http"
@ -213,6 +239,11 @@ retry:
 		builder.WriteString(fmt.Sprintf(" [%s]", title))
 	}

+	serverHeader := resp.GetHeader("Server")
+	if scanopts.OutputServerHeader {
+		builder.WriteString(fmt.Sprintf(" [%s]", serverHeader))
+	}
+
 	// check for virtual host
 	isvhost := false
 	if scanopts.VHost {
@ -224,11 +255,15 @@ retry:

 	// store responses in directory
 	if scanopts.StoreResponse {
-		responsePath := path.Join(scanopts.StoreResponseDirectory, domain+".txt")
-		ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
+		var domainFile = strings.Replace(domain, "/", "_", -1) + ".txt"
+		responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
+		err := ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
+		if err != nil {
+			gologger.Fatalf("Could not write response, at path '%s', to disc.", responsePath)
+		}
 	}

-	output <- Result{URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, Title: title, str: builder.String(), VHost: isvhost}
+	output <- Result{URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, Title: title, str: builder.String(), VHost: isvhost, WebServer: serverHeader}
 }

 // Result of a scan
@ -238,8 +273,9 @@ type Result struct {
 	StatusCode    int    `json:"status-code"`
 	Title         string `json:"title"`
 	str           string
-	Error         error `json:"error"`
+	err           error
 	VHost         bool   `json:"vhost"`
+	WebServer     string `json:"webserver"`
 }

 // JSON the result
--- a/cmd/httpx/options.go
+++ b/cmd/httpx/options.go
@ -36,6 +36,7 @@ type Options struct {
 	Version            bool
 	Verbose            bool
 	NoColor            bool
+	OutputServerHeader bool
 	FollowHostRedirects	 bool
 }

@ -65,6 +66,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
 	flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode")
 	flag.BoolVar(&options.NoColor, "no-color", false, "No Color")
+	flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Prints out the Server header content")
 	flag.Parse()

 	// Read the inputs and configure the logging
--- a/common/httpx/httpx.go
+++ b/common/httpx/httpx.go
@ -101,6 +101,8 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 		return nil, err
 	}

+	resp.Headers = httpresp.Header.Clone()
+
 	rawresp, err := httputil.DumpResponse(httpresp, true)
 	if err != nil {
 		return nil, err
--- a/common/httpx/response.go
+++ b/common/httpx/response.go
@ -1,5 +1,9 @@
 package httpx

+import (
+	"strings"
+)
+
 // Response contains the response to a server
 type Response struct {
 	StatusCode    int
@ -10,3 +14,13 @@ type Response struct {
 	Words         int
 	Lines         int
 }
+
+// GetHeader value
+func (r *Response) GetHeader(name string) string {
+	v, ok := r.Headers[name]
+	if ok {
+		return strings.Join(v, " ")
+	}
+
+	return ""
+}
--- a/common/iputil/iputil.go
+++ b/common/iputil/iputil.go
@ -0,0 +1,42 @@
+package iputil
+
+import "net"
+
+// IsCidr determines if the given ip is a cidr range
+func IsCidr(ip string) bool {
+	_, _, err := net.ParseCIDR(ip)
+	if err != nil {
+		return false
+	}
+
+	return true
+}
+
+// IsIP determines if the given string is a valid ip
+func IsIP(ip string) bool {
+	return net.ParseIP(ip) != nil
+}
+
+// Ips of a cidr
+func Ips(cidr string) ([]string, error) {
+	ip, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return nil, err
+	}
+
+	var ips []string
+	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+		ips = append(ips, ip.String())
+	}
+	// remove network address and broadcast address
+	return ips[1 : len(ips)-1], nil
+}
+
+func inc(ip net.IP) {
+	for j := len(ip) - 1; j >= 0; j-- {
+		ip[j]++
+		if ip[j] > 0 {
+			break
+		}
+	}
+}
--- a/go.mod
+++ b/go.mod
@ -12,5 +12,5 @@ require (
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.2.1
-	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
+	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
 )
--- a/go.sum
+++ b/go.sum
@ -32,8 +32,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
-golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=