mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 05:20:29 +03:00
Merge branch 'master' into local_redirect
This commit is contained in:
Timo Müller
GitHub
commit
b9d73af0d2
17
.github/workflows/dockerhub-push.yml
vendored
Normal file
17
.github/workflows/dockerhub-push.yml
vendored
Normal file
@ -0,0 +1,17 @@
|
||||
# dockerhub-push pushes docker build to dockerhub automatically
|
||||
# on the creation of a new release
|
||||
name: Publish to Dockerhub on creation of a new release
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@master
|
||||
- name: Publish to Dockerhub Registry
|
||||
uses: elgohr/Publish-Docker-Github-Action@master
|
||||
with:
|
||||
name: projectdiscovery/httpx
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
10
Dockerfile
Normal file
10
Dockerfile
Normal file
@ -0,0 +1,10 @@
|
||||
FROM golang:1.14-alpine AS build-env
|
||||
|
||||
RUN apk add --no-cache --upgrade git openssh-client ca-certificates
|
||||
RUN go get -u github.com/golang/dep/cmd/dep
|
||||
WORKDIR /go/src/app
|
||||
|
||||
# Install
|
||||
RUN go get -u github.com/projectdiscovery/httpx/cmd/httpx
|
||||
|
||||
ENTRYPOINT ["httpx"]
|
||||
60
README.md
60
README.md
@ -30,7 +30,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi
|
||||
- Fast And fully configurable flags to probe mutiple elements.
|
||||
- Supports vhost, urls, ports, title, content-length, status-code, response-body probbing.
|
||||
- Smart auto fallback from https to http as default.
|
||||
- Supports hosts and URLs as input.
|
||||
- Supports hosts, URLs and CIDR as input.
|
||||
- Handles edge cases doing retries, backoffs etc for handling WAFs.
|
||||
|
||||
# Usage
|
||||
@ -47,7 +47,8 @@ This will display help for the tool. Here are all the switches it supports.
|
||||
| -follow-redirects | Follow URL redirects (default false) | httpx -follow-redirects |
|
||||
| -follow-host-redirects | Follow URL redirects only when staying on the same host (default false) | httpx -follow-host-redirects |
|
||||
| -http-proxy | URL of the proxy server | httpx -http-proxy hxxp://proxy-host:80 |
|
||||
| -l | File containing host/urls to process | httpx -l hosts.txt |
|
||||
| -l | File containing host/urls to process | httpx -l hosts.txt |
|
||||
| -l | File containing CIDR to process | httpx -l cidr.txt |
|
||||
| -no-color | Disable colors in the output. | httpx -no-color |
|
||||
| -o | File to save output result (optional) | httpx -o output.txt |
|
||||
| -json | Prints all the probes in JSON format (default false) | httpx -json |
|
||||
@ -56,7 +57,8 @@ This will display help for the tool. Here are all the switches it supports.
|
||||
| -ports | Ports ranges to probe (nmap syntax: eg 1,2-10,11) | httpx -ports 80,443,100-200 |
|
||||
| -title | Prints title of page if available | httpx -title |
|
||||
| -content-length | Prints content length in the output | httpx -content-length |
|
||||
| -status-code | Prints status code in the output | httpx -status-code |
|
||||
| -status-code | Prints status code in the output | httpx -status-code |
|
||||
| -web-server | Prints running web sever if available | httpx -web-server |
|
||||
| -store-response | Store response as domain.txt | httpx -store-response |
|
||||
| -store-response-dir| Directory to store response (default current path) | httpx -store-response-dir output |
|
||||
| -retries | Number of retries | httpx -retries |
|
||||
@ -174,6 +176,42 @@ https://api.hackerone.com
|
||||
https://support.hackerone.com
|
||||
```
|
||||
|
||||
### Running httpx with CIDR input
|
||||
|
||||
```bash
|
||||
> echo 173.0.84.0/24 | httpx
|
||||
|
||||
__ __ __ _ __
|
||||
/ /_ / /_/ /_____ | |/ /
|
||||
/ __ \/ __/ __/ __ \| /
|
||||
/ / / / /_/ /_/ /_/ / |
|
||||
/_/ /_/\__/\__/ .___/_/|_|
|
||||
/_/
|
||||
|
||||
projectdiscovery.io
|
||||
|
||||
[WRN] Use with caution. You are responsible for your actions
|
||||
[WRN] Developers assume no liability and are not responsible for any misuse or damage.
|
||||
https://173.0.84.29
|
||||
https://173.0.84.43
|
||||
https://173.0.84.31
|
||||
https://173.0.84.44
|
||||
https://173.0.84.12
|
||||
https://173.0.84.4
|
||||
https://173.0.84.36
|
||||
https://173.0.84.45
|
||||
https://173.0.84.14
|
||||
https://173.0.84.25
|
||||
https://173.0.84.46
|
||||
https://173.0.84.24
|
||||
https://173.0.84.32
|
||||
https://173.0.84.9
|
||||
https://173.0.84.13
|
||||
https://173.0.84.6
|
||||
https://173.0.84.16
|
||||
https://173.0.84.34
|
||||
```
|
||||
|
||||
|
||||
### Using httpX with subfinder/chaos and any other similar tool.
|
||||
|
||||
@ -221,14 +259,14 @@ https://resources.hackerone.com [301] [0] []
|
||||
[WRN] Use with caution. You are responsible for your actions
|
||||
[WRN] Developers assume no liability and are not responsible for any misuse or damage.
|
||||
|
||||
{"url":"https://mta-sts.forwarding.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://mta-sts.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://docs.hackerone.com","content-length":65444,"status-code":200,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://mta-sts.managed.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://support.hackerone.com","content-length":489,"status-code":301,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://resources.hackerone.com","content-length":0,"status-code":301,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://api.hackerone.com","content-length":7791,"status-code":200,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://www.hackerone.com","content-length":54166,"status-code":200,"title":"","error":null,"vhost":false}
|
||||
{"url":"https://mta-sts.managed.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
|
||||
{"url":"https://mta-sts.forwarding.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
|
||||
{"url":"https://mta-sts.hackerone.com","content-length":9339,"status-code":404,"title":"Page not found · GitHub Pages","vhost":false,"webserver":"GitHub.com"}
|
||||
{"url":"https://docs.hackerone.com","content-length":65781,"status-code":200,"title":"HackerOne Platform Documentation","vhost":false,"webserver":"GitHub.com"}
|
||||
{"url":"https://api.hackerone.com","content-length":7791,"status-code":200,"title":"HackerOne API","vhost":false,"webserver":"cloudflare"}
|
||||
{"url":"https://support.hackerone.com","content-length":98,"status-code":301,"title":"","vhost":false,"webserver":"cloudflare"}
|
||||
{"url":"https://resources.hackerone.com","content-length":0,"status-code":301,"title":"","vhost":false,"webserver":""}
|
||||
{"url":"https://www.hackerone.com","content-length":54136,"status-code":200,"title":"Bug Bounty - Hacker Powered Security Testing | HackerOne","vhost":false,"webserver":"cloudflare"}
|
||||
|
||||
```
|
||||
|
||||
|
||||
@ -14,6 +14,7 @@ import (
|
||||
customport "github.com/projectdiscovery/httpx/common/customports"
|
||||
"github.com/projectdiscovery/httpx/common/fileutil"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/httpx/common/iputil"
|
||||
"github.com/projectdiscovery/httpx/common/stringz"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
)
|
||||
@ -54,6 +55,7 @@ func main() {
|
||||
scanopts.StoreResponse = options.StoreResponse
|
||||
scanopts.StoreResponseDirectory = options.StoreResponseDir
|
||||
scanopts.Method = options.Method
|
||||
scanopts.OutputServerHeader = options.OutputServerHeader
|
||||
|
||||
// Try to create output folder if it doesnt exist
|
||||
if options.StoreResponse && options.StoreResponseDir != "" && options.StoreResponseDir != "." {
|
||||
@ -79,7 +81,7 @@ func main() {
|
||||
defer f.Close()
|
||||
}
|
||||
for r := range output {
|
||||
if r.Error != nil {
|
||||
if r.err != nil {
|
||||
continue
|
||||
}
|
||||
row := r.str
|
||||
@ -112,29 +114,29 @@ func main() {
|
||||
}
|
||||
|
||||
for sc.Scan() {
|
||||
target := stringz.TrimProtocol(sc.Text())
|
||||
for target := range targets(stringz.TrimProtocol(sc.Text())) {
|
||||
// if no custom ports specified then test the default ones
|
||||
if len(customport.Ports) == 0 {
|
||||
wg.Add()
|
||||
go func(target string) {
|
||||
defer wg.Done()
|
||||
analyze(hp, protocol, target, 0, &scanopts, output)
|
||||
}(target)
|
||||
}
|
||||
|
||||
// if no custom ports specified then test the default ones
|
||||
if len(customport.Ports) == 0 {
|
||||
wg.Add()
|
||||
go func(target string) {
|
||||
defer wg.Done()
|
||||
analyze(hp, protocol, target, 0, &scanopts, output)
|
||||
}(target)
|
||||
}
|
||||
// the host name shouldn't have any semicolon - in case remove the port
|
||||
semicolonPosition := strings.LastIndex(target, ":")
|
||||
if semicolonPosition > 0 {
|
||||
target = target[:semicolonPosition]
|
||||
}
|
||||
|
||||
// the host name shouldn't have any semicolon - in case remove the port
|
||||
semicolonPosition := strings.LastIndex(target, ":")
|
||||
if semicolonPosition > 0 {
|
||||
target = target[:semicolonPosition]
|
||||
}
|
||||
|
||||
for port := range customport.Ports {
|
||||
wg.Add()
|
||||
go func(port int) {
|
||||
defer wg.Done()
|
||||
analyze(hp, protocol, target, port, &scanopts, output)
|
||||
}(port)
|
||||
for port := range customport.Ports {
|
||||
wg.Add()
|
||||
go func(port int) {
|
||||
defer wg.Done()
|
||||
analyze(hp, protocol, target, port, &scanopts, output)
|
||||
}(port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,6 +147,29 @@ func main() {
|
||||
wgoutput.Wait()
|
||||
}
|
||||
|
||||
// returns all the targets within a cidr range or the single target
|
||||
func targets(target string) chan string {
|
||||
results := make(chan string)
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
// test if the target is a cidr
|
||||
if iputil.IsCidr(target) {
|
||||
cidrIps, err := iputil.Ips(target)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, ip := range cidrIps {
|
||||
results <- ip
|
||||
}
|
||||
} else {
|
||||
results <- target
|
||||
}
|
||||
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
type scanOptions struct {
|
||||
Method string
|
||||
VHost bool
|
||||
@ -153,6 +178,7 @@ type scanOptions struct {
|
||||
OutputContentLength bool
|
||||
StoreResponse bool
|
||||
StoreResponseDirectory string
|
||||
OutputServerHeader bool
|
||||
}
|
||||
|
||||
func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions, output chan Result) {
|
||||
@ -165,7 +191,7 @@ retry:
|
||||
|
||||
req, err := hp.NewRequest(scanopts.Method, URL)
|
||||
if err != nil {
|
||||
output <- Result{URL: URL, Error: err}
|
||||
output <- Result{URL: URL, err: err}
|
||||
return
|
||||
}
|
||||
|
||||
@ -173,7 +199,7 @@ retry:
|
||||
|
||||
resp, err := hp.Do(req)
|
||||
if err != nil {
|
||||
output <- Result{URL: URL, Error: err}
|
||||
output <- Result{URL: URL, err: err}
|
||||
if !retried {
|
||||
if protocol == "https" {
|
||||
protocol = "http"
|
||||
@ -213,6 +239,11 @@ retry:
|
||||
builder.WriteString(fmt.Sprintf(" [%s]", title))
|
||||
}
|
||||
|
||||
serverHeader := resp.GetHeader("Server")
|
||||
if scanopts.OutputServerHeader {
|
||||
builder.WriteString(fmt.Sprintf(" [%s]", serverHeader))
|
||||
}
|
||||
|
||||
// check for virtual host
|
||||
isvhost := false
|
||||
if scanopts.VHost {
|
||||
@ -224,11 +255,15 @@ retry:
|
||||
|
||||
// store responses in directory
|
||||
if scanopts.StoreResponse {
|
||||
responsePath := path.Join(scanopts.StoreResponseDirectory, domain+".txt")
|
||||
ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
|
||||
var domainFile = strings.Replace(domain, "/", "_", -1) + ".txt"
|
||||
responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
|
||||
err := ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not write response, at path '%s', to disc.", responsePath)
|
||||
}
|
||||
}
|
||||
|
||||
output <- Result{URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, Title: title, str: builder.String(), VHost: isvhost}
|
||||
output <- Result{URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, Title: title, str: builder.String(), VHost: isvhost, WebServer: serverHeader}
|
||||
}
|
||||
|
||||
// Result of a scan
|
||||
@ -238,8 +273,9 @@ type Result struct {
|
||||
StatusCode int `json:"status-code"`
|
||||
Title string `json:"title"`
|
||||
str string
|
||||
Error error `json:"error"`
|
||||
VHost bool `json:"vhost"`
|
||||
err error
|
||||
VHost bool `json:"vhost"`
|
||||
WebServer string `json:"webserver"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
||||
@ -12,30 +12,31 @@ import (
|
||||
|
||||
// Options contains configuration options for chaos client.
|
||||
type Options struct {
|
||||
RawRequestFile string
|
||||
VHost bool
|
||||
Smuggling bool
|
||||
ExtractTitle bool
|
||||
StatusCode bool
|
||||
ContentLength bool
|
||||
Retries int
|
||||
Threads int
|
||||
Timeout int
|
||||
CustomHeaders customheader.CustomHeaders
|
||||
CustomPorts customport.CustomPorts
|
||||
Output string
|
||||
FollowRedirects bool
|
||||
StoreResponse bool
|
||||
StoreResponseDir string
|
||||
HttpProxy string
|
||||
SocksProxy string
|
||||
JSONOutput bool
|
||||
InputFile string
|
||||
Method string
|
||||
Silent bool
|
||||
Version bool
|
||||
Verbose bool
|
||||
NoColor bool
|
||||
RawRequestFile string
|
||||
VHost bool
|
||||
Smuggling bool
|
||||
ExtractTitle bool
|
||||
StatusCode bool
|
||||
ContentLength bool
|
||||
Retries int
|
||||
Threads int
|
||||
Timeout int
|
||||
CustomHeaders customheader.CustomHeaders
|
||||
CustomPorts customport.CustomPorts
|
||||
Output string
|
||||
FollowRedirects bool
|
||||
StoreResponse bool
|
||||
StoreResponseDir string
|
||||
HttpProxy string
|
||||
SocksProxy string
|
||||
JSONOutput bool
|
||||
InputFile string
|
||||
Method string
|
||||
Silent bool
|
||||
Version bool
|
||||
Verbose bool
|
||||
NoColor bool
|
||||
OutputServerHeader bool
|
||||
FollowHostRedirects bool
|
||||
}
|
||||
|
||||
@ -65,6 +66,7 @@ func ParseOptions() *Options {
|
||||
flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
|
||||
flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode")
|
||||
flag.BoolVar(&options.NoColor, "no-color", false, "No Color")
|
||||
flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Prints out the Server header content")
|
||||
flag.Parse()
|
||||
|
||||
// Read the inputs and configure the logging
|
||||
|
||||
@ -101,6 +101,8 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp.Headers = httpresp.Header.Clone()
|
||||
|
||||
rawresp, err := httputil.DumpResponse(httpresp, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
package httpx
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Response contains the response to a server
|
||||
type Response struct {
|
||||
StatusCode int
|
||||
@ -10,3 +14,13 @@ type Response struct {
|
||||
Words int
|
||||
Lines int
|
||||
}
|
||||
|
||||
// GetHeader value
|
||||
func (r *Response) GetHeader(name string) string {
|
||||
v, ok := r.Headers[name]
|
||||
if ok {
|
||||
return strings.Join(v, " ")
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
42
common/iputil/iputil.go
Normal file
42
common/iputil/iputil.go
Normal file
@ -0,0 +1,42 @@
|
||||
package iputil
|
||||
|
||||
import "net"
|
||||
|
||||
// IsCidr determines if the given ip is a cidr range
|
||||
func IsCidr(ip string) bool {
|
||||
_, _, err := net.ParseCIDR(ip)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// IsIP determines if the given string is a valid ip
|
||||
func IsIP(ip string) bool {
|
||||
return net.ParseIP(ip) != nil
|
||||
}
|
||||
|
||||
// Ips of a cidr
|
||||
func Ips(cidr string) ([]string, error) {
|
||||
ip, ipnet, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ips []string
|
||||
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
|
||||
ips = append(ips, ip.String())
|
||||
}
|
||||
// remove network address and broadcast address
|
||||
return ips[1 : len(ips)-1], nil
|
||||
}
|
||||
|
||||
func inc(ip net.IP) {
|
||||
for j := len(ip) - 1; j >= 0; j-- {
|
||||
ip[j]++
|
||||
if ip[j] > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
2
go.mod
2
go.mod
@ -12,5 +12,5 @@ require (
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.2.1
|
||||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@ -32,8 +32,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
|
||||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user