commit e974bb52e5d2d32355f18977d0b57b99926b6278 Author: bauthard Date: Fri May 29 09:37:55 2020 +0000 Initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..081b05c --- /dev/null +++ b/README.md @@ -0,0 +1,246 @@ +

+ httpx +
+

+ +[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT) +[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/httpx)](https://goreportcard.com/report/github.com/projectdiscovery/httpx) +[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/httpx/issues) + +httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using [retryablehttp-go library](https://github.com/projectdiscovery/retryablehttp-go) library. + +# Resources +- [Resources](#resources) +- [Features](#features) +- [Usage](#usage) +- [Installation Instructions](#installation-instructions) + - [From Binary](#from-binary) + - [From Source](#from-source) +- [Running httpx](#running-httpx) + - [Running httpx with a single template.](#running-httpx) +- [Thanks](#thanks) + + # Features + +

+ httpx +
+

+ + - Simple and modular code base making it easy to contribute. + - Fast And fully configurable flags to probe mutiple elements + - Supports vhost, urls, ports, title, content-length, status-code, response-body probbing. + - Handles edge cases doing retries, backoffs etc for handling WAFs. + +# Usage + +```bash +httpx -h +``` + +This will display help for the tool. Here are all the switches it supports. + +| Flag | Description | Example | +|------------------- |-------------------------------------------------------|----------------------------------------------------| +| -H | Custom Header input | httpx -H 'x-bug-bounty: hacker' | +| -follow-redirects | Follow URL redirects (default false) | httpx -follow-redirects | +| -http-proxy | URL of the proxy server | httpx -http-proxy hxxp://proxy-host:80 | +| -l | File to save output result (optional) | httpx -o output.txt | +| -no-color | Disable colors in the output. | httpx -no-color | +| -o | File to save output result (optional) | httpx -o output.txt | +| -json | Prints all the probes in JSON format (default false) | httpx -json | +| -vhost | Probes to detect vhost from list of subdomains | httpx -vhost | +| -threads | Number of threads (default 50) | httpx - threads 100 | +| -ports | Ports ranges to probe (nmap syntax: eg 1,2-10,11) | httpx -ports 80,443,100-200 | +| -title | Prints title of page if avaiable | httpx -title | +| -content-length | Prints content length in the output | httpx -content-length | +| -status-code | Prints status code in the output | httpx -status-code | +| -store-response | Store response as domain.txt | httpx -store-response | +| -store-response-dir| Directory to store response (default current path) | httpx -store-response-dir output | +| -retries | Number of retries | httpx -retries | +| -silent | Prints only results in the output | httpx -silent | +| -timeout | Timeout in seconds (default 5) | httpx -timeout 10 | +| -verbose | Verbose Mode | httpx -verbose | +| -version | Prints current version of the httpx | httpx -version | +| -x | Request Method (default 'GET') | httpx -x HEAD | + + +# Installation Instructions + + +### From Binary + +The installation is easy. You can download the pre-built binaries for your platform from the [Releases](https://github.com/projectdiscovery/httpx/releases/) page. Extract them using tar, move it to your `$PATH`and you're ready to go. + +```bash +> tar -xzvf httpx-linux-amd64.tar.gz +> mv httpx-linux-amd64 /usr/bin/httpx +> httpx -h +``` + +### From Source + +httpx requires go1.13+ to install successfully. Run the following command to get the repo - + +```bash +> GO111MODULE=on go get -u -v github.com/projectdiscovery/httpx/cmd/httpx +``` + +In order to update the tool, you can use -u flag with `go get` command. + +# Running httpX to probe `2967` hosts + +```bash +> chaos -d oath.cloud -count -silent +2967 + +> time chaos -d oath.cloud -silent | httpx -status-code -content-length -title -store-response -threads 100 -json | wc + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ + + projectdiscovery.io + +[WRN] Use with caution. You are responsible for your actions +[WRN] Developers assume no liability and are not responsible for any misuse or damage. +196 + +real 0m52.159s +user 0m4.084s +sys 0m3.880s +``` + +### Running httpx with stnin + +This will run the tool against all the hosts in `urls.txt` and returns the matched results. + +```bash +> cat hosts.txt | httpx + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ + + projectdiscovery.io + +[WRN] Use with caution. You are responsible for your actions +[WRN] Developers assume no liability and are not responsible for any misuse or damage. +https://mta-sts.managed.hackerone.com +https://mta-sts.hackerone.com +https://mta-sts.forwarding.hackerone.com +https://docs.hackerone.com +https://www.hackerone.com +https://resources.hackerone.com +https://api.hackerone.com +https://support.hackerone.com +``` + +### Running httpx with file input + +This will run the tool against all the hosts in `urls.txt` and returns the matched results. + +```bash +> httpx -l hosts.txt + +root@b0x:~/httpx# httpx -l hosts.txt + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ + + projectdiscovery.io + +[WRN] Use with caution. You are responsible for your actions +[WRN] Developers assume no liability and are not responsible for any misuse or damage. +https://docs.hackerone.com +https://mta-sts.hackerone.com +https://mta-sts.managed.hackerone.com +https://mta-sts.forwarding.hackerone.com +https://www.hackerone.com +https://resources.hackerone.com +https://api.hackerone.com +https://support.hackerone.com +``` + + +### Using httpX with subfinder/chaos and any other similar tool. + + +```bash +> subfinder -d hackerone.com -silent | httpx httpx -title -content-length -status-code + + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ + + projectdiscovery.io + +[WRN] Use with caution. You are responsible for your actions +[WRN] Developers assume no liability and are not responsible for any misuse or damage. +https://mta-sts.forwarding.hackerone.com [404] [9339] [Page not found · GitHub Pages] +https://mta-sts.hackerone.com [404] [9339] [Page not found · GitHub Pages] +https://mta-sts.managed.hackerone.com [404] [9339] [Page not found · GitHub Pages] +https://docs.hackerone.com [200] [65444] [HackerOne Platform Documentation] +https://www.hackerone.com [200] [54166] [Bug Bounty - Hacker Powered Security Testing | HackerOne] +https://support.hackerone.com [301] [489] [] +https://api.hackerone.com [200] [7791] [HackerOne API] +https://hackerone.com [301] [92] [] +https://resources.hackerone.com [301] [0] [] +``` + +### Running httpX with json output + +```bash +> chaos -d hackerone.com -silent | httpx -json + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ + + projectdiscovery.io + +[WRN] Use with caution. You are responsible for your actions +[WRN] Developers assume no liability and are not responsible for any misuse or damage. + +{"url":"https://mta-sts.forwarding.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false} +{"url":"https://mta-sts.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false} +{"url":"https://docs.hackerone.com","content-length":65444,"status-code":200,"title":"","error":null,"vhost":false} +{"url":"https://mta-sts.managed.hackerone.com","content-length":9339,"status-code":404,"title":"","error":null,"vhost":false} +{"url":"https://support.hackerone.com","content-length":489,"status-code":301,"title":"","error":null,"vhost":false} +{"url":"https://resources.hackerone.com","content-length":0,"status-code":301,"title":"","error":null,"vhost":false} +{"url":"https://api.hackerone.com","content-length":7791,"status-code":200,"title":"","error":null,"vhost":false} +{"url":"https://www.hackerone.com","content-length":54166,"status-code":200,"title":"","error":null,"vhost":false} + +``` + +You can simply use `jq` to filter out the json results as per your interest. + +## Todo + +- [ ] Adding support to probe [http smuggling](https://portswigger.net/web-security/request-smuggling) + + +# Thanks + +httpX is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: + +[https://github.com/tomnomnom/httprobe](https://github.com/tomnomnom/httprobe)
+ + diff --git a/cmd/httpx/banner.go b/cmd/httpx/banner.go new file mode 100644 index 0000000..7e2b877 --- /dev/null +++ b/cmd/httpx/banner.go @@ -0,0 +1,24 @@ +package main + +import "github.com/projectdiscovery/gologger" + +const banner = ` + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ +` + +// Version is the current version of httpx +const Version = `0.0.1` + +// showBanner is used to show the banner to the user +func showBanner() { + gologger.Printf("%s\n", banner) + gologger.Printf("\t\tprojectdiscovery.io\n\n") + + gologger.Labelf("Use with caution. You are responsible for your actions\n") + gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") +} diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go new file mode 100644 index 0000000..fce62f3 --- /dev/null +++ b/cmd/httpx/httpx.go @@ -0,0 +1,252 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + "time" + + "github.com/projectdiscovery/gologger" + customport "github.com/projectdiscovery/httpx/common/customports" + "github.com/projectdiscovery/httpx/common/fileutil" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/httpx/common/stringz" + "github.com/remeh/sizedwaitgroup" +) + +func main() { + options := ParseOptions() + options.validateOptions() + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second + httpxOptions.RetryMax = options.Retries + httpxOptions.FollowRedirects = options.FollowRedirects + + httpxOptions.CustomHeaders = make(map[string]string) + for _, customHeader := range options.CustomHeaders { + tokens := strings.Split(customHeader, ":") + // if it's an invalid header skip it + if len(tokens) < 2 { + continue + } + + httpxOptions.CustomHeaders[tokens[0]] = tokens[1] + } + + hp, err := httpx.New(&httpxOptions) + if err != nil { + gologger.Fatalf("Could not create httpx instance: %s\n", err) + } + + var scanopts scanOptions + scanopts.Method = options.Method + protocol := "https" + scanopts.VHost = options.VHost + scanopts.OutputTitle = options.ExtractTitle + scanopts.OutputStatusCode = options.StatusCode + scanopts.OutputContentLength = options.ContentLength + scanopts.StoreResponse = options.StoreResponse + scanopts.StoreResponseDirectory = options.StoreResponseDir + scanopts.Method = options.Method + + // Try to create output folder if it doesnt exist + if options.StoreResponse && options.StoreResponseDir != "" && options.StoreResponseDir != "." { + if err := os.MkdirAll(options.StoreResponseDir, os.ModePerm); err != nil { + gologger.Fatalf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err) + } + } + + // output routine + wgoutput := sizedwaitgroup.New(1) + wgoutput.Add() + output := make(chan Result) + go func(output chan Result) { + defer wgoutput.Done() + + var f *os.File + if options.Output != "" { + var err error + f, err = os.Create(options.Output) + if err != nil { + gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err) + } + defer f.Close() + } + for r := range output { + if r.Error != nil { + continue + } + row := r.str + if options.JSONOutput { + row = r.JSON() + } + + fmt.Println(row) + if f != nil { + f.WriteString(row + "\n") + } + } + }(output) + + wg := sizedwaitgroup.New(options.Threads) + var sc *bufio.Scanner + + // check if file has been provided + if fileutil.FileExists(options.InputFile) { + finput, err := os.Open(options.InputFile) + if err != nil { + gologger.Fatalf("Could read input file '%s': %s\n", options.InputFile, err) + } + defer finput.Close() + sc = bufio.NewScanner(finput) + } else if fileutil.HasStdin() { + sc = bufio.NewScanner(os.Stdin) + } else { + gologger.Fatalf("No input provided") + } + + for sc.Scan() { + target := stringz.TrimProtocol(sc.Text()) + + // if no custom ports specified then test the default ones + if len(customport.Ports) == 0 { + wg.Add() + go func(target string) { + defer wg.Done() + analyze(hp, protocol, target, 0, &scanopts, output) + }(target) + } + + // the host name shouldn't have any semicolon - in case remove the port + semicolonPosition := strings.LastIndex(target, ":") + if semicolonPosition > 0 { + target = target[:semicolonPosition] + } + + for port := range customport.Ports { + wg.Add() + go func(port int) { + defer wg.Done() + analyze(hp, protocol, target, port, &scanopts, output) + }(port) + } + } + + wg.Wait() + + close(output) + + wgoutput.Wait() +} + +type scanOptions struct { + Method string + VHost bool + OutputTitle bool + OutputStatusCode bool + OutputContentLength bool + StoreResponse bool + StoreResponseDirectory string +} + +func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions, output chan Result) { + retried := false +retry: + URL := fmt.Sprintf("%s://%s", protocol, domain) + if port > 0 { + URL = fmt.Sprintf("%s:%d", URL, port) + } + + req, err := hp.NewRequest(scanopts.Method, URL) + if err != nil { + output <- Result{URL: URL, Error: err} + return + } + + hp.SetCustomHeaders(req, hp.CustomHeaders) + + resp, err := hp.Do(req) + if err != nil { + output <- Result{URL: URL, Error: err} + if !retried { + if protocol == "https" { + protocol = "http" + } else { + protocol = "https" + } + retried = true + goto retry + } + return + } + + var fullURL string + + if resp.StatusCode >= 0 { + if port > 0 { + fullURL = fmt.Sprintf("%s://%s:%d", protocol, domain, port) + } else { + fullURL = fmt.Sprintf("%s://%s", protocol, domain) + } + } + + builder := &strings.Builder{} + + builder.WriteString(fullURL) + + if scanopts.OutputStatusCode { + builder.WriteString(fmt.Sprintf(" [%d]", resp.StatusCode)) + } + + if scanopts.OutputContentLength { + builder.WriteString(fmt.Sprintf(" [%d]", resp.ContentLength)) + } + + title := "" + if scanopts.OutputTitle { + title = httpx.ExtractTitle(resp) + builder.WriteString(fmt.Sprintf(" [%s]", title)) + } + + // check for virtual host + isvhost := false + if scanopts.VHost { + isvhost, _ = hp.IsVirtualHost(req) + if isvhost { + builder.WriteString(" [vhost]") + } + } + + // store responses in directory + if scanopts.StoreResponse { + responsePath := path.Join(scanopts.StoreResponseDirectory, domain+".txt") + ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644) + } + + output <- Result{URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, Title: title, str: builder.String(), VHost: isvhost} +} + +// Result of a scan +type Result struct { + URL string `json:"url"` + ContentLength int `json:"content-length"` + StatusCode int `json:"status-code"` + Title string `json:"title"` + str string + Error error `json:"error"` + VHost bool `json:"vhost"` +} + +// JSON the result +func (r *Result) JSON() string { + if js, err := json.Marshal(r); err == nil { + return string(js) + } + + return "" +} diff --git a/cmd/httpx/options.go b/cmd/httpx/options.go new file mode 100644 index 0000000..b81bf26 --- /dev/null +++ b/cmd/httpx/options.go @@ -0,0 +1,101 @@ +package main + +import ( + "flag" + "os" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/common/customheader" + customport "github.com/projectdiscovery/httpx/common/customports" + "github.com/projectdiscovery/httpx/common/fileutil" +) + +// Options contains configuration options for chaos client. +type Options struct { + RawRequestFile string + VHost bool + Smuggling bool + ExtractTitle bool + StatusCode bool + ContentLength bool + Retries int + Threads int + Timeout int + CustomHeaders customheader.CustomHeaders + CustomPorts customport.CustomPorts + Output string + FollowRedirects bool + StoreResponse bool + StoreResponseDir string + HttpProxy string + SocksProxy string + JSONOutput bool + InputFile string + Method string + Silent bool + Version bool + Verbose bool + NoColor bool +} + +// ParseOptions parses the command line options for application +func ParseOptions() *Options { + options := &Options{} + + flag.IntVar(&options.Threads, "threads", 50, "Number of threads") + flag.IntVar(&options.Retries, "retries", 0, "Number of retries") + flag.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds") + flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") + flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs") + flag.BoolVar(&options.ExtractTitle, "title", false, "Extracts title") + flag.BoolVar(&options.StatusCode, "status-code", false, "Extracts Status Code") + flag.Var(&options.CustomHeaders, "H", "Custom Header") + flag.Var(&options.CustomPorts, "ports", "ports range (nmap syntax: eg 1,2-10,11)") + flag.BoolVar(&options.ContentLength, "content-length", false, "Content Length") + flag.BoolVar(&options.StoreResponse, "store-response", false, "Store Response as domain.txt") + flag.StringVar(&options.StoreResponseDir, "store-response-dir", ".", "Store Response Directory (default current directory)") + flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects") + flag.StringVar(&options.HttpProxy, "http-proxy", "", "Http Proxy") + flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") + flag.StringVar(&options.InputFile, "l", "", "File containing domains") + flag.StringVar(&options.Method, "x", "GET", "Request Method") + flag.BoolVar(&options.Silent, "silent", false, "Silent mode") + flag.BoolVar(&options.Version, "version", false, "Show version of httpx") + flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode") + flag.BoolVar(&options.NoColor, "no-color", true, "No Color") + flag.Parse() + + // Read the inputs and configure the logging + options.configureOutput() + + showBanner() + + if options.Version { + gologger.Infof("Current Version: %s\n", Version) + os.Exit(0) + } + + options.validateOptions() + + return options +} + +func (options *Options) validateOptions() { + if options.InputFile != "" && !fileutil.FileExists(options.InputFile) { + gologger.Fatalf("File %s does not exist!\n", options.InputFile) + } +} + +// configureOutput configures the output on the screen +func (options *Options) configureOutput() { + // If the user desires verbose output, show verbose output + if options.Verbose { + gologger.MaxLevel = gologger.Verbose + } + if options.NoColor { + gologger.UseColors = false + } + if options.Silent { + gologger.MaxLevel = gologger.Silent + } +} diff --git a/common/cache/cache.go b/common/cache/cache.go new file mode 100644 index 0000000..db37126 --- /dev/null +++ b/common/cache/cache.go @@ -0,0 +1,80 @@ +package cache + +import ( + "net" + + "github.com/coocood/freecache" + dns "github.com/projectdiscovery/httpx/common/resolver" +) + +// Cache is a strcture for caching DNS lookups +type Cache struct { + dnsClient Resolver + cache *freecache.Cache + defaultExpirationTime int +} + +// Resolver interface +type Resolver interface { + Resolve(string) (dns.Result, error) +} + +// Options of the cache +type Options struct { + BaseResolvers []string + CacheSize int + ExpirationTime int + MaxRetries int +} + +// DefaultOptions of the cache +var DefaultOptions = Options{ + BaseResolvers: DefaultResolvers, + CacheSize: 10, + ExpirationTime: 5 * 60, + MaxRetries: 5, +} + +// DefaultResolvers trusted +var DefaultResolvers = []string{ + "1.1.1.1:53", + "1.0.0.1:53", + "8.8.8.8:53", + "8.8.4.4:53", +} + +// New creates a new caching dns resolver +func New(options Options) (*Cache, error) { + dnsClient, err := dns.New(options.BaseResolvers, options.MaxRetries) + if err != nil { + return nil, err + } + cache := freecache.NewCache(options.CacheSize * 1024 * 1024) + return &Cache{dnsClient: dnsClient, cache: cache, defaultExpirationTime: options.ExpirationTime}, nil +} + +// Lookup a hostname +func (c *Cache) Lookup(hostname string) ([]string, error) { + if ip := net.ParseIP(hostname); ip != nil { + return []string{hostname}, nil + } + hostnameBytes := []byte(hostname) + value, err := c.cache.Get(hostnameBytes) + if err != nil { + if len(err.Error()) != 15 { + return nil, err + } + results, err := c.dnsClient.Resolve(hostname) + if err != nil { + return nil, err + } + if results.TTL == 0 { + results.TTL = c.defaultExpirationTime + } + c.cache.Set(hostnameBytes, MarshalAddresses(results.IPs), results.TTL) + + return results.IPs, nil + } + + return UnmarshalAddresses(value), nil +} diff --git a/common/cache/dialer.go b/common/cache/dialer.go new file mode 100644 index 0000000..d2b5e90 --- /dev/null +++ b/common/cache/dialer.go @@ -0,0 +1,53 @@ +package cache + +import ( + "context" + "net" + "strings" + "time" +) + +// NoAddressFoundError occurs when no addresses are found for the host +type NoAddressFoundError struct{} + +func (m *NoAddressFoundError) Error() string { + return "no address found for host" +} + +// DialerFunc with signature matching of go net/dial +type DialerFunc func(context.Context, string, string) (net.Conn, error) + +// NewDialer gets a new Dialer instance using a resolver cache +func NewDialer(options Options) (DialerFunc, error) { + cache, err := New(options) + if err != nil { + return nil, err + } + dialer := &net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 10 * time.Second, + DualStack: true, + } + return func(ctx context.Context, network, address string) (conn net.Conn, err error) { + separator := strings.LastIndex(address, ":") + + // we need to filter out empty records + ips, err := cache.Lookup(address[:separator]) + var finalIps []string + for _, ip := range ips { + if ip != "" { + finalIps = append(finalIps, ip) + } + } + if err != nil || len(finalIps) <= 0 { + return nil, &NoAddressFoundError{} + } // Dial to the IPs finally. + for _, ip := range ips { + conn, err = dialer.DialContext(ctx, network, ip+address[separator:]) + if err == nil { + break + } + } + return + }, nil +} diff --git a/common/cache/utils.go b/common/cache/utils.go new file mode 100644 index 0000000..a5facab --- /dev/null +++ b/common/cache/utils.go @@ -0,0 +1,13 @@ +package cache + +import "strings" + +var Separator = "-" + +func MarshalAddresses(ips []string) []byte { + return []byte(strings.Join(ips, Separator)) +} + +func UnmarshalAddresses(data []byte) []string { + return strings.Split(string(data), Separator) +} diff --git a/common/customheader/customheader.go b/common/customheader/customheader.go new file mode 100644 index 0000000..133b6fe --- /dev/null +++ b/common/customheader/customheader.go @@ -0,0 +1,15 @@ +package customheader + +// CustomHeaders valid for all requests +type CustomHeaders []string + +// String returns just a label +func (c *CustomHeaders) String() string { + return "Custom Global Headers" +} + +// Set a new global header +func (c *CustomHeaders) Set(value string) error { + *c = append(*c, value) + return nil +} diff --git a/common/customports/customport.go b/common/customports/customport.go new file mode 100644 index 0000000..8c5dddb --- /dev/null +++ b/common/customports/customport.go @@ -0,0 +1,56 @@ +package customport + +import ( + "strconv" + "strings" +) + +func init() { + Ports = make(map[int]struct{}) +} + +// Ports to scan +var Ports map[int]struct{} + +// CustomPorts definition +type CustomPorts []string + +// String returns just a label +func (c *CustomPorts) String() string { + return "Custom Ports" +} + +// Set a port range +func (c *CustomPorts) Set(value string) error { + // ports can be like nmap -p start-end,port1,port2,port3 + // splits on comma + potentialPorts := strings.Split(value, ",") + + // check if port is a single integer value or needs to be expanded futher + for _, potentialPort := range potentialPorts { + potentialRange := strings.Split(strings.TrimSpace(potentialPort), "-") + // it's a single port? + if len(potentialRange) < 2 { + if p, err := strconv.Atoi(potentialPort); err == nil { + Ports[p] = struct{}{} + } + } else { + // expand range + var lowP, highP int + lowP, err := strconv.Atoi(potentialRange[0]) + if err != nil { + continue + } + highP, err = strconv.Atoi(potentialRange[1]) + if err != nil { + continue + } + for i := lowP; i <= highP; i++ { + Ports[i] = struct{}{} + } + } + } + + *c = append(*c, value) + return nil +} diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go new file mode 100644 index 0000000..60218df --- /dev/null +++ b/common/fileutil/fileutil.go @@ -0,0 +1,24 @@ +package fileutil + +import "os" + +// FileExists checks if a file exists and is not a directory +func FileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +// HasStdin determines if the user has piped input +func HasStdin() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } + return true +} diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go new file mode 100644 index 0000000..14e0aa1 --- /dev/null +++ b/common/httputilz/httputilz.go @@ -0,0 +1,14 @@ +package httputilz + +import ( + "net/http/httputil" + + "github.com/projectdiscovery/retryablehttp-go" +) + +// DumpRequest to string +func DumpRequest(req *retryablehttp.Request) (string, error) { + dump, err := httputil.DumpRequestOut(req.Request, true) + + return string(dump), err +} diff --git a/common/httpx/filter.go b/common/httpx/filter.go new file mode 100644 index 0000000..846b962 --- /dev/null +++ b/common/httpx/filter.go @@ -0,0 +1,67 @@ +package httpx + +import ( + "regexp" + "strings" +) + +// Filter defines a generic filter interface to apply to responses +type Filter interface { + Filter(response *Response) (bool, error) +} + +// FilterString defines a filter of type string +type FilterString struct { + Keywords []string +} + +// Filter a response with strings filtering +func (f FilterString) Filter(response *Response) (bool, error) { + for _, keyword := range f.Keywords { + if strings.Contains(response.Raw, keyword) { + return true, nil + } + } + + return false, nil +} + +// FilterRegex defines a filter of type regex +type FilterRegex struct { + Regexs []string +} + +// Filter a response with regexes +func (f FilterRegex) Filter(response *Response) (bool, error) { + for _, regex := range f.Regexs { + matched, err := regexp.MatchString(regex, response.Raw) + if err != nil { + return false, err + } + if matched { + return true, nil + } + } + + return false, nil +} + +// CustomCallback used in custom filters +type CustomCallback func(response *Response) (bool, error) + +// FilterCustom defines a filter with callback functions applied +type FilterCustom struct { + CallBacks []CustomCallback +} + +// Filter a response with custom callbacks +func (f FilterCustom) Filter(response *Response) (bool, error) { + for _, callback := range f.CallBacks { + ok, err := callback(response) + if ok && err == nil { + return true, err + } + } + + return false, nil +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go new file mode 100644 index 0000000..4418e8c --- /dev/null +++ b/common/httpx/httpx.go @@ -0,0 +1,161 @@ +package httpx + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "unicode/utf8" + + "github.com/microcosm-cc/bluemonday" + "github.com/projectdiscovery/httpx/common/cache" + retryablehttp "github.com/projectdiscovery/retryablehttp-go" +) + +// HTTPX represent an instance of the library client +type HTTPX struct { + client *retryablehttp.Client + Filters []Filter + Options *Options + htmlPolicy *bluemonday.Policy + CustomHeaders map[string]string +} + +// New httpx instance +func New(options *Options) (*HTTPX, error) { + httpx := &HTTPX{} + dialer, err := cache.NewDialer(cache.DefaultOptions) + if err != nil { + return nil, fmt.Errorf("Could not create resolver cache: %s", err) + } + + httpx.Options = options + + var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying + retryablehttpOptions.Timeout = httpx.Options.Timeout + retryablehttpOptions.RetryMax = httpx.Options.RetryMax + + var redirectFunc = func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + } + + if httpx.Options.FollowRedirects { + redirectFunc = nil + } + + transport := &http.Transport{ + DialContext: dialer, + MaxIdleConnsPerHost: -1, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + } + + if httpx.Options.HttpProxy != "" { + proxyURL, err := url.Parse(httpx.Options.HttpProxy) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyURL) + } + + httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ + Transport: transport, + Timeout: httpx.Options.Timeout, + CheckRedirect: redirectFunc, + }, retryablehttpOptions) + + httpx.htmlPolicy = bluemonday.NewPolicy() + httpx.CustomHeaders = httpx.Options.CustomHeaders + + return httpx, nil +} + +// Do http request +func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) { + var ( + resp Response + ) + httpresp, err := h.client.Do(req) + if err != nil { + return nil, err + } + + rawresp, err := httputil.DumpResponse(httpresp, true) + if err != nil { + return nil, err + } + + resp.Raw = string(rawresp) + + respbody, err := ioutil.ReadAll(httpresp.Body) + if err != nil { + return nil, err + } + + respbodystr := string(respbody) + + // check if we need to strip html + if h.Options.VHostStripHTML { + respbodystr = h.htmlPolicy.Sanitize(respbodystr) + } + + resp.ContentLength = utf8.RuneCountInString(respbodystr) + resp.Data = respbody + + // fill metrics + resp.StatusCode = httpresp.StatusCode + // number of words + resp.Words = len(strings.Split(respbodystr, " ")) + // number of lines + resp.Lines = len(strings.Split(respbodystr, "\n")) + + return &resp, nil +} + +// Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true +func (h *HTTPX) Verify(req *retryablehttp.Request) (bool, error) { + resp, err := h.Do(req) + if err != nil { + return false, err + } + + // apply all filters + for _, f := range h.Filters { + ok, err := f.Filter(resp) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } + + return false, nil +} + +// AddFilter cascade +func (h *HTTPX) AddFilter(f Filter) { + h.Filters = append(h.Filters, f) +} + +// NewRequest from url +func (h *HTTPX) NewRequest(method, URL string) (req *retryablehttp.Request, err error) { + req, err = retryablehttp.NewRequest(method, URL, nil) + + // set default user agent + req.Header.Set("User-Agent", h.Options.DefaultUserAgent) + + return +} + +// SetCustomHeaders on the provided request +func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]string) { + for name, value := range headers { + r.Header.Set(name, value) + } +} diff --git a/common/httpx/option.go b/common/httpx/option.go new file mode 100644 index 0000000..7bd6052 --- /dev/null +++ b/common/httpx/option.go @@ -0,0 +1,47 @@ +package httpx + +import ( + "time" +) + +// Options contains configuration options for the client +type Options struct { + Threads int + // Timeout is the maximum time to wait for the request + Timeout time.Duration + // RetryMax is the maximum number of retries + RetryMax int + + CustomHeaders map[string]string + FollowRedirects bool + DefaultUserAgent string + + HttpProxy string + SocksProxy string + + // VHOSTs options + VHostIgnoreStatusCode bool + VHostIgnoreContentLength bool + VHostIgnoreNumberOfWords bool + VHostIgnoreNumberOfLines bool + VHostStripHTML bool + + // VHostimilarityRatio 1 - 100 + VHostSimilarityRatio int +} + +// DefaultOptions contains the default options +var DefaultOptions = Options{ + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + // VHOSTs options + VHostIgnoreStatusCode: false, + VHostIgnoreContentLength: true, + VHostIgnoreNumberOfWords: false, + VHostIgnoreNumberOfLines: false, + VHostStripHTML: false, + VHostSimilarityRatio: 85, + DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)", + // Smuggling Options +} diff --git a/common/httpx/response.go b/common/httpx/response.go new file mode 100644 index 0000000..183aadb --- /dev/null +++ b/common/httpx/response.go @@ -0,0 +1,12 @@ +package httpx + +// Response contains the response to a server +type Response struct { + StatusCode int + Headers map[string][]string + Data []byte + ContentLength int + Raw string + Words int + Lines int +} diff --git a/common/httpx/title.go b/common/httpx/title.go new file mode 100644 index 0000000..4fb62a9 --- /dev/null +++ b/common/httpx/title.go @@ -0,0 +1,24 @@ +package httpx + +import ( + "regexp" + "strings" + + "golang.org/x/net/html" +) + +// ExtractTitle from a response +func ExtractTitle(r *Response) string { + var re = regexp.MustCompile(`(?m)<\s*title[^>]*>(.*?)<\s*/\s*title>`) + for _, match := range re.FindAllString(r.Raw, -1) { + return html.UnescapeString(trimTitleTags(match)) + } + return "" +} + +func trimTitleTags(title string) string { + // trim * + titleBegin := strings.Index(title, ">") + titleEnd := strings.Index(title, "