projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-11-28 22:01:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

manual merging dev

Browse Source
This commit is contained in:
mzack 2021-08-07 13:46:20 +02:00
commit 13fa589288
6 changed files with 176 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
common/httpx/httpx.go
View File

@ -3,6 +3,7 @@ package httpx
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
@ -156,7 +157,7 @@ get_response:
// websockets don't have a readable body
if httpresp.StatusCode != http.StatusSwitchingProtocols {
var err error
respbody, err = ioutil.ReadAll(httpresp.Body)
respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
if err != nil {
return nil, err
}

16
common/httpx/option.go
View File

@ -26,13 +26,15 @@ type Options struct {
Unsafe bool
TLSGrab bool
// VHOSTs options
VHostIgnoreStatusCode bool
VHostIgnoreContentLength bool
VHostIgnoreNumberOfWords bool
VHostIgnoreNumberOfLines bool
VHostStripHTML bool
Allow []string
Deny []string
VHostIgnoreStatusCode bool
VHostIgnoreContentLength bool
VHostIgnoreNumberOfWords bool
VHostIgnoreNumberOfLines bool
VHostStripHTML bool
Allow []string
Deny []string
MaxResponseBodySizeToSave int64
MaxResponseBodySizeToRead int64
}
// DefaultOptions contains the default options

4
go.mod
View File

@ -25,8 +25,8 @@ require (
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.12 // indirect
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d // indirect
github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
github.com/projectdiscovery/wappalyzergo v0.0.7
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0

4
go.sum
View File

@ -167,8 +167,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
github.com/projectdiscovery/wappalyzergo v0.0.7 h1:MvlienkiFUbO3nDvlc5mNy1C5XiHzD2EklLDgnG9Zv4=
github.com/projectdiscovery/wappalyzergo v0.0.7/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=

146
runner/options.go
View File

@ -22,78 +22,81 @@ const (
)
type scanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN bool
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
MaxResponseBodySize int
OutputExtractRegex string
extractRegex *regexp.Regexp
ExcludeCDN bool
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN bool
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
extractRegex *regexp.Regexp
ExcludeCDN bool
}
func (s *scanOptions) Clone() *scanOptions {
return &scanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseInStdout: s.ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseInStdout: s.ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
}
}
@ -171,7 +174,8 @@ type Options struct {
StoreChain bool
Deny customlist.CustomList
Allow customlist.CustomList
MaxResponseBodySize int
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
RateLimit int
Probe bool
@ -182,6 +186,7 @@ type Options struct {
func ParseOptions() *Options {
options := &Options{}
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS data grabbing")
flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection")
flag.IntVar(&options.Threads, "threads", 50, "Number of threads")
@ -244,7 +249,8 @@ func ParseOptions() *Options {
flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')")
flag.Var(&options.Allow, "allow", "Allowlist ip/cidr")
flag.Var(&options.Deny, "deny", "Denylist ip/cidr")
flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size")
flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)")
flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)")
flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second")
flag.BoolVar(&options.Probe, "probe", false, "Display probe status")

116
runner/runner.go
View File

@ -14,14 +14,15 @@ import (
"os"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/clistats"
"github.com/projectdiscovery/stringsutil"
"github.com/projectdiscovery/urlutil"
// automatic fd max increase if running as root
@ -40,6 +41,7 @@ import (
"github.com/projectdiscovery/rawhttp"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/ratelimit"
)
const (
@ -48,12 +50,13 @@ const (
// Runner is a client for running the enumeration process.
type Runner struct {
options *Options
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
scanopts scanOptions
hm *hybrid.HybridMap
stats clistats.StatisticsClient
options *Options
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
scanopts scanOptions
hm *hybrid.HybridMap
stats clistats.StatisticsClient
ratelimiter ratelimit.Limiter
}
// New creates a new client for running enumeration process.
@ -84,6 +87,12 @@ func New(options *Options) (*Runner, error) {
httpxOptions.RandomAgent = options.RandomAgent
httpxOptions.Deny = options.Deny
httpxOptions.Allow = options.Allow
httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave)
httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead)
// adjust response size saved according to the max one read by the server
if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead {
httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead
}
var key, value string
httpxOptions.CustomHeaders = make(map[string]string)
@ -182,7 +191,8 @@ func New(options *Options) (*Runner, error) {
scanopts.NoFallbackScheme = options.NoFallbackScheme
scanopts.TechDetect = options.TechDetect
scanopts.StoreChain = options.StoreChain
scanopts.MaxResponseBodySize = options.MaxResponseBodySize
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
if options.OutputExtractRegex != "" {
if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
return nil, err
@ -210,6 +220,12 @@ func New(options *Options) (*Runner, error) {
}
runner.hm = hm
if options.RateLimit > 0 {
runner.ratelimiter = ratelimit.New(options.RateLimit)
} else {
runner.ratelimiter = ratelimit.NewUnlimited()
}
return runner, nil
}
@ -256,7 +272,7 @@ func (r *Runner) prepareInput() {
}
numTargets += numTargetsStdin
}
// reqLength = numTargets
if r.options.ShowStatistics {
numPorts := len(customport.Ports)
if numPorts == 0 {
@ -280,6 +296,9 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
for scanner.Scan() {
target := strings.TrimSpace(scanner.Text())
// Used just to get the exact number of targets
if target == "" {
continue
}
if _, ok := r.hm.Get(target); ok {
continue
}
@ -340,7 +359,6 @@ func (r *Runner) Close() {
// RunEnumeration on targets for httpx client
func (r *Runner) RunEnumeration() {
//fmt.Println("result : == >> 45646 ")
// Try to create output folder if it doesnt exist
if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) {
if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil {
@ -368,7 +386,7 @@ func (r *Runner) RunEnumeration() {
}
for resp := range output {
if resp.err != nil {
gologger.Debug().Msgf("Failure '%s': %s\n", resp.URL, resp.err)
gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err)
}
if resp.str == "" {
continue
@ -404,7 +422,6 @@ func (r *Runner) RunEnumeration() {
if r.options.JSONOutput {
row = resp.JSON(&r.scanopts)
}
gologger.Silent().Msgf("%s\n", row)
if f != nil {
//nolint:errcheck // this method needs a small refactor to reduce complexity
@ -456,7 +473,6 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
if scanopts.NoFallback {
protocols = []string{httpx.HTTPS, httpx.HTTP}
}
for target := range targets(stringz.TrimProtocol(t, scanopts.NoFallback || scanopts.NoFallbackScheme)) {
// if no custom ports specified then test the default ones
if len(customport.Ports) == 0 {
@ -615,9 +631,46 @@ retry:
req.Body = nil
}
}
r.ratelimiter.Take()
resp, err := hp.Do(req)
fullURL := req.URL.String()
builder := &strings.Builder{}
// if the full url doesn't end with the custom path we pick the original input value
if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
parsedURL, _ := urlutil.Parse(fullURL)
parsedURL.RequestURI = scanopts.RequestURI
fullURL = parsedURL.String()
}
builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
if r.options.Probe {
builder.WriteString(" [")
outputStatus := "SUCCESS"
if err != nil {
outputStatus = "FAILED"
}
if !scanopts.OutputWithNoColor && err != nil {
builder.WriteString(aurora.Red(outputStatus).String())
} else if !scanopts.OutputWithNoColor && err == nil {
builder.WriteString(aurora.Green(outputStatus).String())
} else {
builder.WriteString(outputStatus)
}
builder.WriteRune(']')
}
if err != nil {
errString := ""
errString = err.Error()
splitErr := strings.Split(errString, ":")
errString = strings.TrimSpace(splitErr[len(splitErr)-1])
if !retried && origProtocol == httpx.HTTPorHTTPS {
if protocol == httpx.HTTPS {
protocol = httpx.HTTP
@ -627,19 +680,13 @@ retry:
retried = true
goto retry
}
return Result{URL: URL.String(), err: err}
if r.options.Probe {
return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
} else {
return Result{URL: URL.String(), Input: domain, Timestamp: time.Now(), err: err}
}
}
var fullURL string
if resp.StatusCode >= 0 {
fullURL = req.URL.String()
}
builder := &strings.Builder{}
builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
// portLst = append(portLst, fullURL)
if scanopts.OutputStatusCode {
builder.WriteString(" [")
for i, chainItem := range resp.Chain {
@ -733,6 +780,7 @@ retry:
// check for virtual host
isvhost := false
if scanopts.VHost {
r.ratelimiter.Take()
isvhost, _ = hp.IsVirtualHost(req)
if isvhost {
builder.WriteString(" [vhost]")
@ -748,6 +796,7 @@ retry:
pipeline := false
if scanopts.Pipeline {
port, _ := strconv.Atoi(URL.Port)
r.ratelimiter.Take()
pipeline = hp.SupportPipeline(protocol, method, URL.Host, port)
if pipeline {
builder.WriteString(" [pipeline]")
@ -757,6 +806,7 @@ retry:
var http2 bool
// if requested probes for http2
if scanopts.HTTP2Probe {
r.ratelimiter.Take()
http2 = hp.SupportHTTP2(protocol, method, URL.String())
if http2 {
builder.WriteString(" [http2]")
@ -802,6 +852,7 @@ retry:
}
if len(technologies) > 0 {
sort.Strings(technologies)
technologies := strings.Join(technologies, ",")
builder.WriteString(" [")
@ -852,8 +903,8 @@ retry:
// store response
responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
respRaw := resp.Raw
if len(respRaw) > scanopts.MaxResponseBodySize {
respRaw = respRaw[:scanopts.MaxResponseBodySize]
if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
}
writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644)
if writeErr != nil {
@ -903,6 +954,7 @@ retry:
if scanopts.ChainInStdout && resp.HasChain() {
chainItems = append(chainItems, resp.GetChainAsSlice()...)
}
return Result{
Timestamp: time.Now(),
Request: request,
@ -914,6 +966,7 @@ retry:
HeaderSHA256: headersSha,
raw: resp.Raw,
URL: fullURL,
Input: domain,
ContentLength: resp.ContentLength,
ChainStatusCodes: chainStatusCodes,
Chain: chainItems,
@ -938,8 +991,6 @@ retry:
ResponseTime: resp.Duration.String(),
Technologies: technologies,
FinalURL: finalURL,
// UniqueUrl: uniqueUrlStrngs,
// PortLst: portLst,
}
}
@ -957,10 +1008,12 @@ type Result struct {
CNAMEs []string `json:"cnames,omitempty"`
raw string
URL string `json:"url,omitempty"`
Input string `json:"input,omitempty"`
Location string `json:"location,omitempty"`
Title string `json:"title,omitempty"`
str string
err error
Error string `json:"error,omitempty"`
WebServer string `json:"webserver,omitempty"`
ResponseBody string `json:"response-body,omitempty"`
ContentType string `json:"content-type,omitempty"`
@ -980,12 +1033,13 @@ type Result struct {
Technologies []string `json:"technologies,omitempty"`
Chain []httpx.ChainItem `json:"chain,omitempty"`
FinalURL string `json:"final-url,omitempty"`
Failed bool `json:"failed"`
}
// JSON the result
func (r Result) JSON(scanopts *scanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySize {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySize]
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}
if js, err := json.Marshal(r); err == nil {