mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 05:20:29 +03:00
Added wappalyzer based technology detection
This commit is contained in:
Ice3man543
parent
88927cfe06
commit
07bff31c45
@ -9,11 +9,10 @@ func main() {
|
||||
// Parse the command line flags and read config files
|
||||
options := runner.ParseOptions()
|
||||
|
||||
runner, err := runner.New(options)
|
||||
r, err := runner.New(options)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not create runner: %s\n", err)
|
||||
}
|
||||
|
||||
runner.RunEnumeration()
|
||||
runner.Close()
|
||||
r.RunEnumeration()
|
||||
r.Close()
|
||||
}
|
||||
|
||||
@ -41,3 +41,23 @@ func (r *Response) GetHeaderPart(name, sep string) string {
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetHeadersMap returns a map[string]string of response headers
|
||||
func (r *Response) GetHeadersMap() map[string]string {
|
||||
headers := make(map[string]string, len(r.Headers))
|
||||
|
||||
builder := &strings.Builder{}
|
||||
for key, value := range r.Headers {
|
||||
for i, v := range value {
|
||||
builder.WriteString(v)
|
||||
if i != len(value)-1 {
|
||||
builder.WriteString(", ")
|
||||
}
|
||||
}
|
||||
headerValue := builder.String()
|
||||
|
||||
headers[key] = headerValue
|
||||
builder.Reset()
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
5
go.mod
5
go.mod
@ -8,14 +8,17 @@ require (
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/microcosm-cc/bluemonday v1.0.4
|
||||
github.com/miekg/dns v1.1.33
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935
|
||||
github.com/projectdiscovery/fdmax v0.0.2
|
||||
github.com/projectdiscovery/gologger v1.0.1
|
||||
github.com/projectdiscovery/mapcidr v0.0.4
|
||||
github.com/projectdiscovery/rawhttp v0.0.4
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.1
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.2.1
|
||||
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
|
||||
golang.org/x/text v0.3.3
|
||||
)
|
||||
|
||||
34
go.sum
34
go.sum
@ -1,3 +1,4 @@
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
@ -8,19 +9,26 @@ github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpR
|
||||
github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc=
|
||||
github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
|
||||
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk=
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/microcosm-cc/bluemonday v1.0.4 h1:p0L+CTpo/PLFdkoPcJemLXG+fpMD7pYOoDEq1axMbGg=
|
||||
github.com/microcosm-cc/bluemonday v1.0.4/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
|
||||
github.com/miekg/dns v1.1.32 h1:MDaYYzWOYscpvDOEgPMT1c1mebCZmIdxZI/J161OdJU=
|
||||
github.com/miekg/dns v1.1.32/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.33 h1:8KUVEKrUw2dmu1Ys0aWnkEJgoRaLAzNysfCh2KSMWiI=
|
||||
github.com/miekg/dns v1.1.33/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 h1:YCM8XrQXo9llkohuXw3CBoz2S/T3kVMnb1pM04X23t8=
|
||||
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw=
|
||||
@ -30,19 +38,22 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
|
||||
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
|
||||
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
|
||||
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
|
||||
github.com/projectdiscovery/rawhttp v0.0.3 h1:UCNHNnRDHixtPd75kUOWi8QtIlxFnkSa7ugrKUB5Eto=
|
||||
github.com/projectdiscovery/rawhttp v0.0.3/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
|
||||
github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E=
|
||||
github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.1 h1:DMjAfleVjA1FxmIe2RIsDYspPlfN/uyAFbLC4AbZvyM=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.1/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/yl2chen/cidranger v1.0.0 h1:9tdo0orHQJvXsX6mf+1Goou/R4kq21AfpbYeTcpXs2Q=
|
||||
github.com/yl2chen/cidranger v1.0.0/go.mod h1:L7Msw4X7EQK7zMVjOtv7o8xMyjv1rJcNlYlMgGwP7ko=
|
||||
@ -55,21 +66,24 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg=
|
||||
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxFileNameLenght = 255
|
||||
maxFileNameLength = 255
|
||||
two = 2
|
||||
)
|
||||
|
||||
@ -35,6 +35,7 @@ type scanOptions struct {
|
||||
ResponseInStdout bool
|
||||
TLSProbe bool
|
||||
CSPProbe bool
|
||||
VHostInput bool
|
||||
OutputContentType bool
|
||||
Unsafe bool
|
||||
Pipeline bool
|
||||
@ -45,9 +46,10 @@ type scanOptions struct {
|
||||
OutputResponseTime bool
|
||||
PreferHTTPS bool
|
||||
NoFallback bool
|
||||
TechDetect bool
|
||||
}
|
||||
|
||||
// Options contains configuration options for chaos client.
|
||||
// Options contains configuration options for httpx
|
||||
type Options struct {
|
||||
CustomHeaders customheader.CustomHeaders
|
||||
CustomPorts customport.CustomPorts
|
||||
@ -79,6 +81,7 @@ type Options struct {
|
||||
filterRegex *regexp.Regexp
|
||||
matchRegex *regexp.Regexp
|
||||
VHost bool
|
||||
VHostInput bool
|
||||
Smuggling bool
|
||||
ExtractTitle bool
|
||||
StatusCode bool
|
||||
@ -108,6 +111,7 @@ type Options struct {
|
||||
OutputCDN bool
|
||||
OutputResponseTime bool
|
||||
NoFallback bool
|
||||
TechDetect bool
|
||||
protocol string
|
||||
}
|
||||
|
||||
@ -115,11 +119,13 @@ type Options struct {
|
||||
func ParseOptions() *Options {
|
||||
options := &Options{}
|
||||
|
||||
flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection")
|
||||
flag.IntVar(&options.Threads, "threads", 50, "Number of threads")
|
||||
flag.IntVar(&options.Retries, "retries", 0, "Number of retries")
|
||||
flag.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds")
|
||||
flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
|
||||
flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs")
|
||||
flag.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input")
|
||||
flag.BoolVar(&options.ExtractTitle, "title", false, "Extracts title")
|
||||
flag.BoolVar(&options.StatusCode, "status-code", false, "Extracts status code")
|
||||
flag.BoolVar(&options.Location, "location", false, "Extracts location header")
|
||||
|
||||
@ -13,6 +13,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
// Automatically set max file descriptors
|
||||
_ "github.com/projectdiscovery/fdmax/autofdmax"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/httpx/common/cache"
|
||||
@ -25,14 +28,16 @@ import (
|
||||
"github.com/projectdiscovery/httpx/common/stringz"
|
||||
"github.com/projectdiscovery/mapcidr"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
)
|
||||
|
||||
// Runner is a client for running the enumeration process.
|
||||
type Runner struct {
|
||||
options *Options
|
||||
hp *httpx.HTTPX
|
||||
scanopts *scanOptions
|
||||
options *Options
|
||||
hp *httpx.HTTPX
|
||||
wappalyzer *wappalyzer.Wappalyze
|
||||
scanopts *scanOptions
|
||||
}
|
||||
|
||||
// New creates a new client for running enumeration process.
|
||||
@ -40,6 +45,13 @@ func New(options *Options) (*Runner, error) {
|
||||
runner := &Runner{
|
||||
options: options,
|
||||
}
|
||||
var err error
|
||||
if options.TechDetect {
|
||||
runner.wappalyzer, err = wappalyzer.New()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create wappalyzer client")
|
||||
}
|
||||
|
||||
httpxOptions := httpx.DefaultOptions
|
||||
httpxOptions.Timeout = time.Duration(options.Timeout) * time.Second
|
||||
@ -70,7 +82,6 @@ func New(options *Options) (*Runner, error) {
|
||||
httpxOptions.CustomHeaders[key] = value
|
||||
}
|
||||
|
||||
var err error
|
||||
runner.hp, err = httpx.New(&httpxOptions)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not create httpx instance: %s\n", err)
|
||||
@ -133,6 +144,7 @@ func New(options *Options) (*Runner, error) {
|
||||
if options.RequestURI != "" {
|
||||
scanopts.RequestURI = options.RequestURI
|
||||
}
|
||||
scanopts.VHostInput = options.VHostInput
|
||||
scanopts.OutputContentType = options.OutputContentType
|
||||
scanopts.RequestBody = options.RequestBody
|
||||
scanopts.Unsafe = options.Unsafe
|
||||
@ -144,6 +156,7 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.OutputCDN = options.OutputCDN
|
||||
scanopts.OutputResponseTime = options.OutputResponseTime
|
||||
scanopts.NoFallback = options.NoFallback
|
||||
scanopts.TechDetect = options.TechDetect
|
||||
|
||||
// output verb if more than one is specified
|
||||
if len(scanopts.Methods) > 1 && !options.Silent {
|
||||
@ -155,15 +168,17 @@ func New(options *Options) (*Runner, error) {
|
||||
return runner, nil
|
||||
}
|
||||
|
||||
func (runner *Runner) Close() {
|
||||
// Close closes the httpx runner
|
||||
func (r *Runner) Close() {
|
||||
// not implemented
|
||||
}
|
||||
|
||||
func (runner *Runner) RunEnumeration() {
|
||||
// RunEnumeration performs httpx enumeration process on input recursively
|
||||
func (r *Runner) RunEnumeration() {
|
||||
// Try to create output folder if it doesnt exist
|
||||
if runner.options.StoreResponse && !fileutil.FolderExists(runner.options.StoreResponseDir) {
|
||||
if err := os.MkdirAll(runner.options.StoreResponseDir, os.ModePerm); err != nil {
|
||||
gologger.Fatalf("Could not create output directory '%s': %s\n", runner.options.StoreResponseDir, err)
|
||||
if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) {
|
||||
if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil {
|
||||
gologger.Fatalf("Could not create output directory '%s': %s\n", r.options.StoreResponseDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -175,50 +190,50 @@ func (runner *Runner) RunEnumeration() {
|
||||
defer wgoutput.Done()
|
||||
|
||||
var f *os.File
|
||||
if runner.options.Output != "" {
|
||||
if r.options.Output != "" {
|
||||
var err error
|
||||
f, err = os.Create(runner.options.Output)
|
||||
f, err = os.Create(r.options.Output)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could not create output file '%s': %s\n", runner.options.Output, err)
|
||||
gologger.Fatalf("Could not create output file '%s': %s\n", r.options.Output, err)
|
||||
}
|
||||
//nolint:errcheck // this method needs a small refactor to reduce complexity
|
||||
defer f.Close()
|
||||
}
|
||||
for r := range output {
|
||||
if r.err != nil {
|
||||
gologger.Debugf("Failure '%s': %s\n", r.URL, r.err)
|
||||
for resp := range output {
|
||||
if resp.err != nil {
|
||||
gologger.Debugf("Failure '%s': %s\n", resp.URL, resp.err)
|
||||
continue
|
||||
}
|
||||
|
||||
// apply matchers and filters
|
||||
if len(runner.options.filterStatusCode) > 0 && slice.IntSliceContains(runner.options.filterStatusCode, r.StatusCode) {
|
||||
if len(r.options.filterStatusCode) > 0 && slice.IntSliceContains(r.options.filterStatusCode, resp.StatusCode) {
|
||||
continue
|
||||
}
|
||||
if len(runner.options.filterContentLength) > 0 && slice.IntSliceContains(runner.options.filterContentLength, r.ContentLength) {
|
||||
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
|
||||
continue
|
||||
}
|
||||
if runner.options.filterRegex != nil && runner.options.filterRegex.MatchString(r.raw) {
|
||||
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
|
||||
continue
|
||||
}
|
||||
if runner.options.OutputFilterString != "" && strings.Contains(strings.ToLower(r.raw), strings.ToLower(runner.options.OutputFilterString)) {
|
||||
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) {
|
||||
continue
|
||||
}
|
||||
if len(runner.options.matchStatusCode) > 0 && !slice.IntSliceContains(runner.options.matchStatusCode, r.StatusCode) {
|
||||
if len(r.options.matchStatusCode) > 0 && !slice.IntSliceContains(r.options.matchStatusCode, resp.StatusCode) {
|
||||
continue
|
||||
}
|
||||
if len(runner.options.matchContentLength) > 0 && !slice.IntSliceContains(runner.options.matchContentLength, r.ContentLength) {
|
||||
if len(r.options.matchContentLength) > 0 && !slice.IntSliceContains(r.options.matchContentLength, resp.ContentLength) {
|
||||
continue
|
||||
}
|
||||
if runner.options.matchRegex != nil && !runner.options.matchRegex.MatchString(r.raw) {
|
||||
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.raw) {
|
||||
continue
|
||||
}
|
||||
if runner.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(r.raw), strings.ToLower(runner.options.OutputMatchString)) {
|
||||
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
|
||||
continue
|
||||
}
|
||||
|
||||
row := r.str
|
||||
if runner.options.JSONOutput {
|
||||
row = r.JSON()
|
||||
row := resp.str
|
||||
if r.options.JSONOutput {
|
||||
row = resp.JSON()
|
||||
}
|
||||
|
||||
gologger.Silentf("%s\n", row)
|
||||
@ -229,20 +244,20 @@ func (runner *Runner) RunEnumeration() {
|
||||
}
|
||||
}(output)
|
||||
|
||||
wg := sizedwaitgroup.New(runner.options.Threads)
|
||||
wg := sizedwaitgroup.New(r.options.Threads)
|
||||
var scanner *bufio.Scanner
|
||||
|
||||
// check if file has been provided
|
||||
if fileutil.FileExists(runner.options.InputFile) {
|
||||
finput, err := os.Open(runner.options.InputFile)
|
||||
if fileutil.FileExists(r.options.InputFile) {
|
||||
finput, err := os.Open(r.options.InputFile)
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could read input file '%s': %s\n", runner.options.InputFile, err)
|
||||
gologger.Fatalf("Could read input file '%s': %s\n", r.options.InputFile, err)
|
||||
}
|
||||
scanner = bufio.NewScanner(finput)
|
||||
defer func() {
|
||||
err := finput.Close()
|
||||
if err != nil {
|
||||
gologger.Fatalf("Could close input file '%s': %s\n", runner.options.InputFile, err)
|
||||
gologger.Fatalf("Could close input file '%s': %s\n", r.options.InputFile, err)
|
||||
}
|
||||
}()
|
||||
} else if fileutil.HasStdin() {
|
||||
@ -252,7 +267,7 @@ func (runner *Runner) RunEnumeration() {
|
||||
}
|
||||
|
||||
for scanner.Scan() {
|
||||
process(scanner.Text(), &wg, runner.hp, runner.options.protocol, runner.scanopts, output)
|
||||
r.process(scanner.Text(), &wg, r.hp, r.options.protocol, r.scanopts, output)
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
@ -266,7 +281,7 @@ func (runner *Runner) RunEnumeration() {
|
||||
wgoutput.Wait()
|
||||
}
|
||||
|
||||
func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
|
||||
func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
|
||||
protocols := []string{protocol}
|
||||
if scanopts.NoFallback {
|
||||
protocols = []string{httpx.HTTPS, httpx.HTTP}
|
||||
@ -279,21 +294,21 @@ func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, proto
|
||||
wg.Add()
|
||||
go func(target, method, protocol string) {
|
||||
defer wg.Done()
|
||||
r := analyze(hp, protocol, target, 0, method, scanopts)
|
||||
output <- r
|
||||
if scanopts.TLSProbe && r.TLSData != nil {
|
||||
result := r.analyze(hp, protocol, target, 0, method, scanopts)
|
||||
output <- result
|
||||
if scanopts.TLSProbe && result.TLSData != nil {
|
||||
scanopts.TLSProbe = false
|
||||
for _, tt := range r.TLSData.DNSNames {
|
||||
process(tt, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.DNSNames {
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
for _, tt := range r.TLSData.CommonName {
|
||||
process(tt, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.CommonName {
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
}
|
||||
if scanopts.CSPProbe && r.CSPData != nil {
|
||||
if scanopts.CSPProbe && result.CSPData != nil {
|
||||
scanopts.CSPProbe = false
|
||||
for _, tt := range r.CSPData.Domains {
|
||||
process(tt, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.CSPData.Domains {
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
}
|
||||
}(target, method, prot)
|
||||
@ -312,15 +327,15 @@ func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, proto
|
||||
wg.Add()
|
||||
go func(port int, method, protocol string) {
|
||||
defer wg.Done()
|
||||
r := analyze(hp, protocol, target, port, method, scanopts)
|
||||
output <- r
|
||||
if scanopts.TLSProbe && r.TLSData != nil {
|
||||
result := r.analyze(hp, protocol, target, port, method, scanopts)
|
||||
output <- result
|
||||
if scanopts.TLSProbe && result.TLSData != nil {
|
||||
scanopts.TLSProbe = false
|
||||
for _, tt := range r.TLSData.DNSNames {
|
||||
process(tt, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.DNSNames {
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
for _, tt := range r.TLSData.CommonName {
|
||||
process(tt, wg, hp, protocol, scanopts, output)
|
||||
for _, tt := range result.TLSData.CommonName {
|
||||
r.process(tt, wg, hp, protocol, scanopts, output)
|
||||
}
|
||||
}
|
||||
}(port, method, wantedProtocol)
|
||||
@ -358,13 +373,24 @@ func targets(target string) chan string {
|
||||
return results
|
||||
}
|
||||
|
||||
func analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
|
||||
func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
|
||||
origProtocol := protocol
|
||||
if protocol == httpx.HTTPorHTTPS {
|
||||
protocol = httpx.HTTPS
|
||||
}
|
||||
retried := false
|
||||
retry:
|
||||
var customHost string
|
||||
if scanopts.VHostInput {
|
||||
parts := strings.Split(domain, ",")
|
||||
//nolint:gomnd // not a magic number
|
||||
if len(parts) != 2 {
|
||||
return Result{}
|
||||
}
|
||||
domain = parts[0]
|
||||
customHost = parts[1]
|
||||
}
|
||||
|
||||
URL := fmt.Sprintf("%s://%s", protocol, domain)
|
||||
if port > 0 {
|
||||
URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
|
||||
@ -378,6 +404,9 @@ retry:
|
||||
if err != nil {
|
||||
return Result{URL: URL, err: err}
|
||||
}
|
||||
if customHost != "" {
|
||||
req.Host = customHost
|
||||
}
|
||||
|
||||
hp.SetCustomHeaders(req, hp.CustomHeaders)
|
||||
if scanopts.RequestBody != "" {
|
||||
@ -558,6 +587,26 @@ retry:
|
||||
builder.WriteString(fmt.Sprintf(" [%s]", resp.Duration))
|
||||
}
|
||||
|
||||
var technologies []string
|
||||
if scanopts.TechDetect {
|
||||
matches := r.wappalyzer.Fingerprint(resp.GetHeadersMap(), resp.Data)
|
||||
for match := range matches {
|
||||
technologies = append(technologies, match)
|
||||
}
|
||||
|
||||
if len(technologies) > 0 {
|
||||
technologies := strings.Join(technologies, ",")
|
||||
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Magenta(technologies).String())
|
||||
} else {
|
||||
builder.WriteString(technologies)
|
||||
}
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
}
|
||||
|
||||
// store responses in directory
|
||||
if scanopts.StoreResponse {
|
||||
domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
|
||||
@ -566,9 +615,9 @@ retry:
|
||||
}
|
||||
// On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux
|
||||
// Truncating length at 255
|
||||
if len(domainFile) >= maxFileNameLenght {
|
||||
if len(domainFile) >= maxFileNameLength {
|
||||
// leaving last 4 bytes free to append ".txt"
|
||||
domainFile = domainFile[:maxFileNameLenght-1]
|
||||
domainFile = domainFile[:maxFileNameLength-1]
|
||||
}
|
||||
|
||||
domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt"
|
||||
@ -602,6 +651,7 @@ retry:
|
||||
CNAMEs: cnames,
|
||||
CDN: isCDN,
|
||||
ResponseTime: resp.Duration.String(),
|
||||
Technologies: technologies,
|
||||
}
|
||||
}
|
||||
|
||||
@ -630,6 +680,7 @@ type Result struct {
|
||||
HTTP2 bool `json:"http2"`
|
||||
CDN bool `json:"cdn,omitempty"`
|
||||
ResponseTime string `json:"response-time"`
|
||||
Technologies []string `json:"technologies"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
||||
Loading…
Reference in New Issue
Block a user