From c7456929b437fd600fc2decf18daf2185b1fffc2 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 8 Feb 2023 19:50:03 +0530 Subject: [PATCH] fix integration tests and upgrade deps --- cmd/httpx/httpx.go | 7 +++++ cmd/integration-test/library.go | 3 ++- common/httpx/httpx.go | 9 +++++-- go.mod | 42 ++++++++++++++--------------- go.sum | 46 ++++++++++++++++++++++++++++++++ runner/runner.go | 47 ++++++++++++++++++--------------- 6 files changed, 108 insertions(+), 46 deletions(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 6890365..1a233e9 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/runner" + errorutil "github.com/projectdiscovery/utils/errors" ) func main() { @@ -58,3 +59,9 @@ func main() { httpxRunner.RunEnumeration() httpxRunner.Close() } + +func init() { + if os.Getenv("DEBUG") != "" { + errorutil.ShowStackTrace = true + } +} diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index a81432a..289192d 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -44,7 +44,8 @@ func (h *httpxLibrary) Execute() error { httpxRunner.RunEnumeration() - expected := "https://scanme.sh:443" + // httpx removes default ports for simplicity Ref: https://pkg.go.dev/github.com/projectdiscovery/httpx/common/stringz#RemoveURLDefaultPort + expected := "https://scanme.sh" if got != expected { return errIncorrectResult(expected, got) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 0d0d952..56224d9 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -19,6 +19,7 @@ import ( retryablehttp "github.com/projectdiscovery/retryablehttp-go" pdhttputil "github.com/projectdiscovery/utils/http" stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "golang.org/x/net/context" "golang.org/x/net/http2" ) @@ -330,11 +331,15 @@ func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request // NewRequest from url func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) { - req, err = retryablehttp.NewRequestWithContext(ctx, method, targetURL, nil) + urlx, err := urlutil.ParseURL(targetURL, h.Options.Unsafe) if err != nil { - return + return nil, err } + req, err = retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil) + if err != nil { + return nil, err + } // Skip if unsafe is used if !h.Options.Unsafe { // set default user agent diff --git a/go.mod b/go.mod index a352278..80430a6 100644 --- a/go.mod +++ b/go.mod @@ -22,16 +22,16 @@ require ( github.com/projectdiscovery/hmap v0.0.7 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 - github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e - github.com/projectdiscovery/retryablehttp-go v1.0.8 + github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed + github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208133157-acb223323d89 github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/projectdiscovery/wappalyzergo v0.0.80 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 - go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/net v0.5.0 - golang.org/x/sys v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.6.0 ) @@ -48,10 +48,10 @@ require ( github.com/projectdiscovery/fastdialer v0.0.22 github.com/projectdiscovery/ratelimit v0.0.5 github.com/projectdiscovery/tlsx v1.0.3 - github.com/projectdiscovery/utils v0.0.8-0.20230207142824-2343fd8d6d0b + github.com/projectdiscovery/utils v0.0.8-0.20230207220018-0e486b130065 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.9.0 - golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 ) require ( @@ -64,19 +64,19 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/cockroachdb/errors v1.9.0 // indirect - github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect - github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20230207164304-7d1e4ba7ffd0 // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect - github.com/getsentry/sentry-go v0.16.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.13 // indirect + github.com/klauspost/compress v1.15.15 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -96,27 +96,27 @@ require ( github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/tidwall/btree v1.4.3 // indirect + github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.2.10 // indirect - github.com/tidwall/gjson v1.14.3 // indirect + github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect - github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect + github.com/weppos/publicsuffix-go v0.20.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect - github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect + github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect - github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/tools v0.2.0 // indirect + github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect + golang.org/x/crypto v0.5.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/tools v0.5.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index ebd520b..0c42afd 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,7 @@ github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0 github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -47,15 +48,22 @@ github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3 github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b h1:6TtBvyITBhpkL8EvEEZdS9om8egSxP/mUML01aM3pyY= github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b/go.mod h1:JsehdjcR1QgLZkqBeYrbVdE3cdxbdrycA/PN+Cg+RNw= +github.com/cockroachdb/pebble v0.0.0-20230207164304-7d1e4ba7ffd0 h1:m2/PDCdtaaNbs1EujshCCTIiwp2WA8WDokNgtcHaLMA= +github.com/cockroachdb/pebble v0.0.0-20230207164304-7d1e4ba7ffd0/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= @@ -98,6 +106,8 @@ github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo= github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -202,6 +212,8 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -324,6 +336,8 @@ github.com/projectdiscovery/ratelimit v0.0.5 h1:QAWtECH5MJtqen448fHr6ikV8NTBdebm github.com/projectdiscovery/ratelimit v0.0.5/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed h1:XbW1NY+l1B4qnDJyAqFKB7b7+hG+S6pDA4hkFsB5s20= +github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed/go.mod h1:GoReErVAHUI24tw3AoEB/2C1xkvb5zHDI5qQwibJrik= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= github.com/projectdiscovery/retryabledns v1.0.20 h1:grRyh4EzuyqsaK07iNkJKgrGLu/qDJwfDJ+83SBo6yo= @@ -331,6 +345,10 @@ github.com/projectdiscovery/retryabledns v1.0.20/go.mod h1:97Et22Kw2iPyvz/Vn41/i github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.8 h1:F4UZ+3NkIPDj+tobHZY+sgvDVmiVCX/CUUh/jrKq7Oc= github.com/projectdiscovery/retryablehttp-go v1.0.8/go.mod h1:tD5TNOXKNuvqrUHHGoP6xJmVZTjSkLGsqcyD7gCsc3I= +github.com/projectdiscovery/retryablehttp-go v1.0.10 h1:A8tsJZhzpijgnvHT41JNrhiXtn8F8s7TWMC5ZxLV/Cg= +github.com/projectdiscovery/retryablehttp-go v1.0.10/go.mod h1:a5bmSbaxgHvC0P80csOymMOwKaJirMnsS6otRUH/vcU= +github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208133157-acb223323d89 h1:KkTW2sNFZfRH7DPAArTHpVH6ltxSkIaVfVVZBARFJ08= +github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208133157-acb223323d89/go.mod h1:RWViUDjf9NTx1j8HatkstoSj2hE4xrrDIum1SsQqZfE= github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= @@ -342,6 +360,8 @@ github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDk github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/utils v0.0.8-0.20230207142824-2343fd8d6d0b h1:Nu3lpNgNKU+npZzn5bGWQljWd94XgdkH+q5XwC3wwoY= github.com/projectdiscovery/utils v0.0.8-0.20230207142824-2343fd8d6d0b/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.8-0.20230207220018-0e486b130065 h1:nguDaV3Ye3j3vLah56pRI+wK25JSJF0eGE7+19ZZ0dM= +github.com/projectdiscovery/utils v0.0.8-0.20230207220018-0e486b130065/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/wappalyzergo v0.0.80 h1:QWE1Nrxwc3bo5dYMgO1NMADZRzPWY0l25bcAPFjKBB4= github.com/projectdiscovery/wappalyzergo v0.0.80/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= @@ -365,6 +385,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -400,11 +422,15 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -412,6 +438,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= @@ -435,6 +463,8 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go v0.20.0 h1:59ypvSUbW3Dunc6zVm+v+MmXf2Q6cGiNDkxgRIzEnaA= +github.com/weppos/publicsuffix-go v0.20.0/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= @@ -453,12 +483,18 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= +github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw= +github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -476,9 +512,13 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -489,6 +529,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -561,6 +603,8 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -591,6 +635,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= +golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/runner/runner.go b/runner/runner.go index 1166b8c..b721ef8 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -11,7 +11,6 @@ import ( "net" "net/http" "net/http/httputil" - "net/url" "os" "path" "path/filepath" @@ -31,6 +30,7 @@ import ( "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/mapcidr/asn" + errorutil "github.com/projectdiscovery/utils/errors" mapsutil "github.com/projectdiscovery/utils/maps" "github.com/bluele/gcache" @@ -645,7 +645,7 @@ func (r *Runner) RunEnumeration() { for resp := range output { if resp.err != nil { // Change the error message if any port value passed explicitly - if url, err := url.Parse(resp.URL); err == nil && url.Port() != "" { + if url, err := r.parseURL(resp.URL); err == nil && url.Port() != "" { resp.err = errors.New(strings.ReplaceAll(resp.err.Error(), "address", "port")) } gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err) @@ -837,7 +837,7 @@ func (r *Runner) RunEnumeration() { protocol := r.options.protocol // attempt to parse url as is - if u, err := url.Parse(k); err == nil { + if u, err := r.parseURL(k); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -937,7 +937,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. wg.Add() go func(port int, target httpx.Target, method, protocol string) { defer wg.Done() - if urlx, err := urlutil.Parse(target.Host); err != nil { + if urlx, err := r.parseURL(target.Host); err != nil { gologger.Warning().Msgf("failed to update port of %v got %v", target.Host, err) } else { urlx.UpdatePort(fmt.Sprint(port)) @@ -1001,7 +1001,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results <- httpx.Target{Host: ip} } case r.options.ProbeAllIPS: - URL, err := urlutil.Parse(target) + URL, err := r.parseURL(target) if err != nil { results <- httpx.Target{Host: target} } @@ -1032,7 +1032,7 @@ retry: if scanopts.VHostInput && target.CustomHost == "" { return Result{Input: origInput} } - URL, err := urlutil.Parse(target.Host) + URL, err := r.parseURL(target.Host) if err != nil { return Result{URL: target.Host, Input: origInput, err: err} } @@ -1140,7 +1140,7 @@ retry: } // fix the final output url fullURL := req.URL.String() - if parsedURL, errParse := urlutil.Parse(fullURL); errParse != nil { + if parsedURL, errParse := r.parseURL(fullURL); errParse != nil { return Result{URL: URL.String(), Input: origInput, err: errParse} } else { if r.options.Unsafe { @@ -1478,7 +1478,7 @@ retry: } builder.WriteRune(']') } else { - gologger.Warning().Msgf("could not calculate favicon hash: %s", err) + gologger.Warning().Msgf("could not calculate favicon hash for path %v : %s", faviconPath, err) } } @@ -1592,7 +1592,7 @@ retry: } } - parsed, err := urlutil.Parse(fullURL) + parsed, err := r.parseURL(fullURL) if err != nil { return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")} } @@ -1688,22 +1688,16 @@ func (r *Runner) handleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, // pick the first - we want only one request if len(potentialURLs) > 0 { - URL, err := url.Parse(potentialURLs[0]) + URL, err := r.parseURL(potentialURLs[0]) if err != nil { return "", "", err } - if URL.IsAbs() { - req.URL = URL - } else { - if strings.HasPrefix(URL.Path, "/") { - req.URL.Path = URL.Path - } else { - req.URL.Path = "/" + URL.Path - } - } - req.Host = URL.Host + req.URL = URL } else { - req.URL = req.URL.JoinPath("favicon.ico") + err := req.URL.MergePath("/favicon.ico", false) + if err != nil { + return "", "", errorutil.NewWithTag("favicon", "failed to add /favicon.ico to url got %v", err) + } } resp, err := hp.Do(req, httpx.UnsafeOptions{}) @@ -1717,7 +1711,7 @@ func (r *Runner) handleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) { hashNum, err := stringz.FaviconHash(data) if err != nil { - return "", errors.Wrap(err, "could not calculate favicon hash") + return "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) } return fmt.Sprintf("%d", hashNum), nil } @@ -1848,6 +1842,15 @@ func (r *Runner) skipCDNPort(host string, port string) bool { return false } +// parseURL parses url based on cli option(unsafe) +func (r *Runner) parseURL(url string) (*urlutil.URL, error) { + urlx, err := urlutil.ParseURL(url, r.options.Unsafe) + if err != nil { + gologger.Debug().Msgf("failed to parse url %v got %v in unsafe:%v", url, err, r.options.Unsafe) + } + return urlx, err +} + func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames []string, err error) { dnsData, err := hp.Dialer.GetDNSData(hostname) if err != nil {