mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 05:20:29 +03:00
include dsl tag for all json keys (#812)
* include dsl tag for all json keys * removed dsl tag and built list from json tag * asn interface changed * refactoring expression engine - Using mapstructure for uniform mapping - Flat key:value in expressions/variables list * adding fields to ignore Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
This commit is contained in:
Sami
GitHub
parent
5a806699df
commit
12fee4f3ee
@ -486,7 +486,7 @@ func main() {
|
||||
|
||||
httpxRunner, err := runner.New(&options)
|
||||
if err != nil {
|
||||
log.Fatal()
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer httpxRunner.Close()
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@ var CSPHeaders = []string{
|
||||
|
||||
// CSPData contains the Content-Security-Policy domain list
|
||||
type CSPData struct {
|
||||
Domains []string `json:"domains,omitempty" dsl:"domains"`
|
||||
Domains []string `json:"domains,omitempty"`
|
||||
}
|
||||
|
||||
// CSPGrab fills the CSPData
|
||||
|
||||
13
go.mod
13
go.mod
@ -10,7 +10,7 @@ require (
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/microcosm-cc/bluemonday v1.0.20
|
||||
github.com/microcosm-cc/bluemonday v1.0.21
|
||||
github.com/miekg/dns v1.1.50 // indirect
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1
|
||||
@ -37,8 +37,8 @@ require (
|
||||
github.com/smartystreets/assertions v1.0.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
golang.org/x/net v0.0.0-20220927171203-f486391704dc
|
||||
golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect
|
||||
golang.org/x/net v0.1.0
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
golang.org/x/text v0.4.0
|
||||
)
|
||||
|
||||
@ -47,12 +47,14 @@ require github.com/spaolacci/murmur3 v1.1.0
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0
|
||||
github.com/ammario/ipisp/v2 v2.0.0
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3
|
||||
github.com/hdm/jarm-go v0.0.7
|
||||
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/projectdiscovery/dsl v0.0.3
|
||||
github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655
|
||||
github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa
|
||||
github.com/projectdiscovery/tlsx v0.0.8
|
||||
github.com/projectdiscovery/tlsx v0.0.9
|
||||
go.uber.org/multierr v1.8.0
|
||||
golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0
|
||||
)
|
||||
@ -97,8 +99,9 @@ require (
|
||||
github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect
|
||||
github.com/yl2chen/cidranger v1.0.2 // indirect
|
||||
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
|
||||
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
|
||||
github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect
|
||||
golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
golang.org/x/tools v0.1.12 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
|
||||
23
go.sum
23
go.sum
@ -38,6 +38,8 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3
|
||||
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k=
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
@ -217,8 +219,9 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
|
||||
github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
|
||||
github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
|
||||
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
|
||||
@ -306,6 +309,8 @@ github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJ
|
||||
github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw=
|
||||
github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY=
|
||||
github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU=
|
||||
github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 h1:6FTqSAohPHeBnzTypby8Gk88zv03PfrRMKQtRCFELrU=
|
||||
github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655/go.mod h1:NIgEkt1yYqHWYi1Razavwa1J2hinHfSwD287ZtKBQjU=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
|
||||
@ -332,8 +337,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mo
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU=
|
||||
github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ=
|
||||
github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM=
|
||||
github.com/projectdiscovery/tlsx v0.0.8 h1:+YsPi1sQ6UZUPA6eA8t1RRdbuNzy2cFb9DvvlJFCq9o=
|
||||
github.com/projectdiscovery/tlsx v0.0.8/go.mod h1:TlIEi84Vfhs5MVGbb41UAiLkDcSIF8feca6XhyNU2KY=
|
||||
github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU=
|
||||
github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.65 h1:M1/r75lJI5rezDmfrARQiNNm+Ii9o3jsV+Q+F69Lcso=
|
||||
@ -413,6 +418,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M=
|
||||
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
|
||||
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY=
|
||||
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
|
||||
github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw=
|
||||
github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ=
|
||||
@ -435,8 +441,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig=
|
||||
golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to=
|
||||
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
|
||||
@ -485,8 +491,8 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ=
|
||||
golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -533,8 +539,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
|
||||
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
|
||||
@ -415,7 +415,11 @@ func ParseOptions() *Options {
|
||||
gologger.Fatal().Msgf("%s\n", err)
|
||||
}
|
||||
if options.ListDSLVariable {
|
||||
for _, dsl := range dslVariables() {
|
||||
dslVars, err := dslVariables()
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("%s\n", err)
|
||||
}
|
||||
for _, dsl := range dslVars {
|
||||
gologger.Print().Msg(dsl)
|
||||
}
|
||||
os.Exit(0)
|
||||
|
||||
@ -28,6 +28,7 @@ import (
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
"github.com/projectdiscovery/httpx/common/customextract"
|
||||
"github.com/projectdiscovery/httpx/common/hashes/jarm"
|
||||
"github.com/projectdiscovery/mapsutil"
|
||||
|
||||
"github.com/ammario/ipisp/v2"
|
||||
"github.com/bluele/gcache"
|
||||
@ -42,8 +43,8 @@ import (
|
||||
"github.com/projectdiscovery/stringsutil"
|
||||
"github.com/projectdiscovery/urlutil"
|
||||
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
|
||||
// automatic fd max increase if running as root
|
||||
_ "github.com/projectdiscovery/fdmax/autofdmax"
|
||||
@ -608,29 +609,36 @@ func (r *Runner) RunEnumeration() {
|
||||
|
||||
// apply matchers and filters
|
||||
if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" {
|
||||
respObj := make(map[string]interface{})
|
||||
if err := decodeResponse(resp, respObj); err != nil {
|
||||
rawMap, err := ResultToMap(resp)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not decode response: %s\n", err)
|
||||
} else {
|
||||
if r.options.OutputMatchCondition != "" {
|
||||
res, err := dsl.EvalExpr(r.options.OutputMatchCondition, respObj)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not evaluate match condition: %s\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
} else {
|
||||
if res == false {
|
||||
continue
|
||||
}
|
||||
flatMap := make(map[string]any)
|
||||
mapsutil.Walk(rawMap, func(k string, v any) {
|
||||
flatMap[k] = v
|
||||
})
|
||||
|
||||
if r.options.OutputMatchCondition != "" {
|
||||
res, err := dsl.EvalExpr(r.options.OutputMatchCondition, flatMap)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not evaluate match condition: %s\n", err)
|
||||
continue
|
||||
} else {
|
||||
if res == false {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.options.OutputFilterCondition != "" {
|
||||
res, err := dsl.EvalExpr(r.options.OutputFilterCondition, respObj)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err)
|
||||
} else {
|
||||
if res == true {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if r.options.OutputFilterCondition != "" {
|
||||
res, err := dsl.EvalExpr(r.options.OutputFilterCondition, flatMap)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err)
|
||||
continue
|
||||
} else {
|
||||
if res == true {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1232,7 +1240,7 @@ retry:
|
||||
var serverResponseRaw string
|
||||
var request string
|
||||
var rawResponseHeader string
|
||||
var responseHeader map[string]string
|
||||
var responseHeader map[string]interface{}
|
||||
if scanopts.ResponseInStdout {
|
||||
serverResponseRaw = string(resp.Data)
|
||||
request = string(requestDump)
|
||||
@ -1290,7 +1298,7 @@ retry:
|
||||
ip = hp.Dialer.GetDialedIP(URL.Host)
|
||||
}
|
||||
|
||||
var asnResponse interface{ String() string }
|
||||
var asnResponse *AsnResponse
|
||||
if r.options.Asn {
|
||||
lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip))
|
||||
if err != nil {
|
||||
@ -1298,7 +1306,7 @@ retry:
|
||||
}
|
||||
if lookupResult != nil {
|
||||
lookupResult.ISPName = stringsutil.TrimSuffixAny(strings.ReplaceAll(lookupResult.ISPName, lookupResult.Country, ""), ", ", " ")
|
||||
asnResponse = AsnResponse{
|
||||
asnResponse = &AsnResponse{
|
||||
AsNumber: lookupResult.ASN.String(),
|
||||
AsName: lookupResult.ISPName,
|
||||
AsCountry: lookupResult.Country,
|
||||
@ -1406,7 +1414,7 @@ retry:
|
||||
if r.options.JSONOutput && len(scanopts.Hashes) == 0 {
|
||||
scanopts.Hashes = "md5,mmh3,sha256,simhash"
|
||||
}
|
||||
var hashesMap = map[string]string{}
|
||||
hashesMap := make(map[string]interface{})
|
||||
if scanopts.Hashes != "" {
|
||||
hs := strings.Split(scanopts.Hashes, ",")
|
||||
builder.WriteString(" [")
|
||||
@ -1706,8 +1714,8 @@ func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames []string, err err
|
||||
return
|
||||
}
|
||||
|
||||
func normalizeHeaders(headers map[string][]string) map[string]string {
|
||||
normalized := make(map[string]string, len(headers))
|
||||
func normalizeHeaders(headers map[string][]string) map[string]interface{} {
|
||||
normalized := make(map[string]interface{}, len(headers))
|
||||
for k, v := range headers {
|
||||
normalized[strings.ReplaceAll(strings.ToLower(k), "-", "_")] = strings.Join(v, ", ")
|
||||
}
|
||||
|
||||
144
runner/types.go
144
runner/types.go
@ -2,20 +2,21 @@ package runner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bxcodec/faker/v4"
|
||||
"github.com/bxcodec/faker/v4/pkg/options"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/projectdiscovery/httpx/common/httpx"
|
||||
"github.com/projectdiscovery/mapsutil"
|
||||
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
||||
)
|
||||
|
||||
type AsnResponse struct {
|
||||
AsNumber string `json:"as-number" csv:"as-number"`
|
||||
AsName string `json:"as-name" csv:"as-name"`
|
||||
AsCountry string `json:"as-country" csv:"as-country"`
|
||||
AsRange string `json:"as-range" csv:"as-range"`
|
||||
AsNumber string `json:"as_number" csv:"as_number"`
|
||||
AsName string `json:"as_name" csv:"as_name"`
|
||||
AsCountry string `json:"as_country" csv:"as_country"`
|
||||
AsRange string `json:"as_range" csv:"as_range"`
|
||||
}
|
||||
|
||||
func (o AsnResponse) String() string {
|
||||
@ -24,89 +25,86 @@ func (o AsnResponse) String() string {
|
||||
|
||||
// Result of a scan
|
||||
type Result struct {
|
||||
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" dsl:"timestamp"`
|
||||
ASN interface{} `json:"asn,omitempty" csv:"asn"`
|
||||
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
|
||||
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
|
||||
err error
|
||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" dsl:"csp"`
|
||||
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
||||
Hashes map[string]string `json:"hash,omitempty" csv:"hash" dsl:"hash,header_md5,header_mmh3,header_sha256,header_simhash,body_md5,body_mmh3,body_sha256,body_simhash"`
|
||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
|
||||
Port string `json:"port,omitempty" csv:"port" dsl:"port"`
|
||||
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
|
||||
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
|
||||
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
|
||||
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
|
||||
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
|
||||
Port string `json:"port,omitempty" csv:"port"`
|
||||
raw string
|
||||
URL string `json:"url,omitempty" csv:"url" dsl:"url"`
|
||||
Input string `json:"input,omitempty" csv:"input" dsl:"input"`
|
||||
Location string `json:"location,omitempty" csv:"location" dsl:"location"`
|
||||
URL string `json:"url,omitempty" csv:"url"`
|
||||
Input string `json:"input,omitempty" csv:"input"`
|
||||
Location string `json:"location,omitempty" csv:"location"`
|
||||
Title string `json:"title,omitempty" csv:"title"`
|
||||
str string
|
||||
Scheme string `json:"scheme,omitempty" csv:"scheme" dsl:"scheme"`
|
||||
Error string `json:"error,omitempty" csv:"error"`
|
||||
WebServer string `json:"webserver,omitempty" csv:"webserver" dsl:"webserver"`
|
||||
ResponseBody string `json:"body,omitempty" csv:"body" dsl:"body"`
|
||||
ContentType string `json:"content_type,omitempty" csv:"content_type" dsl:"content_type"`
|
||||
Method string `json:"method,omitempty" csv:"method" dsl:"method"`
|
||||
Host string `json:"host,omitempty" csv:"host" dsl:"host"`
|
||||
Path string `json:"path,omitempty" csv:"path" dsl:"path"`
|
||||
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"`
|
||||
FinalURL string `json:"final_url,omitempty" csv:"final_url"`
|
||||
ResponseHeader map[string]string `json:"header,omitempty" csv:"header" dsl:"header"`
|
||||
RawHeader string `json:"raw_header,omitempty" csv:"raw_header"`
|
||||
Request string `json:"request,omitempty" csv:"request" dsl:"request"`
|
||||
ResponseTime string `json:"time,omitempty" csv:"time" dsl:"time"`
|
||||
Jarm string `json:"jarm,omitempty" csv:"jarm"`
|
||||
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"`
|
||||
A []string `json:"a,omitempty" csv:"a" dsl:"a"`
|
||||
CNAMEs []string `json:"cname,omitempty" csv:"cname"`
|
||||
Technologies []string `json:"tech,omitempty" csv:"tech"`
|
||||
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"`
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
Words int `json:"words" csv:"words" dsl:"words"`
|
||||
Lines int `json:"lines" csv:"lines" dsl:"lines"`
|
||||
StatusCode int `json:"status_code,omitempty" csv:"status_code" dsl:"status_code"`
|
||||
ContentLength int `json:"content_length,omitempty" csv:"content_length" dsl:"content_length"`
|
||||
Failed bool `json:"failed" csv:"failed" dsl:"failed"`
|
||||
VHost bool `json:"vhost,omitempty" csv:"vhost"`
|
||||
WebSocket bool `json:"websocket,omitempty" csv:"websocket"`
|
||||
CDN bool `json:"cdn,omitempty" csv:"cdn"`
|
||||
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
|
||||
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
|
||||
Scheme string `json:"scheme,omitempty" csv:"scheme"`
|
||||
Error string `json:"error,omitempty" csv:"error"`
|
||||
WebServer string `json:"webserver,omitempty" csv:"webserver"`
|
||||
ResponseBody string `json:"body,omitempty" csv:"body"`
|
||||
ContentType string `json:"content_type,omitempty" csv:"content_type"`
|
||||
Method string `json:"method,omitempty" csv:"method"`
|
||||
Host string `json:"host,omitempty" csv:"host"`
|
||||
Path string `json:"path,omitempty" csv:"path"`
|
||||
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"`
|
||||
FinalURL string `json:"final_url,omitempty" csv:"final_url"`
|
||||
ResponseHeader map[string]interface{} `json:"header,omitempty" csv:"header"`
|
||||
RawHeader string `json:"raw_header,omitempty" csv:"raw_header"`
|
||||
Request string `json:"request,omitempty" csv:"request"`
|
||||
ResponseTime string `json:"time,omitempty" csv:"time"`
|
||||
Jarm string `json:"jarm,omitempty" csv:"jarm"`
|
||||
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"`
|
||||
A []string `json:"a,omitempty" csv:"a"`
|
||||
CNAMEs []string `json:"cname,omitempty" csv:"cname"`
|
||||
Technologies []string `json:"tech,omitempty" csv:"tech"`
|
||||
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"`
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
StatusCode int `json:"status_code,omitempty" csv:"status_code"`
|
||||
ContentLength int `json:"content_length,omitempty" csv:"content_length"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
VHost bool `json:"vhost,omitempty" csv:"vhost"`
|
||||
WebSocket bool `json:"websocket,omitempty" csv:"websocket"`
|
||||
CDN bool `json:"cdn,omitempty" csv:"cdn"`
|
||||
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
|
||||
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
|
||||
}
|
||||
|
||||
// function to get dsl variables from result struct
|
||||
func dslVariables() []string {
|
||||
dslVarList := []string{}
|
||||
t := reflect.TypeOf(Result{})
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
field := t.Field(i)
|
||||
if field.Tag.Get("dsl") != "" {
|
||||
dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...)
|
||||
}
|
||||
|
||||
if field.Name == "CSPData" {
|
||||
u := reflect.TypeOf(httpx.CSPData{})
|
||||
for j := 0; j < u.NumField(); j++ {
|
||||
field := u.Field(j)
|
||||
if field.Tag.Get("dsl") != "" {
|
||||
dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...)
|
||||
}
|
||||
}
|
||||
}
|
||||
func dslVariables() ([]string, error) {
|
||||
fakeResult := Result{}
|
||||
fieldsToIgnore := []string{"Hashes", "ResponseHeader"}
|
||||
if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dslVarList
|
||||
m, err := ResultToMap(fakeResult)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vars := []string{"header_md5", "header_mmh3", "header_sha256", "header_simhash", "body_md5", "body_mmh3", "body_sha256", "body_simhash"}
|
||||
mapsutil.Walk(m, func(k string, v any) {
|
||||
vars = append(vars, k)
|
||||
})
|
||||
|
||||
return vars, nil
|
||||
}
|
||||
|
||||
func decodeResponse(resp Result, respObj interface{}) error {
|
||||
func ResultToMap(resp Result) (map[string]any, error) {
|
||||
m := make(map[string]any)
|
||||
config := &mapstructure.DecoderConfig{
|
||||
TagName: "dsl",
|
||||
Result: &respObj,
|
||||
TagName: "json",
|
||||
Result: &m,
|
||||
}
|
||||
decoder, err := mapstructure.NewDecoder(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating decoder: %v", err)
|
||||
return nil, fmt.Errorf("error creating decoder: %v", err)
|
||||
}
|
||||
err = decoder.Decode(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error decoding: %v", err)
|
||||
return nil, fmt.Errorf("error decoding: %v", err)
|
||||
}
|
||||
return nil
|
||||
return m, nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user