diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7004d49..c253ab9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -31,6 +31,10 @@ jobs: run: go test ./... working-directory: . + - name: Running example + run: go run . + working-directory: examples/ + - name: Integration Tests Linux, macOS if: runner.os == 'Linux' || runner.os == 'macOS' env: diff --git a/Dockerfile b/Dockerfile index e3c78de..0935236 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN go build ./cmd/httpx FROM alpine:3.17.3 RUN apk -U upgrade --no-cache \ - && apk add --no-cache bind-tools ca-certificates + && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/httpx /usr/local/bin/ ENTRYPOINT ["httpx"] \ No newline at end of file diff --git a/README.md b/README.md index 9dd2503..89ab22c 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,19 @@ PROBES: -cdn display cdn in use -probe display probe status +HEADLESS: + -ss, -screenshot enable saving screenshot of the page using headless browser + -system-chrome enable using local installed chrome for screenshot + MATCHERS: -mc, -match-code string match response with specified status code (-mc 200,302) -ml, -match-length string match response with specified content length (-ml 100,102) -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (case insensitive) (-ms admin) + -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) + -mcdn, -match-cdn string[] match host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition @@ -133,7 +137,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition @@ -154,6 +158,10 @@ MISCELLANEOUS: -vhost probe and display server supporting VHOST -ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter +UPDATE: + -up, -update update httpx to latest version + -duc, -disable-update-check disable automatic httpx update check + OUTPUT: -o, -output string file to write output results -sr, -store-response store http response to output directory @@ -184,7 +192,7 @@ CONFIGURATIONS: -body string post body to include in http request -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443 -ztls use ztls library with autofallback to standard one for tls13 -no-decode avoid decoding body @@ -472,55 +480,75 @@ https://docs.hackerone.com https://support.hackerone.com ``` -### Using `httpx` as a library -`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: +### Screenshot -```go -package main +Latest addition to the project, the addition of the `-screenshot` option in httpx, a powerful new feature that allows users to take screenshots of target URLs, pages, or endpoints along with the rendered DOM. This functionality enables the **visual content discovery process**, providing a comprehensive view of the target's visual appearance. -import ( - "log" +Rendered DOM body is also included in json line output when `-screenshot` option is used with `-json` option. - "github.com/projectdiscovery/goflags" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" - "github.com/projectdiscovery/httpx/runner" -) +#### 🚩 Usage -func main() { - gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional) +To use the screenshot feature, simply add the `-screenshot` flag to your httpx command: - options := runner.Options{ - Methods: "GET", - InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"}, - //InputFile: "./targetDomains.txt", // path to file containing the target domains list - } - - if err := options.ValidateOptions(); err != nil { - log.Fatal(err) - } - - httpxRunner, err := runner.New(&options) - if err != nil { - log.Fatal(err) - } - defer httpxRunner.Close() - - httpxRunner.RunEnumeration() -} +```console +httpx -screenshot -u https://example.com ``` +🎯 Domain, Subdomain, and Path Support +The `-screenshot` option is versatile and can be used to capture screenshots for domains, subdomains, and even specific paths when used in conjunction with the `-path` option: + +```console +httpx -screenshot -u example.com +httpx -screenshot -u https://example.com/login +httpx -screenshot -path fuzz_path.txt -u https://example.com +``` + +Using with other tools: + +```console +subfinder -d example.com | httpx -screenshot +``` + +#### 🌐 System Chrome + +By default, httpx will use the go-rod library to install and manage Chrome for taking screenshots. However, if you prefer to use your locally installed system Chrome, add the `-system-chrome` flag: + +```console +httpx -screenshot -system-chrome -u https://example.com +``` + +#### 📁 Output Directory + +Screenshots are stored in the output/screenshot directory by default. To specify a custom output directory, use the `-srd` option: + +```console +httpx -screenshot -srd /path/to/custom/directory -u https://example.com +``` + +#### ⏳ Performance Considerations + +Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option. + +### Using `httpx` as a library +`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder # Notes -- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable. -- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results +- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. +- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` -- The following flags should be used for specific use cases instead of running them as default with other probes: - * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path` -- When using the `-json` flag, all the default probe results are included in the JSON output. - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) -- Invalid custom resolvers/files are ignored. +- The following flags should be used for specific use cases instead of running them as default with other probes: + - `-ports` + - `-path` + - `-vhost` + - `-screenshot` + - `-csp-probe` + - `-tls-probe` + - `-favicon` + - `-http2` + - `-pipeline` + # Acknowledgement diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 8efe89f..994587a 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -80,3 +80,10 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m return } + +func AbsPathOrDefault(p string) string { + if absPath, err := filepath.Abs(p); err == nil { + return absPath + } + return p +} diff --git a/examples/example.go b/examples/example.go new file mode 100644 index 0000000..355b9ab --- /dev/null +++ b/examples/example.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/httpx/runner" +) + +func main() { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional) + + options := runner.Options{ + Methods: "GET", + InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"}, + //InputFile: "./targetDomains.txt", // path to file containing the target domains list + } + + if err := options.ValidateOptions(); err != nil { + log.Fatal(err) + } + + httpxRunner, err := runner.New(&options) + if err != nil { + log.Fatal(err) + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() +} diff --git a/go.mod b/go.mod index a1994fa..feed8b8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.19 require ( github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.2.0 github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf @@ -37,8 +36,10 @@ require ( require github.com/spaolacci/murmur3 v1.1.0 require ( + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/PuerkitoBio/goquery v1.8.1 github.com/bxcodec/faker/v4 v4.0.0-beta.3 + github.com/go-rod/rod v0.112.8 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 @@ -55,6 +56,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect + cloud.google.com/go/compute/metadata v0.2.0 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect @@ -86,6 +88,7 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fullstorydev/grpcurl v1.8.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.5.0 // indirect @@ -107,6 +110,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -121,6 +125,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.0-20230328171319-f24b18d05b64 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/projectdiscovery/networkpolicy v0.0.4 // indirect @@ -133,6 +138,8 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/sashabaranov/go-openai v1.8.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.3 // indirect + github.com/shoenig/go-m1cpu v0.1.4 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/cobra v1.1.3 // indirect @@ -146,6 +153,8 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect @@ -154,8 +163,12 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/gson v0.7.3 // indirect + github.com/ysmood/leakless v0.8.0 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect diff --git a/go.sum b/go.sum index 4263a0b..dec9f84 100644 --- a/go.sum +++ b/go.sum @@ -24,7 +24,6 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -33,6 +32,7 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.2.0 h1:nBbNSZyDpkNlo3DepaaLKVuO7ClyifSAmNloSCZrHnQ= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -76,6 +76,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -144,8 +146,6 @@ github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMf github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= -github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= -github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= @@ -282,7 +282,11 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-rod/rod v0.112.8 h1:lYFnHv/lFyjW/Ye0IhyKLeHw/zfhHbSTqawoCi2z/nI= +github.com/go-rod/rod v0.112.8/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -359,6 +363,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -542,6 +547,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -683,6 +690,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/asnmap v1.0.2 h1:2+8tqzJeFVpJS7u27YH7kMK7edDAr7OsmSxs92aWFNc= github.com/projectdiscovery/asnmap v1.0.2/go.mod h1:64YfriVxyRQvqc+1iPMHMf+i/of2jr+Qx7geCIm4ZsU= github.com/projectdiscovery/blackrock v0.0.0-20230328171319-f24b18d05b64 h1:3oOT3yauepbOp84gz67JQLu/y9uyyIeGakpi+rYw1Cc= @@ -802,6 +811,12 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= +github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= +github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVbozs= +github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= +github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= +github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -890,6 +905,10 @@ github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLD github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -925,6 +944,16 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/got v0.32.0 h1:aAHdQgfgMb/lo4v+OekM+SSqEJYFI035h5YYvLXsVyU= +github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= +github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak= +github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -936,6 +965,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= @@ -1176,6 +1207,7 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1206,6 +1238,7 @@ golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1225,7 +1258,9 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/runner/headless.go b/runner/headless.go new file mode 100644 index 0000000..cdf082e --- /dev/null +++ b/runner/headless.go @@ -0,0 +1,129 @@ +package runner + +import ( + "fmt" + "os" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" + "github.com/pkg/errors" + fileutil "github.com/projectdiscovery/utils/file" + osutils "github.com/projectdiscovery/utils/os" + processutil "github.com/projectdiscovery/utils/process" +) + +// MustDisableSandbox determines if the current os and user needs sandbox mode disabled +func MustDisableSandbox() bool { + // linux with root user needs "--no-sandbox" option + // https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209 + return osutils.IsLinux() && os.Geteuid() == 0 +} + +type Browser struct { + tempDir string + engine *rod.Browser + pids map[int32]struct{} +} + +func NewBrowser(proxy string, useLocal bool) (*Browser, error) { + dataStore, err := os.MkdirTemp("", "nuclei-*") + if err != nil { + return nil, errors.Wrap(err, "could not create temporary directory") + } + + pids := processutil.FindProcesses(processutil.IsChromeProcess) + + chromeLauncher := launcher.New(). + Leakless(false). + Set("disable-gpu", "true"). + Set("ignore-certificate-errors", "true"). + Set("ignore-certificate-errors", "1"). + Set("disable-crash-reporter", "true"). + Set("disable-notifications", "true"). + Set("hide-scrollbars", "true"). + Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). + Set("mute-audio", "true"). + Set("incognito", "true"). + Delete("use-mock-keychain"). + Headless(true). + UserDataDir(dataStore) + + if MustDisableSandbox() { + chromeLauncher = chromeLauncher.NoSandbox(true) + } + + executablePath, err := os.Executable() + if err != nil { + return nil, err + } + + // if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome + useMusl, _ := fileutil.UseMusl(executablePath) + if useLocal || useMusl { + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errors.New("the chrome browser is not installed") + } + } + + if proxy != "" { + chromeLauncher = chromeLauncher.Proxy(proxy) + } + launcherURL, err := chromeLauncher.Launch() + if err != nil { + return nil, err + } + + browser := rod.New().ControlURL(launcherURL) + if browserErr := browser.Connect(); browserErr != nil { + return nil, browserErr + } + + engine := &Browser{ + tempDir: dataStore, + engine: browser, + pids: pids, + } + return engine, nil +} + +func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration) ([]byte, string, error) { + page, err := b.engine.Page(proto.TargetCreateTarget{}) + if err != nil { + return nil, "", err + } + page = page.Timeout(timeout) + defer page.Close() + + if err := page.Navigate(url); err != nil { + return nil, "", err + } + + page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)() + + if err := page.WaitLoad(); err != nil { + return nil, "", err + } + _ = page.WaitIdle(1 * time.Second) + + screenshot, err := page.Screenshot(true, &proto.PageCaptureScreenshot{}) + if err != nil { + return nil, "", err + } + + body, err := page.HTML() + if err != nil { + return screenshot, "", err + } + + return screenshot, body, nil +} + +func (b *Browser) Close() { + b.engine.Close() + os.RemoveAll(b.tempDir) + processutil.CloseProcesses(processutil.IsChromeProcess, b.pids) +} diff --git a/runner/options.go b/runner/options.go index bbaa8f2..b3069a4 100644 --- a/runner/options.go +++ b/runner/options.go @@ -85,6 +85,8 @@ type scanOptions struct { OutputLinesCount bool OutputWordsCount bool Hashes string + Screenshot bool + UseInstalledChrome bool } func (s *scanOptions) Clone() *scanOptions { @@ -131,6 +133,8 @@ func (s *scanOptions) Clone() *scanOptions { OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, Hashes: s.Hashes, + Screenshot: s.Screenshot, + UseInstalledChrome: s.UseInstalledChrome, } } @@ -263,6 +267,8 @@ type Options struct { OnResult OnResultCallback DisableUpdateCheck bool NoDecode bool + Screenshot bool + UseInstalledChrome bool } // ParseOptions parses the command line options for application @@ -301,6 +307,11 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), ) + flagSet.CreateGroup("headless", "Headless", + flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"), + flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"), + ) + flagSet.CreateGroup("matchers", "Matchers", flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"), flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), @@ -412,7 +423,7 @@ func ParseOptions() *Options { flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"), flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), - flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"), + flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), @@ -557,6 +568,10 @@ func (options *Options) ValidateOptions() error { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) } + if options.Screenshot && !options.StoreResponse { + gologger.Debug().Msgf("automatically enabling store response") + options.StoreResponse = true + } if options.StoreResponse && options.StoreResponseDir == "" { gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory) options.StoreResponseDir = DefaultOutputDirectory @@ -565,6 +580,7 @@ func (options *Options) ValidateOptions() error { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } + if options.Hashes != "" { for _, hashType := range strings.Split(options.Hashes, ",") { if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { diff --git a/runner/runner.go b/runner/runner.go index 201a7f4..2760ca1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -32,7 +32,7 @@ import ( "github.com/projectdiscovery/mapcidr/asn" errorutil "github.com/projectdiscovery/utils/errors" - "github.com/bluele/gcache" + "github.com/Mzack9999/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -75,7 +75,8 @@ type Runner struct { hm *hybrid.HybridMap stats clistats.StatisticsClient ratelimiter ratelimit.Limiter - HostErrorsCache gcache.Cache + HostErrorsCache gcache.Cache[string, int] + browser *Browser } // New creates a new client for running enumeration process. @@ -91,7 +92,8 @@ func New(options *Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create wappalyzer client") } if options.StoreResponseDir != "" { - os.RemoveAll(filepath.Join(options.StoreResponseDir, "index.txt")) + os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) + os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) } dialerOpts := fastdialer.DefaultOptions dialerOpts.WithDialerHistory = true @@ -242,6 +244,15 @@ func New(options *Options) (*Runner, error) { scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead scanopts.extractRegexps = make(map[string]*regexp.Regexp) + if options.Screenshot { + browser, err := NewBrowser(options.HTTPProxy, options.UseInstalledChrome) + if err != nil { + return nil, err + } + runner.browser = browser + } + scanopts.Screenshot = options.Screenshot + scanopts.UseInstalledChrome = options.UseInstalledChrome if options.OutputExtractRegexs != nil { for _, regex := range options.OutputExtractRegexs { @@ -306,7 +317,7 @@ func New(options *Options) (*Runner, error) { } if options.HostMaxErrors >= 0 { - gc := gcache.New(1000). + gc := gcache.New[string, int](1000). ARC(). Build() runner.HostErrorsCache = gc @@ -554,15 +565,29 @@ func (r *Runner) Close() { if r.options.HostMaxErrors >= 0 { r.HostErrorsCache.Purge() } + if r.options.Screenshot { + r.browser.Close() + } } // RunEnumeration on targets for httpx client func (r *Runner) RunEnumeration() { - // Try to create output folder if it doesn't exist + // Try to create output folders if it doesn't exist if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) { + // main folder if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", r.options.StoreResponseDir, err) } + // response folder + responseFolder := filepath.Join(r.options.StoreResponseDir, "response") + if err := os.MkdirAll(responseFolder, os.ModePerm); err != nil { + gologger.Fatal().Msgf("Could not create output response directory '%s': %s\n", r.options.StoreResponseDir, err) + } + // screenshot folder + screenshotFolder := filepath.Join(r.options.StoreResponseDir, "screenshot") + if err := os.MkdirAll(screenshotFolder, os.ModePerm); err != nil { + gologger.Fatal().Msgf("Could not create output screenshot directory '%s': %s\n", r.options.StoreResponseDir, err) + } } r.prepareInputPaths() @@ -590,7 +615,7 @@ func (r *Runner) RunEnumeration() { go func(output chan Result) { defer wgoutput.Done() - var f, indexFile *os.File + var f, indexFile, indexScreenshotFile *os.File if r.options.Output != "" { var err error @@ -626,7 +651,7 @@ func (r *Runner) RunEnumeration() { } if r.options.StoreResponseDir != "" { var err error - indexPath := filepath.Join(r.options.StoreResponseDir, "index.txt") + indexPath := filepath.Join(r.options.StoreResponseDir, "response", "index.txt") if r.options.Resume { indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) } else { @@ -637,6 +662,19 @@ func (r *Runner) RunEnumeration() { } defer indexFile.Close() //nolint } + if r.options.Screenshot { + var err error + indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt") + if r.options.Resume { + indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + indexScreenshotFile, err = os.Create(indexScreenshotPath) + } + if err != nil { + gologger.Fatal().Msgf("Could not open/create index screenshot file '%s': %s\n", r.options.Output, err) + } + defer indexScreenshotFile.Close() //nolint + } for resp := range output { if resp.err != nil { @@ -654,6 +692,10 @@ func (r *Runner) RunEnumeration() { indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) _, _ = indexFile.WriteString(indexData) } + if indexScreenshotFile != nil { + indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + _, _ = indexScreenshotFile.WriteString(indexData) + } // apply matchers and filters if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { @@ -1039,7 +1081,7 @@ retry: hostPort := net.JoinHostPort(URL.Host, URL.Port()) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) - if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { + if err == nil && numberOfErrors >= r.options.HostMaxErrors { return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")} } } @@ -1199,10 +1241,10 @@ retry: // mark the host:port as failed to avoid further checks if r.options.HostMaxErrors >= 0 { errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort) - if err != nil || errorCount == nil { + if err != nil || errorCount == 0 { _ = r.HostErrorsCache.Set(hostPort, 1) - } else if errorCount != nil { - _ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1) + } else if errorCount > 0 { + _ = r.HostErrorsCache.Set(hostPort, errorCount+1) } } @@ -1573,16 +1615,21 @@ retry: } // store responses or chain in directory - var responsePath string + domainFile := URL.EscapedString() + hash := hashes.Sha1([]byte(domainFile)) + domainResponseFile := fmt.Sprintf("%s.txt", hash) + screenshotResponseFile := fmt.Sprintf("%s.png", hash) + hostFilename := strings.ReplaceAll(URL.Host, ":", "_") + domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response") + domainScreenshotBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "screenshot") + responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) + screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) + + var responsePath, screenshotPath string + // store response if scanopts.StoreResponse || scanopts.StoreChain { + responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) // URL.EscapedString returns that can be used as filename - domainFile := URL.EscapedString() - hash := hashes.Sha1([]byte(domainFile)) - domainFile = fmt.Sprintf("%s.txt", hash) - host := strings.ReplaceAll(URL.Host, ":", "_") - domainBaseDir := filepath.Join(scanopts.StoreResponseDirectory, host) - // store response - responsePath = filepath.Join(domainBaseDir, domainFile) respRaw := resp.Raw reqRaw := requestDump if len(respRaw) > scanopts.MaxResponseBodySizeToSave { @@ -1590,14 +1637,12 @@ retry: } data := append([]byte(fullURL), append([]byte("\n\n"), reqRaw...)...) data = append(data, append([]byte("\n"), respRaw...)...) - _ = fileutil.CreateFolder(domainBaseDir) + _ = fileutil.CreateFolder(responseBaseDir) writeErr := os.WriteFile(responsePath, data, 0644) if writeErr != nil { gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { - domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") - responsePath = filepath.Join(domainBaseDir, domainFile) writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) @@ -1631,6 +1676,26 @@ retry: chainItems = append(chainItems, resp.GetChainAsSlice()...) } + // screenshot + var ( + screenshotBytes []byte + headlessBody string + ) + if scanopts.Screenshot { + screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile)) + var err error + screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, r.hp.Options.Timeout) + if err != nil { + gologger.Warning().Msgf("Could not take screenshot '%s': %s", fullURL, err) + } else { + _ = fileutil.CreateFolder(screenshotBaseDir) + err := os.WriteFile(screenshotPath, screenshotBytes, 0644) + if err != nil { + gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err) + } + } + } + result := Result{ Timestamp: time.Now(), Request: request, @@ -1677,6 +1742,9 @@ retry: ASN: asnResponse, ExtractRegex: extractRegex, StoredResponsePath: responsePath, + ScreenshotBytes: screenshotBytes, + ScreenshotPath: screenshotPath, + HeadlessBody: headlessBody, } if r.options.OnResult != nil { r.options.OnResult(result) diff --git a/runner/types.go b/runner/types.go index 3cf7414..d04b543 100644 --- a/runner/types.go +++ b/runner/types.go @@ -72,7 +72,10 @@ type Result struct { CDN bool `json:"cdn,omitempty" csv:"cdn"` HTTP2 bool `json:"http2,omitempty" csv:"http2"` Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"` + ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"` StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"` + ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"` } // function to get dsl variables from result struct