mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 13:30:22 +03:00
Adding Screenshot support (#1097)
* Adding Screenshot support * adding headless body * moving example into compilable file * updating docs * enabling store response with screenshot * fixing output dir path * fixing mod * using native leakless * splitting response+screenshot folders * better handling json output * readme update * rel => abs path * Added chromium into docker * “Wipe them out. All of them.” – Darth Sidious * utils bump * go mod tidy --------- Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Mzack9999
GitHub
parent
1b12a94988
commit
00d0977963
4
.github/workflows/build-test.yml
vendored
4
.github/workflows/build-test.yml
vendored
@ -31,6 +31,10 @@ jobs:
|
||||
run: go test ./...
|
||||
working-directory: .
|
||||
|
||||
- name: Running example
|
||||
run: go run .
|
||||
working-directory: examples/
|
||||
|
||||
- name: Integration Tests Linux, macOS
|
||||
if: runner.os == 'Linux' || runner.os == 'macOS'
|
||||
env:
|
||||
|
||||
@ -9,7 +9,7 @@ RUN go build ./cmd/httpx
|
||||
|
||||
FROM alpine:3.17.3
|
||||
RUN apk -U upgrade --no-cache \
|
||||
&& apk add --no-cache bind-tools ca-certificates
|
||||
&& apk add --no-cache bind-tools ca-certificates chromium
|
||||
COPY --from=builder /app/httpx /usr/local/bin/
|
||||
|
||||
ENTRYPOINT ["httpx"]
|
||||
110
README.md
110
README.md
@ -109,15 +109,19 @@ PROBES:
|
||||
-cdn display cdn in use
|
||||
-probe display probe status
|
||||
|
||||
HEADLESS:
|
||||
-ss, -screenshot enable saving screenshot of the page using headless browser
|
||||
-system-chrome enable using local installed chrome for screenshot
|
||||
|
||||
MATCHERS:
|
||||
-mc, -match-code string match response with specified status code (-mc 200,302)
|
||||
-ml, -match-length string match response with specified content length (-ml 100,102)
|
||||
-mlc, -match-line-count string match response body with specified line count (-mlc 423,532)
|
||||
-mwc, -match-word-count string match response body with specified word count (-mwc 43,55)
|
||||
-mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000)
|
||||
-ms, -match-string string match response with specified string (case insensitive) (-ms admin)
|
||||
-ms, -match-string string match response with specified string (-ms admin)
|
||||
-mr, -match-regex string match response with specified regex (-mr admin)
|
||||
-mcdn, -match-cdn string[] match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
|
||||
-mcdn, -match-cdn string[] match host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb)
|
||||
-mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1')
|
||||
-mdc, -match-condition string match response with dsl expression condition
|
||||
|
||||
@ -133,7 +137,7 @@ FILTERS:
|
||||
-ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000)
|
||||
-fs, -filter-string string filter response with specified string (-fs admin)
|
||||
-fe, -filter-regex string filter response with specified regex (-fe admin)
|
||||
-fcdn, -filter-cdn string[] filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
|
||||
-fcdn, -filter-cdn string[] filter host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb)
|
||||
-frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1')
|
||||
-fdc, -filter-condition string filter response with dsl expression condition
|
||||
|
||||
@ -154,6 +158,10 @@ MISCELLANEOUS:
|
||||
-vhost probe and display server supporting VHOST
|
||||
-ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter
|
||||
|
||||
UPDATE:
|
||||
-up, -update update httpx to latest version
|
||||
-duc, -disable-update-check disable automatic httpx update check
|
||||
|
||||
OUTPUT:
|
||||
-o, -output string file to write output results
|
||||
-sr, -store-response store http response to output directory
|
||||
@ -184,7 +192,7 @@ CONFIGURATIONS:
|
||||
-body string post body to include in http request
|
||||
-s, -stream stream mode - start elaborating input targets without sorting
|
||||
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
|
||||
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443
|
||||
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
|
||||
-ztls use ztls library with autofallback to standard one for tls13
|
||||
-no-decode avoid decoding body
|
||||
|
||||
@ -472,55 +480,75 @@ https://docs.hackerone.com
|
||||
https://support.hackerone.com
|
||||
```
|
||||
|
||||
### Using `httpx` as a library
|
||||
`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
|
||||
### Screenshot
|
||||
|
||||
```go
|
||||
package main
|
||||
Latest addition to the project, the addition of the `-screenshot` option in httpx, a powerful new feature that allows users to take screenshots of target URLs, pages, or endpoints along with the rendered DOM. This functionality enables the **visual content discovery process**, providing a comprehensive view of the target's visual appearance.
|
||||
|
||||
import (
|
||||
"log"
|
||||
Rendered DOM body is also included in json line output when `-screenshot` option is used with `-json` option.
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/httpx/runner"
|
||||
)
|
||||
#### 🚩 Usage
|
||||
|
||||
func main() {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
|
||||
To use the screenshot feature, simply add the `-screenshot` flag to your httpx command:
|
||||
|
||||
options := runner.Options{
|
||||
Methods: "GET",
|
||||
InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
|
||||
//InputFile: "./targetDomains.txt", // path to file containing the target domains list
|
||||
}
|
||||
|
||||
if err := options.ValidateOptions(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
httpxRunner, err := runner.New(&options)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer httpxRunner.Close()
|
||||
|
||||
httpxRunner.RunEnumeration()
|
||||
}
|
||||
```console
|
||||
httpx -screenshot -u https://example.com
|
||||
```
|
||||
|
||||
🎯 Domain, Subdomain, and Path Support
|
||||
The `-screenshot` option is versatile and can be used to capture screenshots for domains, subdomains, and even specific paths when used in conjunction with the `-path` option:
|
||||
|
||||
```console
|
||||
httpx -screenshot -u example.com
|
||||
httpx -screenshot -u https://example.com/login
|
||||
httpx -screenshot -path fuzz_path.txt -u https://example.com
|
||||
```
|
||||
|
||||
Using with other tools:
|
||||
|
||||
```console
|
||||
subfinder -d example.com | httpx -screenshot
|
||||
```
|
||||
|
||||
#### 🌐 System Chrome
|
||||
|
||||
By default, httpx will use the go-rod library to install and manage Chrome for taking screenshots. However, if you prefer to use your locally installed system Chrome, add the `-system-chrome` flag:
|
||||
|
||||
```console
|
||||
httpx -screenshot -system-chrome -u https://example.com
|
||||
```
|
||||
|
||||
#### 📁 Output Directory
|
||||
|
||||
Screenshots are stored in the output/screenshot directory by default. To specify a custom output directory, use the `-srd` option:
|
||||
|
||||
```console
|
||||
httpx -screenshot -srd /path/to/custom/directory -u https://example.com
|
||||
```
|
||||
|
||||
#### ⏳ Performance Considerations
|
||||
|
||||
Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option.
|
||||
|
||||
### Using `httpx` as a library
|
||||
`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder
|
||||
|
||||
# Notes
|
||||
|
||||
- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable.
|
||||
- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results
|
||||
- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
|
||||
- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
|
||||
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
|
||||
- The following flags should be used for specific use cases instead of running them as default with other probes:
|
||||
* `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path`
|
||||
- When using the `-json` flag, all the default probe results are included in the JSON output.
|
||||
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
|
||||
- Invalid custom resolvers/files are ignored.
|
||||
- The following flags should be used for specific use cases instead of running them as default with other probes:
|
||||
- `-ports`
|
||||
- `-path`
|
||||
- `-vhost`
|
||||
- `-screenshot`
|
||||
- `-csp-probe`
|
||||
- `-tls-probe`
|
||||
- `-favicon`
|
||||
- `-http2`
|
||||
- `-pipeline`
|
||||
|
||||
|
||||
# Acknowledgement
|
||||
|
||||
|
||||
@ -80,3 +80,10 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func AbsPathOrDefault(p string) string {
|
||||
if absPath, err := filepath.Abs(p); err == nil {
|
||||
return absPath
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
32
examples/example.go
Normal file
32
examples/example.go
Normal file
@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/httpx/runner"
|
||||
)
|
||||
|
||||
func main() {
|
||||
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
|
||||
|
||||
options := runner.Options{
|
||||
Methods: "GET",
|
||||
InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
|
||||
//InputFile: "./targetDomains.txt", // path to file containing the target domains list
|
||||
}
|
||||
|
||||
if err := options.ValidateOptions(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
httpxRunner, err := runner.New(&options)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer httpxRunner.Close()
|
||||
|
||||
httpxRunner.RunEnumeration()
|
||||
}
|
||||
15
go.mod
15
go.mod
@ -4,7 +4,6 @@ go 1.19
|
||||
|
||||
require (
|
||||
github.com/akrylysov/pogreb v0.10.1 // indirect
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/corpix/uarand v0.2.0
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
|
||||
@ -37,8 +36,10 @@ require (
|
||||
require github.com/spaolacci/murmur3 v1.1.0
|
||||
|
||||
require (
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
|
||||
github.com/PuerkitoBio/goquery v1.8.1
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3
|
||||
github.com/go-rod/rod v0.112.8
|
||||
github.com/hdm/jarm-go v0.0.7
|
||||
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
@ -55,6 +56,7 @@ require (
|
||||
|
||||
require (
|
||||
aead.dev/minisign v0.2.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.0 // indirect
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
|
||||
github.com/Masterminds/semver/v3 v3.2.1 // indirect
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
|
||||
@ -86,6 +88,7 @@ require (
|
||||
github.com/fatih/color v1.14.1 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
|
||||
github.com/fullstorydev/grpcurl v1.8.1 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/mock v1.5.0 // indirect
|
||||
@ -107,6 +110,7 @@ require (
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kataras/jwt v0.1.8 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
@ -121,6 +125,7 @@ require (
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/projectdiscovery/blackrock v0.0.0-20230328171319-f24b18d05b64 // indirect
|
||||
github.com/projectdiscovery/freeport v0.0.4 // indirect
|
||||
github.com/projectdiscovery/networkpolicy v0.0.4 // indirect
|
||||
@ -133,6 +138,8 @@ require (
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/sashabaranov/go-openai v1.8.0 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.3 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.4 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/soheilhy/cmux v0.1.5 // indirect
|
||||
github.com/spf13/cobra v1.1.3 // indirect
|
||||
@ -146,6 +153,8 @@ require (
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/rtred v0.1.2 // indirect
|
||||
github.com/tidwall/tinyqueue v0.1.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
|
||||
@ -154,8 +163,12 @@ require (
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
github.com/yl2chen/cidranger v1.0.2 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.8.0 // indirect
|
||||
github.com/yuin/goldmark v1.5.4 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
|
||||
github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect
|
||||
|
||||
41
go.sum
41
go.sum
@ -24,7 +24,6 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP
|
||||
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
|
||||
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
|
||||
cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
|
||||
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
@ -33,6 +32,7 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute/metadata v0.2.0 h1:nBbNSZyDpkNlo3DepaaLKVuO7ClyifSAmNloSCZrHnQ=
|
||||
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
||||
@ -76,6 +76,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
|
||||
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
@ -144,8 +146,6 @@ github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMf
|
||||
github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k=
|
||||
github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho=
|
||||
github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw=
|
||||
@ -282,7 +282,11 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-rod/rod v0.112.8 h1:lYFnHv/lFyjW/Ye0IhyKLeHw/zfhHbSTqawoCi2z/nI=
|
||||
github.com/go-rod/rod v0.112.8/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
@ -359,6 +363,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
@ -542,6 +547,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
@ -683,6 +690,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/projectdiscovery/asnmap v1.0.2 h1:2+8tqzJeFVpJS7u27YH7kMK7edDAr7OsmSxs92aWFNc=
|
||||
github.com/projectdiscovery/asnmap v1.0.2/go.mod h1:64YfriVxyRQvqc+1iPMHMf+i/of2jr+Qx7geCIm4ZsU=
|
||||
github.com/projectdiscovery/blackrock v0.0.0-20230328171319-f24b18d05b64 h1:3oOT3yauepbOp84gz67JQLu/y9uyyIeGakpi+rYw1Cc=
|
||||
@ -802,6 +811,12 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE=
|
||||
github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU=
|
||||
github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVbozs=
|
||||
github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
|
||||
github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=
|
||||
github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
@ -890,6 +905,10 @@ github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLD
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
|
||||
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
|
||||
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
|
||||
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
@ -925,6 +944,16 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
|
||||
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/got v0.32.0 h1:aAHdQgfgMb/lo4v+OekM+SSqEJYFI035h5YYvLXsVyU=
|
||||
github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
|
||||
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@ -936,6 +965,8 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
|
||||
github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os=
|
||||
github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
|
||||
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
|
||||
@ -1176,6 +1207,7 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -1206,6 +1238,7 @@ golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -1225,7 +1258,9 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
|
||||
129
runner/headless.go
Normal file
129
runner/headless.go
Normal file
@ -0,0 +1,129 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/pkg/errors"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
osutils "github.com/projectdiscovery/utils/os"
|
||||
processutil "github.com/projectdiscovery/utils/process"
|
||||
)
|
||||
|
||||
// MustDisableSandbox determines if the current os and user needs sandbox mode disabled
|
||||
func MustDisableSandbox() bool {
|
||||
// linux with root user needs "--no-sandbox" option
|
||||
// https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209
|
||||
return osutils.IsLinux() && os.Geteuid() == 0
|
||||
}
|
||||
|
||||
type Browser struct {
|
||||
tempDir string
|
||||
engine *rod.Browser
|
||||
pids map[int32]struct{}
|
||||
}
|
||||
|
||||
func NewBrowser(proxy string, useLocal bool) (*Browser, error) {
|
||||
dataStore, err := os.MkdirTemp("", "nuclei-*")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create temporary directory")
|
||||
}
|
||||
|
||||
pids := processutil.FindProcesses(processutil.IsChromeProcess)
|
||||
|
||||
chromeLauncher := launcher.New().
|
||||
Leakless(false).
|
||||
Set("disable-gpu", "true").
|
||||
Set("ignore-certificate-errors", "true").
|
||||
Set("ignore-certificate-errors", "1").
|
||||
Set("disable-crash-reporter", "true").
|
||||
Set("disable-notifications", "true").
|
||||
Set("hide-scrollbars", "true").
|
||||
Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
|
||||
Set("mute-audio", "true").
|
||||
Set("incognito", "true").
|
||||
Delete("use-mock-keychain").
|
||||
Headless(true).
|
||||
UserDataDir(dataStore)
|
||||
|
||||
if MustDisableSandbox() {
|
||||
chromeLauncher = chromeLauncher.NoSandbox(true)
|
||||
}
|
||||
|
||||
executablePath, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome
|
||||
useMusl, _ := fileutil.UseMusl(executablePath)
|
||||
if useLocal || useMusl {
|
||||
if chromePath, hasChrome := launcher.LookPath(); hasChrome {
|
||||
chromeLauncher.Bin(chromePath)
|
||||
} else {
|
||||
return nil, errors.New("the chrome browser is not installed")
|
||||
}
|
||||
}
|
||||
|
||||
if proxy != "" {
|
||||
chromeLauncher = chromeLauncher.Proxy(proxy)
|
||||
}
|
||||
launcherURL, err := chromeLauncher.Launch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
browser := rod.New().ControlURL(launcherURL)
|
||||
if browserErr := browser.Connect(); browserErr != nil {
|
||||
return nil, browserErr
|
||||
}
|
||||
|
||||
engine := &Browser{
|
||||
tempDir: dataStore,
|
||||
engine: browser,
|
||||
pids: pids,
|
||||
}
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration) ([]byte, string, error) {
|
||||
page, err := b.engine.Page(proto.TargetCreateTarget{})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
page = page.Timeout(timeout)
|
||||
defer page.Close()
|
||||
|
||||
if err := page.Navigate(url); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
page.Timeout(2 * time.Second).WaitNavigation(proto.PageLifecycleEventNameFirstMeaningfulPaint)()
|
||||
|
||||
if err := page.WaitLoad(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
_ = page.WaitIdle(1 * time.Second)
|
||||
|
||||
screenshot, err := page.Screenshot(true, &proto.PageCaptureScreenshot{})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
body, err := page.HTML()
|
||||
if err != nil {
|
||||
return screenshot, "", err
|
||||
}
|
||||
|
||||
return screenshot, body, nil
|
||||
}
|
||||
|
||||
func (b *Browser) Close() {
|
||||
b.engine.Close()
|
||||
os.RemoveAll(b.tempDir)
|
||||
processutil.CloseProcesses(processutil.IsChromeProcess, b.pids)
|
||||
}
|
||||
@ -85,6 +85,8 @@ type scanOptions struct {
|
||||
OutputLinesCount bool
|
||||
OutputWordsCount bool
|
||||
Hashes string
|
||||
Screenshot bool
|
||||
UseInstalledChrome bool
|
||||
}
|
||||
|
||||
func (s *scanOptions) Clone() *scanOptions {
|
||||
@ -131,6 +133,8 @@ func (s *scanOptions) Clone() *scanOptions {
|
||||
OutputLinesCount: s.OutputLinesCount,
|
||||
OutputWordsCount: s.OutputWordsCount,
|
||||
Hashes: s.Hashes,
|
||||
Screenshot: s.Screenshot,
|
||||
UseInstalledChrome: s.UseInstalledChrome,
|
||||
}
|
||||
}
|
||||
|
||||
@ -263,6 +267,8 @@ type Options struct {
|
||||
OnResult OnResultCallback
|
||||
DisableUpdateCheck bool
|
||||
NoDecode bool
|
||||
Screenshot bool
|
||||
UseInstalledChrome bool
|
||||
}
|
||||
|
||||
// ParseOptions parses the command line options for application
|
||||
@ -301,6 +307,11 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("headless", "Headless",
|
||||
flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"),
|
||||
flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("matchers", "Matchers",
|
||||
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"),
|
||||
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"),
|
||||
@ -412,7 +423,7 @@ func ParseOptions() *Options {
|
||||
flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
|
||||
flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"),
|
||||
flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
|
||||
flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"),
|
||||
flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
|
||||
flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),
|
||||
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"),
|
||||
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"),
|
||||
@ -557,6 +568,10 @@ func (options *Options) ValidateOptions() error {
|
||||
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
|
||||
}
|
||||
|
||||
if options.Screenshot && !options.StoreResponse {
|
||||
gologger.Debug().Msgf("automatically enabling store response")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
if options.StoreResponse && options.StoreResponseDir == "" {
|
||||
gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory)
|
||||
options.StoreResponseDir = DefaultOutputDirectory
|
||||
@ -565,6 +580,7 @@ func (options *Options) ValidateOptions() error {
|
||||
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
|
||||
options.StoreResponse = true
|
||||
}
|
||||
|
||||
if options.Hashes != "" {
|
||||
for _, hashType := range strings.Split(options.Hashes, ",") {
|
||||
if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
|
||||
|
||||
112
runner/runner.go
112
runner/runner.go
@ -32,7 +32,7 @@ import (
|
||||
"github.com/projectdiscovery/mapcidr/asn"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/Mzack9999/gcache"
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
@ -75,7 +75,8 @@ type Runner struct {
|
||||
hm *hybrid.HybridMap
|
||||
stats clistats.StatisticsClient
|
||||
ratelimiter ratelimit.Limiter
|
||||
HostErrorsCache gcache.Cache
|
||||
HostErrorsCache gcache.Cache[string, int]
|
||||
browser *Browser
|
||||
}
|
||||
|
||||
// New creates a new client for running enumeration process.
|
||||
@ -91,7 +92,8 @@ func New(options *Options) (*Runner, error) {
|
||||
return nil, errors.Wrap(err, "could not create wappalyzer client")
|
||||
}
|
||||
if options.StoreResponseDir != "" {
|
||||
os.RemoveAll(filepath.Join(options.StoreResponseDir, "index.txt"))
|
||||
os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
|
||||
os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
|
||||
}
|
||||
dialerOpts := fastdialer.DefaultOptions
|
||||
dialerOpts.WithDialerHistory = true
|
||||
@ -242,6 +244,15 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
|
||||
scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
|
||||
scanopts.extractRegexps = make(map[string]*regexp.Regexp)
|
||||
if options.Screenshot {
|
||||
browser, err := NewBrowser(options.HTTPProxy, options.UseInstalledChrome)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runner.browser = browser
|
||||
}
|
||||
scanopts.Screenshot = options.Screenshot
|
||||
scanopts.UseInstalledChrome = options.UseInstalledChrome
|
||||
|
||||
if options.OutputExtractRegexs != nil {
|
||||
for _, regex := range options.OutputExtractRegexs {
|
||||
@ -306,7 +317,7 @@ func New(options *Options) (*Runner, error) {
|
||||
}
|
||||
|
||||
if options.HostMaxErrors >= 0 {
|
||||
gc := gcache.New(1000).
|
||||
gc := gcache.New[string, int](1000).
|
||||
ARC().
|
||||
Build()
|
||||
runner.HostErrorsCache = gc
|
||||
@ -554,15 +565,29 @@ func (r *Runner) Close() {
|
||||
if r.options.HostMaxErrors >= 0 {
|
||||
r.HostErrorsCache.Purge()
|
||||
}
|
||||
if r.options.Screenshot {
|
||||
r.browser.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// RunEnumeration on targets for httpx client
|
||||
func (r *Runner) RunEnumeration() {
|
||||
// Try to create output folder if it doesn't exist
|
||||
// Try to create output folders if it doesn't exist
|
||||
if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) {
|
||||
// main folder
|
||||
if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil {
|
||||
gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", r.options.StoreResponseDir, err)
|
||||
}
|
||||
// response folder
|
||||
responseFolder := filepath.Join(r.options.StoreResponseDir, "response")
|
||||
if err := os.MkdirAll(responseFolder, os.ModePerm); err != nil {
|
||||
gologger.Fatal().Msgf("Could not create output response directory '%s': %s\n", r.options.StoreResponseDir, err)
|
||||
}
|
||||
// screenshot folder
|
||||
screenshotFolder := filepath.Join(r.options.StoreResponseDir, "screenshot")
|
||||
if err := os.MkdirAll(screenshotFolder, os.ModePerm); err != nil {
|
||||
gologger.Fatal().Msgf("Could not create output screenshot directory '%s': %s\n", r.options.StoreResponseDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
r.prepareInputPaths()
|
||||
@ -590,7 +615,7 @@ func (r *Runner) RunEnumeration() {
|
||||
go func(output chan Result) {
|
||||
defer wgoutput.Done()
|
||||
|
||||
var f, indexFile *os.File
|
||||
var f, indexFile, indexScreenshotFile *os.File
|
||||
|
||||
if r.options.Output != "" {
|
||||
var err error
|
||||
@ -626,7 +651,7 @@ func (r *Runner) RunEnumeration() {
|
||||
}
|
||||
if r.options.StoreResponseDir != "" {
|
||||
var err error
|
||||
indexPath := filepath.Join(r.options.StoreResponseDir, "index.txt")
|
||||
indexPath := filepath.Join(r.options.StoreResponseDir, "response", "index.txt")
|
||||
if r.options.Resume {
|
||||
indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
|
||||
} else {
|
||||
@ -637,6 +662,19 @@ func (r *Runner) RunEnumeration() {
|
||||
}
|
||||
defer indexFile.Close() //nolint
|
||||
}
|
||||
if r.options.Screenshot {
|
||||
var err error
|
||||
indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt")
|
||||
if r.options.Resume {
|
||||
indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
|
||||
} else {
|
||||
indexScreenshotFile, err = os.Create(indexScreenshotPath)
|
||||
}
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not open/create index screenshot file '%s': %s\n", r.options.Output, err)
|
||||
}
|
||||
defer indexScreenshotFile.Close() //nolint
|
||||
}
|
||||
|
||||
for resp := range output {
|
||||
if resp.err != nil {
|
||||
@ -654,6 +692,10 @@ func (r *Runner) RunEnumeration() {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexFile.WriteString(indexData)
|
||||
}
|
||||
if indexScreenshotFile != nil {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexScreenshotFile.WriteString(indexData)
|
||||
}
|
||||
|
||||
// apply matchers and filters
|
||||
if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" {
|
||||
@ -1039,7 +1081,7 @@ retry:
|
||||
hostPort := net.JoinHostPort(URL.Host, URL.Port())
|
||||
if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
|
||||
numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
|
||||
if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors {
|
||||
if err == nil && numberOfErrors >= r.options.HostMaxErrors {
|
||||
return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")}
|
||||
}
|
||||
}
|
||||
@ -1199,10 +1241,10 @@ retry:
|
||||
// mark the host:port as failed to avoid further checks
|
||||
if r.options.HostMaxErrors >= 0 {
|
||||
errorCount, err := r.HostErrorsCache.GetIFPresent(hostPort)
|
||||
if err != nil || errorCount == nil {
|
||||
if err != nil || errorCount == 0 {
|
||||
_ = r.HostErrorsCache.Set(hostPort, 1)
|
||||
} else if errorCount != nil {
|
||||
_ = r.HostErrorsCache.Set(hostPort, errorCount.(int)+1)
|
||||
} else if errorCount > 0 {
|
||||
_ = r.HostErrorsCache.Set(hostPort, errorCount+1)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1573,16 +1615,21 @@ retry:
|
||||
}
|
||||
|
||||
// store responses or chain in directory
|
||||
var responsePath string
|
||||
domainFile := URL.EscapedString()
|
||||
hash := hashes.Sha1([]byte(domainFile))
|
||||
domainResponseFile := fmt.Sprintf("%s.txt", hash)
|
||||
screenshotResponseFile := fmt.Sprintf("%s.png", hash)
|
||||
hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
|
||||
domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response")
|
||||
domainScreenshotBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "screenshot")
|
||||
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
|
||||
screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename)
|
||||
|
||||
var responsePath, screenshotPath string
|
||||
// store response
|
||||
if scanopts.StoreResponse || scanopts.StoreChain {
|
||||
responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
|
||||
// URL.EscapedString returns that can be used as filename
|
||||
domainFile := URL.EscapedString()
|
||||
hash := hashes.Sha1([]byte(domainFile))
|
||||
domainFile = fmt.Sprintf("%s.txt", hash)
|
||||
host := strings.ReplaceAll(URL.Host, ":", "_")
|
||||
domainBaseDir := filepath.Join(scanopts.StoreResponseDirectory, host)
|
||||
// store response
|
||||
responsePath = filepath.Join(domainBaseDir, domainFile)
|
||||
respRaw := resp.Raw
|
||||
reqRaw := requestDump
|
||||
if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
|
||||
@ -1590,14 +1637,12 @@ retry:
|
||||
}
|
||||
data := append([]byte(fullURL), append([]byte("\n\n"), reqRaw...)...)
|
||||
data = append(data, append([]byte("\n"), respRaw...)...)
|
||||
_ = fileutil.CreateFolder(domainBaseDir)
|
||||
_ = fileutil.CreateFolder(responseBaseDir)
|
||||
writeErr := os.WriteFile(responsePath, data, 0644)
|
||||
if writeErr != nil {
|
||||
gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
|
||||
}
|
||||
if scanopts.StoreChain && resp.HasChain() {
|
||||
domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt")
|
||||
responsePath = filepath.Join(domainBaseDir, domainFile)
|
||||
writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644)
|
||||
if writeErr != nil {
|
||||
gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
|
||||
@ -1631,6 +1676,26 @@ retry:
|
||||
chainItems = append(chainItems, resp.GetChainAsSlice()...)
|
||||
}
|
||||
|
||||
// screenshot
|
||||
var (
|
||||
screenshotBytes []byte
|
||||
headlessBody string
|
||||
)
|
||||
if scanopts.Screenshot {
|
||||
screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile))
|
||||
var err error
|
||||
screenshotBytes, headlessBody, err = r.browser.ScreenshotWithBody(fullURL, r.hp.Options.Timeout)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("Could not take screenshot '%s': %s", fullURL, err)
|
||||
} else {
|
||||
_ = fileutil.CreateFolder(screenshotBaseDir)
|
||||
err := os.WriteFile(screenshotPath, screenshotBytes, 0644)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result := Result{
|
||||
Timestamp: time.Now(),
|
||||
Request: request,
|
||||
@ -1677,6 +1742,9 @@ retry:
|
||||
ASN: asnResponse,
|
||||
ExtractRegex: extractRegex,
|
||||
StoredResponsePath: responsePath,
|
||||
ScreenshotBytes: screenshotBytes,
|
||||
ScreenshotPath: screenshotPath,
|
||||
HeadlessBody: headlessBody,
|
||||
}
|
||||
if r.options.OnResult != nil {
|
||||
r.options.OnResult(result)
|
||||
|
||||
@ -72,7 +72,10 @@ type Result struct {
|
||||
CDN bool `json:"cdn,omitempty" csv:"cdn"`
|
||||
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
|
||||
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
|
||||
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"`
|
||||
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"`
|
||||
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"`
|
||||
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"`
|
||||
}
|
||||
|
||||
// function to get dsl variables from result struct
|
||||
|
||||
Loading…
Reference in New Issue
Block a user