projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-09-11 09:35:28 +03:00
Code Issues Packages Projects Releases Wiki Activity
72f4c2cef4
httpx/runner/options.go

790 lines
36 KiB
Go
Raw Blame History

package runner
import (
"fmt"
"math"
"os"
"regexp"
"strings"
"time"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/customheader"
"github.com/projectdiscovery/httpx/common/customlist"
customport "github.com/projectdiscovery/httpx/common/customports"
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/networkpolicy"
"github.com/projectdiscovery/utils/auth/pdcp"
"github.com/projectdiscovery/utils/env"
fileutil "github.com/projectdiscovery/utils/file"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
updateutils "github.com/projectdiscovery/utils/update"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
const (
two = 2
defaultThreads = 50
DefaultResumeFile = "resume.cfg"
DefaultOutputDirectory = "output"
)
var PDCPApiKey = ""
// OnResultCallback (hostResult)
type OnResultCallback func(Result)
type ScanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
RequestBody string
VHost bool
OutputTitle bool
OutputStatusCode bool
OutputLocation bool
OutputContentLength bool
StoreResponse bool
OmitBody bool
OutputServerHeader bool
OutputWebSocket bool
OutputWithNoColor bool
OutputMethod bool
ResponseHeadersInStdout bool
ResponseInStdout bool
Base64ResponseInStdout bool
ChainInStdout bool
TLSProbe bool
CSPProbe bool
VHostInput bool
OutputContentType bool
Unsafe bool
Pipeline bool
HTTP2Probe bool
OutputIP bool
OutputCName bool
OutputCDN string
OutputResponseTime bool
PreferHTTPS bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
StoreChain bool
StoreVisionReconClusters bool
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
OutputExtractRegex string
extractRegexps map[string]*regexp.Regexp
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
Favicon bool
LeaveDefaultPorts bool
OutputLinesCount bool
OutputWordsCount bool
Hashes string
Screenshot bool
UseInstalledChrome bool
DisableStdin bool
NoScreenshotBytes bool
NoHeadlessBody bool
ScreenshotTimeout int
}
func (s *ScanOptions) Clone() *ScanOptions {
return &ScanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
RequestBody: s.RequestBody,
VHost: s.VHost,
OutputTitle: s.OutputTitle,
OutputStatusCode: s.OutputStatusCode,
OutputLocation: s.OutputLocation,
OutputContentLength: s.OutputContentLength,
StoreResponse: s.StoreResponse,
OmitBody: s.OmitBody,
OutputServerHeader: s.OutputServerHeader,
OutputWebSocket: s.OutputWebSocket,
OutputWithNoColor: s.OutputWithNoColor,
OutputMethod: s.OutputMethod,
ResponseHeadersInStdout: s.ResponseHeadersInStdout,
ResponseInStdout: s.ResponseInStdout,
Base64ResponseInStdout: s.Base64ResponseInStdout,
ChainInStdout: s.ChainInStdout,
TLSProbe: s.TLSProbe,
CSPProbe: s.CSPProbe,
OutputContentType: s.OutputContentType,
Unsafe: s.Unsafe,
Pipeline: s.Pipeline,
HTTP2Probe: s.HTTP2Probe,
OutputIP: s.OutputIP,
OutputCName: s.OutputCName,
OutputCDN: s.OutputCDN,
OutputResponseTime: s.OutputResponseTime,
PreferHTTPS: s.PreferHTTPS,
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
Favicon: s.Favicon,
extractRegexps: s.extractRegexps,
LeaveDefaultPorts: s.LeaveDefaultPorts,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
Hashes: s.Hashes,
Screenshot: s.Screenshot,
UseInstalledChrome: s.UseInstalledChrome,
NoScreenshotBytes: s.NoScreenshotBytes,
NoHeadlessBody: s.NoHeadlessBody,
ScreenshotTimeout: s.ScreenshotTimeout,
}
}
// Options contains configuration options for httpx.
type Options struct {
CustomHeaders customheader.CustomHeaders
CustomPorts customport.CustomPorts
matchStatusCode []int
matchContentLength []int
filterStatusCode []int
filterContentLength []int
Output string
OutputAll bool
StoreResponseDir string
OmitBody bool
HTTPProxy string
SocksProxy string
InputFile string
InputTargetHost goflags.StringSlice
Methods string
RequestURI string
RequestURIs string
requestURIs []string
OutputMatchStatusCode string
OutputMatchContentLength string
OutputFilterStatusCode string
OutputFilterErrorPage bool
OutputFilterContentLength string
InputRawRequest string
rawRequest string
RequestBody string
OutputFilterString goflags.StringSlice
OutputMatchString goflags.StringSlice
OutputFilterRegex goflags.StringSlice
OutputMatchRegex goflags.StringSlice
Retries int
Threads int
Timeout int
Delay time.Duration
filterRegexes []*regexp.Regexp
matchRegexes []*regexp.Regexp
VHost bool
VHostInput bool
Smuggling bool
ExtractTitle bool
StatusCode bool
Location bool
ContentLength bool
FollowRedirects bool
RespectHSTS bool
StoreResponse bool
JSONOutput bool
CSVOutput bool
CSVOutputEncoding string
PdcpAuth string
Silent bool
Version bool
Verbose bool
NoColor bool
OutputServerHeader bool
OutputWebSocket bool
ResponseHeadersInStdout bool
ResponseInStdout bool
Base64ResponseInStdout bool
chainInStdout bool
FollowHostRedirects bool
MaxRedirects int
OutputMethod bool
TLSProbe bool
CSPProbe bool
OutputContentType bool
OutputIP bool
OutputCName bool
ExtractFqdn bool
Unsafe bool
Debug bool
DebugRequests bool
DebugResponse bool
Pipeline bool
HTTP2Probe bool
OutputCDN string
OutputResponseTime bool
NoFallback bool
NoFallbackScheme bool
TechDetect bool
TLSGrab bool
protocol string
ShowStatistics bool
StatsInterval int
RandomAgent bool
StoreChain bool
StoreVisionReconClusters bool
Deny customlist.CustomList
Allow customlist.CustomList
MaxResponseBodySizeToSave int
MaxResponseBodySizeToRead int
ResponseBodyPreviewSize int
OutputExtractRegexs goflags.StringSlice
OutputExtractPresets goflags.StringSlice
RateLimit int
RateLimitMinute int
Probe bool
Resume bool
resumeCfg *ResumeCfg
Exclude goflags.StringSlice
HostMaxErrors int
Stream bool
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.StringSlice
Favicon bool
OutputFilterFavicon goflags.StringSlice
OutputMatchFavicon goflags.StringSlice
LeaveDefaultPorts bool
ZTLS bool
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
OutputFilterLinesCount string
Memprofile string
filterLinesCount []int
OutputWordsCount bool
OutputMatchWordsCount string
matchWordsCount []int
OutputFilterWordsCount string
filterWordsCount []int
Hashes string
Jarm bool
Asn bool
OutputMatchCdn goflags.StringSlice
OutputFilterCdn goflags.StringSlice
SniName string
OutputMatchResponseTime string
OutputFilterResponseTime string
HealthCheck bool
ListDSLVariable bool
OutputFilterCondition string
OutputMatchCondition string
StripFilter string
//The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err.
OnResult OnResultCallback
DisableUpdateCheck bool
NoDecode bool
Screenshot bool
UseInstalledChrome bool
TlsImpersonate bool
DisableStdin bool
HttpApiEndpoint string
NoScreenshotBytes bool
NoHeadlessBody bool
ScreenshotTimeout int
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
HeadlessOptionalArguments goflags.StringSlice
Protocol string
// Optional pre-created objects to reduce allocations
Wappalyzer *wappalyzer.Wappalyze
Networkpolicy *networkpolicy.NetworkPolicy
}
// ParseOptions parses the command line options for application
func ParseOptions() *Options {
options := &Options{}
var cfgFile string
flagSet := goflags.NewFlagSet()
flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.`)
flagSet.CreateGroup("input", "Input",
flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions),
)
flagSet.CreateGroup("Probes", "Probes",
flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "display response status-code"),
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "display response content-length"),
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "display response content-type"),
flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"),
flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"),
flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"),
flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"),
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"),
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"),
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"),
flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"),
flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"),
flagSet.BoolVarP(&options.ExtractFqdn, "efqdn", "extract-fqdn", false, "get domain and subdomains from response body and header in jsonl/csv output"),
flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"),
flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"),
flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
)
flagSet.CreateGroup("headless", "Headless",
flagSet.BoolVarP(&options.Screenshot, "screenshot", "ss", false, "enable saving screenshot of the page using headless browser"),
flagSet.BoolVar(&options.UseInstalledChrome, "system-chrome", false, "enable using local installed chrome for screenshot"),
flagSet.StringSliceVarP(&options.HeadlessOptionalArguments, "headless-options", "ho", nil, "start headless chrome with additional options", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.NoScreenshotBytes, "exclude-screenshot-bytes", "esb", false, "enable excluding screenshot bytes from json output"),
flagSet.BoolVarP(&options.NoHeadlessBody, "exclude-headless-body", "ehb", false, "enable excluding headless header from json output"),
flagSet.IntVarP(&options.ScreenshotTimeout, "screenshot-timeout", "st", 10, "set timeout for screenshot in seconds"),
)
flagSet.CreateGroup("matchers", "Matchers",
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"),
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"),
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
)
flagSet.CreateGroup("extractor", "Extractor",
flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "display response content with matched regex", goflags.StringSliceOptions),
flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, fmt.Sprintf("display response content matched by a pre-defined regex (%s)", strings.Join(maps.Keys(customextract.ExtractPresets), ",")), goflags.StringSliceOptions),
)
flagSet.CreateGroup("filters", "Filters",
flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
flagSet.BoolVarP(&options.OutputFilterErrorPage, "filter-error-page", "fep", false, "filter response with ML based error page detection"),
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
flagSet.DynamicVar(&options.StripFilter, "strip", "html", "strips all tags in response. supported formats: html,xml"),
)
flagSet.CreateGroup("rate-limit", "Rate-Limit",
flagSet.IntVarP(&options.Threads, "threads", "t", defaultThreads, "number of threads to use"),
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum requests to send per second"),
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
)
flagSet.CreateGroup("Misc", "Miscellaneous",
flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"),
flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg http:1,2-10,11,https:80)"),
flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"),
flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"),
flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"),
flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"),
flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"),
flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"),
flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"),
flagSet.BoolVarP(&options.ListDSLVariable, "list-dsl-variables", "ldv", false, "list json output field keys name that support dsl matcher/filter"),
)
flagSet.CreateGroup("update", "Update",
flagSet.CallbackVarP(GetUpdateCallback(), "update", "up", "update httpx to latest version"),
flagSet.BoolVarP(&options.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic httpx update check"),
)
flagSet.CreateGroup("output", "Output",
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"),
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body in output"),
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"),
flagSet.BoolVarP(&options.JSONOutput, "json", "j", false, "store output in JSONL(ines) format"),
flagSet.BoolVarP(&options.ResponseHeadersInStdout, "include-response-header", "irh", false, "include http response (headers) in JSON output (-json only)"),
flagSet.BoolVarP(&options.ResponseInStdout, "include-response", "irr", false, "include http request/response (headers + body) in JSON output (-json only)"),
flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"),
flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"),
flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"),
)
flagSet.CreateGroup("configs", "Configurations",
flagSet.StringVar(&cfgFile, "config", "", "path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)"),
flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions),
flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"),
flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"),
flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"),
flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"),
flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"),
flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"),
flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"),
flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"),
flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"),
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "max number of redirects to follow per host"),
flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
flagSet.BoolVarP(&options.RespectHSTS, "respect-hsts", "rhsts", false, "respect HSTS response headers for redirect requests"),
flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "get a list of vhosts as input"),
flagSet.StringVar(&options.Methods, "x", "", "request methods to probe, use 'all' to probe all HTTP methods"),
flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"),
flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"),
flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"),
flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https://host:443"),
flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"),
flagSet.BoolVar(&options.NoDecode, "no-decode", false, "avoid decoding body"),
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
)
flagSet.CreateGroup("debug", "Debug",
flagSet.BoolVarP(&options.HealthCheck, "hc", "health-check", false, "run diagnostic check up"),
flagSet.BoolVar(&options.Debug, "debug", false, "display request/response content in cli"),
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "display request content in cli"),
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"),
flagSet.BoolVar(&options.Version, "version", false, "display httpx version"),
flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"),
flagSet.StringVar(&options.Memprofile, "profile-mem", "", "optional httpx memory profile dump file"),
flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"),
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"),
)
flagSet.CreateGroup("Optimizations", "Optimizations",
flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"),
flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "),
flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
flagSet.StringSliceVarP(&options.Exclude, "exclude", "e", nil, "exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)", goflags.CommaSeparatedStringSliceOptions),
flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"),
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"),
)
_ = flagSet.Parse()
if options.OutputAll && options.Output == "" {
gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all")
}
if options.OutputAll {
options.JSONOutput = true
options.CSVOutput = true
}
if cfgFile != "" {
if !fileutil.FileExists(cfgFile) {
gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
}
// merge config file with flags
if err := flagSet.MergeConfigFile(cfgFile); err != nil {
gologger.Fatal().Msgf("Could not read config: %s\n", err)
}
}
// api key hierarchy: cli flag > env var > .pdcp/credential file
if options.PdcpAuth == "true" {
AuthWithPDCP()
} else if len(options.PdcpAuth) == 36 {
PDCPApiKey = options.PdcpAuth
ph := pdcp.PDCPCredHandler{}
if _, err := ph.GetCreds(); err == pdcp.ErrNoCreds {
apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcp.DefaultApiServer)
if validatedCreds, err := ph.ValidateAPIKey(PDCPApiKey, apiServer, "httpx"); err == nil {
_ = ph.SaveCreds(validatedCreds)
}
}
}
if options.HealthCheck {
gologger.Print().Msgf("%s\n", DoHealthCheck(options, flagSet))
os.Exit(0)
}
if options.StatsInterval != 0 {
options.ShowStatistics = true
}
if options.ResponseBodyPreviewSize > 0 && options.StripFilter == "" {
options.StripFilter = "html"
}
// Read the inputs and configure the logging
options.configureOutput()
err := options.configureResume()
if err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
if options.ListDSLVariable {
dslVars, err := dslVariables()
if err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
for _, dsl := range dslVars {
gologger.Print().Msg(dsl)
}
os.Exit(0)
}
showBanner()
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", version)
os.Exit(0)
}
if !options.DisableUpdateCheck {
latestVersion, err := updateutils.GetToolVersionCallback("httpx", version)()
if err != nil {
if options.Verbose {
gologger.Error().Msgf("httpx version check failed: %v", err.Error())
}
} else {
gologger.Info().Msgf("Current httpx version %v %v", version, updateutils.GetVersionDescription(version, latestVersion))
}
}
if err := options.ValidateOptions(); err != nil {
gologger.Fatal().Msgf("%s\n", err)
}
return options
}
func (options *Options) ValidateOptions() error {
if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
return fmt.Errorf("file '%s' does not exist", options.InputFile)
}
if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) {
return fmt.Errorf("file '%s' does not exist", options.InputRawRequest)
}
if options.Silent {
incompatibleFlagsList := flagsIncompatibleWithSilent(options)
if len(incompatibleFlagsList) > 0 {
last := incompatibleFlagsList[len(incompatibleFlagsList)-1]
first := incompatibleFlagsList[:len(incompatibleFlagsList)-1]
msg := ""
if len(incompatibleFlagsList) > 1 {
msg += fmt.Sprintf("%s and %s flags are", strings.Join(first, ", "), last)
} else {
msg += fmt.Sprintf("%s flag is", last)
}
msg += " incompatible with silent flag"
return fmt.Errorf(msg)
}
}
var err error
if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil {
return errors.Wrap(err, "Invalid value for match status code option")
}
if options.matchContentLength, err = stringz.StringToSliceInt(options.OutputMatchContentLength); err != nil {
return errors.Wrap(err, "Invalid value for match content length option")
}
if options.filterStatusCode, err = stringz.StringToSliceInt(options.OutputFilterStatusCode); err != nil {
return errors.Wrap(err, "Invalid value for filter status code option")
}
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
return errors.Wrap(err, "Invalid value for filter content length option")
}
for _, filterRegexStr := range options.OutputFilterRegex {
filterRegex, err := regexp.Compile(filterRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for regex filter option")
}
options.filterRegexes = append(options.filterRegexes, filterRegex)
}
for _, matchRegexStr := range options.OutputMatchRegex {
matchRegex, err := regexp.Compile(matchRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for match regex option")
}
options.matchRegexes = append(options.matchRegexes, matchRegex)
}
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
return errors.Wrap(err, "Invalid value for match lines count option")
}
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
return errors.Wrap(err, "Invalid value for match words count option")
}
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
return errors.Wrap(err, "Invalid value for filter lines count option")
}
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
return errors.Wrap(err, "Invalid value for filter words count option")
}
var resolvers []string
for _, resolver := range options.Resolvers {
if fileutil.FileExists(resolver) {
chFile, err := fileutil.ReadFile(resolver)
if err != nil {
return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver)
}
for line := range chFile {
resolvers = append(resolvers, line)
}
} else {
resolvers = append(resolvers, resolver)
}
}
options.Resolvers = resolvers
if len(options.Resolvers) > 0 {
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
}
if options.Screenshot && !options.StoreResponse {
gologger.Debug().Msgf("automatically enabling store response")
options.StoreResponse = true
}
if options.StoreResponse && options.StoreResponseDir == "" {
gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory)
options.StoreResponseDir = DefaultOutputDirectory
}
if options.StoreResponseDir != "" && !options.StoreResponse {
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
options.StoreResponse = true
}
if options.Hashes != "" {
for _, hashType := range strings.Split(options.Hashes, ",") {
if !sliceutil.Contains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
gologger.Error().Msgf("Unsupported hash type: %s\n", hashType)
}
}
}
if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 {
options.OutputCDN = "true"
}
if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) {
return fmt.Errorf("invalid protocol: %s", options.Protocol)
}
if options.Threads == 0 {
gologger.Info().Msgf("Threads automatically set to %d", defaultThreads)
options.Threads = defaultThreads
}
return nil
}
// redundant with katana
func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
var (
lastKey string
optionalArguments = make(map[string]string)
)
for _, v := range options.HeadlessOptionalArguments {
if v == "" {
continue
}
if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 {
key := strings.TrimSpace(argParts[0])
value := strings.TrimSpace(argParts[1])
if key != "" && value != "" {
optionalArguments[key] = value
lastKey = key
}
} else if !strings.HasPrefix(v, "--") {
optionalArguments[lastKey] += "," + v
} else {
optionalArguments[v] = ""
}
}
return optionalArguments
}
// configureOutput configures the output on the screen
func (options *Options) configureOutput() {
// If the user desires verbose output, show verbose output
if options.Verbose {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
}
if options.Debug {
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
}
if options.NoColor {
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
}
if options.Silent {
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
}
if len(options.OutputMatchResponseTime) > 0 || len(options.OutputFilterResponseTime) > 0 {
options.OutputResponseTime = true
}
if options.CSVOutputEncoding != "" {
options.CSVOutput = true
}
}
func (options *Options) configureResume() error {
options.resumeCfg = &ResumeCfg{}
if options.Resume && fileutil.FileExists(DefaultResumeFile) {
return goconfig.Load(&options.resumeCfg, DefaultResumeFile)
}
return nil
}
// ShouldLoadResume resume file
func (options *Options) ShouldLoadResume() bool {
return options.Resume && fileutil.FileExists(DefaultResumeFile)
}
// ShouldSaveResume file
func (options *Options) ShouldSaveResume() bool {
return true
}
func flagsIncompatibleWithSilent(options *Options) []string {
var incompatibleFlagsList []string
for k, v := range map[string]bool{
"debug": options.Debug,
"debug-request": options.DebugRequests,
"debug-response": options.DebugResponse,
"verbose": options.Verbose,
} {
if v {
incompatibleFlagsList = append(incompatibleFlagsList, k)
}
}
return incompatibleFlagsList
}
Reference in New Issue View Git Blame Copy Permalink