2020-10-20 21:17:02 +03:00
|
|
|
package runner
|
|
|
|
|
|
|
|
import (
|
2021-04-27 16:37:30 +03:00
|
|
|
"math"
|
2020-10-20 21:17:02 +03:00
|
|
|
"os"
|
|
|
|
"regexp"
|
2021-12-28 09:46:42 +03:00
|
|
|
"strings"
|
2022-04-08 16:20:03 +03:00
|
|
|
"github.com/projectdiscovery/httpx/common/slice"
|
2021-08-22 22:07:23 +03:00
|
|
|
"github.com/projectdiscovery/fileutil"
|
2021-08-06 13:38:42 +03:00
|
|
|
"github.com/projectdiscovery/goconfig"
|
2021-09-29 21:43:54 +03:00
|
|
|
"github.com/projectdiscovery/goflags"
|
2020-10-20 21:17:02 +03:00
|
|
|
"github.com/projectdiscovery/gologger"
|
2021-02-17 18:15:37 +03:00
|
|
|
"github.com/projectdiscovery/gologger/formatter"
|
2021-02-17 01:40:01 +03:00
|
|
|
"github.com/projectdiscovery/gologger/levels"
|
2020-10-20 21:17:02 +03:00
|
|
|
"github.com/projectdiscovery/httpx/common/customheader"
|
2021-04-09 00:11:41 +03:00
|
|
|
"github.com/projectdiscovery/httpx/common/customlist"
|
2020-10-20 21:17:02 +03:00
|
|
|
customport "github.com/projectdiscovery/httpx/common/customports"
|
2021-08-22 22:07:23 +03:00
|
|
|
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
|
2020-10-20 21:17:02 +03:00
|
|
|
"github.com/projectdiscovery/httpx/common/stringz"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2021-12-23 21:48:53 +03:00
|
|
|
// The maximum file length is 251 (255 - 4 bytes for ".ext" suffix)
|
2022-01-04 11:47:06 +03:00
|
|
|
maxFileNameLength = 251
|
|
|
|
two = 2
|
|
|
|
DefaultResumeFile = "resume.cfg"
|
|
|
|
DefaultOutputDirectory = "output"
|
2020-10-20 21:17:02 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
type scanOptions struct {
|
2021-08-07 09:39:23 +03:00
|
|
|
Methods []string
|
|
|
|
StoreResponseDirectory string
|
|
|
|
RequestURI string
|
|
|
|
RequestBody string
|
|
|
|
VHost bool
|
|
|
|
OutputTitle bool
|
|
|
|
OutputStatusCode bool
|
|
|
|
OutputLocation bool
|
|
|
|
OutputContentLength bool
|
|
|
|
StoreResponse bool
|
|
|
|
OutputServerHeader bool
|
|
|
|
OutputWebSocket bool
|
|
|
|
OutputWithNoColor bool
|
|
|
|
OutputMethod bool
|
|
|
|
ResponseInStdout bool
|
|
|
|
ChainInStdout bool
|
|
|
|
TLSProbe bool
|
|
|
|
CSPProbe bool
|
|
|
|
VHostInput bool
|
|
|
|
OutputContentType bool
|
|
|
|
Unsafe bool
|
|
|
|
Pipeline bool
|
|
|
|
HTTP2Probe bool
|
|
|
|
OutputIP bool
|
|
|
|
OutputCName bool
|
|
|
|
OutputCDN bool
|
|
|
|
OutputResponseTime bool
|
|
|
|
PreferHTTPS bool
|
|
|
|
NoFallback bool
|
|
|
|
NoFallbackScheme bool
|
|
|
|
TechDetect bool
|
|
|
|
StoreChain bool
|
|
|
|
MaxResponseBodySizeToSave int
|
|
|
|
MaxResponseBodySizeToRead int
|
|
|
|
OutputExtractRegex string
|
|
|
|
extractRegex *regexp.Regexp
|
2021-08-07 14:46:20 +03:00
|
|
|
ExcludeCDN bool
|
2021-08-15 17:04:24 +03:00
|
|
|
HostMaxErrors int
|
2021-12-01 10:10:53 +03:00
|
|
|
ProbeAllIPS bool
|
2022-01-07 14:58:07 +03:00
|
|
|
Favicon bool
|
2022-01-03 11:27:04 +03:00
|
|
|
LeaveDefaultPorts bool
|
2021-12-31 11:26:39 +03:00
|
|
|
OutputLinesCount bool
|
|
|
|
OutputWordsCount bool
|
2022-02-15 15:02:33 +03:00
|
|
|
Hashes string
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
|
2020-12-22 02:32:46 +03:00
|
|
|
func (s *scanOptions) Clone() *scanOptions {
|
|
|
|
return &scanOptions{
|
2021-08-07 09:39:23 +03:00
|
|
|
Methods: s.Methods,
|
|
|
|
StoreResponseDirectory: s.StoreResponseDirectory,
|
|
|
|
RequestURI: s.RequestURI,
|
|
|
|
RequestBody: s.RequestBody,
|
|
|
|
VHost: s.VHost,
|
|
|
|
OutputTitle: s.OutputTitle,
|
|
|
|
OutputStatusCode: s.OutputStatusCode,
|
|
|
|
OutputLocation: s.OutputLocation,
|
|
|
|
OutputContentLength: s.OutputContentLength,
|
|
|
|
StoreResponse: s.StoreResponse,
|
|
|
|
OutputServerHeader: s.OutputServerHeader,
|
|
|
|
OutputWebSocket: s.OutputWebSocket,
|
|
|
|
OutputWithNoColor: s.OutputWithNoColor,
|
|
|
|
OutputMethod: s.OutputMethod,
|
|
|
|
ResponseInStdout: s.ResponseInStdout,
|
|
|
|
ChainInStdout: s.ChainInStdout,
|
|
|
|
TLSProbe: s.TLSProbe,
|
|
|
|
CSPProbe: s.CSPProbe,
|
|
|
|
OutputContentType: s.OutputContentType,
|
|
|
|
Unsafe: s.Unsafe,
|
|
|
|
Pipeline: s.Pipeline,
|
|
|
|
HTTP2Probe: s.HTTP2Probe,
|
|
|
|
OutputIP: s.OutputIP,
|
|
|
|
OutputCName: s.OutputCName,
|
|
|
|
OutputCDN: s.OutputCDN,
|
|
|
|
OutputResponseTime: s.OutputResponseTime,
|
|
|
|
PreferHTTPS: s.PreferHTTPS,
|
|
|
|
NoFallback: s.NoFallback,
|
|
|
|
NoFallbackScheme: s.NoFallbackScheme,
|
|
|
|
TechDetect: s.TechDetect,
|
|
|
|
StoreChain: s.StoreChain,
|
|
|
|
OutputExtractRegex: s.OutputExtractRegex,
|
|
|
|
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
|
|
|
|
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
|
2021-08-15 17:04:24 +03:00
|
|
|
HostMaxErrors: s.HostMaxErrors,
|
2022-01-07 14:58:07 +03:00
|
|
|
Favicon: s.Favicon,
|
2022-01-03 11:27:04 +03:00
|
|
|
LeaveDefaultPorts: s.LeaveDefaultPorts,
|
2021-12-31 11:26:39 +03:00
|
|
|
OutputLinesCount: s.OutputLinesCount,
|
|
|
|
OutputWordsCount: s.OutputWordsCount,
|
2022-02-15 15:02:33 +03:00
|
|
|
Hashes: s.Hashes,
|
2020-12-22 02:32:46 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-09 00:11:41 +03:00
|
|
|
// Options contains configuration options for httpx.
|
2020-10-20 21:17:02 +03:00
|
|
|
type Options struct {
|
|
|
|
CustomHeaders customheader.CustomHeaders
|
|
|
|
CustomPorts customport.CustomPorts
|
|
|
|
matchStatusCode []int
|
|
|
|
matchContentLength []int
|
|
|
|
filterStatusCode []int
|
|
|
|
filterContentLength []int
|
|
|
|
Output string
|
|
|
|
StoreResponseDir string
|
|
|
|
HTTPProxy string
|
|
|
|
SocksProxy string
|
|
|
|
InputFile string
|
|
|
|
Methods string
|
|
|
|
RequestURI string
|
2020-12-20 04:37:45 +03:00
|
|
|
RequestURIs string
|
|
|
|
requestURIs []string
|
2020-10-20 21:17:02 +03:00
|
|
|
OutputMatchStatusCode string
|
|
|
|
OutputMatchContentLength string
|
|
|
|
OutputFilterStatusCode string
|
|
|
|
OutputFilterContentLength string
|
|
|
|
InputRawRequest string
|
|
|
|
rawRequest string
|
|
|
|
RequestBody string
|
|
|
|
OutputFilterString string
|
|
|
|
OutputMatchString string
|
|
|
|
OutputFilterRegex string
|
|
|
|
OutputMatchRegex string
|
|
|
|
Retries int
|
|
|
|
Threads int
|
|
|
|
Timeout int
|
|
|
|
filterRegex *regexp.Regexp
|
|
|
|
matchRegex *regexp.Regexp
|
|
|
|
VHost bool
|
2021-04-06 15:43:55 +03:00
|
|
|
VHostInput bool
|
2020-10-20 21:17:02 +03:00
|
|
|
Smuggling bool
|
|
|
|
ExtractTitle bool
|
|
|
|
StatusCode bool
|
|
|
|
Location bool
|
|
|
|
ContentLength bool
|
|
|
|
FollowRedirects bool
|
|
|
|
StoreResponse bool
|
|
|
|
JSONOutput bool
|
2021-09-09 13:55:31 +03:00
|
|
|
CSVOutput bool
|
2020-10-20 21:17:02 +03:00
|
|
|
Silent bool
|
|
|
|
Version bool
|
|
|
|
Verbose bool
|
|
|
|
NoColor bool
|
|
|
|
OutputServerHeader bool
|
|
|
|
OutputWebSocket bool
|
|
|
|
responseInStdout bool
|
2021-05-06 14:41:35 +03:00
|
|
|
chainInStdout bool
|
2020-10-20 21:17:02 +03:00
|
|
|
FollowHostRedirects bool
|
2021-08-16 15:25:30 +03:00
|
|
|
MaxRedirects int
|
2020-10-20 21:17:02 +03:00
|
|
|
OutputMethod bool
|
|
|
|
TLSProbe bool
|
|
|
|
CSPProbe bool
|
|
|
|
OutputContentType bool
|
|
|
|
OutputIP bool
|
|
|
|
OutputCName bool
|
|
|
|
Unsafe bool
|
|
|
|
Debug bool
|
2021-10-23 13:59:51 +03:00
|
|
|
DebugRequests bool
|
|
|
|
DebugResponse bool
|
2020-10-20 21:17:02 +03:00
|
|
|
Pipeline bool
|
|
|
|
HTTP2Probe bool
|
|
|
|
OutputCDN bool
|
|
|
|
OutputResponseTime bool
|
|
|
|
NoFallback bool
|
2021-05-26 20:28:40 +03:00
|
|
|
NoFallbackScheme bool
|
2021-04-06 15:43:55 +03:00
|
|
|
TechDetect bool
|
2021-04-07 18:40:59 +03:00
|
|
|
TLSGrab bool
|
2020-10-20 21:17:02 +03:00
|
|
|
protocol string
|
2020-11-15 23:21:17 +03:00
|
|
|
ShowStatistics bool
|
2022-01-15 03:43:25 +03:00
|
|
|
StatsInterval int
|
2020-12-21 02:13:27 +03:00
|
|
|
RandomAgent bool
|
2021-04-28 07:39:09 +03:00
|
|
|
StoreChain bool
|
2021-04-09 15:53:27 +03:00
|
|
|
Deny customlist.CustomList
|
|
|
|
Allow customlist.CustomList
|
2021-08-07 09:39:23 +03:00
|
|
|
MaxResponseBodySizeToSave int
|
|
|
|
MaxResponseBodySizeToRead int
|
2021-04-28 16:11:13 +03:00
|
|
|
OutputExtractRegex string
|
2021-07-26 20:56:41 +03:00
|
|
|
RateLimit int
|
2022-01-12 02:32:09 +03:00
|
|
|
RateLimitMinute int
|
2021-07-24 08:52:31 +03:00
|
|
|
Probe bool
|
2021-08-06 13:38:42 +03:00
|
|
|
Resume bool
|
|
|
|
resumeCfg *ResumeCfg
|
2021-08-03 22:12:11 +03:00
|
|
|
ExcludeCDN bool
|
2021-08-15 17:04:24 +03:00
|
|
|
HostMaxErrors int
|
2021-10-04 14:18:00 +03:00
|
|
|
Stream bool
|
|
|
|
SkipDedupe bool
|
2021-12-01 10:10:53 +03:00
|
|
|
ProbeAllIPS bool
|
2021-12-28 09:46:42 +03:00
|
|
|
Resolvers goflags.NormalizedStringSlice
|
2022-01-07 14:58:07 +03:00
|
|
|
Favicon bool
|
|
|
|
OutputFilterFavicon goflags.NormalizedStringSlice
|
|
|
|
OutputMatchFavicon goflags.NormalizedStringSlice
|
2022-01-03 11:27:04 +03:00
|
|
|
LeaveDefaultPorts bool
|
2021-12-31 11:26:39 +03:00
|
|
|
OutputLinesCount bool
|
|
|
|
OutputMatchLinesCount string
|
|
|
|
matchLinesCount []int
|
|
|
|
OutputFilterLinesCount string
|
|
|
|
filterLinesCount []int
|
|
|
|
OutputWordsCount bool
|
|
|
|
OutputMatchWordsCount string
|
|
|
|
matchWordsCount []int
|
|
|
|
OutputFilterWordsCount string
|
|
|
|
filterWordsCount []int
|
2022-02-15 15:02:33 +03:00
|
|
|
Hashes string
|
2022-04-08 16:20:03 +03:00
|
|
|
Jarm bool
|
2022-04-07 10:12:09 +03:00
|
|
|
Asn bool
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// ParseOptions parses the command line options for application
|
|
|
|
func ParseOptions() *Options {
|
|
|
|
options := &Options{}
|
|
|
|
|
2021-09-29 21:43:54 +03:00
|
|
|
flagSet := goflags.NewFlagSet()
|
2021-10-02 23:23:13 +03:00
|
|
|
flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`)
|
2020-10-20 21:17:02 +03:00
|
|
|
|
2021-10-02 04:26:30 +03:00
|
|
|
createGroup(flagSet, "input", "Input",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
|
|
|
|
flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
|
2021-09-29 21:43:54 +03:00
|
|
|
)
|
2020-10-20 21:17:02 +03:00
|
|
|
|
2021-10-02 04:26:30 +03:00
|
|
|
createGroup(flagSet, "Probes", "Probes",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "display response status-code"),
|
|
|
|
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "display response content-length"),
|
|
|
|
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "display response content-type"),
|
|
|
|
flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"),
|
|
|
|
flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"),
|
|
|
|
flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"),
|
2022-04-08 16:20:03 +03:00
|
|
|
flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"),
|
|
|
|
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"),
|
|
|
|
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"),
|
|
|
|
flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"),
|
|
|
|
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
|
|
|
|
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
|
|
|
|
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
|
|
|
|
flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
|
|
|
|
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
|
|
|
|
flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"),
|
|
|
|
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"),
|
|
|
|
flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
|
2021-10-02 04:26:30 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
createGroup(flagSet, "matchers", "Matchers",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"),
|
|
|
|
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"),
|
|
|
|
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
|
|
|
|
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
|
2022-03-01 12:07:41 +03:00
|
|
|
flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"),
|
|
|
|
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"),
|
|
|
|
)
|
|
|
|
|
|
|
|
createGroup(flagSet, "extractor", "Extractor",
|
|
|
|
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "display response content for specified regex"),
|
2021-10-02 04:26:30 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
createGroup(flagSet, "filters", "Filters",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"),
|
|
|
|
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"),
|
|
|
|
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
|
|
|
|
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
|
|
|
|
flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"),
|
2022-03-01 12:07:41 +03:00
|
|
|
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"),
|
|
|
|
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"),
|
2021-10-02 04:26:30 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
createGroup(flagSet, "rate-limit", "Rate-Limit",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.IntVarP(&options.Threads, "threads", "t", 50, "number of threads to use"),
|
|
|
|
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum requests to send per second"),
|
2022-01-12 02:32:09 +03:00
|
|
|
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"),
|
2021-10-02 04:26:30 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
createGroup(flagSet, "Misc", "Miscellaneous",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"),
|
2022-03-01 12:07:41 +03:00
|
|
|
flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"),
|
|
|
|
flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"),
|
|
|
|
flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"),
|
2022-03-01 12:07:41 +03:00
|
|
|
flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"),
|
|
|
|
flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"),
|
|
|
|
flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"),
|
2021-10-02 04:26:30 +03:00
|
|
|
)
|
|
|
|
|
2021-10-02 23:33:48 +03:00
|
|
|
createGroup(flagSet, "output", "Output",
|
2022-01-03 11:22:45 +03:00
|
|
|
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
|
|
|
|
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
|
2022-01-04 11:47:06 +03:00
|
|
|
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
|
2022-01-03 11:22:45 +03:00
|
|
|
flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"),
|
|
|
|
flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"),
|
|
|
|
flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
|
|
|
|
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
|
2022-04-07 10:12:09 +03:00
|
|
|
flagSet.BoolVar(&options.Asn, "asn", false, "displays asn information"),
|
2021-10-02 23:33:48 +03:00
|
|
|
)
|
|
|
|
|
2021-10-02 04:26:30 +03:00
|
|
|
createGroup(flagSet, "configs", "Configurations",
|
2022-03-01 12:13:30 +03:00
|
|
|
flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"),
|
|
|
|
flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"),
|
2021-10-02 23:23:13 +03:00
|
|
|
flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"),
|
|
|
|
flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"),
|
|
|
|
flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"),
|
|
|
|
flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"),
|
|
|
|
flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"),
|
|
|
|
flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "max number of redirects to follow per host"),
|
|
|
|
flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
|
|
|
|
flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "get a list of vhosts as input"),
|
|
|
|
flagSet.StringVar(&options.Methods, "x", "", "request methods to probe, use 'all' to probe all HTTP methods"),
|
|
|
|
flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"),
|
|
|
|
flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"),
|
|
|
|
flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"),
|
2022-03-01 12:07:41 +03:00
|
|
|
flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"),
|
2021-09-29 21:43:54 +03:00
|
|
|
)
|
|
|
|
|
2021-10-02 04:26:30 +03:00
|
|
|
createGroup(flagSet, "debug", "Debug",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVar(&options.Debug, "debug", false, "display request/response content in cli"),
|
|
|
|
flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "display request content in cli"),
|
|
|
|
flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"),
|
|
|
|
flagSet.BoolVar(&options.Version, "version", false, "display httpx version"),
|
|
|
|
flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"),
|
|
|
|
flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"),
|
|
|
|
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
|
2022-01-16 15:04:17 +03:00
|
|
|
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"),
|
2021-09-29 21:43:54 +03:00
|
|
|
)
|
|
|
|
|
2021-10-02 23:33:48 +03:00
|
|
|
createGroup(flagSet, "Optimizations", "Optimizations",
|
2022-03-01 11:58:37 +03:00
|
|
|
flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"),
|
|
|
|
flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "),
|
|
|
|
flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
|
|
|
|
flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"),
|
|
|
|
flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
|
|
|
|
flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"),
|
|
|
|
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"),
|
|
|
|
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"),
|
2021-09-29 21:43:54 +03:00
|
|
|
)
|
2021-10-02 23:33:48 +03:00
|
|
|
|
2021-09-29 21:57:52 +03:00
|
|
|
_ = flagSet.Parse()
|
2022-01-16 15:04:17 +03:00
|
|
|
|
|
|
|
if options.StatsInterval != 0 {
|
|
|
|
options.ShowStatistics = true
|
|
|
|
}
|
|
|
|
|
2020-10-20 21:17:02 +03:00
|
|
|
// Read the inputs and configure the logging
|
|
|
|
options.configureOutput()
|
|
|
|
|
2021-08-06 13:38:42 +03:00
|
|
|
err := options.configureResume()
|
|
|
|
if err != nil {
|
|
|
|
gologger.Fatal().Msgf("%s\n", err)
|
|
|
|
}
|
|
|
|
|
2020-10-20 21:17:02 +03:00
|
|
|
showBanner()
|
|
|
|
|
|
|
|
if options.Version {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Info().Msgf("Current Version: %s\n", Version)
|
2020-10-20 21:17:02 +03:00
|
|
|
os.Exit(0)
|
|
|
|
}
|
|
|
|
|
|
|
|
options.validateOptions()
|
|
|
|
|
|
|
|
return options
|
|
|
|
}
|
|
|
|
|
|
|
|
func (options *Options) validateOptions() {
|
2021-08-22 22:07:23 +03:00
|
|
|
if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
|
2021-09-14 20:06:35 +03:00
|
|
|
gologger.Fatal().Msgf("File %s does not exist.\n", options.InputFile)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) {
|
2021-09-14 20:06:35 +03:00
|
|
|
gologger.Fatal().Msgf("File %s does not exist.\n", options.InputRawRequest)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
|
2021-09-09 13:55:31 +03:00
|
|
|
multiOutput := options.CSVOutput && options.JSONOutput
|
|
|
|
if multiOutput {
|
|
|
|
gologger.Fatal().Msg("Results can only be displayed in one format: 'JSON' or 'CSV'\n")
|
|
|
|
}
|
|
|
|
|
2020-10-20 21:17:02 +03:00
|
|
|
var err error
|
|
|
|
if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for match status code option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.matchContentLength, err = stringz.StringToSliceInt(options.OutputMatchContentLength); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for match content length option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.filterStatusCode, err = stringz.StringToSliceInt(options.OutputFilterStatusCode); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for filter status code option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for filter content length option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.OutputFilterRegex != "" {
|
|
|
|
if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for regex filter option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if options.OutputMatchRegex != "" {
|
|
|
|
if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
}
|
2021-12-31 11:26:39 +03:00
|
|
|
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
|
|
|
|
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
|
|
|
|
}
|
|
|
|
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
|
|
|
|
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
|
|
|
|
}
|
|
|
|
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
|
|
|
|
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
|
|
|
|
}
|
|
|
|
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
|
|
|
|
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
|
|
|
|
}
|
2021-12-28 09:46:42 +03:00
|
|
|
|
|
|
|
var resolvers []string
|
|
|
|
for _, resolver := range options.Resolvers {
|
|
|
|
if fileutil.FileExists(resolver) {
|
|
|
|
chFile, err := fileutil.ReadFile(resolver)
|
|
|
|
if err != nil {
|
|
|
|
gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err)
|
|
|
|
}
|
|
|
|
for line := range chFile {
|
|
|
|
resolvers = append(resolvers, line)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
resolvers = append(resolvers, resolver)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
options.Resolvers = resolvers
|
|
|
|
if len(options.Resolvers) > 0 {
|
|
|
|
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
|
2022-01-03 10:53:15 +03:00
|
|
|
}
|
2021-12-28 09:46:42 +03:00
|
|
|
|
2022-01-04 11:47:06 +03:00
|
|
|
if options.StoreResponse && options.StoreResponseDir == "" {
|
|
|
|
gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory)
|
|
|
|
options.StoreResponseDir = DefaultOutputDirectory
|
|
|
|
}
|
2022-01-03 10:53:15 +03:00
|
|
|
if options.StoreResponseDir != "" && !options.StoreResponse {
|
|
|
|
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
|
|
|
|
options.StoreResponse = true
|
2021-12-28 09:46:42 +03:00
|
|
|
}
|
2022-01-07 14:58:07 +03:00
|
|
|
|
|
|
|
if options.Favicon {
|
|
|
|
gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n")
|
|
|
|
options.RequestURIs = "/favicon.ico"
|
|
|
|
}
|
2022-02-15 15:02:33 +03:00
|
|
|
|
|
|
|
if options.Hashes != "" {
|
|
|
|
for _, hashType := range strings.Split(options.Hashes, ",") {
|
|
|
|
if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) {
|
|
|
|
gologger.Error().Msgf("Unsupported hash type: %s\n", hashType)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// configureOutput configures the output on the screen
|
|
|
|
func (options *Options) configureOutput() {
|
|
|
|
// If the user desires verbose output, show verbose output
|
|
|
|
if options.Verbose {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.Debug {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.NoColor {
|
2021-02-17 18:15:37 +03:00
|
|
|
gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
if options.Silent {
|
2021-02-17 01:40:01 +03:00
|
|
|
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
|
2020-10-20 21:17:02 +03:00
|
|
|
}
|
|
|
|
}
|
2021-08-06 13:38:42 +03:00
|
|
|
|
|
|
|
func (options *Options) configureResume() error {
|
|
|
|
options.resumeCfg = &ResumeCfg{}
|
|
|
|
if options.Resume && fileutil.FileExists(DefaultResumeFile) {
|
|
|
|
return goconfig.Load(&options.resumeCfg, DefaultResumeFile)
|
|
|
|
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ShouldLoadResume resume file
|
|
|
|
func (options *Options) ShouldLoadResume() bool {
|
|
|
|
return options.Resume && fileutil.FileExists(DefaultResumeFile)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ShouldSaveResume file
|
|
|
|
func (options *Options) ShouldSaveResume() bool {
|
|
|
|
return true
|
|
|
|
}
|
2021-09-29 21:43:54 +03:00
|
|
|
|
|
|
|
func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
|
|
|
|
flagSet.SetGroup(groupName, description)
|
|
|
|
for _, currentFlag := range flags {
|
|
|
|
currentFlag.Group(groupName)
|
|
|
|
}
|
|
|
|
}
|