projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-12-01 12:13:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #471 from projectdiscovery/issue-464-default-ports

Browse Source 
Adding CLI option to trim default HTTP/HTTPS ports from Host header
This commit is contained in:
Sandeep Singh 2022-01-04 15:05:08 +05:30 committed by GitHub
commit 47915603a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
runner/options.go
View File

@ -21,9 +21,10 @@ import (
const (
// The maximum file length is 251 (255 - 4 bytes for ".ext" suffix)
maxFileNameLength = 251
two = 2
DefaultResumeFile = "resume.cfg"
maxFileNameLength = 251
two = 2
DefaultResumeFile = "resume.cfg"
DefaultOutputDirectory = "output"
)
type scanOptions struct {
@ -66,6 +67,7 @@ type scanOptions struct {
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
LeaveDefaultPorts bool
OutputLinesCount bool
OutputWordsCount bool
}
@ -107,6 +109,7 @@ func (s *scanOptions) Clone() *scanOptions {
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
LeaveDefaultPorts: s.LeaveDefaultPorts,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
}
@ -203,6 +206,7 @@ type Options struct {
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.NormalizedStringSlice
LeaveDefaultPorts bool
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
@ -285,7 +289,7 @@ func ParseOptions() *Options {
createGroup(flagSet, "output", "Output",
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "store http response to custom directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in CSV format"),
flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"),
flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"),
@ -311,6 +315,7 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"),
flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"),
flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"),
flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "Leave default HTTP/HTTPS ports (eg. http://host:80 - https//host:443"),
)
createGroup(flagSet, "debug", "Debug",
@ -425,6 +430,10 @@ func (options *Options) validateOptions() {
gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ","))
}
if options.StoreResponse && options.StoreResponseDir == "" {
gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory)
options.StoreResponseDir = DefaultOutputDirectory
}
if options.StoreResponseDir != "" && !options.StoreResponse {
gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n")
options.StoreResponse = true

10
runner/runner.go
View File

@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) {
scanopts.ExcludeCDN = options.ExcludeCDN
scanopts.HostMaxErrors = options.HostMaxErrors
scanopts.ProbeAllIPS = options.ProbeAllIPS
scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts
scanopts.OutputLinesCount = options.OutputLinesCount
scanopts.OutputWordsCount = options.OutputWordsCount
runner.scanopts = scanopts
@ -853,6 +854,15 @@ retry:
req.Host = customHost
}
if !scanopts.LeaveDefaultPorts {
switch {
case protocol == httpx.HTTP && strings.HasSuffix(req.Host, ":80"):
req.Host = strings.TrimSuffix(req.Host, ":80")
case protocol == httpx.HTTPS && strings.HasSuffix(req.Host, ":443"):
req.Host = strings.TrimSuffix(req.Host, ":443")
}
}
hp.SetCustomHeaders(req, hp.CustomHeaders)
// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
if scanopts.RequestBody != "" {