Merge pull request #121 from vzamanillo/golangci-lint#117

Add golangci-lint to the current GitHub workflow

.github/workflows/build.yaml

@@ -4,11 +4,33 @@ on:
     branches:
       - master
   pull_request:
-  
-jobs:          
+
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2.2.0
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.29
+          args: --timeout 5m
+          working-directory: cmd/httpx/
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
   build:
     name: Build
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2
@@ -24,4 +46,4 @@ jobs:
 
       - name: Build
         run: go build .
-        working-directory: cmd/httpx/
+        working-directory: cmd/httpx/

.golangci.yml

@@ -0,0 +1,122 @@
+linters-settings:
+  dupl:
+    threshold: 100
+  exhaustive:
+    default-signifies-exhaustive: false
+  # funlen:
+  #   lines: 100
+  #   statements: 50
+  goconst:
+    min-len: 2
+    min-occurrences: 2
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - dupImport # https://github.com/go-critic/go-critic/issues/845
+      - ifElseChain
+  # gocyclo:
+  #   min-complexity: 15
+  goimports:
+    local-prefixes: github.com/golangci/golangci-lint
+  golint:
+    min-confidence: 0
+  gomnd:
+    settings:
+      mnd:
+        # don't include the "operation" and "assign"
+        checks: argument,case,condition,return
+  govet:
+    check-shadowing: true
+    settings:
+      printf:
+        funcs:
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  # lll:
+  #   line-length: 140
+  maligned:
+    suggest-new: true
+  misspell:
+    locale: US
+  nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
+    allow-unused: false # report any unused nolint directives
+    require-explanation: false # don't require an explanation for nolint directives
+    require-specific: false # don't require nolint directives to be specific about which linter is being skipped
+
+linters:
+  # please, do not use `enable-all`: it's deprecated and will be removed soon.
+  # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - dogsled
+    - dupl
+    - errcheck
+    - exhaustive
+    - gochecknoinits
+    - goconst
+    - gocritic
+    - gofmt
+    - goimports
+    - golint
+    - gomnd
+    - goprintffuncname
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - maligned
+    - misspell
+    - nakedret
+    - noctx
+    - nolintlint
+    - rowserrcheck
+    - scopelint
+    - staticcheck
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    - whitespace
+
+  # don't enable:
+  # - depguard
+  # - asciicheck
+  # - funlen
+  # - gochecknoglobals
+  # - gocognit
+  # - gocyclo
+  # - godot
+  # - godox
+  # - goerr113
+  # - gosec
+  # - lll
+  # - nestif
+  # - prealloc
+  # - testpackage
+  # - wsl
+
+issues:
+  exclude-use-default: false
+  exclude:
+    # should have a package comment, unless it's in another file for this package (golint)
+    - 'in another file for this package'
+
+# golangci.com configuration
+# https://github.com/golangci/golangci/wiki/Configuration
+service:
+  golangci-lint-version: 1.29.x # use the fixed version to not introduce new linters unexpectedly
+  prepare:
+    - echo "here I can run custom commands, but no preparation needed for this repo"

cmd/httpx/httpx.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"regexp"
@@ -30,6 +31,11 @@ import (
 	"github.com/remeh/sizedwaitgroup"
 )
 
+const (
+	maxFileNameLenght = 255
+	tokenParts        = 2
+)
+
 func main() {
 	options := ParseOptions()
 
@@ -38,16 +44,16 @@ func main() {
 	httpxOptions.RetryMax = options.Retries
 	httpxOptions.FollowRedirects = options.FollowRedirects
 	httpxOptions.FollowHostRedirects = options.FollowHostRedirects
-	httpxOptions.HttpProxy = options.HttpProxy
+	httpxOptions.HTTPProxy = options.HTTPProxy
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)
 	for _, customHeader := range options.CustomHeaders {
-		tokens := strings.SplitN(customHeader, ":", 2)
+		tokens := strings.SplitN(customHeader, ":", tokenParts)
 		// if it's an invalid header skip it
-		if len(tokens) < 2 {
+		if len(tokens) < tokenParts {
 			continue
 		}
 		key = strings.TrimSpace(tokens[0])
@@ -90,15 +96,15 @@ func main() {
 			rawhttp.AutomaticHostHeader(false)
 		}
 	}
-	if strings.ToLower(options.Methods) == "all" {
+	if strings.EqualFold(options.Methods, "all") {
 		scanopts.Methods = httputilz.AllHTTPMethods()
 	} else if options.Methods != "" {
 		scanopts.Methods = append(scanopts.Methods, stringz.SplitByCharAndTrimSpace(options.Methods, ",")...)
 	}
 	if len(scanopts.Methods) == 0 {
-		scanopts.Methods = append(scanopts.Methods, "GET")
+		scanopts.Methods = append(scanopts.Methods, http.MethodGet)
 	}
-	protocol := "https"
+	protocol := httpx.HTTPS
 	scanopts.VHost = options.VHost
 	scanopts.OutputTitle = options.ExtractTitle
 	scanopts.OutputStatusCode = options.StatusCode
@@ -110,8 +116,8 @@ func main() {
 	scanopts.OutputWithNoColor = options.NoColor
 	scanopts.ResponseInStdout = options.responseInStdout
 	scanopts.OutputWebSocket = options.OutputWebSocket
-	scanopts.TlsProbe = options.TLSProbe
-	scanopts.CspProbe = options.CSPProbe
+	scanopts.TLSProbe = options.TLSProbe
+	scanopts.CSPProbe = options.CSPProbe
 	if options.RequestURI != "" {
 		scanopts.RequestURI = options.RequestURI
 	}
@@ -150,6 +156,7 @@ func main() {
 			if err != nil {
 				gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
 			}
+			//nolint:errcheck // this method needs a small refactor to reduce complexity
 			defer f.Close()
 		}
 		for r := range output {
@@ -191,6 +198,7 @@ func main() {
 
 			gologger.Silentf("%s\n", row)
 			if f != nil {
+				//nolint:errcheck // this method needs a small refactor to reduce complexity
 				f.WriteString(row + "\n")
 			}
 		}
@@ -205,8 +213,11 @@ func main() {
 		if err != nil {
 			gologger.Fatalf("Could read input file '%s': %s\n", options.InputFile, err)
 		}
-		defer finput.Close()
 		sc = bufio.NewScanner(finput)
+		err = finput.Close()
+		if err != nil {
+			gologger.Fatalf("Could close input file '%s': %s\n", options.InputFile, err)
+		}
 	} else if fileutil.HasStdin() {
 		sc = bufio.NewScanner(os.Stdin)
 	} else {
@@ -214,7 +225,7 @@ func main() {
 	}
 
 	for sc.Scan() {
-		process(sc.Text(), &wg, hp, protocol, scanopts, output)
+		process(sc.Text(), &wg, hp, protocol, &scanopts, output)
 	}
 
 	wg.Wait()
@@ -222,10 +233,9 @@ func main() {
 	close(output)
 
 	wgoutput.Wait()
-
 }
 
-func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts scanOptions, output chan Result) {
+func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
 	for target := range targets(stringz.TrimProtocol(t)) {
 		// if no custom ports specified then test the default ones
 		if len(customport.Ports) == 0 {
@@ -233,20 +243,20 @@ func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, proto
 				wg.Add()
 				go func(target, method string) {
 					defer wg.Done()
-					r := analyze(hp, protocol, target, 0, method, &scanopts)
+					r := analyze(hp, protocol, target, 0, method, scanopts)
 					output <- r
-					if scanopts.TlsProbe && r.TlsData != nil {
-						scanopts.TlsProbe = false
-						for _, tt := range r.TlsData.DNSNames {
+					if scanopts.TLSProbe && r.TLSData != nil {
+						scanopts.TLSProbe = false
+						for _, tt := range r.TLSData.DNSNames {
 							process(tt, wg, hp, protocol, scanopts, output)
 						}
-						for _, tt := range r.TlsData.CommonName {
+						for _, tt := range r.TLSData.CommonName {
 							process(tt, wg, hp, protocol, scanopts, output)
 						}
 					}
-					if scanopts.CspProbe && r.CspData != nil {
-						scanopts.CspProbe = false
-						for _, tt := range r.CspData.Domains {
+					if scanopts.CSPProbe && r.CSPData != nil {
+						scanopts.CSPProbe = false
+						for _, tt := range r.CSPData.Domains {
 							process(tt, wg, hp, protocol, scanopts, output)
 						}
 					}
@@ -265,14 +275,14 @@ func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, proto
 				wg.Add()
 				go func(port int, method string) {
 					defer wg.Done()
-					r := analyze(hp, protocol, target, port, method, &scanopts)
+					r := analyze(hp, protocol, target, port, method, scanopts)
 					output <- r
-					if scanopts.TlsProbe && r.TlsData != nil {
-						scanopts.TlsProbe = false
-						for _, tt := range r.TlsData.DNSNames {
+					if scanopts.TLSProbe && r.TLSData != nil {
+						scanopts.TLSProbe = false
+						for _, tt := range r.TLSData.DNSNames {
 							process(tt, wg, hp, protocol, scanopts, output)
 						}
-						for _, tt := range r.TlsData.CommonName {
+						for _, tt := range r.TLSData.CommonName {
 							process(tt, wg, hp, protocol, scanopts, output)
 						}
 					}
@@ -307,30 +317,29 @@ func targets(target string) chan string {
 		} else {
 			results <- target
 		}
-
 	}()
 	return results
 }
 
 type scanOptions struct {
 	Methods                []string
+	StoreResponseDirectory string
+	RequestURI             string
+	RequestBody            string
 	VHost                  bool
 	OutputTitle            bool
 	OutputStatusCode       bool
 	OutputLocation         bool
 	OutputContentLength    bool
 	StoreResponse          bool
-	StoreResponseDirectory string
 	OutputServerHeader     bool
 	OutputWebSocket        bool
 	OutputWithNoColor      bool
 	OutputMethod           bool
 	ResponseInStdout       bool
-	TlsProbe               bool
-	CspProbe               bool
-	RequestURI             string
+	TLSProbe               bool
+	CSPProbe               bool
 	OutputContentType      bool
-	RequestBody            string
 	Unsafe                 bool
 	Pipeline               bool
 	HTTP2Probe             bool
@@ -339,7 +348,7 @@ type scanOptions struct {
 	OutputCDN              bool
 }
 
-func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, method string, scanopts *scanOptions) Result {
+func analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
 	retried := false
 retry:
 	URL := fmt.Sprintf("%s://%s", protocol, domain)
@@ -364,10 +373,10 @@ retry:
 	resp, err := hp.Do(req)
 	if err != nil {
 		if !retried {
-			if protocol == "https" {
-				protocol = "http"
+			if protocol == httpx.HTTPS {
+				protocol = httpx.HTTP
 			} else {
-				protocol = "https"
+				protocol = httpx.HTTPS
 			}
 			retried = true
 			goto retry
@@ -394,13 +403,13 @@ retry:
 		if !scanopts.OutputWithNoColor {
 			// Color the status code based on its value
 			switch {
-			case resp.StatusCode >= 200 && resp.StatusCode < 300:
+			case resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices:
 				builder.WriteString(aurora.Green(strconv.Itoa(resp.StatusCode)).String())
-			case resp.StatusCode >= 300 && resp.StatusCode < 400:
+			case resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest:
 				builder.WriteString(aurora.Yellow(strconv.Itoa(resp.StatusCode)).String())
-			case resp.StatusCode >= 400 && resp.StatusCode < 500:
+			case resp.StatusCode >= http.StatusBadRequest && resp.StatusCode < http.StatusInternalServerError:
 				builder.WriteString(aurora.Red(strconv.Itoa(resp.StatusCode)).String())
-			case resp.StatusCode > 500:
+			case resp.StatusCode > http.StatusInternalServerError:
 				builder.WriteString(aurora.Bold(aurora.Yellow(strconv.Itoa(resp.StatusCode))).String())
 			}
 		} else {
@@ -537,11 +546,12 @@ retry:
 		}
 		// On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux
 		// Truncating length at 255
-		if len(domainFile) >= 255 {
+		if len(domainFile) >= maxFileNameLenght {
 			// leaving last 4 bytes free to append ".txt"
-			domainFile = domainFile[:251]
+			domainFile = domainFile[:maxFileNameLenght-1]
 		}
-		domainFile = strings.Replace(domainFile, "/", "_", -1) + ".txt"
+
+		domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt"
 		responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
 		err := ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
 		if err != nil {
@@ -562,8 +572,8 @@ retry:
 		WebServer:     serverHeader,
 		Response:      serverResponseRaw,
 		WebSocket:     isWebSocket,
-		TlsData:       resp.TlsData,
-		CspData:       resp.CspData,
+		TLSData:       resp.TLSData,
+		CSPData:       resp.CSPData,
 		Pipeline:      pipeline,
 		HTTP2:         http2,
 		Method:        method,
@@ -576,27 +586,27 @@ retry:
 
 // Result of a scan
 type Result struct {
+	IPs           []string `json:"ips"`
+	CNAMEs        []string `json:"cnames,omitempty"`
 	raw           string
 	URL           string `json:"url"`
-	ContentLength int    `json:"content-length"`
-	StatusCode    int    `json:"status-code"`
 	Location      string `json:"location"`
 	Title         string `json:"title"`
 	str           string
 	err           error
-	VHost         bool           `json:"vhost"`
 	WebServer     string         `json:"webserver"`
 	Response      string         `json:"serverResponse,omitempty"`
-	WebSocket     bool           `json:"websocket,omitempty"`
 	ContentType   string         `json:"content-type,omitempty"`
-	TlsData       *httpx.TlsData `json:"tls,omitempty"`
-	CspData       *httpx.CspData `json:"csp,omitempty"`
-	Pipeline      bool           `json:"pipeline,omitempty"`
-	HTTP2         bool           `json:"http2"`
 	Method        string         `json:"method"`
 	IP            string         `json:"ip"`
-	IPs           []string       `json:"ips"`
-	CNAMEs        []string       `json:"cnames,omitempty"`
+	ContentLength int            `json:"content-length"`
+	StatusCode    int            `json:"status-code"`
+	TLSData       *httpx.TLSData `json:"tls,omitempty"`
+	CSPData       *httpx.CSPData `json:"csp,omitempty"`
+	VHost         bool           `json:"vhost"`
+	WebSocket     bool           `json:"websocket,omitempty"`
+	Pipeline      bool           `json:"pipeline,omitempty"`
+	HTTP2         bool           `json:"http2"`
 	CDN           bool           `json:"cdn"`
 }
 
@@ -611,26 +621,44 @@ func (r *Result) JSON() string {
 
 // Options contains configuration options for chaos client.
 type Options struct {
+	CustomHeaders             customheader.CustomHeaders
+	CustomPorts               customport.CustomPorts
+	matchStatusCode           []int
+	matchContentLength        []int
+	filterStatusCode          []int
+	filterContentLength       []int
+	Output                    string
+	StoreResponseDir          string
+	HTTPProxy                 string
+	SocksProxy                string
+	InputFile                 string
+	Methods                   string
+	RequestURI                string
+	OutputMatchStatusCode     string
+	OutputMatchContentLength  string
+	OutputFilterStatusCode    string
+	OutputFilterContentLength string
+	InputRawRequest           string
+	rawRequest                string
+	RequestBody               string
+	OutputFilterString        string
+	OutputMatchString         string
+	OutputFilterRegex         string
+	OutputMatchRegex          string
+	Retries                   int
+	Threads                   int
+	Timeout                   int
+	filterRegex               *regexp.Regexp
+	matchRegex                *regexp.Regexp
 	VHost                     bool
 	Smuggling                 bool
 	ExtractTitle              bool
 	StatusCode                bool
 	Location                  bool
 	ContentLength             bool
-	Retries                   int
-	Threads                   int
-	Timeout                   int
-	CustomHeaders             customheader.CustomHeaders
-	CustomPorts               customport.CustomPorts
-	Output                    string
 	FollowRedirects           bool
 	StoreResponse             bool
-	StoreResponseDir          string
-	HttpProxy                 string
-	SocksProxy                string
 	JSONOutput                bool
-	InputFile                 string
-	Methods                   string
 	Silent                    bool
 	Version                   bool
 	Verbose                   bool
@@ -642,31 +670,13 @@ type Options struct {
 	OutputMethod              bool
 	TLSProbe                  bool
 	CSPProbe                  bool
-	RequestURI                string
 	OutputContentType         bool
-	OutputMatchStatusCode     string
-	matchStatusCode           []int
-	OutputMatchContentLength  string
-	matchContentLength        []int
-	OutputFilterStatusCode    string
-	filterStatusCode          []int
-	OutputFilterContentLength string
 	OutputIP                  bool
 	OutputCName               bool
-	filterContentLength       []int
-	InputRawRequest           string
-	rawRequest                string
 	Unsafe                    bool
-	RequestBody               string
 	Debug                     bool
 	Pipeline                  bool
 	HTTP2Probe                bool
-	OutputFilterString        string
-	OutputMatchString         string
-	OutputFilterRegex         string
-	filterRegex               *regexp.Regexp
-	OutputMatchRegex          string
-	matchRegex                *regexp.Regexp
 	OutputCDN                 bool
 }
 
@@ -689,7 +699,7 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.StoreResponseDir, "srd", "output", "Save response directory")
 	flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow Redirects")
 	flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only follow redirects on the same host")
-	flag.StringVar(&options.HttpProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
+	flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080")
 	flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
 	flag.StringVar(&options.InputFile, "l", "", "File containing domains")
 	flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
@@ -794,9 +804,9 @@ func (options *Options) configureOutput() {
 const banner = `
     __    __  __       _  __
    / /_  / /_/ /_____ | |/ /
-  / __ \/ __/ __/ __ \|   / 
- / / / / /_/ /_/ /_/ /   |  
-/_/ /_/\__/\__/ .___/_/|_|  
+  / __ \/ __/ __/ __ \|   /
+ / / / / /_/ /_/ /_/ /   |
+/_/ /_/\__/\__/ .___/_/|_|
              /_/              v1.0.2
 `
 

common/cache/cache.go

@@ -4,10 +4,12 @@ import (
 	"net"
 
 	"github.com/coocood/freecache"
-	dns "github.com/projectdiscovery/httpx/common/resolver"
+	dns "github.com/projectdiscovery/httpx/common/resolve"
 )
 
-// Cache is a strcture for caching DNS lookups
+const megaByteBytes = 1048576
+
+// Cache is a structure for caching DNS lookups
 type Cache struct {
 	dnsClient             Resolver
 	cache                 *freecache.Cache
@@ -49,7 +51,7 @@ func New(options Options) (*Cache, error) {
 	if err != nil {
 		return nil, err
 	}
-	cache := freecache.NewCache(options.CacheSize * 1024 * 1024)
+	cache := freecache.NewCache(options.CacheSize * megaByteBytes)
 	return &Cache{dnsClient: dnsClient, cache: cache, defaultExpirationTime: options.ExpirationTime}, nil
 }
 
@@ -65,21 +67,29 @@ func (c *Cache) Lookup(hostname string) (*dns.Result, error) {
 		if err != freecache.ErrNotFound {
 			return nil, err
 		}
-		result, err := c.dnsClient.Resolve(hostname)
-		if err != nil {
-			return nil, err
+		result, resolveErr := c.dnsClient.Resolve(hostname)
+		if resolveErr != nil {
+			return nil, resolveErr
 		}
 		if result.TTL == 0 {
 			result.TTL = c.defaultExpirationTime
 		}
 		b, _ := result.Marshal()
-		c.cache.Set(hostnameBytes, b, result.TTL)
+
+		err = c.cache.Set(hostnameBytes, b, result.TTL)
+		if err != nil {
+			return nil, err
+		}
 
 		return &result, nil
 	}
 
 	var result dns.Result
-	result.Unmarshal(value)
+
+	err = result.Unmarshal(value)
+	if err != nil {
+		return nil, err
+	}
 
 	return &result, nil
 }

common/cache/dialer.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/coocood/freecache"
-	dns "github.com/projectdiscovery/httpx/common/resolver"
+	dns "github.com/projectdiscovery/httpx/common/resolve"
 )
 
 var (
@@ -32,7 +32,7 @@ func NewDialer(options Options) (DialerFunc, error) {
 	if err != nil {
 		return nil, err
 	}
-	dialerHistory = freecache.NewCache(options.CacheSize * 1024 * 1024)
+	dialerHistory = freecache.NewCache(options.CacheSize * megaByteBytes)
 	dialer := &net.Dialer{
 		Timeout:   10 * time.Second,
 		KeepAlive: 10 * time.Second,
@@ -50,7 +50,10 @@ func NewDialer(options Options) (DialerFunc, error) {
 		for _, ip := range dnsResult.IPs {
 			conn, err = dialer.DialContext(ctx, network, ip+address[separator:])
 			if err == nil {
-				dialerHistory.Set([]byte(hostname), []byte(ip), 0)
+				setErr := dialerHistory.Set([]byte(hostname), []byte(ip), 0)
+				if setErr != nil {
+					return nil, err
+				}
 				break
 			}
 		}

common/cache/doc.go

@@ -0,0 +1,2 @@
+// Package cache contains the logic for caching dns resolver
+package cache

common/customheader/doc.go

@@ -0,0 +1,2 @@
+// Package customheader contains all the funcionality to deal with Custom Global Headers
+package customheader

common/customports/customport.go

@@ -5,10 +5,13 @@ import (
 	"strings"
 )
 
+//nolint:gochecknoinits // this flag var needs a small refactor to avoid the use of the init function
 func init() {
 	Ports = make(map[int]struct{})
 }
 
+const portRangeParts = 2
+
 // Ports to scan
 var Ports map[int]struct{}
 
@@ -30,7 +33,7 @@ func (c *CustomPorts) Set(value string) error {
 	for _, potentialPort := range potentialPorts {
 		potentialRange := strings.Split(strings.TrimSpace(potentialPort), "-")
 		// it's a single port?
-		if len(potentialRange) < 2 {
+		if len(potentialRange) < portRangeParts {
 			if p, err := strconv.Atoi(potentialPort); err == nil {
 				Ports[p] = struct{}{}
 			}

common/customports/doc.go

@@ -0,0 +1,2 @@
+// Package customport contains all the funcionality to deal with HTTP ports
+package customport

common/fileutil/doc.go

@@ -0,0 +1,2 @@
+// Package fileutil contains all the funcionality related to deal with files
+package fileutil

common/httputilz/doc.go

@@ -0,0 +1,2 @@
+// Package httputilz contains all the funcionality related to common HTTP operations, dump, define methods...
+package httputilz

common/httputilz/httputilz.go

@@ -12,6 +12,11 @@ import (
 	"github.com/projectdiscovery/retryablehttp-go"
 )
 
+const (
+	headerParts  = 2
+	requestParts = 3
+)
+
 // DumpRequest to string
 func DumpRequest(req *retryablehttp.Request) (string, error) {
 	dump, err := httputil.DumpRequestOut(req.Request, true)
@@ -22,7 +27,7 @@ func DumpRequest(req *retryablehttp.Request) (string, error) {
 // DumpResponse to string
 func DumpResponse(resp *http.Response) (string, error) {
 	// httputil.DumpResponse does not work with websockets
-	if resp.StatusCode == 101 {
+	if resp.StatusCode == http.StatusContinue {
 		raw := resp.Status + "\n"
 		for h, v := range resp.Header {
 			raw += fmt.Sprintf("%s: %s\n", h, v)
@@ -35,7 +40,7 @@ func DumpResponse(resp *http.Response) (string, error) {
 }
 
 // ParseRequest from raw string
-func ParseRequest(req string) (method string, path string, headers map[string]string, body string, err error) {
+func ParseRequest(req string) (method, path string, headers map[string]string, body string, err error) {
 	headers = make(map[string]string)
 	reader := bufio.NewReader(strings.NewReader(req))
 	s, err := reader.ReadString('\n')
@@ -44,22 +49,22 @@ func ParseRequest(req string) (method string, path string, headers map[string]st
 		return
 	}
 	parts := strings.Split(s, " ")
-	if len(parts) < 3 {
+	if len(parts) < requestParts {
 		err = fmt.Errorf("malformed request supplied")
 		return
 	}
 	method = parts[0]
 
 	for {
-		line, err := reader.ReadString('\n')
+		line, readErr := reader.ReadString('\n')
 		line = strings.TrimSpace(line)
 
-		if err != nil || line == "" {
+		if readErr != nil || line == "" {
 			break
 		}
 
-		p := strings.SplitN(line, ":", 2)
-		if len(p) != 2 {
+		p := strings.SplitN(line, ":", headerParts)
+		if len(p) != headerParts {
 			continue
 		}
 
@@ -95,5 +100,6 @@ func ParseRequest(req string) (method string, path string, headers map[string]st
 		return
 	}
 	body = string(b)
-	return
+
+	return method, path, headers, body, nil
 }

common/httputilz/methods.go

@@ -1,28 +1,18 @@
 package httputilz
 
-// HTTP methods were copied from net/http - fasthttp
-const (
-	MethodGet     = "GET"     // RFC 7231, 4.3.1
-	MethodHead    = "HEAD"    // RFC 7231, 4.3.2
-	MethodPost    = "POST"    // RFC 7231, 4.3.3
-	MethodPut     = "PUT"     // RFC 7231, 4.3.4
-	MethodPatch   = "PATCH"   // RFC 5789
-	MethodDelete  = "DELETE"  // RFC 7231, 4.3.5
-	MethodConnect = "CONNECT" // RFC 7231, 4.3.6
-	MethodOptions = "OPTIONS" // RFC 7231, 4.3.7
-	MethodTrace   = "TRACE"   // RFC 7231, 4.3.8
-)
+import "net/http"
 
+// AllHTTPMethods contains all available HTTP methods
 func AllHTTPMethods() []string {
 	return []string{
-		MethodGet,
-		MethodHead,
-		MethodPost,
-		MethodPut,
-		MethodPatch,
-		MethodDelete,
-		MethodConnect,
-		MethodOptions,
-		MethodTrace,
+		http.MethodGet,
+		http.MethodHead,
+		http.MethodPost,
+		http.MethodPut,
+		http.MethodPatch,
+		http.MethodDelete,
+		http.MethodConnect,
+		http.MethodOptions,
+		http.MethodTrace,
 	}
 }

common/httpx/csp.go

@@ -9,17 +9,19 @@ import (
 
 // CSPHeaders is an incomplete list of most common CSP headers
 var CSPHeaders []string = []string{
-	"Content-Security-Policy",               //standard
-	"Content-Security-Policy-Report-Only",   //standard
+	"Content-Security-Policy",               // standard
+	"Content-Security-Policy-Report-Only",   // standard
 	"X-Content-Security-Policy-Report-Only", // non - standard
 	"X-Webkit-Csp-Report-Only",              // non - standard
 }
 
-type CspData struct {
+// CSPData contains the Content-Security-Policy domain list
+type CSPData struct {
 	Domains []string `json:"domains,omitempty"`
 }
 
-func (h *HTTPX) CspGrab(r *http.Response) *CspData {
+// CSPGrab fills the CSPData
+func (h *HTTPX) CSPGrab(r *http.Response) *CSPData {
 	domains := make(map[string]struct{})
 	for _, cspHeader := range CSPHeaders {
 		cspRaw := r.Header.Get(cspHeader)
@@ -39,7 +41,7 @@ func (h *HTTPX) CspGrab(r *http.Response) *CspData {
 	}
 
 	if len(domains) > 0 {
-		return &CspData{Domains: slice.ToSlice(domains)}
+		return &CSPData{Domains: slice.ToSlice(domains)}
 	}
 	return nil
 }

common/httpx/doc.go

@@ -0,0 +1,2 @@
+// Package httpx containst the httpx common funcionality
+package httpx

common/httpx/encodings.go

@@ -11,7 +11,7 @@ import (
 
 // Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5
 
-//convert GBK to UTF-8
+// Decodegbk converts GBK to UTF-8
 func Decodegbk(s []byte) ([]byte, error) {
 	I := bytes.NewReader(s)
 	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
@@ -22,7 +22,7 @@ func Decodegbk(s []byte) ([]byte, error) {
 	return d, nil
 }
 
-//convert BIG5 to UTF-8
+// Decodebig5 converts BIG5 to UTF-8
 func Decodebig5(s []byte) ([]byte, error) {
 	I := bytes.NewReader(s)
 	O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder())
@@ -33,7 +33,7 @@ func Decodebig5(s []byte) ([]byte, error) {
 	return d, nil
 }
 
-//convert UTF-8 to BIG5
+// Encodebig5 converts UTF-8 to BIG5
 func Encodebig5(s []byte) ([]byte, error) {
 	I := bytes.NewReader(s)
 	O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder())

common/httpx/http2.go

@@ -1,18 +1,28 @@
 package httpx
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/retryablehttp-go"
 )
 
+const (
+	// HTTP defines the plain http scheme
+	HTTP = "http"
+	// HTTPS defines the secure http scheme
+	HTTPS = "https"
+)
+
 // SupportHTTP2 checks if the target host supports HTTP2
-func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool {
+func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool {
 	// http => supports HTTP1.1 => HTTP/2 (H2C)
-	if protocol == "http" {
-		req, err := retryablehttp.NewRequest(method, URL, nil)
+	if protocol == HTTP {
+		req, err := retryablehttp.NewRequest(method, targetURL, nil)
 		if err != nil {
 			return false
 		}
@@ -23,22 +33,46 @@ func (h *HTTPX) SupportHTTP2(protocol, method, URL string) bool {
 		if err != nil {
 			return false
 		}
-		io.Copy(ioutil.Discard, httpresp.Body)
-		httpresp.Body.Close()
 
-		return httpresp.StatusCode == 101
+		err = freeHTTPResources(httpresp)
+		if err != nil {
+			gologger.Errorf("%s", err)
+			return false
+		}
+
+		return httpresp.StatusCode == http.StatusSwitchingProtocols
 	}
 
 	// attempts a direct http2 connection
-	req, err := http.NewRequest(method, URL, nil)
+	req, err := http.NewRequestWithContext(context.Background(), method, targetURL, nil)
 	if err != nil {
 		return false
 	}
+
 	httpresp, err := h.client2.Do(req)
 	if err != nil {
 		return false
 	}
-	io.Copy(ioutil.Discard, httpresp.Body)
-	httpresp.Body.Close()
+
+	err = freeHTTPResources(httpresp)
+	if err != nil {
+		gologger.Errorf("%s", err)
+		return false
+	}
+
 	return httpresp.Proto == "HTTP/2.0"
 }
+
+func freeHTTPResources(response *http.Response) error {
+	_, err := io.Copy(ioutil.Discard, response.Body)
+	if err != nil {
+		return fmt.Errorf("could not discard response body: %s", err)
+	}
+
+	err = response.Body.Close()
+	if err != nil {
+		return fmt.Errorf("could not close response body: %s", err)
+	}
+
+	return nil
+}

common/httpx/httpx.go

@@ -35,7 +35,7 @@ func New(options *Options) (*HTTPX, error) {
 	httpx := &HTTPX{}
 	dialer, err := cache.NewDialer(cache.DefaultOptions)
 	if err != nil {
-		return nil, fmt.Errorf("Could not create resolver cache: %s", err)
+		return nil, fmt.Errorf("could not create resolver cache: %s", err)
 	}
 
 	httpx.Options = options
@@ -63,7 +63,6 @@ func New(options *Options) (*HTTPX, error) {
 				return http.ErrUseLastResponse // Tell the http client to not follow redirect
 			}
 			return nil
-
 		}
 	}
 
@@ -76,10 +75,10 @@ func New(options *Options) (*HTTPX, error) {
 		DisableKeepAlives: true,
 	}
 
-	if httpx.Options.HttpProxy != "" {
-		proxyURL, err := url.Parse(httpx.Options.HttpProxy)
-		if err != nil {
-			return nil, err
+	if httpx.Options.HTTPProxy != "" {
+		proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy)
+		if parseErr != nil {
+			return nil, parseErr
 		}
 		transport.Proxy = http.ProxyURL(proxyURL)
 	}
@@ -105,7 +104,7 @@ func New(options *Options) (*HTTPX, error) {
 	httpx.RequestOverride = &options.RequestOverride
 	httpx.cdn, err = cdncheck.New()
 	if err != nil {
-		return nil, fmt.Errorf("Could not create cdn check: %s", err)
+		return nil, fmt.Errorf("could not create cdn check: %s", err)
 	}
 
 	return httpx, nil
@@ -113,15 +112,8 @@ func New(options *Options) (*HTTPX, error) {
 
 // Do http request
 func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
-	var (
-		httpresp *http.Response
-		err      error
-	)
-	if h.Options.Unsafe {
-		httpresp, err = h.doUnsafe(req)
-	} else {
-		httpresp, err = h.client.Do(req)
-	}
+	httpresp, err := h.getResponse(req)
+
 	if err != nil {
 		return nil, err
 	}
@@ -139,13 +131,19 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 
 	var respbody []byte
 	// websockets don't have a readable body
-	if httpresp.StatusCode != 101 {
+	if httpresp.StatusCode != http.StatusSwitchingProtocols {
 		var err error
 		respbody, err = ioutil.ReadAll(httpresp.Body)
 		if err != nil {
 			return nil, err
 		}
 	}
+
+	closeErr := httpresp.Body.Close()
+	if closeErr != nil {
+		return nil, closeErr
+	}
+
 	respbodystr := string(respbody)
 
 	// check if we need to strip html
@@ -165,25 +163,35 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 
 	if !h.Options.Unsafe {
 		// extracts TLS data if any
-		resp.TlsData = h.TlsGrab(httpresp)
+		resp.TLSData = h.TLSGrab(httpresp)
 	}
 
-	resp.CspData = h.CspGrab(httpresp)
+	resp.CSPData = h.CSPGrab(httpresp)
 
 	return &resp, nil
 }
 
+// RequestOverride contains the URI path to override the request
 type RequestOverride struct {
 	URIPath string
 }
 
-// Do http request
+// getResponse returns response from safe / unsafe request
+func (h *HTTPX) getResponse(req *retryablehttp.Request) (*http.Response, error) {
+	if h.Options.Unsafe {
+		return h.doUnsafe(req)
+	}
+
+	return h.client.Do(req)
+}
+
+// doUnsafe does an unsafe http request
 func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) {
 	method := req.Method
 	headers := req.Header
-	url := req.URL.String()
+	targetURL := req.URL.String()
 	body := req.Body
-	return rawhttp.DoRaw(method, url, h.RequestOverride.URIPath, headers, body)
+	return rawhttp.DoRaw(method, targetURL, h.RequestOverride.URIPath, headers, body)
 }
 
 // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true
@@ -213,8 +221,8 @@ func (h *HTTPX) AddFilter(f Filter) {
 }
 
 // NewRequest from url
-func (h *HTTPX) NewRequest(method, URL string) (req *retryablehttp.Request, err error) {
-	req, err = retryablehttp.NewRequest(method, URL, nil)
+func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error) {
+	req, err = retryablehttp.NewRequest(method, targetURL, nil)
 	if err != nil {
 		return
 	}
@@ -231,7 +239,7 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]st
 	for name, value := range headers {
 		r.Header.Set(name, value)
 		// host header is particular
-		if strings.ToLower(name) == "host" {
+		if strings.EqualFold(name, "host") {
 			r.Host = value
 		}
 	}

common/httpx/option.go

@@ -6,31 +6,27 @@ import (
 
 // Options contains configuration options for the client
 type Options struct {
-	Threads int
+	DefaultUserAgent string
+	RequestOverride  RequestOverride
+	HTTPProxy        string
+	SocksProxy       string
+	Threads          int
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
 	// RetryMax is the maximum number of retries
-	RetryMax int
-
-	CustomHeaders       map[string]string
-	FollowRedirects     bool
-	FollowHostRedirects bool
-	DefaultUserAgent    string
-	Unsafe              bool
-	RequestOverride     RequestOverride
-
-	HttpProxy  string
-	SocksProxy string
-
+	RetryMax      int
+	CustomHeaders map[string]string
+	// VHostSimilarityRatio 1 - 100
+	VHostSimilarityRatio int
+	FollowRedirects      bool
+	FollowHostRedirects  bool
+	Unsafe               bool
 	// VHOSTs options
 	VHostIgnoreStatusCode    bool
 	VHostIgnoreContentLength bool
 	VHostIgnoreNumberOfWords bool
 	VHostIgnoreNumberOfLines bool
 	VHostStripHTML           bool
-
-	// VHostimilarityRatio 1 - 100
-	VHostSimilarityRatio int
 }
 
 // DefaultOptions contains the default options

common/httpx/pipeline.go

@@ -39,7 +39,11 @@ func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool {
 	gotReplies := 0
 	reply := make([]byte, 1024)
 	for i := 0; i < nprobes; i++ {
-		conn.SetReadDeadline(time.Now().Add(1 * time.Second))
+		err := conn.SetReadDeadline(time.Now().Add(1 * time.Second))
+		if err != nil {
+			return false
+		}
+
 		if _, err := conn.Read(reply); err != nil {
 			break
 		}
@@ -50,7 +54,6 @@ func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool {
 				gotReplies++
 			}
 		}
-
 	}
 
 	// expect at least 2 replies

common/httpx/response.go

@@ -13,9 +13,9 @@ type Response struct {
 	Raw           string
 	Words         int
 	Lines         int
-	TlsData       *TlsData
-	CspData       *CspData
-	Http2         bool
+	TLSData       *TLSData
+	CSPData       *CSPData
+	HTTP2         bool
 	Pipeline      bool
 }
 
@@ -30,7 +30,7 @@ func (r *Response) GetHeader(name string) string {
 }
 
 // GetHeaderPart with offset
-func (r *Response) GetHeaderPart(name string, sep string) string {
+func (r *Response) GetHeaderPart(name, sep string) string {
 	v, ok := r.Headers[name]
 	if ok && len(v) > 0 {
 		tokens := strings.Split(strings.Join(v, " "), sep)

common/httpx/tls.go

@@ -4,7 +4,8 @@ import (
 	"net/http"
 )
 
-type TlsData struct {
+// TLSData contains the relevant Transport Layer Security information
+type TLSData struct {
 	DNSNames         []string `json:"dns_names,omitempty"`
 	Emails           []string `json:"emails,omitempty"`
 	CommonName       []string `json:"common_name,omitempty"`
@@ -13,9 +14,10 @@ type TlsData struct {
 	IssuerOrg        []string `json:"issuer_organization,omitempty"`
 }
 
-func (h *HTTPX) TlsGrab(r *http.Response) *TlsData {
+// TLSGrab fills the TLSData
+func (h *HTTPX) TLSGrab(r *http.Response) *TLSData {
 	if r.TLS != nil {
-		var tlsdata TlsData
+		var tlsdata TLSData
 		for _, certificate := range r.TLS.PeerCertificates {
 			tlsdata.DNSNames = append(tlsdata.DNSNames, certificate.DNSNames...)
 			tlsdata.Emails = append(tlsdata.Emails, certificate.EmailAddresses...)

common/httpx/virtualhost.go

@@ -8,6 +8,8 @@ import (
 	"github.com/rs/xid"
 )
 
+const simMultiplier = 100
+
 // IsVirtualHost checks if the target endpoint is a virtual host
 func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) {
 	httpresp1, err := h.Do(req)
@@ -44,7 +46,7 @@ func (h *HTTPX) IsVirtualHost(req *retryablehttp.Request) (bool, error) {
 	}
 
 	// Similarity Ratio - if similarity is under threshold we consider it a valid vHost
-	if int(strsim.Compare(httpresp1.Raw, httpresp2.Raw)*100) <= h.Options.VHostSimilarityRatio {
+	if int(strsim.Compare(httpresp1.Raw, httpresp2.Raw)*simMultiplier) <= h.Options.VHostSimilarityRatio {
 		return true, nil
 	}
 

common/iputil/doc.go

@@ -0,0 +1,2 @@
+// Package iputil containst common utils for IP addresses
+package iputil

common/resolve/client.go

@@ -1,4 +1,4 @@
-package dns
+package resolve
 
 import (
 	"bytes"

common/resolve/doc.go

@@ -0,0 +1,3 @@
+// Package resolve is used to handle resolving records
+// It also handles wildcard subdomains and rotating resolvers.
+package resolve

common/resolve/utils.go

@@ -1,4 +1,4 @@
-package dns
+package resolve
 
 import (
 	"net"

common/slice/doc.go

@@ -0,0 +1,2 @@
+// Package slice contains a set of utilities to deal with slices
+package slice

common/stringz/doc.go

@@ -0,0 +1,2 @@
+// Package stringz contains a set of utilities to deal with strings
+package stringz

common/stringz/stringz.go

@@ -5,8 +5,9 @@ import (
 	"strings"
 )
 
-func TrimProtocol(URL string) string {
-	URL = strings.TrimSpace(URL)
+// TrimProtocol removes the HTTP scheme from an URI
+func TrimProtocol(targetURL string) string {
+	URL := strings.TrimSpace(targetURL)
 	if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") {
 		URL = URL[strings.Index(URL, "//")+2:]
 	}
@@ -21,8 +22,8 @@ func StringToSliceInt(s string) ([]int, error) {
 		return r, nil
 	}
 	for _, v := range strings.Split(s, ",") {
-		v := strings.TrimSpace(v)
-		if i, err := strconv.Atoi(v); err == nil {
+		vTrim := strings.TrimSpace(v)
+		if i, err := strconv.Atoi(vTrim); err == nil {
 			r = append(r, i)
 		} else {
 			return r, err
@@ -32,7 +33,8 @@ func StringToSliceInt(s string) ([]int, error) {
 	return r, nil
 }
 
-func SplitByCharAndTrimSpace(s string, splitchar string) (result []string) {
+// SplitByCharAndTrimSpace splits string by a character and remove spaces
+func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
 	for _, token := range strings.Split(s, splitchar) {
 		result = append(result, strings.TrimSpace(token))
 	}