[Feature] Adding multiple path support

cmd/httpx/paths.txt

@@ -0,0 +1,2 @@
+/c
+/d

cmd/httpx/targets.txt

@@ -0,0 +1,3 @@
+https://chaos.projectdiscovery.io/a
+https://chaos.projectdiscovery.io/b
+192.168.1.1

common/fileutil/fileutil.go

@@ -1,6 +1,9 @@
 package fileutil
 
-import "os"
+import (
+	"bufio"
+	"os"
+)
 
 // FileExists checks if a file exists and is not a directory
 func FileExists(filename string) bool {
@@ -31,3 +34,18 @@ func HasStdin() bool {
 
 	return isPipedFromChrDev || isPipedFromFIFO
 }
+
+// LoadFile content to slice
+func LoadFile(filename string) (lines []string) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		lines = append(lines, s.Text())
+	}
+
+	return
+}

internal/runner/options.go

@@ -63,6 +63,8 @@ type Options struct {
 	InputFile                 string
 	Methods                   string
 	RequestURI                string
+	RequestURIs               string
+	requestURIs               []string
 	OutputMatchStatusCode     string
 	OutputMatchContentLength  string
 	OutputFilterStatusCode    string
@@ -147,6 +149,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains")
 	flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains")
 	flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')")
+	flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')")
 	flag.BoolVar(&options.OutputContentType, "content-type", false, "Extracts content-type")
 	flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match status code")
 	flag.StringVar(&options.OutputMatchStatusCode, "ml", "", "Match content length")

internal/runner/runner.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
@@ -196,6 +197,13 @@ func (runner *Runner) prepareInput() {
 		gologger.Fatalf("No input provided")
 	}
 
+	// Check if the user requested multiple paths
+	if fileutil.FileExists(runner.options.RequestURIs) {
+		runner.options.requestURIs = fileutil.LoadFile(runner.options.RequestURIs)
+	} else if runner.options.RequestURIs != "" {
+		runner.options.requestURIs = strings.Split(runner.options.RequestURIs, ",")
+	}
+
 	numTargets := 0
 	for scanner.Scan() {
 		target := strings.TrimSpace(scanner.Text())
@@ -203,9 +211,33 @@ func (runner *Runner) prepareInput() {
 		if _, ok := runner.hm.Get(target); ok {
 			continue
 		}
+
+		// base path
 		numTargets++
-		// nolint:errcheck // ignore
 		runner.hm.Set(target, nil)
+
+		// Combine multiple paths
+		// Not RFC compliant - we just append
+		for _, p := range runner.options.requestURIs {
+			newTarget := target + p
+			numTargets++
+			runner.hm.Set(newTarget, nil)
+		}
+
+		// RFC compliant
+		baseURL, err := url.Parse(target)
+		if err != nil || baseURL.Host == "" {
+			continue
+		}
+		for _, p := range runner.options.requestURIs {
+			newPath, err := url.Parse(p)
+			if err != nil {
+				continue
+			}
+			newTarget := baseURL.ResolveReference(newPath)
+			numTargets++
+			runner.hm.Set(newTarget.String(), nil)
+		}
 	}
 
 	if runner.options.InputFile != "" {
@@ -239,7 +271,7 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
 		builder.WriteRune('[')
 		startedAt, _ := stats.GetStatic("startedAt")
 		duration := time.Since(startedAt.(time.Time))
-		builder.WriteString(fmtDuration(duration))
+		builder.WriteString(clistats.FmtDuration(duration))
 		builder.WriteRune(']')
 
 		hosts, _ := stats.GetStatic("hosts")

internal/runner/util.go

@@ -1,16 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"time"
-)
-
-func fmtDuration(d time.Duration) string {
-	d = d.Round(time.Second)
-	h := d / time.Hour
-	d -= h * time.Hour
-	m := d / time.Minute
-	d -= m * time.Minute
-	s := d / time.Second
-	return fmt.Sprintf("%d:%02d:%02d", h, m, s)
-}