Adding integration/regression tests

2024-08-17 05:20:29 +03:00 · 2021-08-23 12:50:37 +02:00 · 2021-08-23 12:50:37 +02:00 · 020b719e4d
commit 020b719e4d
parent 4f7f352677
8 changed files with 270 additions and 6 deletions
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@ -19,8 +19,11 @@ jobs:
        uses: actions/checkout@v2

      - name: Test
-        run: go test .
-        working-directory: cmd/httpx/
+        run: go test ./...
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/

      - name: Build
        run: go build .
--- a/.gitignore
+++ b/.gitignore
@ -1,2 +1,4 @@

 cmd/httpx/httpx
+integration_tests/httpx
+integration_tests/integration-test
--- a/cmd/integration-test/http.go
+++ b/cmd/integration-test/http.go
@ -0,0 +1,138 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"Standard HTTP GET Request":  &standardHttpGet{},
+	"Standard HTTPS GET Request": &standardHttpGet{tls: true},
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{},
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{},
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{},
+}
+
+type standardHttpGet struct {
+	tls bool
+}
+
+func (h *standardHttpGet) Execute() error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is a test")
+	}))
+	var ts *httptest.Server
+	if h.tls {
+		ts = httptest.NewTLSServer(router)
+	} else {
+		ts = httptest.NewServer(router)
+	}
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
+type issue276 struct{}
+
+func (h *issue276) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.Header().Add("Location", ts.URL+"/redirect")
+		w.WriteHeader(302)
+		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-status-code", "-title", "-no-color")
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	// status code
+	// title
+	expected := ts.URL + "/redirect" + " [302] [Object moved]"
+	// log.Fatal(results[0], expected)
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
+
+type issue277 struct{}
+
+func (h *issue277) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		if p.ByName("pp") == `%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E` {
+			w.WriteHeader(http.StatusOK)
+		}
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+	uripath := "/hpp/?pp=%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E"
+	results, err := testutils.RunHttpxAndGetResults(ts.URL+uripath, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	// status code
+	// title
+	expected := ts.URL + uripath
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
+
+type issue303 struct{}
+
+func (h *issue303) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		// mimic a misconfigured web server behavior declaring gzip body
+		w.Header().Add("Content-Encoding", "gzip")
+		// but sending it uncompressed
+		fmt.Fprint(w, "<html><body>This is a test</body></html>")
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	expected := ts.URL
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
--- a/cmd/integration-test/integration-test.go
+++ b/cmd/integration-test/integration-test.go
@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+
+	errored = false
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	tests := map[string]map[string]testutils.TestCase{
+		"http": httpTestcases,
+	}
+	for proto, tests := range tests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(proto))
+
+			for name, test := range tests {
+				if customTest != "" && !strings.Contains(name, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute()
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err)
+					errored = true
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, name)
+				}
+			}
+		}
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}
+
+func errIncorrectResult(expected, got string) error {
+	return fmt.Errorf("incorrect result: expected \"%s\" got \"%s\"", expected, got)
+}
--- a/go.mod
+++ b/go.mod
@ -10,6 +10,7 @@ require (
 	github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
+	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.15
 	github.com/miekg/dns v1.1.43 // indirect
@ -29,11 +30,12 @@ require (
 	github.com/projectdiscovery/retryabledns v1.0.12 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
 	github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea
-	github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
+	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
 	github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
 	github.com/projectdiscovery/wappalyzergo v0.0.10
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
+	github.com/smartystreets/assertions v1.0.0 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/ratelimit v0.2.0
--- a/go.sum
+++ b/go.sum
@ -87,6 +87,8 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@ -179,8 +181,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA=
 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
-github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/wappalyzergo v0.0.10 h1:bmAjI2V99mq5mz5QikkpHQ0tXENEtaNJ5//Opr8vb5g=
@ -190,8 +192,9 @@ github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNC
 github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
+github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
--- a/integration_tests/run.sh
+++ b/integration_tests/run.sh
@ -0,0 +1,18 @@
+#!/bin/bash
+
+rm integration-test httpx 2>/dev/null
+cd ../cmd/httpx
+go build
+mv httpx ../../integration_tests/httpx
+cd ../integration-test
+go build
+mv integration-test ../../integration_tests/integration-test 
+cd ../../integration_tests
+./integration-test
+rm integration-test httpx 2>/dev/null
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi
--- a/internal/testutils/integration.go
+++ b/internal/testutils/integration.go
@ -0,0 +1,42 @@
+package testutils
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// RunNucleiAndGetResults returns a list of results for a template
+func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) {
+	sh := os.Getenv("SHELL")
+	cmd := exec.Command(sh, "-c")
+	cmdLine := `echo ` + url + ` | ./httpx `
+	cmdLine += strings.Join(extra, " ")
+	if debug {
+		cmdLine += " -debug"
+		cmd.Stderr = os.Stderr
+	} else {
+		cmdLine += " -silent"
+	}
+
+	cmd.Args = append(cmd.Args, cmdLine)
+
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	// Execute executes a test case and returns any errors if occurred
+	Execute() error
+}