mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-08-17 05:20:29 +03:00
Adding integration/regression tests
This commit is contained in:
parent
4f7f352677
commit
020b719e4d
7
.github/workflows/build-test.yml
vendored
7
.github/workflows/build-test.yml
vendored
@ -19,8 +19,11 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Test
|
||||
run: go test .
|
||||
working-directory: cmd/httpx/
|
||||
run: go test ./...
|
||||
|
||||
- name: Integration Tests
|
||||
run: bash run.sh
|
||||
working-directory: integration_tests/
|
||||
|
||||
- name: Build
|
||||
run: go build .
|
||||
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,2 +1,4 @@
|
||||
|
||||
cmd/httpx/httpx
|
||||
integration_tests/httpx
|
||||
integration_tests/integration-test
|
138
cmd/integration-test/http.go
Normal file
138
cmd/integration-test/http.go
Normal file
@ -0,0 +1,138 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/projectdiscovery/httpx/internal/testutils"
|
||||
)
|
||||
|
||||
var httpTestcases = map[string]testutils.TestCase{
|
||||
"Standard HTTP GET Request": &standardHttpGet{},
|
||||
"Standard HTTPS GET Request": &standardHttpGet{tls: true},
|
||||
"Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{},
|
||||
"Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{},
|
||||
"Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{},
|
||||
}
|
||||
|
||||
type standardHttpGet struct {
|
||||
tls bool
|
||||
}
|
||||
|
||||
func (h *standardHttpGet) Execute() error {
|
||||
router := httprouter.New()
|
||||
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
fmt.Fprintf(w, "This is a test")
|
||||
}))
|
||||
var ts *httptest.Server
|
||||
if h.tls {
|
||||
ts = httptest.NewTLSServer(router)
|
||||
} else {
|
||||
ts = httptest.NewServer(router)
|
||||
}
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 1 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type issue276 struct{}
|
||||
|
||||
func (h *issue276) Execute() error {
|
||||
var ts *httptest.Server
|
||||
router := httprouter.New()
|
||||
router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
w.Header().Add("Location", ts.URL+"/redirect")
|
||||
w.WriteHeader(302)
|
||||
fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
|
||||
}))
|
||||
ts = httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-status-code", "-title", "-no-color")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 1 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
// check if we have all the items on the cli
|
||||
// full url with port
|
||||
// status code
|
||||
// title
|
||||
expected := ts.URL + "/redirect" + " [302] [Object moved]"
|
||||
// log.Fatal(results[0], expected)
|
||||
if !strings.EqualFold(results[0], expected) {
|
||||
return errIncorrectResult(results[0], expected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type issue277 struct{}
|
||||
|
||||
func (h *issue277) Execute() error {
|
||||
var ts *httptest.Server
|
||||
router := httprouter.New()
|
||||
router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
if p.ByName("pp") == `%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E` {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
}))
|
||||
ts = httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
uripath := "/hpp/?pp=%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E"
|
||||
results, err := testutils.RunHttpxAndGetResults(ts.URL+uripath, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 1 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
// check if we have all the items on the cli
|
||||
// full url with port
|
||||
// status code
|
||||
// title
|
||||
expected := ts.URL + uripath
|
||||
if !strings.EqualFold(results[0], expected) {
|
||||
return errIncorrectResult(results[0], expected)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type issue303 struct{}
|
||||
|
||||
func (h *issue303) Execute() error {
|
||||
var ts *httptest.Server
|
||||
router := httprouter.New()
|
||||
router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||
// mimic a misconfigured web server behavior declaring gzip body
|
||||
w.Header().Add("Content-Encoding", "gzip")
|
||||
// but sending it uncompressed
|
||||
fmt.Fprint(w, "<html><body>This is a test</body></html>")
|
||||
}))
|
||||
ts = httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) != 1 {
|
||||
return errIncorrectResultsCount(results)
|
||||
}
|
||||
// check if we have all the items on the cli
|
||||
// full url with port
|
||||
expected := ts.URL
|
||||
if !strings.EqualFold(results[0], expected) {
|
||||
return errIncorrectResult(results[0], expected)
|
||||
}
|
||||
return nil
|
||||
}
|
56
cmd/integration-test/integration-test.go
Normal file
56
cmd/integration-test/integration-test.go
Normal file
@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
"github.com/projectdiscovery/httpx/internal/testutils"
|
||||
)
|
||||
|
||||
var (
|
||||
debug = os.Getenv("DEBUG") == "true"
|
||||
customTest = os.Getenv("TEST")
|
||||
protocol = os.Getenv("PROTO")
|
||||
|
||||
errored = false
|
||||
)
|
||||
|
||||
func main() {
|
||||
success := aurora.Green("[✓]").String()
|
||||
failed := aurora.Red("[✘]").String()
|
||||
|
||||
tests := map[string]map[string]testutils.TestCase{
|
||||
"http": httpTestcases,
|
||||
}
|
||||
for proto, tests := range tests {
|
||||
if protocol == "" || protocol == proto {
|
||||
fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(proto))
|
||||
|
||||
for name, test := range tests {
|
||||
if customTest != "" && !strings.Contains(name, customTest) {
|
||||
continue // only run tests user asked
|
||||
}
|
||||
err := test.Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err)
|
||||
errored = true
|
||||
} else {
|
||||
fmt.Printf("%s Test \"%s\" passed!\n", success, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if errored {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func errIncorrectResultsCount(results []string) error {
|
||||
return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
|
||||
}
|
||||
|
||||
func errIncorrectResult(expected, got string) error {
|
||||
return fmt.Errorf("incorrect result: expected \"%s\" got \"%s\"", expected, got)
|
||||
}
|
4
go.mod
4
go.mod
@ -10,6 +10,7 @@ require (
|
||||
github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/microcosm-cc/bluemonday v1.0.15
|
||||
github.com/miekg/dns v1.1.43 // indirect
|
||||
@ -29,11 +30,12 @@ require (
|
||||
github.com/projectdiscovery/retryabledns v1.0.12 // indirect
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.10
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.3.0
|
||||
github.com/smartystreets/assertions v1.0.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/ratelimit v0.2.0
|
||||
|
9
go.sum
9
go.sum
@ -87,6 +87,8 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
@ -179,8 +181,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA=
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d h1:nlOAex7twmrEqD5i6WLnugF9uO3DQ6jDEKN9gevrTAk=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210617141317-00728870f68d/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
|
||||
github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.10 h1:bmAjI2V99mq5mz5QikkpHQ0tXENEtaNJ5//Opr8vb5g=
|
||||
@ -190,8 +192,9 @@ github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNC
|
||||
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
|
||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
|
18
integration_tests/run.sh
Executable file
18
integration_tests/run.sh
Executable file
@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm integration-test httpx 2>/dev/null
|
||||
cd ../cmd/httpx
|
||||
go build
|
||||
mv httpx ../../integration_tests/httpx
|
||||
cd ../integration-test
|
||||
go build
|
||||
mv integration-test ../../integration_tests/integration-test
|
||||
cd ../../integration_tests
|
||||
./integration-test
|
||||
rm integration-test httpx 2>/dev/null
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
42
internal/testutils/integration.go
Normal file
42
internal/testutils/integration.go
Normal file
@ -0,0 +1,42 @@
|
||||
package testutils
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RunNucleiAndGetResults returns a list of results for a template
|
||||
func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, error) {
|
||||
sh := os.Getenv("SHELL")
|
||||
cmd := exec.Command(sh, "-c")
|
||||
cmdLine := `echo ` + url + ` | ./httpx `
|
||||
cmdLine += strings.Join(extra, " ")
|
||||
if debug {
|
||||
cmdLine += " -debug"
|
||||
cmd.Stderr = os.Stderr
|
||||
} else {
|
||||
cmdLine += " -silent"
|
||||
}
|
||||
|
||||
cmd.Args = append(cmd.Args, cmdLine)
|
||||
|
||||
data, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parts := []string{}
|
||||
items := strings.Split(string(data), "\n")
|
||||
for _, i := range items {
|
||||
if i != "" {
|
||||
parts = append(parts, i)
|
||||
}
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
// TestCase is a single integration test case
|
||||
type TestCase interface {
|
||||
// Execute executes a test case and returns any errors if occurred
|
||||
Execute() error
|
||||
}
|
Loading…
Reference in New Issue
Block a user