projectdiscovery/httpx
1
1
Fork 0
mirror of https://github.com/projectdiscovery/httpx.git synced 2024-11-28 04:24:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update helpers libraries #850 (#851)

Browse Source 
Use utils helpers libraries (#850)
This commit is contained in:
vrenzolaverace 2022-11-06 21:22:47 +01:00 committed by GitHub
parent cbfc6ac347
commit 686a2cced8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 48 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/functional-test/main.go
View File

@ -22,7 +22,7 @@ var (
mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary")
devHttpxBinary = flag.String("dev", "", "Dev Branch Httpx Binary")
testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests")
testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests")
)
func main() {

3
common/customheader/customheader.go
View File

@ -2,7 +2,8 @@ package customheader
import (
"strings"
"github.com/projectdiscovery/stringsutil"
stringsutil "github.com/projectdiscovery/utils/strings"
)
// CustomHeaders valid for all requests

2
common/customports/customport.go
View File

@ -6,7 +6,7 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/sliceutil"
sliceutil "github.com/projectdiscovery/utils/slice"
"github.com/projectdiscovery/httpx/common/httpx"
)

2
common/fileutil/fileutil.go
View File

@ -9,8 +9,8 @@ import (
"path/filepath"
"regexp"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/httpx/common/stringz"
fileutil "github.com/projectdiscovery/utils/file"
)
// HasStdin determines if the user has piped input

2
common/httputilz/httputilz.go
View File

@ -8,7 +8,7 @@ import (
"strings"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/urlutil"
urlutil "github.com/projectdiscovery/utils/url"
)
const (

2
common/httpx/csp.go
View File

@ -6,7 +6,7 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/projectdiscovery/httpx/common/slice"
"github.com/projectdiscovery/stringsutil"
stringsutil "github.com/projectdiscovery/utils/strings"
)
// CSPHeaders is an incomplete list of most common CSP headers

2
common/httpx/encodings.go
View File

@ -6,7 +6,7 @@ import (
"net/http"
"strings"
"github.com/projectdiscovery/stringsutil"
stringsutil "github.com/projectdiscovery/utils/strings"
"golang.org/x/text/encoding/korean"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/encoding/traditionalchinese"

4
common/httpx/httpx.go
View File

@ -16,10 +16,10 @@ import (
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/gologger"
pdhttputil "github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/stringsutil"
pdhttputil "github.com/projectdiscovery/utils/http"
stringsutil "github.com/projectdiscovery/utils/strings"
"golang.org/x/net/context"
"golang.org/x/net/http2"
)

2
common/httpx/response.go
View File

@ -4,8 +4,8 @@ import (
"strings"
"time"
"github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
httputil "github.com/projectdiscovery/utils/http"
)
// Response contains the response to a server

2
common/httpx/title.go
View File

@ -7,7 +7,7 @@ import (
"regexp"
"strings"
"github.com/projectdiscovery/stringsutil"
stringsutil "github.com/projectdiscovery/utils/strings"
"golang.org/x/net/html"
)

2
common/stringz/stringz.go
View File

@ -7,7 +7,7 @@ import (
"strconv"
"strings"
"github.com/projectdiscovery/urlutil"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/spaolacci/murmur3"
)

13
go.mod
View File

@ -57,7 +57,7 @@ require (
github.com/projectdiscovery/tlsx v0.0.9
github.com/stretchr/testify v1.8.1
go.uber.org/multierr v1.8.0
golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
)
require (
@ -95,18 +95,19 @@ require (
github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect
github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect
github.com/projectdiscovery/retryabledns v1.0.15 // indirect
github.com/projectdiscovery/utils v0.0.1 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect
github.com/yl2chen/cidranger v1.0.2 // indirect
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/tools v0.1.12 // indirect
github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/tools v0.2.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

18
go.sum
View File

@ -142,6 +142,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@ -340,6 +341,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb
github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI=
github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA=
github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o=
github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0=
github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI=
github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ=
github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -400,9 +403,13 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 h1:vMJBP3PQViZsF6cOINtvyMC8ptpLsyJ4EwyFnzuWNxc=
github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ=
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
@ -424,6 +431,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t
github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw=
github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ=
github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c/go.mod h1:egdRkzUylATvPkWMpebZbXhv0FMEMJGX/ur0D3Csk2s=
github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY=
github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
@ -444,11 +453,16 @@ golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to=
golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 h1:17k44ji3KFYG94XS5QEFC8pyuOlMh3IoR+vkmTZmJJs=
golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -462,6 +476,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -572,6 +588,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

2
runner/healthcheck.go
View File

@ -6,8 +6,8 @@ import (
"runtime"
"strings"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/goflags"
fileutil "github.com/projectdiscovery/utils/file"
)
func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string {

2
runner/options.go
View File

@ -11,7 +11,6 @@ import (
"golang.org/x/exp/maps"
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
@ -24,6 +23,7 @@ import (
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/slice"
"github.com/projectdiscovery/httpx/common/stringz"
fileutil "github.com/projectdiscovery/utils/file"
)
const (

14
runner/runner.go
View File

@ -29,7 +29,7 @@ import (
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/hashes/jarm"
"github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/mapsutil"
mapsutil "github.com/projectdiscovery/utils/maps"
"github.com/bluele/gcache"
"github.com/logrusorgru/aurora"
@ -39,28 +39,28 @@ import (
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/httpx/common/hashes"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/sliceutil"
"github.com/projectdiscovery/stringsutil"
"github.com/projectdiscovery/urlutil"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/projectdiscovery/ratelimit"
"github.com/remeh/sizedwaitgroup"
// automatic fd max increase if running as root
_ "github.com/projectdiscovery/fdmax/autofdmax"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
pdhttputil "github.com/projectdiscovery/httputil"
customport "github.com/projectdiscovery/httpx/common/customports"
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httputilz"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/slice"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/iputil"
"github.com/projectdiscovery/mapcidr"
"github.com/projectdiscovery/rawhttp"
fileutil "github.com/projectdiscovery/utils/file"
pdhttputil "github.com/projectdiscovery/utils/http"
iputil "github.com/projectdiscovery/utils/ip"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)

2
runner/types.go
View File

@ -8,8 +8,8 @@ import (
"github.com/bxcodec/faker/v4/pkg/options"
"github.com/mitchellh/mapstructure"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/mapsutil"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
mapsutil "github.com/projectdiscovery/utils/maps"
)
type AsnResponse struct {