AS input support +asnmap integration (#821)

* Add asnmap integration

- Use PD's asnmap lib to get asn details

* Add AS input support [WIP]

* Add AS input support

- Add AS Number input support
- Update README.md to add AS input example
- Add test cases to test `targets()` function
- Refactor target function remove if-else with switch case

* Resolve merge conflicts

* Replace - with _ in asnResponse for json

* Replace - with _ in asnResponse for csv
This commit is contained in:
Shubham Rasal 2022-10-20 17:54:52 +05:30 committed by GitHub
parent 12fee4f3ee
commit bab68a487f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 480 additions and 45 deletions

View File

@ -289,7 +289,16 @@ https://173.0.84.6
https://173.0.84.16
https://173.0.84.34
```
### AS Number Input
```console
echo AS14421 | httpx -silent
https://216.101.17.248
https://216.101.17.249
https://216.101.17.250
https://216.101.17.251
https://216.101.17.252
```
### Tool Chain
@ -387,15 +396,15 @@ subfinder -d hackerone.com -silent | httpx -asn
Use with caution. You are responsible for your actions.
Developers assume no liability and are not responsible for any misuse or damage.
https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24]
https://gslink.hackerone.com [AS16509, AMAZON-02, US, 13.33.168.0/22]
https://www.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20]
https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24]
https://resources.hackerone.com [AS16509, AMAZON-02, US, 3.98.0.0/15]
https://support.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.48.0/20]
https://mta-sts.hackerone.com [AS54113, FASTLY, US, 185.199.111.0/24]
https://docs.hackerone.com [AS54113, FASTLY, US, 185.199.109.0/24]
https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20]
https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US]
https://gslink.hackerone.com [AS16509, AMAZON-02, US]
https://www.hackerone.com [AS13335, CLOUDFLARENET, US]
https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US]
https://resources.hackerone.com [AS16509, AMAZON-02, US]
https://support.hackerone.com [AS13335, CLOUDFLARENET, US]
https://mta-sts.hackerone.com [AS54113, FASTLY, US]
https://docs.hackerone.com [AS54113, FASTLY, US]
https://api.hackerone.com [AS13335, CLOUDFLARENET, US]
```

5
go.mod
View File

@ -46,15 +46,16 @@ require github.com/spaolacci/murmur3 v1.1.0
require (
github.com/PuerkitoBio/goquery v1.8.0
github.com/ammario/ipisp/v2 v2.0.0
github.com/bxcodec/faker/v4 v4.0.0-beta.3
github.com/hdm/jarm-go v0.0.7
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
github.com/mitchellh/mapstructure v1.5.0
github.com/projectdiscovery/asnmap v0.0.1
github.com/projectdiscovery/dsl v0.0.3
github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655
github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa
github.com/projectdiscovery/tlsx v0.0.9
github.com/stretchr/testify v1.8.0
go.uber.org/multierr v1.8.0
golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0
)
@ -75,6 +76,7 @@ require (
github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae // indirect
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/badger v1.6.2 // indirect
github.com/dgraph-io/ristretto v0.0.3 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
@ -88,6 +90,7 @@ require (
github.com/kr/text v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect
github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect
github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect

4
go.sum
View File

@ -26,8 +26,6 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY
github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU=
github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@ -265,6 +263,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/asnmap v0.0.1 h1:n4YCz1ljUaDA3dOUCkjI/bUOtiS7ge1KJ39qpURCd/o=
github.com/projectdiscovery/asnmap v0.0.1/go.mod h1:CjCVDhQPVtmlE247L6YFeIVX9c4m8pOX8V8BmB0JkX8=
github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs=
github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04=
github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8=

View File

@ -24,13 +24,14 @@ import (
"golang.org/x/exp/maps"
asnmap "github.com/projectdiscovery/asnmap/libs"
dsl "github.com/projectdiscovery/dsl"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/hashes/jarm"
"github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/mapsutil"
"github.com/ammario/ipisp/v2"
"github.com/bluele/gcache"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
@ -75,12 +76,14 @@ type Runner struct {
stats clistats.StatisticsClient
ratelimiter ratelimit.Limiter
HostErrorsCache gcache.Cache
asnClinet asn.ASNClient
}
// New creates a new client for running enumeration process.
func New(options *Options) (*Runner, error) {
runner := &Runner{
options: options,
asnClinet: asn.New(),
}
var err error
if options.TechDetect {
@ -475,11 +478,20 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) {
return 0
}
expandedTarget := 1
if iputil.IsCIDR(rawTarget) {
expandedTarget := 0
switch {
case iputil.IsCIDR(rawTarget):
if ipsCount, err := mapcidr.AddressCount(rawTarget); err == nil && ipsCount > 0 {
expandedTarget = int(ipsCount)
}
case asn.IsASN(rawTarget):
asn := asn.New()
cidrs, _ := asn.GetCIDRsForASNNum(rawTarget)
for _, cidr := range cidrs {
expandedTarget += int(mapcidr.AddressCountIpnet(cidr))
}
default:
expandedTarget = 1
}
return expandedTarget
}
@ -910,28 +922,34 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
results := make(chan httpx.Target)
go func() {
defer close(results)
switch {
case strings.ContainsAny(target, "*") || strings.HasPrefix(target, "."):
// A valid target does not contain:
// *
// spaces
if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") {
// trim * and/or . (prefix) from the target to return the domain instead of wilcard
target = strings.TrimPrefix(strings.Trim(target, "*"), ".")
if !r.testAndSet(target) {
return
}
}
// test if the target is a cidr
if iputil.IsCIDR(target) {
cidrIps, err := mapcidr.IPAddresses(target)
results <- httpx.Target{Host: target}
case asn.IsASN(target):
cidrIps, err := r.asnClinet.GetIPAddressesAsStream(target)
if err != nil {
return
}
for _, ip := range cidrIps {
for ip := range cidrIps {
results <- httpx.Target{Host: ip}
}
} else if r.options.ProbeAllIPS {
case iputil.IsCIDR(target):
cidrIps, err := mapcidr.IPAddressesAsStream(target)
if err != nil {
return
}
for ip := range cidrIps {
results <- httpx.Target{Host: ip}
}
case r.options.ProbeAllIPS:
URL, err := urlutil.Parse(target)
if err != nil {
results <- httpx.Target{Host: target}
@ -943,9 +961,10 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
for _, ip := range ips {
results <- httpx.Target{Host: target, CustomIP: ip}
}
} else if idxComma := strings.Index(target, ","); idxComma > 0 {
case strings.Index(target, ",") > 0:
idxComma := strings.Index(target, ",")
results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]}
} else {
default:
results <- httpx.Target{Host: target}
}
}()
@ -1300,17 +1319,17 @@ retry:
var asnResponse *AsnResponse
if r.options.Asn {
lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip))
if err != nil {
gologger.Warning().Msg(err.Error())
results := asnmap.NewClient().GetData(asnmap.IP(ip))
if len(results) > 0 {
var cidrs []string
for _, cidr := range asnmap.GetCIDR(results) {
cidrs = append(cidrs, cidr.String())
}
if lookupResult != nil {
lookupResult.ISPName = stringsutil.TrimSuffixAny(strings.ReplaceAll(lookupResult.ISPName, lookupResult.Country, ""), ", ", " ")
asnResponse = &AsnResponse{
AsNumber: lookupResult.ASN.String(),
AsName: lookupResult.ISPName,
AsCountry: lookupResult.Country,
AsRange: lookupResult.Range.String(),
AsNumber: fmt.Sprintf("AS%v", results[0].ASN),
AsName: results[0].Org,
AsCountry: results[0].Country,
AsRange: cidrs,
}
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {

148
runner/runner_test.go Normal file
View File

@ -0,0 +1,148 @@
package runner
import (
"os"
"strings"
"testing"
_ "github.com/projectdiscovery/fdmax/autofdmax"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/stretchr/testify/require"
)
func TestRunner_domain_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := []string{"example.com", "*.example.com", "example.com,one.one.one.one"}
expected := []httpx.Target{{
Host: "example.com",
}, {
Host: "example.com",
}, {
Host: "one.one.one.one",
CustomHost: "example.com",
}}
got := []httpx.Target{}
for _, inp := range input {
for target := range r.targets(r.hp, inp) {
got = append(got, target)
}
}
require.ElementsMatch(t, expected, got, "could not exepcted output")
}
func TestRunner_probeall_targets(t *testing.T) {
options := &Options{
ProbeAllIPS: true,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "one.one.one.one"
expected := []httpx.Target{{
Host: "one.one.one.one",
CustomIP: "2606:4700:4700::1111",
},
{
Host: "one.one.one.one",
CustomIP: "2606:4700:4700::1001",
},
{
Host: "one.one.one.one",
CustomIP: "1.0.0.1",
},
{
Host: "one.one.one.one",
CustomIP: "1.1.1.1",
}}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not exepcted output")
}
func TestRunner_cidr_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "173.0.84.0/30"
expected := []httpx.Target{
{
Host: "173.0.84.0",
}, {
Host: "173.0.84.1",
},
{
Host: "173.0.84.2",
},
{
Host: "173.0.84.3",
}}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not exepcted output")
}
func TestRunner_asn_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "AS14421"
expected := []httpx.Target{}
expectedOutputFile := "tests/AS14421.txt"
// read the expected IPs from the file
fileContent, err := os.ReadFile(expectedOutputFile)
require.Nil(t, err, "could not read the expectedOutputFile file")
ips := strings.Split(strings.ReplaceAll(string(fileContent), "\r\n", "\n"), "\n")
for _, ip := range ips {
expected = append(expected, httpx.Target{Host: ip})
}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not exepcted output")
}
func TestRunner_countTargetFromRawTarget(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "example.com"
expected := 1
got := r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
input = "example.com"
expected = 0
err = r.hm.Set(input, nil)
require.Nil(t, err, "could not set value to hm")
got = r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
input = ""
expected = 0
got = r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
input = "AS14421"
expected = 256
got = r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
input = "AS15133"
expected = 153088
got = r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
input = "173.0.84.0/24"
expected = 256
got = r.countTargetFromRawTarget(input)
require.Equal(t, expected, got, "got wrong output")
}

256
runner/tests/AS14421.txt Normal file
View File

@ -0,0 +1,256 @@
216.101.17.0
216.101.17.1
216.101.17.2
216.101.17.3
216.101.17.4
216.101.17.5
216.101.17.6
216.101.17.7
216.101.17.8
216.101.17.9
216.101.17.10
216.101.17.11
216.101.17.12
216.101.17.13
216.101.17.14
216.101.17.15
216.101.17.16
216.101.17.17
216.101.17.18
216.101.17.19
216.101.17.20
216.101.17.21
216.101.17.22
216.101.17.23
216.101.17.24
216.101.17.25
216.101.17.26
216.101.17.27
216.101.17.28
216.101.17.29
216.101.17.30
216.101.17.31
216.101.17.32
216.101.17.33
216.101.17.34
216.101.17.35
216.101.17.36
216.101.17.37
216.101.17.38
216.101.17.39
216.101.17.40
216.101.17.41
216.101.17.42
216.101.17.43
216.101.17.44
216.101.17.45
216.101.17.46
216.101.17.47
216.101.17.48
216.101.17.49
216.101.17.50
216.101.17.51
216.101.17.52
216.101.17.53
216.101.17.54
216.101.17.55
216.101.17.56
216.101.17.57
216.101.17.58
216.101.17.59
216.101.17.60
216.101.17.61
216.101.17.62
216.101.17.63
216.101.17.64
216.101.17.65
216.101.17.66
216.101.17.67
216.101.17.68
216.101.17.69
216.101.17.70
216.101.17.71
216.101.17.72
216.101.17.73
216.101.17.74
216.101.17.75
216.101.17.76
216.101.17.77
216.101.17.78
216.101.17.79
216.101.17.80
216.101.17.81
216.101.17.82
216.101.17.83
216.101.17.84
216.101.17.85
216.101.17.86
216.101.17.87
216.101.17.88
216.101.17.89
216.101.17.90
216.101.17.91
216.101.17.92
216.101.17.93
216.101.17.94
216.101.17.95
216.101.17.96
216.101.17.97
216.101.17.98
216.101.17.99
216.101.17.100
216.101.17.101
216.101.17.102
216.101.17.103
216.101.17.104
216.101.17.105
216.101.17.106
216.101.17.107
216.101.17.108
216.101.17.109
216.101.17.110
216.101.17.111
216.101.17.112
216.101.17.113
216.101.17.114
216.101.17.115
216.101.17.116
216.101.17.117
216.101.17.118
216.101.17.119
216.101.17.120
216.101.17.121
216.101.17.122
216.101.17.123
216.101.17.124
216.101.17.125
216.101.17.126
216.101.17.127
216.101.17.128
216.101.17.129
216.101.17.130
216.101.17.131
216.101.17.132
216.101.17.133
216.101.17.134
216.101.17.135
216.101.17.136
216.101.17.137
216.101.17.138
216.101.17.139
216.101.17.140
216.101.17.141
216.101.17.142
216.101.17.143
216.101.17.144
216.101.17.145
216.101.17.146
216.101.17.147
216.101.17.148
216.101.17.149
216.101.17.150
216.101.17.151
216.101.17.152
216.101.17.153
216.101.17.154
216.101.17.155
216.101.17.156
216.101.17.157
216.101.17.158
216.101.17.159
216.101.17.160
216.101.17.161
216.101.17.162
216.101.17.163
216.101.17.164
216.101.17.165
216.101.17.166
216.101.17.167
216.101.17.168
216.101.17.169
216.101.17.170
216.101.17.171
216.101.17.172
216.101.17.173
216.101.17.174
216.101.17.175
216.101.17.176
216.101.17.177
216.101.17.178
216.101.17.179
216.101.17.180
216.101.17.181
216.101.17.182
216.101.17.183
216.101.17.184
216.101.17.185
216.101.17.186
216.101.17.187
216.101.17.188
216.101.17.189
216.101.17.190
216.101.17.191
216.101.17.192
216.101.17.193
216.101.17.194
216.101.17.195
216.101.17.196
216.101.17.197
216.101.17.198
216.101.17.199
216.101.17.200
216.101.17.201
216.101.17.202
216.101.17.203
216.101.17.204
216.101.17.205
216.101.17.206
216.101.17.207
216.101.17.208
216.101.17.209
216.101.17.210
216.101.17.211
216.101.17.212
216.101.17.213
216.101.17.214
216.101.17.215
216.101.17.216
216.101.17.217
216.101.17.218
216.101.17.219
216.101.17.220
216.101.17.221
216.101.17.222
216.101.17.223
216.101.17.224
216.101.17.225
216.101.17.226
216.101.17.227
216.101.17.228
216.101.17.229
216.101.17.230
216.101.17.231
216.101.17.232
216.101.17.233
216.101.17.234
216.101.17.235
216.101.17.236
216.101.17.237
216.101.17.238
216.101.17.239
216.101.17.240
216.101.17.241
216.101.17.242
216.101.17.243
216.101.17.244
216.101.17.245
216.101.17.246
216.101.17.247
216.101.17.248
216.101.17.249
216.101.17.250
216.101.17.251
216.101.17.252
216.101.17.253
216.101.17.254
216.101.17.255

View File

@ -16,11 +16,11 @@ type AsnResponse struct {
AsNumber string `json:"as_number" csv:"as_number"`
AsName string `json:"as_name" csv:"as_name"`
AsCountry string `json:"as_country" csv:"as_country"`
AsRange string `json:"as_range" csv:"as_range"`
AsRange []string `json:"as_range" csv:"as_range"`
}
func (o AsnResponse) String() string {
return fmt.Sprintf("%v, %v, %v, %v", o.AsNumber, o.AsName, o.AsCountry, o.AsRange)
return fmt.Sprintf("%v, %v, %v", o.AsNumber, o.AsName, o.AsCountry)
}
// Result of a scan