mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-10-26 18:08:27 +03:00
Merge pull request #1372 from projectdiscovery/introduce_visual_recon_clusters
introduce visual recon clusters
This commit is contained in:
commit
3e1bd2aadc
@ -185,6 +185,7 @@ OUTPUT:
|
||||
-irrb, -include-response-base64 include base64 encoded http request/response in JSON output (-json only)
|
||||
-include-chain include redirect http chain in JSON output (-json only)
|
||||
-store-chain include http redirect chain in responses (-sr only)
|
||||
-svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only)
|
||||
|
||||
CONFIGURATIONS:
|
||||
-config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
|
||||
|
@ -72,6 +72,7 @@ type ScanOptions struct {
|
||||
NoFallbackScheme bool
|
||||
TechDetect bool
|
||||
StoreChain bool
|
||||
StoreVisionReconClusters bool
|
||||
MaxResponseBodySizeToSave int
|
||||
MaxResponseBodySizeToRead int
|
||||
OutputExtractRegex string
|
||||
@ -228,6 +229,7 @@ type Options struct {
|
||||
StatsInterval int
|
||||
RandomAgent bool
|
||||
StoreChain bool
|
||||
StoreVisionReconClusters bool
|
||||
Deny customlist.CustomList
|
||||
Allow customlist.CustomList
|
||||
MaxResponseBodySizeToSave int
|
||||
@ -402,6 +404,7 @@ func ParseOptions() *Options {
|
||||
flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
|
||||
flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
|
||||
flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("configs", "Configurations",
|
||||
|
@ -82,6 +82,19 @@ type Runner struct {
|
||||
HostErrorsCache gcache.Cache[string, int]
|
||||
browser *Browser
|
||||
errorPageClassifier *errorpageclassifier.ErrorPageClassifier
|
||||
pHashClusters []pHashCluster
|
||||
}
|
||||
|
||||
// picked based on try-fail but it seems to close to one it's used https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html#c1992
|
||||
var hammingDistanceThreshold int = 22
|
||||
|
||||
type pHashCluster struct {
|
||||
BasePHash uint64 `json:"base_phash,omitempty" csv:"base_phash"`
|
||||
Hashes []pHashUrl `json:"hashes,omitempty" csv:"hashes"`
|
||||
}
|
||||
type pHashUrl struct {
|
||||
PHash uint64 `json:"phash,omitempty" csv:"phash"`
|
||||
Url string `json:"url,omitempty" csv:"url"`
|
||||
}
|
||||
|
||||
// New creates a new client for running enumeration process.
|
||||
@ -237,6 +250,7 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.NoFallbackScheme = options.NoFallbackScheme
|
||||
scanopts.TechDetect = options.TechDetect
|
||||
scanopts.StoreChain = options.StoreChain
|
||||
scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters
|
||||
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
|
||||
scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
|
||||
scanopts.extractRegexps = make(map[string]*regexp.Regexp)
|
||||
@ -881,6 +895,27 @@ func (r *Runner) RunEnumeration() {
|
||||
}
|
||||
}
|
||||
|
||||
if r.scanopts.StoreVisionReconClusters {
|
||||
foundCluster := false
|
||||
pHash, _ := resp.KnowledgeBase["pHash"].(uint64)
|
||||
for i, cluster := range r.pHashClusters {
|
||||
distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash))
|
||||
if distance <= hammingDistanceThreshold {
|
||||
r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL})
|
||||
foundCluster = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !foundCluster {
|
||||
newCluster := pHashCluster{
|
||||
BasePHash: pHash,
|
||||
Hashes: []pHashUrl{{PHash: pHash, Url: resp.URL}},
|
||||
}
|
||||
r.pHashClusters = append(r.pHashClusters, newCluster)
|
||||
}
|
||||
}
|
||||
|
||||
if !jsonOrCsv || jsonAndCsv || r.options.OutputAll {
|
||||
gologger.Silent().Msgf("%s\n", resp.str)
|
||||
}
|
||||
@ -1016,6 +1051,24 @@ func (r *Runner) RunEnumeration() {
|
||||
close(output)
|
||||
|
||||
wgoutput.Wait()
|
||||
|
||||
if r.scanopts.StoreVisionReconClusters {
|
||||
visionReconClusters := filepath.Join(r.options.StoreResponseDir, "vision_recon_clusters.json")
|
||||
clusterReportJSON, err := json.Marshal(r.pHashClusters)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Failed to marshal report to JSON: %v", err)
|
||||
}
|
||||
file, err := os.Create(visionReconClusters)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Failed to create JSON file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.Write(clusterReportJSON)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Failed to write to JSON file: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func logFilteredErrorPage(url string) {
|
||||
|
Loading…
Reference in New Issue
Block a user