Merge pull request #1372 from projectdiscovery/introduce_visual_recon_clusters

introduce visual recon clusters

README.md

@@ -185,6 +185,7 @@ OUTPUT:
    -irrb, -include-response-base64     include base64 encoded http request/response in JSON output (-json only)
    -include-chain                      include redirect http chain in JSON output (-json only)
    -store-chain                        include http redirect chain in responses (-sr only)
+   -svrc, -store-vision-recon-cluster  include visual recon clusters (-ss and -sr only)
 
 CONFIGURATIONS:
    -config string                path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)

runner/options.go

@@ -72,6 +72,7 @@ type ScanOptions struct {
 	NoFallbackScheme          bool
 	TechDetect                bool
 	StoreChain                bool
+	StoreVisionReconClusters  bool
 	MaxResponseBodySizeToSave int
 	MaxResponseBodySizeToRead int
 	OutputExtractRegex        string
@@ -228,6 +229,7 @@ type Options struct {
 	StatsInterval             int
 	RandomAgent               bool
 	StoreChain                bool
+	StoreVisionReconClusters  bool
 	Deny                      customlist.CustomList
 	Allow                     customlist.CustomList
 	MaxResponseBodySizeToSave int
@@ -402,6 +404,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"),
 		flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
 		flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
+		flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"),
 	)
 
 	flagSet.CreateGroup("configs", "Configurations",

runner/runner.go

@@ -82,6 +82,19 @@ type Runner struct {
 	HostErrorsCache     gcache.Cache[string, int]
 	browser             *Browser
 	errorPageClassifier *errorpageclassifier.ErrorPageClassifier
+	pHashClusters       []pHashCluster
+}
+
+// picked based on try-fail but it seems to close to one it's used https://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html#c1992
+var hammingDistanceThreshold int = 22
+
+type pHashCluster struct {
+	BasePHash uint64     `json:"base_phash,omitempty" csv:"base_phash"`
+	Hashes    []pHashUrl `json:"hashes,omitempty" csv:"hashes"`
+}
+type pHashUrl struct {
+	PHash uint64 `json:"phash,omitempty" csv:"phash"`
+	Url   string `json:"url,omitempty" csv:"url"`
 }
 
 // New creates a new client for running enumeration process.
@@ -237,6 +250,7 @@ func New(options *Options) (*Runner, error) {
 	scanopts.NoFallbackScheme = options.NoFallbackScheme
 	scanopts.TechDetect = options.TechDetect
 	scanopts.StoreChain = options.StoreChain
+	scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters
 	scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
 	scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
 	scanopts.extractRegexps = make(map[string]*regexp.Regexp)
@@ -881,6 +895,27 @@ func (r *Runner) RunEnumeration() {
 				}
 			}
 
+			if r.scanopts.StoreVisionReconClusters {
+				foundCluster := false
+				pHash, _ := resp.KnowledgeBase["pHash"].(uint64)
+				for i, cluster := range r.pHashClusters {
+					distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash))
+					if distance <= hammingDistanceThreshold {
+						r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL})
+						foundCluster = true
+						break
+					}
+				}
+
+				if !foundCluster {
+					newCluster := pHashCluster{
+						BasePHash: pHash,
+						Hashes:    []pHashUrl{{PHash: pHash, Url: resp.URL}},
+					}
+					r.pHashClusters = append(r.pHashClusters, newCluster)
+				}
+			}
+
 			if !jsonOrCsv || jsonAndCsv || r.options.OutputAll {
 				gologger.Silent().Msgf("%s\n", resp.str)
 			}
@@ -1016,6 +1051,24 @@ func (r *Runner) RunEnumeration() {
 	close(output)
 
 	wgoutput.Wait()
+
+	if r.scanopts.StoreVisionReconClusters {
+		visionReconClusters := filepath.Join(r.options.StoreResponseDir, "vision_recon_clusters.json")
+		clusterReportJSON, err := json.Marshal(r.pHashClusters)
+		if err != nil {
+			gologger.Fatal().Msgf("Failed to marshal report to JSON: %v", err)
+		}
+		file, err := os.Create(visionReconClusters)
+		if err != nil {
+			gologger.Fatal().Msgf("Failed to create JSON file: %v", err)
+		}
+		defer file.Close()
+
+		_, err = file.Write(clusterReportJSON)
+		if err != nil {
+			gologger.Fatal().Msgf("Failed to write to JSON file: %v", err)
+		}
+	}
 }
 
 func logFilteredErrorPage(url string) {