mirror of
https://github.com/projectdiscovery/httpx.git
synced 2024-09-11 09:35:28 +03:00
Introducing CanHaveTitleTag Function for MIME Type Validation (#1608)
* chore(deps): bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#1493) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.14.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 (#1526) Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.3.3 to 1.3.7. - [Release notes](https://github.com/cloudflare/circl/releases) - [Commits](https://github.com/cloudflare/circl/compare/v1.3.3...v1.3.7) --- updated-dependencies: - dependency-name: github.com/cloudflare/circl dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/quic-go/quic-go from 0.37.4 to 0.37.7 (#1531) Bumps [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go) from 0.37.4 to 0.37.7. - [Release notes](https://github.com/quic-go/quic-go/releases) - [Changelog](https://github.com/quic-go/quic-go/blob/master/Changelog.md) - [Commits](https://github.com/quic-go/quic-go/compare/v0.37.4...v0.37.7) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update Title Extraction * Make Requsted Changes * Import slices * optional asn --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>
This commit is contained in:
parent
a4b026111a
commit
c72d98628e
@ -9,12 +9,22 @@ import (
|
||||
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
"golang.org/x/net/html"
|
||||
"slices"
|
||||
)
|
||||
|
||||
var (
|
||||
cutset = "\n\t\v\f\r"
|
||||
reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
|
||||
reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
|
||||
cutset = "\n\t\v\f\r"
|
||||
reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
|
||||
reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
|
||||
supportedTitleMimeTypes = []string{
|
||||
"text/html",
|
||||
"application/xhtml+xml",
|
||||
"application/xml",
|
||||
"application/rss+xml",
|
||||
"application/atom+xml",
|
||||
"application/xhtml+xml",
|
||||
"application/vnd.wap.xhtml+xml",
|
||||
}
|
||||
)
|
||||
|
||||
// ExtractTitle from a response
|
||||
@ -40,6 +50,10 @@ func ExtractTitle(r *Response) (title string) {
|
||||
return title
|
||||
}
|
||||
|
||||
func CanHaveTitleTag(mimeType string) bool {
|
||||
return slices.Contains(supportedTitleMimeTypes, mimeType)
|
||||
}
|
||||
|
||||
func getTitleWithDom(r *Response) (*html.Node, error) {
|
||||
var title *html.Node
|
||||
var crawler func(*html.Node)
|
||||
|
@ -1590,8 +1590,12 @@ retry:
|
||||
builder.WriteRune(']')
|
||||
}
|
||||
|
||||
title := httpx.ExtractTitle(resp)
|
||||
if scanopts.OutputTitle {
|
||||
var title string
|
||||
if httpx.CanHaveTitleTag(resp.GetHeaderPart("Content-Type", ";")) {
|
||||
title = httpx.ExtractTitle(resp)
|
||||
}
|
||||
|
||||
if scanopts.OutputTitle && title != "" {
|
||||
builder.WriteString(" [")
|
||||
if !scanopts.OutputWithNoColor {
|
||||
builder.WriteString(aurora.Cyan(title).String())
|
||||
|
@ -89,6 +89,10 @@ func TestRunner_cidr_targets(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunner_asn_targets(t *testing.T) {
|
||||
if os.Getenv("PDCP_API_KEY") == "" {
|
||||
return
|
||||
}
|
||||
|
||||
options := &Options{}
|
||||
r, err := New(options)
|
||||
require.Nil(t, err, "could not create httpx runner")
|
||||
@ -131,10 +135,12 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
|
||||
got = r.countTargetFromRawTarget(input)
|
||||
require.Equal(t, expected, got, "got wrong output")
|
||||
|
||||
input = "AS14421"
|
||||
expected = 256
|
||||
got = r.countTargetFromRawTarget(input)
|
||||
require.Equal(t, expected, got, "got wrong output")
|
||||
if os.Getenv("PDCP_API_KEY") != "" {
|
||||
input = "AS14421"
|
||||
expected = 256
|
||||
got = r.countTargetFromRawTarget(input)
|
||||
require.Equal(t, expected, got, "got wrong output")
|
||||
}
|
||||
|
||||
input = "173.0.84.0/24"
|
||||
expected = 256
|
||||
|
Loading…
Reference in New Issue
Block a user