From e4e2aab40211f03062a87ff67609b2340c283c93 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 28 Jun 2022 12:49:46 +0200 Subject: [PATCH] Adding httpx as a library minimal example (#670) --- README.md | 39 ++++++++++++++++++++++ cmd/integration-test/integration-test.go | 3 +- cmd/integration-test/library.go | 41 ++++++++++++++++++++++++ runner/options.go | 39 +++++++++++++--------- 4 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 cmd/integration-test/library.go diff --git a/README.md b/README.md index ae5ae2d..30297a7 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,45 @@ https://docs.hackerone.com https://support.hackerone.com ``` +### Using httpx as a library +`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to close at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: + +```go +package main + +import ( + "log" + "os" + + "github.com/projectdiscovery/httpx/runner" +) + +func main() { + inputFile := "test.txt" + err := os.WriteFile(inputFile, []byte("scanme.sh"), 0644) + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(inputFile) + + options := runner.Options{ + Methods: "GET", + InputFile: inputFile, + } + if err := options.ValidateOptions(); err != nil { + log.Fatal(err) + } + + httpxRunner, err := runner.New(&options) + if err != nil { + log.Fatal() + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() +} +``` + # 📋 Notes diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 0b5a57e..d5115e0 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -22,7 +22,8 @@ func main() { failed := aurora.Red("[✘]").String() tests := map[string]map[string]testutils.TestCase{ - "http": httpTestcases, + "http": httpTestcases, + "library": libraryTestcases, } for proto, tests := range tests { if protocol == "" || protocol == proto { diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go new file mode 100644 index 0000000..f20e4bf --- /dev/null +++ b/cmd/integration-test/library.go @@ -0,0 +1,41 @@ +package main + +import ( + "os" + + "github.com/projectdiscovery/httpx/internal/testutils" + "github.com/projectdiscovery/httpx/runner" +) + +var libraryTestcases = map[string]testutils.TestCase{ + "Httpx as library": &httpxLibrary{}, +} + +type httpxLibrary struct { +} + +func (h *httpxLibrary) Execute() error { + testFile := "test.txt" + err := os.WriteFile(testFile, []byte("scanme.sh"), 0644) + if err != nil { + return err + } + defer os.RemoveAll(testFile) + + options := runner.Options{ + Methods: "GET", + InputFile: testFile, + } + if err := options.ValidateOptions(); err != nil { + return err + } + + httpxRunner, err := runner.New(&options) + if err != nil { + return err + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() + return nil +} diff --git a/runner/options.go b/runner/options.go index 5b8e9a4..a47db04 100644 --- a/runner/options.go +++ b/runner/options.go @@ -7,6 +7,8 @@ import ( "regexp" "strings" + "github.com/pkg/errors" + "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" @@ -407,59 +409,61 @@ func ParseOptions() *Options { os.Exit(0) } - options.validateOptions() + if err := options.ValidateOptions(); err != nil { + gologger.Fatal().Msgf("%s\n", err) + } return options } -func (options *Options) validateOptions() { +func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - gologger.Fatal().Msgf("File %s does not exist.\n", options.InputFile) + return fmt.Errorf("File %s does not exist.", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - gologger.Fatal().Msgf("File %s does not exist.\n", options.InputRawRequest) + return fmt.Errorf("File %s does not exist.", options.InputRawRequest) } multiOutput := options.CSVOutput && options.JSONOutput if multiOutput { - gologger.Fatal().Msg("Results can only be displayed in one format: 'JSON' or 'CSV'\n") + return fmt.Errorf("Results can only be displayed in one format: 'JSON' or 'CSV'") } var err error if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil { - gologger.Fatal().Msgf("Invalid value for match status code option: %s\n", err) + return errors.Wrap(err, "Invalid value for match status code option") } if options.matchContentLength, err = stringz.StringToSliceInt(options.OutputMatchContentLength); err != nil { - gologger.Fatal().Msgf("Invalid value for match content length option: %s\n", err) + return errors.Wrap(err, "Invalid value for match content length option") } if options.filterStatusCode, err = stringz.StringToSliceInt(options.OutputFilterStatusCode); err != nil { - gologger.Fatal().Msgf("Invalid value for filter status code option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter status code option") } if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil { - gologger.Fatal().Msgf("Invalid value for filter content length option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter content length option") } if options.OutputFilterRegex != "" { if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil { - gologger.Fatal().Msgf("Invalid value for regex filter option: %s\n", err) + return errors.Wrap(err, "Invalid value for regex filter option") } } if options.OutputMatchRegex != "" { if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil { - gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err) + return errors.Wrap(err, "Invalid value for match regex option") } } if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil { - gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err) + return errors.Wrap(err, "Invalid value for match lines count option") } if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil { - gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err) + return errors.Wrap(err, "Invalid value for match words count option") } if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil { - gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter lines count option") } if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil { - gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter words count option") } var resolvers []string @@ -467,7 +471,7 @@ func (options *Options) validateOptions() { if fileutil.FileExists(resolver) { chFile, err := fileutil.ReadFile(resolver) if err != nil { - gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err) + return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver) } for line := range chFile { resolvers = append(resolvers, line) @@ -476,6 +480,7 @@ func (options *Options) validateOptions() { resolvers = append(resolvers, resolver) } } + options.Resolvers = resolvers if len(options.Resolvers) > 0 { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) @@ -505,6 +510,8 @@ func (options *Options) validateOptions() { if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { options.OutputCDN = true } + + return nil } // configureOutput configures the output on the screen