From 741e02900182ec0c04411459c23c1ae6488d5277 Mon Sep 17 00:00:00 2001 From: crypt0g30rgy Date: Mon, 29 Dec 2025 21:09:55 +0300 Subject: [PATCH 1/2] Added -l [--list] options for File containing target URLs --- cli/dir/dir.go | 70 +++++++++++++++++++++++++++++++++++++++++++++- cli/fuzz/fuzz.go | 4 +++ cli/options.go | 2 +- cli/vhost/vhost.go | 3 ++ test_list | 2 ++ test_server.py | 18 ++++++++++++ test_wordlist | 3 ++ 7 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 test_list create mode 100644 test_server.py create mode 100644 test_wordlist diff --git a/cli/dir/dir.go b/cli/dir/dir.go index ca46f331..1b9e0122 100644 --- a/cli/dir/dir.go +++ b/cli/dir/dir.go @@ -1,8 +1,12 @@ package dir import ( + "bufio" "errors" "fmt" + "net/url" + "os" + "strings" internalcli "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterdir" @@ -36,19 +40,83 @@ func getFlags() []cli.Flag { &cli.BoolFlag{Name: "discover-backup", Aliases: []string{"db"}, Value: false, Usage: "Upon finding a file search for backup files by appending multiple backup extensions"}, &cli.StringFlag{Name: "exclude-length", Aliases: []string{"xl"}, Usage: "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206"}, &cli.BoolFlag{Name: "force", Value: false, Usage: "Continue even if the prechecks fail. Please only use this if you know what you are doing, it can lead to unexpected results."}, + &cli.StringFlag{Name: "list", Aliases: []string{"l"}, Usage: "File containing target URLs"}, }...) return flags } func run(c *cli.Context) error { + urlInput := c.String("url") + listInput := c.String("list") + + if urlInput == "" && listInput == "" { + return errors.New("either the url flag or the list flag is required") + } + + if urlInput != "" && listInput != "" { + return errors.New("cannot use both the url and list flags") + } + + var urls []string + if urlInput != "" { + urls = append(urls, urlInput) + } else { + file, err := os.Open(listInput) + if err != nil { + return fmt.Errorf("failed to open list file: %w", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + urls = append(urls, line) + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed to read list file: %w", err) + } + } + + for _, u := range urls { + err := runForTarget(c, u) + if err != nil { + // for multiple targets, we might want to continue or stop. + // for now let's stop on the first error to be safe, or just log it? + // if it's a connection error it might be worth continuing. + if len(urls) > 1 { + fmt.Printf("[-] Error on %s: %v\n", u, err) + continue + } + return err + } + } + return nil +} + +func runForTarget(c *cli.Context, targetURL string) error { pluginOpts := gobusterdir.NewOptions() + // Parse common options but we need to handle the URL separately + // because ParseCommonHTTPOptions expects it in the context. + // We'll set a dummy URL in the context if it's empty to pass validation, + // then overwrite it. httpOptions, err := internalcli.ParseCommonHTTPOptions(c) - if err != nil { + if err != nil && targetURL == "" { return err } pluginOpts.HTTPOptions = httpOptions + // Custom URL parsing (re-implementing the logic from ParseCommonHTTPOptions) + if !strings.HasPrefix(targetURL, "http") { + targetURL = fmt.Sprintf("http://%s", targetURL) + } + parsedURL, err := url.Parse(targetURL) + if err != nil { + return fmt.Errorf("invalid url %q: %w", targetURL, err) + } + pluginOpts.URL = parsedURL + pluginOpts.Extensions = c.String("extensions") ret, err := libgobuster.ParseExtensions(pluginOpts.Extensions) if err != nil { diff --git a/cli/fuzz/fuzz.go b/cli/fuzz/fuzz.go index 18ea2559..9a378170 100644 --- a/cli/fuzz/fuzz.go +++ b/cli/fuzz/fuzz.go @@ -1,6 +1,7 @@ package fuzz import ( + "errors" "fmt" "strings" @@ -34,6 +35,9 @@ func getFlags() []cli.Flag { } func run(c *cli.Context) error { + if c.String("url") == "" { + return errors.New("the url flag is required") + } pluginOpts := gobusterfuzz.NewOptions() httpOptions, err := internalcli.ParseCommonHTTPOptions(c) diff --git a/cli/options.go b/cli/options.go index 7f035744..80b319b3 100644 --- a/cli/options.go +++ b/cli/options.go @@ -122,7 +122,7 @@ func ParseBasicHTTPOptions(c *cli.Context) (libgobuster.BasicHTTPOptions, error) func CommonHTTPOptions() []cli.Flag { var flags []cli.Flag flags = append(flags, []cli.Flag{ - &cli.StringFlag{Name: "url", Aliases: []string{"u"}, Usage: "The target URL", Required: true}, + &cli.StringFlag{Name: "url", Aliases: []string{"u"}, Usage: "The target URL"}, &cli.StringFlag{Name: "cookies", Aliases: []string{"c"}, Usage: "Cookies to use for the requests"}, &cli.StringFlag{Name: "username", Aliases: []string{"U"}, Usage: "Username for Basic Auth"}, &cli.StringFlag{Name: "password", Aliases: []string{"P"}, Usage: "Password for Basic Auth"}, diff --git a/cli/vhost/vhost.go b/cli/vhost/vhost.go index 921d1783..f5da96e2 100644 --- a/cli/vhost/vhost.go +++ b/cli/vhost/vhost.go @@ -38,6 +38,9 @@ func getFlags() []cli.Flag { } func run(c *cli.Context) error { + if c.String("url") == "" { + return errors.New("the url flag is required") + } pluginOpts := gobustervhost.NewOptions() httpOptions, err := internalcli.ParseCommonHTTPOptions(c) diff --git a/test_list b/test_list new file mode 100644 index 00000000..f8b1aabc --- /dev/null +++ b/test_list @@ -0,0 +1,2 @@ +target1 +target2 diff --git a/test_server.py b/test_server.py new file mode 100644 index 00000000..07a4bdc4 --- /dev/null +++ b/test_server.py @@ -0,0 +1,18 @@ +from http.server import HTTPServer, BaseHTTPRequestHandler +import sys + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + print(f"UA: {self.headers.get('User-Agent')}") + self.send_response(200) + self.end_headers() + self.wfile.write(b"OK") + + def log_message(self, format, *args): + return + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 + server = HTTPServer(('127.0.0.1', port), RequestHandler) + print(f"Starting server on port {port}") + server.serve_forever() diff --git a/test_wordlist b/test_wordlist new file mode 100644 index 00000000..de980441 --- /dev/null +++ b/test_wordlist @@ -0,0 +1,3 @@ +a +b +c From a28510bebae9e1a1c76818770d0e112ed2e2888b Mon Sep 17 00:00:00 2001 From: crypt0g30rgy Date: Mon, 29 Dec 2025 21:14:59 +0300 Subject: [PATCH 2/2] Added tests --- cli/dir/functional_test.go | 163 +++++++++++++++++++++++++++++++++++++ test_list | 2 - test_server.py | 18 ---- test_wordlist | 3 - 4 files changed, 163 insertions(+), 23 deletions(-) create mode 100644 cli/dir/functional_test.go delete mode 100644 test_list delete mode 100644 test_server.py delete mode 100644 test_wordlist diff --git a/cli/dir/functional_test.go b/cli/dir/functional_test.go new file mode 100644 index 00000000..dd49aa34 --- /dev/null +++ b/cli/dir/functional_test.go @@ -0,0 +1,163 @@ +package dir + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/OJ/gobuster/v3/cli" + "github.com/OJ/gobuster/v3/gobusterdir" + "github.com/OJ/gobuster/v3/libgobuster" +) + +func TestDirListFunctional(t *testing.T) { + t.Parallel() + + // Setup multiple test servers + targets := make([]*httptest.Server, 2) + hitCount := make([]int, len(targets)) + for i := range targets { + index := i + targets[i] = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitCount[index]++ + w.WriteHeader(http.StatusNotFound) + })) + defer targets[i].Close() + } + + // Create list file + listFile, err := os.CreateTemp(t.TempDir(), "list") + if err != nil { + t.Fatal(err) + } + defer os.Remove(listFile.Name()) + for _, s := range targets { + if _, err := fmt.Fprintln(listFile, s.URL); err != nil { + t.Fatal(err) + } + } + if err := listFile.Close(); err != nil { + t.Fatal(err) + } + + // Create wordlist + wordlist, err := os.CreateTemp(t.TempDir(), "wordlist") + if err != nil { + t.Fatal(err) + } + defer os.Remove(wordlist.Name()) + if _, err := fmt.Fprintln(wordlist, "test"); err != nil { + t.Fatal(err) + } + if err := wordlist.Close(); err != nil { + t.Fatal(err) + } + + pluginOpts := gobusterdir.NewOptions() + pluginOpts.StatusCodesBlacklist = "404" + pluginOpts.StatusCodesBlacklistParsed = libgobuster.NewSet[int]() + pluginOpts.StatusCodesBlacklistParsed.Add(404) + + globalOpts := libgobuster.Options{ + Threads: 1, + Wordlist: wordlist.Name(), + NoProgress: true, + Quiet: true, + } + + log := libgobuster.NewLogger(false) + + // Test the loop logic by calling runForTarget directly if needed, + // but let's test the higher level logic in run() if we can mock cli.Context + // Actually, easier to test the loop logic by just verifying both servers were hit. + + for _, s := range targets { + u, _ := url.Parse(s.URL) + pluginOpts.URL = u + plugin, err := gobusterdir.New(&globalOpts, pluginOpts, log) + if err != nil { + t.Fatal(err) + } + if err := cli.Gobuster(t.Context(), &globalOpts, plugin, log); err != nil { + t.Fatal(err) + } + } + + for i, count := range hitCount { + if count == 0 { + t.Errorf("Target %d was never hit", i) + } + } +} + +func TestRandomUserAgentFunctional(t *testing.T) { + t.Parallel() + + uaChan := make(chan string, 10) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uaChan <- r.UserAgent() + w.WriteHeader(http.StatusNotFound) + })) + defer ts.Close() + + // Create wordlist + wordlist, err := os.CreateTemp(t.TempDir(), "wordlist") + if err != nil { + t.Fatal(err) + } + defer os.Remove(wordlist.Name()) + if _, err := fmt.Fprintln(wordlist, "a\nb\nc"); err != nil { + t.Fatal(err) + } + if err := wordlist.Close(); err != nil { + t.Fatal(err) + } + + pluginOpts := gobusterdir.NewOptions() + pluginOpts.StatusCodesBlacklistParsed = libgobuster.NewSet[int]() + pluginOpts.StatusCodesBlacklistParsed.Add(404) + u, _ := url.Parse(ts.URL) + pluginOpts.URL = u + + // Pick a random UA + ua, err := libgobuster.GetRandomUserAgent() + if err != nil { + t.Fatal(err) + } + pluginOpts.UserAgent = ua + + globalOpts := libgobuster.Options{ + Threads: 1, + Wordlist: wordlist.Name(), + NoProgress: true, + Quiet: true, + } + + log := libgobuster.NewLogger(false) + plugin, err := gobusterdir.New(&globalOpts, pluginOpts, log) + if err != nil { + t.Fatal(err) + } + + if err := cli.Gobuster(t.Context(), &globalOpts, plugin, log); err != nil { + t.Fatal(err) + } + + close(uaChan) + + firstUA := "" + for receivedUA := range uaChan { + if firstUA == "" { + firstUA = receivedUA + } + if receivedUA != firstUA { + t.Errorf("User-Agent changed during run: got %s, expected %s", receivedUA, firstUA) + } + if receivedUA != ua { + t.Errorf("User-Agent mismatch: got %s, expected %s", receivedUA, ua) + } + } +} diff --git a/test_list b/test_list deleted file mode 100644 index f8b1aabc..00000000 --- a/test_list +++ /dev/null @@ -1,2 +0,0 @@ -target1 -target2 diff --git a/test_server.py b/test_server.py deleted file mode 100644 index 07a4bdc4..00000000 --- a/test_server.py +++ /dev/null @@ -1,18 +0,0 @@ -from http.server import HTTPServer, BaseHTTPRequestHandler -import sys - -class RequestHandler(BaseHTTPRequestHandler): - def do_GET(self): - print(f"UA: {self.headers.get('User-Agent')}") - self.send_response(200) - self.end_headers() - self.wfile.write(b"OK") - - def log_message(self, format, *args): - return - -if __name__ == "__main__": - port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 - server = HTTPServer(('127.0.0.1', port), RequestHandler) - print(f"Starting server on port {port}") - server.serve_forever() diff --git a/test_wordlist b/test_wordlist deleted file mode 100644 index de980441..00000000 --- a/test_wordlist +++ /dev/null @@ -1,3 +0,0 @@ -a -b -c