blacklist path + fix blacklist extension

README.md

@@ -28,6 +28,7 @@ $ gau -h
 | Flag | Description | Example |
 |------|-------------|---------|
 |`--blacklist`| list of extensions to skip | gau --blacklist ttf,woff,svg,png|
+|`--blacklist_path`| list of paths to skip | gau --blacklist_path https://example.com/path/to/something or gau --blacklist_path path/to/something|
 |`--fc`| list of status codes to filter | gau --fc 404,302 |
 |`--from`| fetch urls from date (format: YYYYMM) | gau --from 202101 |
 |`--ft`| list of mime-types to filter | gau --ft text/plain|

cmd/gau/main.go

@@ -46,8 +46,8 @@ func main() {
 	go func(out io.Writer, JSON bool) {
 		defer writeWg.Done()
 		if JSON {
-			output.WriteURLsJSON(out, results, config.Blacklist, config.RemoveParameters)
-		} else if err = output.WriteURLs(out, results, config.Blacklist, config.RemoveParameters); err != nil {
+			output.WriteURLsJSON(out, results, config.Blacklist, config.BlacklistPaths, config.RemoveParameters)
+		} else if err = output.WriteURLs(out, results, config.Blacklist, config.BlacklistPaths, config.RemoveParameters); err != nil {
 			log.Fatalf("error writing results: %v\n", err)
 		}
 	}(out, config.JSON)

pkg/output/output.go

@@ -1,28 +1,44 @@
 package output
 
 import (
-	mapset "github.com/deckarep/golang-set/v2"
-	jsoniter "github.com/json-iterator/go"
-	"github.com/valyala/bytebufferpool"
 	"io"
 	"net/url"
 	"path"
 	"strings"
+
+	mapset "github.com/deckarep/golang-set/v2"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/valyala/bytebufferpool"
 )
 
 type JSONResult struct {
 	Url string `json:"url"`
 }
 
-func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], RemoveParameters bool) error {
+func Blacklisted(u *url.URL, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string]) bool {
+	if path.Ext(u.Path) != "" {
+		if blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) || blacklistMap.Contains(strings.ToLower(path.Ext(u.RawQuery))) {
+			return true
+		}
+		for path := range blacklistpathsMap.Iter() {
+			if strings.Contains(u.Path, path) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string], RemoveParameters bool) error {
 	lastURL := mapset.NewThreadUnsafeSet[string]()
 	for result := range results {
 		buf := bytebufferpool.Get()
 		u, err := url.Parse(result)
 		if err != nil {
 			continue
 		}
-		if path.Ext(u.Path) != "" && blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) {
+
+		if Blacklisted(u, blacklistMap, blacklistpathsMap) {
 			continue
 		}
 
@@ -42,15 +58,15 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[
 	return nil
 }
 
-func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], RemoveParameters bool) {
+func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string], RemoveParameters bool) {
 	var jr JSONResult
 	enc := jsoniter.NewEncoder(writer)
 	for result := range results {
 		u, err := url.Parse(result)
 		if err != nil {
 			continue
 		}
-		if blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) {
+		if Blacklisted(u, blacklistMap, blacklistpathsMap) {
 			continue
 		}
 		jr.Url = result

pkg/providers/providers.go

@@ -2,6 +2,7 @@ package providers
 
 import (
 	"context"
+
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/valyala/fasthttp"
 )
@@ -28,6 +29,7 @@ type Config struct {
 	Client            *fasthttp.Client
 	Providers         []string
 	Blacklist         mapset.Set[string]
+	BlacklistPaths    mapset.Set[string]
 	Output            string
 	JSON              bool
 	URLScan           URLScan

runner/flags/flags.go

@@ -35,6 +35,7 @@ type Config struct {
 	RemoveParameters  bool              `mapstructure:"parameters"`
 	Providers         []string          `mapstructure:"providers"`
 	Blacklist         []string          `mapstructure:"blacklist"`
+	BlacklistPaths    []string          `mapstructure:"blacklist_paths"`
 	JSON              bool              `mapstructure:"json"`
 	URLScan           URLScanConfig     `mapstructure:"urlscan"`
 	OTX               string            `mapstructure:"otx"`
@@ -87,6 +88,9 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
 	}
 	pc.Blacklist = mapset.NewThreadUnsafeSet(c.Blacklist...)
 	pc.Blacklist.Add("")
+
+	pc.BlacklistPaths = mapset.NewThreadUnsafeSet(c.BlacklistPaths...)
+	pc.BlacklistPaths.Add("")
 	return pc, nil
 }
 
@@ -103,6 +107,7 @@ func New() *Options {
 	pflag.Uint("retries", 0, "retries for HTTP client")
 	pflag.String("proxy", "", "http proxy to use")
 	pflag.StringSlice("blacklist", []string{}, "list of extensions to skip")
+	pflag.StringSlice("blacklist_paths", []string{}, "list of paths to skip")
 	pflag.StringSlice("providers", []string{}, "list of providers to use (wayback,commoncrawl,otx,urlscan)")
 	pflag.Bool("subs", false, "include subdomains of target domain")
 	pflag.Bool("fp", false, "remove different parameters of the same endpoint")
@@ -172,6 +177,7 @@ func (o *Options) DefaultConfig() *Config {
 		RemoveParameters:  false,
 		Providers:         []string{"wayback", "commoncrawl", "otx", "urlscan"},
 		Blacklist:         []string{},
+		BlacklistPaths:    []string{},
 		JSON:              false,
 		Outfile:           "",
 	}
@@ -191,6 +197,7 @@ func (o *Options) getFlagValues(c *Config) {
 	fetchers := o.viper.GetStringSlice("providers")
 	threads := o.viper.GetUint("threads")
 	blacklist := o.viper.GetStringSlice("blacklist")
+	blacklist_paths := o.viper.GetStringSlice("blacklist_paths")
 	subs := o.viper.GetBool("subs")
 	fp := o.viper.GetBool("fp")
 
@@ -213,7 +220,27 @@ func (o *Options) getFlagValues(c *Config) {
 
 	// set if --blacklist flag is specified, otherwise use default
 	if len(blacklist) > 0 {
-		c.Blacklist = blacklist
+		for _, value := range blacklist {
+			if strings.HasPrefix(value, ".") {
+				c.Blacklist = append(c.Blacklist, value)
+			} else {
+				c.Blacklist = append(c.Blacklist, "."+value)
+			}
+		}
+	}
+
+	// set if --blacklist_path flag is specified, otherwise use default
+	if len(blacklist_paths) > 0 {
+		for _, path := range blacklist_paths {
+			if !strings.HasPrefix(path, "http") {
+				if !strings.HasPrefix(path, "/") {
+					c.BlacklistPaths = append(c.BlacklistPaths, "/"+path)
+				}
+			} else {
+				u, _ := url.Parse(path)
+				c.BlacklistPaths = append(c.BlacklistPaths, u.Path)
+			}
+		}
 	}
 
 	// set if --providers flag is specified, otherwise use default