diff --git a/README.md b/README.md index 3ddd33b..f7f42da 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ $ gau -h | Flag | Description | Example | |------|-------------|---------| |`--blacklist`| list of extensions to skip | gau --blacklist ttf,woff,svg,png| +|`--blacklist_path`| list of paths to skip | gau --blacklist_path https://example.com/path/to/something or gau --blacklist_path path/to/something| |`--fc`| list of status codes to filter | gau --fc 404,302 | |`--from`| fetch urls from date (format: YYYYMM) | gau --from 202101 | |`--ft`| list of mime-types to filter | gau --ft text/plain| diff --git a/cmd/gau/main.go b/cmd/gau/main.go index bed608b..436284d 100644 --- a/cmd/gau/main.go +++ b/cmd/gau/main.go @@ -46,8 +46,8 @@ func main() { go func(out io.Writer, JSON bool) { defer writeWg.Done() if JSON { - output.WriteURLsJSON(out, results, config.Blacklist, config.RemoveParameters) - } else if err = output.WriteURLs(out, results, config.Blacklist, config.RemoveParameters); err != nil { + output.WriteURLsJSON(out, results, config.Blacklist, config.BlacklistPaths, config.RemoveParameters) + } else if err = output.WriteURLs(out, results, config.Blacklist, config.BlacklistPaths, config.RemoveParameters); err != nil { log.Fatalf("error writing results: %v\n", err) } }(out, config.JSON) diff --git a/pkg/output/output.go b/pkg/output/output.go index fa2c1bc..12f00fe 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,20 +1,35 @@ package output import ( - mapset "github.com/deckarep/golang-set/v2" - jsoniter "github.com/json-iterator/go" - "github.com/valyala/bytebufferpool" "io" "net/url" "path" "strings" + + mapset "github.com/deckarep/golang-set/v2" + jsoniter "github.com/json-iterator/go" + "github.com/valyala/bytebufferpool" ) type JSONResult struct { Url string `json:"url"` } -func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], RemoveParameters bool) error { +func Blacklisted(u *url.URL, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string]) bool { + if path.Ext(u.Path) != "" { + if blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) || blacklistMap.Contains(strings.ToLower(path.Ext(u.RawQuery))) { + return true + } + for path := range blacklistpathsMap.Iter() { + if strings.Contains(u.Path, path) { + return true + } + } + } + return false +} + +func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string], RemoveParameters bool) error { lastURL := mapset.NewThreadUnsafeSet[string]() for result := range results { buf := bytebufferpool.Get() @@ -22,7 +37,8 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[ if err != nil { continue } - if path.Ext(u.Path) != "" && blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) { + + if Blacklisted(u, blacklistMap, blacklistpathsMap) { continue } @@ -42,7 +58,7 @@ func WriteURLs(writer io.Writer, results <-chan string, blacklistMap mapset.Set[ return nil } -func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], RemoveParameters bool) { +func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap mapset.Set[string], blacklistpathsMap mapset.Set[string], RemoveParameters bool) { var jr JSONResult enc := jsoniter.NewEncoder(writer) for result := range results { @@ -50,7 +66,7 @@ func WriteURLsJSON(writer io.Writer, results <-chan string, blacklistMap mapset. if err != nil { continue } - if blacklistMap.Contains(strings.ToLower(path.Ext(u.Path))) { + if Blacklisted(u, blacklistMap, blacklistpathsMap) { continue } jr.Url = result diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index 9e5b283..da42033 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -2,6 +2,7 @@ package providers import ( "context" + mapset "github.com/deckarep/golang-set/v2" "github.com/valyala/fasthttp" ) @@ -28,6 +29,7 @@ type Config struct { Client *fasthttp.Client Providers []string Blacklist mapset.Set[string] + BlacklistPaths mapset.Set[string] Output string JSON bool URLScan URLScan diff --git a/runner/flags/flags.go b/runner/flags/flags.go index cac4947..4c193e0 100644 --- a/runner/flags/flags.go +++ b/runner/flags/flags.go @@ -35,6 +35,7 @@ type Config struct { RemoveParameters bool `mapstructure:"parameters"` Providers []string `mapstructure:"providers"` Blacklist []string `mapstructure:"blacklist"` + BlacklistPaths []string `mapstructure:"blacklist_paths"` JSON bool `mapstructure:"json"` URLScan URLScanConfig `mapstructure:"urlscan"` OTX string `mapstructure:"otx"` @@ -87,6 +88,9 @@ func (c *Config) ProviderConfig() (*providers.Config, error) { } pc.Blacklist = mapset.NewThreadUnsafeSet(c.Blacklist...) pc.Blacklist.Add("") + + pc.BlacklistPaths = mapset.NewThreadUnsafeSet(c.BlacklistPaths...) + pc.BlacklistPaths.Add("") return pc, nil } @@ -103,6 +107,7 @@ func New() *Options { pflag.Uint("retries", 0, "retries for HTTP client") pflag.String("proxy", "", "http proxy to use") pflag.StringSlice("blacklist", []string{}, "list of extensions to skip") + pflag.StringSlice("blacklist_paths", []string{}, "list of paths to skip") pflag.StringSlice("providers", []string{}, "list of providers to use (wayback,commoncrawl,otx,urlscan)") pflag.Bool("subs", false, "include subdomains of target domain") pflag.Bool("fp", false, "remove different parameters of the same endpoint") @@ -172,6 +177,7 @@ func (o *Options) DefaultConfig() *Config { RemoveParameters: false, Providers: []string{"wayback", "commoncrawl", "otx", "urlscan"}, Blacklist: []string{}, + BlacklistPaths: []string{}, JSON: false, Outfile: "", } @@ -191,6 +197,7 @@ func (o *Options) getFlagValues(c *Config) { fetchers := o.viper.GetStringSlice("providers") threads := o.viper.GetUint("threads") blacklist := o.viper.GetStringSlice("blacklist") + blacklist_paths := o.viper.GetStringSlice("blacklist_paths") subs := o.viper.GetBool("subs") fp := o.viper.GetBool("fp") @@ -213,7 +220,27 @@ func (o *Options) getFlagValues(c *Config) { // set if --blacklist flag is specified, otherwise use default if len(blacklist) > 0 { - c.Blacklist = blacklist + for _, value := range blacklist { + if strings.HasPrefix(value, ".") { + c.Blacklist = append(c.Blacklist, value) + } else { + c.Blacklist = append(c.Blacklist, "."+value) + } + } + } + + // set if --blacklist_path flag is specified, otherwise use default + if len(blacklist_paths) > 0 { + for _, path := range blacklist_paths { + if !strings.HasPrefix(path, "http") { + if !strings.HasPrefix(path, "/") { + c.BlacklistPaths = append(c.BlacklistPaths, "/"+path) + } + } else { + u, _ := url.Parse(path) + c.BlacklistPaths = append(c.BlacklistPaths, u.Path) + } + } } // set if --providers flag is specified, otherwise use default