Merge pull request #808 from epi052/fix-wildcard-directory-redirect-v2

Fix wildcard directory redirect v2

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.8.0"
+version = "2.9.0"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,28 +22,28 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = { version = "4.1.6", features = ["wrap_help", "cargo"] }
-clap_complete = "4.1.3"
+clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
+clap_complete = "4.1.4"
 regex = "1.5.5"
 lazy_static = "1.4.0"
 dirs = "4.0.0"
 
 [dependencies]
-scraper = "0.14.0"
+scraper = "0.15.0"
 futures = "0.3.26"
-tokio = { version = "1.25.0", features = ["full"] }
+tokio = { version = "1.26.0", features = ["full"] }
 tokio-util = { version = "0.7.7", features = ["codec"] }
 log = "0.4.17"
 env_logger = "0.10.0"
 reqwest = { version = "0.11.10", features = ["socks"] }
 # uses feature unification to add 'serde' to reqwest::Url
 url = { version = "2.2.2", features = ["serde"] }
 serde_regex = "1.1.0"
-clap = { version = "4.1.6", features = ["wrap_help", "cargo"] }
+clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
 lazy_static = "1.4.0"
 toml = "0.7.2"
 serde = { version = "1.0.137", features = ["derive", "rc"] }
-serde_json = "1.0.93"
+serde_json = "1.0.94"
 uuid = { version = "1.3.0", features = ["v4"] }
 indicatif = "0.15"
 console = "0.15.2"

Makefile.toml

@@ -23,3 +23,10 @@ clear = true
 script = """
 cargo clippy --all-targets --all-features -- -D warnings
 """
+
+# tests
+[tasks.test]
+clear = true
+script = """
+cargo nextest run --all-features --all-targets --retries 10
+"""

shell_completions/_feroxbuster

@@ -24,8 +24,8 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
@@ -72,8 +72,8 @@ _feroxbuster() {
 '(-u --url)--stdin[Read url(s) from STDIN]' \
 '(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \
 '(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \
-'--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
-'--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
+'(--rate-limit --auto-bail)--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
+'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
 '-A[Use a random User-Agent]' \
 '--random-agent[Use a random User-Agent]' \
 '-f[Append / to each request'\''s URL]' \
@@ -117,4 +117,8 @@ _feroxbuster_commands() {
     _describe -t commands 'feroxbuster commands' commands "$@"
 }
 
-_feroxbuster "$@"
+if [ "$funcstack[1]" = "_feroxbuster" ]; then
+    _feroxbuster "$@"
+else
+    compdef _feroxbuster feroxbuster
+fi

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')

shell_completions/feroxbuster.elv

@@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.8.0)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.8.0)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.9.0)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.0)'
             cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'

src/event_handlers/outputs.rs

@@ -248,7 +248,7 @@ impl TermOutHandler {
                 .unwrap()
                 .filters
                 .data
-                .should_filter_response(&resp);
+                .should_filter_response(&resp, tx_stats.clone());
 
             let contains_sentry = if !self.config.filter_status.is_empty() {
                 // -C was used, meaning -s was not and we should ignore the defaults

src/event_handlers/scans.rs

@@ -6,7 +6,7 @@ use tokio::sync::{mpsc, Semaphore};
 use crate::{
     response::FeroxResponse,
     scan_manager::{FeroxScan, FeroxScans, ScanOrder},
-    scanner::FeroxScanner,
+    scanner::{FeroxScanner, RESPONSES},
     statistics::StatField::TotalScans,
     url::FeroxUrl,
     utils::should_deny_url,
@@ -395,6 +395,58 @@ impl ScanHandler {
             return Ok(());
         }
 
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for maybe_wild in responses.iter() {
+                if !maybe_wild.wildcard() || !maybe_wild.is_directory() {
+                    // if the stored response isn't a wildcard, skip it
+                    // if the stored response isn't a directory, skip it
+                    // we're only interested in preventing recursion into wildcard directories
+                    continue;
+                }
+
+                if maybe_wild.method() != response.method() {
+                    // methods don't match, skip it
+                    continue;
+                }
+
+                // methods match and is a directory wildcard
+                // need to check the wildcard's parent directory
+                // for equality with the incoming response's parent directory
+                //
+                // if the parent directories match, we need to prevent recursion
+                // into the wildcard directory
+
+                match (
+                    maybe_wild.url().path_segments(),
+                    response.url().path_segments(),
+                ) {
+                    // both urls must have path segments
+                    (Some(mut maybe_wild_segments), Some(mut response_segments)) => {
+                        match (
+                            maybe_wild_segments.nth_back(1),
+                            response_segments.nth_back(1),
+                        ) {
+                            // both urls must have at least 2 path segments, the next to last being the parent
+                            (Some(maybe_wild_parent), Some(response_parent)) => {
+                                if maybe_wild_parent == response_parent {
+                                    // the parent directories match, so we need to prevent recursion
+                                    return Ok(());
+                                }
+                            }
+                            _ => {
+                                // we couldn't get the parent directory, so we'll skip this
+                                continue;
+                            }
+                        }
+                    }
+                    _ => {
+                        // we couldn't get the path segments, so we'll skip this
+                        continue;
+                    }
+                }
+            }
+        }
+
         let targets = vec![response.url().to_string()];
         self.ordered_scan_url(targets, ScanOrder::Latest).await?;
 

src/extractor/container.rs

@@ -144,7 +144,12 @@ impl<'a> Extractor<'a> {
             };
 
             // filter if necessary
-            if self.handles.filters.data.should_filter_response(&resp) {
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&resp, self.handles.stats.tx.clone())
+            {
                 continue;
             }
 

src/filters/container.rs

@@ -7,9 +7,12 @@ use crate::response::FeroxResponse;
 
 use super::{
     FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
-    WordsFilter,
+    WildcardFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
 };
-
 /// Container around a collection of `FeroxFilters`s
 #[derive(Debug, Default)]
 pub struct FeroxFilters {
@@ -64,12 +67,21 @@ impl FeroxFilters {
 
     /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
     /// to the user or not.
-    pub fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
         if let Ok(filters) = self.filters.read() {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
                 if filter.should_filter_response(response) {
                     log::debug!("filtering response due to: {:?}", filter);
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
                     return true;
                 }
             }
@@ -93,6 +105,10 @@ impl Serialize for FeroxFilters {
                     seq.serialize_element(word_filter).unwrap_or_default();
                 } else if let Some(size_filter) = filter.as_any().downcast_ref::<SizeFilter>() {
                     seq.serialize_element(size_filter).unwrap_or_default();
+                } else if let Some(wildcard_filter) =
+                    filter.as_any().downcast_ref::<WildcardFilter>()
+                {
+                    seq.serialize_element(wildcard_filter).unwrap_or_default();
                 } else if let Some(status_filter) =
                     filter.as_any().downcast_ref::<StatusCodeFilter>()
                 {

src/filters/mod.rs

@@ -15,6 +15,7 @@ pub use self::similarity::{SimilarityFilter, SIM_HASHER};
 pub use self::size::SizeFilter;
 pub use self::status_code::StatusCodeFilter;
 pub(crate) use self::utils::{create_similarity_filter, filter_lookup};
+pub use self::wildcard::WildcardFilter;
 pub use self::words::WordsFilter;
 
 mod status_code;
@@ -28,4 +29,5 @@ mod container;
 mod tests;
 mod init;
 mod utils;
+mod wildcard;
 mod empty;

src/filters/similarity.rs

@@ -37,7 +37,9 @@ impl FeroxFilter for SimilarityFilter {
 
     /// Compare one SimilarityFilter to another
     fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+        other
+            .downcast_ref::<Self>()
+            .map_or(false, |a| self.hash == a.hash)
     }
 
     /// Return self as Any for dynamic dispatch purposes

src/filters/tests.rs

@@ -1,7 +1,41 @@
 use super::*;
 use crate::nlp::preprocess;
+use crate::DEFAULT_METHOD;
 use ::regex::Regex;
 
+#[test]
+/// simply test the default values for wildcardfilter
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.content_length, None);
+    assert_eq!(wcf.line_count, None);
+    assert_eq!(wcf.word_count, None);
+    assert_eq!(wcf.method, DEFAULT_METHOD.to_string());
+    assert_eq!(wcf.status_code, 0);
+    assert!(!wcf.dont_filter);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let mut filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+
+    filter.content_length = Some(1);
+
+    assert_ne!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+}
+
 #[test]
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn lines_filter_as_any() {
@@ -86,6 +120,68 @@ fn regex_filter_as_any() {
     );
 }
 
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let body =
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus";
+
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(body);
+
+    let filter = WildcardFilter {
+        content_length: Some(body.len() as u64),
+        line_count: Some(1),
+        word_count: Some(10),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter {
+        content_length: Some(0),
+        line_count: Some(0),
+        word_count: Some(0),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
+
+    let filter = WildcardFilter {
+        content_length: None,
+        line_count: None,
+        word_count: Some(8),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
 #[test]
 /// test should_filter on RegexFilter where regex matches body
 fn regexfilter_should_filter_when_regex_matches_on_response_body() {