Merge pull request #750 from epi052/742-748-state-file-bug-fixes

multiple bug fixes / small improvements

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.7.2"
+version = "2.7.3"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -29,12 +29,12 @@ lazy_static = "1.4.0"
 dirs = "4.0.0"
 
 [dependencies]
-scraper = "0.13.0"
+scraper = "0.14.0"
 futures = "0.3.21"
 tokio = { version = "1.18.2", features = ["full"] }
 tokio-util = { version = "0.7.1", features = ["codec"] }
 log = "0.4.17"
-env_logger = "0.9.0"
+env_logger = "0.10.0"
 reqwest = { version = "0.11.10", features = ["socks"] }
 # uses feature unification to add 'serde' to reqwest::Url
 url = { version = "2.2.2", features = ["serde"] }
@@ -51,7 +51,7 @@ openssl = { version = "0.10.40", features = ["vendored"] }
 dirs = "4.0.0"
 regex = "1.5.5"
 crossterm = "0.25.0"
-rlimit = "0.8.3"
+rlimit = "0.9.0"
 ctrlc = "3.2.2"
 fuzzyhash = "0.2.1"
 anyhow = "1.0.57"

shell_completions/_feroxbuster

@@ -24,8 +24,8 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default: feroxbuster/2.7.2)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default: feroxbuster/2.7.2)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default: feroxbuster/2.7.3)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/2.7.3)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.7.2)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.7.2)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.7.3)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.7.3)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')

shell_completions/feroxbuster.elv

@@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.7.2)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.7.2)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.7.3)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.7.3)'
             cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'

src/config/container.rs

@@ -790,7 +790,7 @@ impl Configuration {
         if args.get_count("verbosity") > 0 {
             // occurrences_of returns 0 if none are found; this is protected in
             // an if block for the same reason as the quiet option
-            config.verbosity = args.get_count("verbosity") as u8;
+            config.verbosity = args.get_count("verbosity");
         }
 
         if came_from_cli!(args, "no_recursion") {

src/filters/similarity.rs

@@ -22,7 +22,7 @@ impl FeroxFilter for SimilarityFilter {
     fn should_filter_response(&self, response: &FeroxResponse) -> bool {
         let other = FuzzyHash::new(response.text());
 
-        if let Ok(result) = FuzzyHash::compare(&self.hash, &other.to_string()) {
+        if let Ok(result) = FuzzyHash::compare(&self.hash, other.to_string()) {
             return result >= self.threshold;
         }
 

src/logger.rs

@@ -65,7 +65,7 @@ pub fn initialize(config: Arc<Configuration>) -> Result<()> {
                 kind: "log".to_string(),
             };
 
-            PROGRESS_PRINTER.println(&log_entry.as_str());
+            PROGRESS_PRINTER.println(log_entry.as_str());
 
             if let Some(buffered_file) = file.clone() {
                 if let Ok(mut unlocked) = buffered_file.write() {

src/main.rs

@@ -44,9 +44,10 @@ lazy_static! {
     static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
 }
 
-/// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
+/// Create a Vec of Strings from the given wordlist then stores it inside an Arc
 fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
+    let mut trimmed_word = false;
 
     let file = File::open(path).with_context(|| format!("Could not open {}", path))?;
 
@@ -61,12 +62,21 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     for line in reader.lines() {
         line.map(|result| {
             if !result.starts_with('#') && !result.is_empty() {
-                words.push(result);
+                if result.starts_with('/') {
+                    words.push(result.trim_start_matches('/').to_string());
+                    trimmed_word = true;
+                } else {
+                    words.push(result);
+                }
             }
         })
         .ok();
     }
 
+    if trimmed_word {
+        log::warn!("Some words in the wordlist started with a leading forward-slash; those words were trimmed (i.e. /word -> word)");
+    }
+
     log::trace!(
         "exit: get_unique_words_from_wordlist -> Arc<wordlist[{} words...]>",
         words.len()
@@ -328,7 +338,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
             let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
 
-            let final_path = output_path.with_file_name(&new_folder);
+            let final_path = output_path.with_file_name(new_folder);
 
             // create the directory or fail silently, assuming the reason for failure is that
             // the path exists already

src/response.rs

@@ -415,7 +415,18 @@ impl FeroxSerialize for FeroxResponse {
                     .get("Location")
                     .unwrap() // known Some() already
                     .to_str()
-                    .unwrap_or("Unknown");
+                    .unwrap_or("Unknown")
+                    .to_string();
+
+                let loc = if loc.starts_with('/') {
+                    if let Ok(joined) = self.url().join(&loc) {
+                        joined.to_string()
+                    } else {
+                        loc
+                    }
+                } else {
+                    loc
+                };
 
                 // prettify the redirect target
                 let loc = style(loc).yellow();

src/scan_manager/scan.rs

@@ -233,6 +233,14 @@ impl FeroxScan {
         false
     }
 
+    /// small wrapper to inspect ScanStatus and see if it's Cancelled
+    pub fn is_cancelled(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(*guard, ScanStatus::Cancelled);
+        }
+        false
+    }
+
     /// await a task's completion, similar to a thread's join; perform necessary bookkeeping
     pub async fn join(&self) {
         log::trace!("enter join({:?})", self);

src/scan_manager/scan_container.rs

@@ -138,6 +138,15 @@ impl FeroxScans {
                     let mut deser_scan: FeroxScan =
                         serde_json::from_value(scan.clone()).unwrap_or_default();
 
+                    if deser_scan.is_cancelled() {
+                        // if the scan was cancelled by the user, mark it as complete. This will
+                        // prevent the scan from being resumed as well as prevent the wordlist
+                        // from requesting it again
+                        if let Ok(mut guard) = deser_scan.status.lock() {
+                            *guard = ScanStatus::Complete;
+                        }
+                    }
+
                     // FeroxScans gets -q value from config as usual; the FeroxScans themselves
                     // rely on that value being passed in. If the user starts a scan without -q
                     // and resumes the scan but adds -q, FeroxScan will not have the proper value

src/scanner/ferox_scanner.rs

@@ -328,7 +328,7 @@ impl FeroxScanner {
             log::info!(
                 "requesting {} collected words: {:?}...",
                 new_words_len,
-                &new_words[..new_words_len.min(3) as usize]
+                &new_words[..new_words_len.min(3)]
             );
 
             self.stream_requests(

tests/test_deny_list.rs

@@ -45,7 +45,7 @@ fn deny_list_works_during_extraction() {
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/LICENSE");
         then.status(200)
-            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+            .body(srv.url("'/homepage/assets/img/icons/handshake.svg'"));
     });
 
     let mock_two = srv.mock(|when, then| {
@@ -90,17 +90,17 @@ fn deny_list_works_during_recursion() {
 
     let js_mock = srv.mock(|when, then| {
         when.method(GET).path("/js");
-        then.status(301).header("Location", &srv.url("/js/"));
+        then.status(301).header("Location", srv.url("/js/"));
     });
 
     let js_prod_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/prod");
-        then.status(301).header("Location", &srv.url("/js/prod/"));
+        then.status(301).header("Location", srv.url("/js/prod/"));
     });
 
     let js_dev_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/dev");
-        then.status(301).header("Location", &srv.url("/js/dev/"));
+        then.status(301).header("Location", srv.url("/js/dev/"));
     });
 
     let js_dev_file_mock = srv.mock(|when, then| {
@@ -155,7 +155,7 @@ fn deny_list_works_during_recursion_with_inverted_parents() {
 
     let js_mock = srv.mock(|when, then| {
         when.method(GET).path("/js");
-        then.status(301).header("Location", &srv.url("/js/"));
+        then.status(301).header("Location", srv.url("/js/"));
     });
 
     let api_mock = srv.mock(|when, then| {
@@ -165,12 +165,12 @@ fn deny_list_works_during_recursion_with_inverted_parents() {
 
     let js_prod_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/prod");
-        then.status(301).header("Location", &srv.url("/js/prod/"));
+        then.status(301).header("Location", srv.url("/js/prod/"));
     });
 
     let js_dev_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/dev");
-        then.status(301).header("Location", &srv.url("/js/dev/"));
+        then.status(301).header("Location", srv.url("/js/dev/"));
     });
 
     let js_dev_file_mock = srv.mock(|when, then| {

tests/test_extractor.rs

@@ -16,7 +16,7 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/LICENSE");
         then.status(200)
-            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+            .body(srv.url("'/homepage/assets/img/icons/handshake.svg'"));
     });
 
     let mock_two = srv.mock(|when, then| {
@@ -136,13 +136,13 @@ fn extractor_finds_same_relative_url_twice() {
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/LICENSE");
         then.status(200)
-            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+            .body(srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
     });
 
     let mock_two = srv.mock(|when, then| {
         when.method(GET).path("/README");
         then.status(200)
-            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+            .body(srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
     });
 
     let mock_three = srv.mock(|when, then| {
@@ -185,7 +185,7 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/LICENSE");
         then.status(200)
-            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+            .body(srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
     });
 
     let mock_two = srv.mock(|when, then| {
@@ -413,7 +413,7 @@ fn extractor_finds_directory_listing_links_and_displays_files() {
 
     let mock_dir_redir = srv.mock(|when, then| {
         when.method(GET).path("/misc");
-        then.status(301).header("Location", &srv.url("/misc/"));
+        then.status(301).header("Location", srv.url("/misc/"));
     });
     let mock_dir = srv.mock(|when, then| {
         when.method(GET).path("/misc/");
@@ -522,7 +522,7 @@ fn extractor_finds_directory_listing_links_and_displays_files_non_recursive() {
 
     let mock_dir_redir = srv.mock(|when, then| {
         when.method(GET).path("/misc");
-        then.status(301).header("Location", &srv.url("/misc/"));
+        then.status(301).header("Location", srv.url("/misc/"));
     });
     let mock_dir = srv.mock(|when, then| {
         when.method(GET).path("/misc/");
@@ -600,7 +600,7 @@ fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::E
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/LICENSE");
         then.status(200)
-            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+            .body(srv.url("'/homepage/assets/img/icons/handshake.svg'"));
     });
 
     let mock_two = srv.mock(|when, then| {

tests/test_main.rs

@@ -196,7 +196,7 @@ fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Er
     let file_regex = Regex::new("ferox-[a-zA-Z_:0-9]+-[0-9]+.log").unwrap();
     let dir_regex = Regex::new("output-file-[0-9]+.logs").unwrap();
 
-    let sub_dir = output_dir.as_ref().join(&sub_dir);
+    let sub_dir = output_dir.as_ref().join(sub_dir);
 
     // created directory like output-file-1627845741.logs/
     assert!(dir_regex.is_match(&sub_dir.to_string_lossy()));

tests/test_scanner.rs

@@ -53,17 +53,17 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
 
     let js_mock = srv.mock(|when, then| {
         when.method(GET).path("/js");
-        then.status(301).header("Location", &srv.url("/js/"));
+        then.status(301).header("Location", srv.url("/js/"));
     });
 
     let js_prod_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/prod");
-        then.status(301).header("Location", &srv.url("/js/prod/"));
+        then.status(301).header("Location", srv.url("/js/prod/"));
     });
 
     let js_dev_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/dev");
-        then.status(301).header("Location", &srv.url("/js/dev/"));
+        then.status(301).header("Location", srv.url("/js/dev/"));
     });
 
     let js_dev_file_mock = srv.mock(|when, then| {
@@ -116,17 +116,17 @@ fn scanner_recursive_request_scan_using_only_success_responses(
 
     let js_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/");
-        then.status(200).header("Location", &srv.url("/js/"));
+        then.status(200).header("Location", srv.url("/js/"));
     });
 
     let js_prod_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/prod/");
-        then.status(200).header("Location", &srv.url("/js/prod/"));
+        then.status(200).header("Location", srv.url("/js/prod/"));
     });
 
     let js_dev_mock = srv.mock(|when, then| {
         when.method(GET).path("/js/dev/");
-        then.status(200).header("Location", &srv.url("/js/dev/"));
+        then.status(200).header("Location", srv.url("/js/dev/"));
     });
 
     let js_dev_file_mock = srv.mock(|when, then| {
@@ -864,7 +864,7 @@ fn scanner_forced_recursion_ignores_normal_redirect_logic() -> Result<(), Box<dy
         when.method(GET).path("/LICENSE");
         then.status(301)
             .body("this is a test")
-            .header("Location", &srv.url("/LICENSE"));
+            .header("Location", srv.url("/LICENSE"));
     });
 
     let mock2 = srv.mock(|when, then| {