From 2416110daca82ae7f4a20aec97cd57029a84fe95 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:47:49 +0700 Subject: [PATCH 01/12] Fix exception handling --- config.ini | 2 +- lib/core/fuzzer.py | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/config.ini b/config.ini index d2ebc13d6..2db94760a 100644 --- a/config.ini +++ b/config.ini @@ -14,6 +14,7 @@ exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/ random-user-agents = False max-time = 0 exit-on-error = False +skip-on-status=429 #subdirs = /,api/ #include-status = 200-299,401 #exclude-status = 400,500-999 @@ -25,7 +26,6 @@ exit-on-error = False #exclude-regex = "^403$" #exclude-redirect = "*/error.html" #exclude-response = 404.html -#skip-on-status = 429,999 [dictionary] default-extensions = php,asp,aspx,jsp,html,htm diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py index 4eec7821d..85cc1304b 100755 --- a/lib/core/fuzzer.py +++ b/lib/core/fuzzer.py @@ -232,7 +232,12 @@ def quit(self) -> None: def scan(self, path: str) -> None: scanners = self.get_scanners_for(path) - response = self._requester.request(path) + try: + response = self._requester.request(path) + except RequestException as e: + for callback in self.error_callbacks: + callback(e) + return if self.is_excluded(response): for callback in self.not_found_callbacks: @@ -246,11 +251,8 @@ def scan(self, path: str) -> None: callback(response) return - try: - for callback in self.match_callbacks: - callback(response) - except Exception as e: - self.exc = e + for callback in self.match_callbacks: + callback(response) def thread_proc(self) -> None: logger.info(f'THREAD-{threading.get_ident()} started"') @@ -263,11 +265,8 @@ def thread_proc(self) -> None: except StopIteration: break - except RequestException as e: - for callback in self.error_callbacks: - callback(e) - - continue + except Exception as e: + self.exc = e finally: time.sleep(options["delay"]) From e54259308390e3a6f3613ed2bd1c069822f86269 Mon Sep 17 00:00:00 2001 From: Pham Sy Minh <59408894+shelld3v@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:49:31 +0700 Subject: [PATCH 02/12] Lint --- config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ini b/config.ini index 2db94760a..e09b3f660 100644 --- a/config.ini +++ b/config.ini @@ -14,7 +14,7 @@ exclude-subdirs = %%ff/,.;/,..;/,;/,./,../,%%2e/,%%2e%%2e/ random-user-agents = False max-time = 0 exit-on-error = False -skip-on-status=429 +skip-on-status = 429 #subdirs = /,api/ #include-status = 200-299,401 #exclude-status = 400,500-999 From f7760554308ef39b7faa3aa0ae856b676c65cc21 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:49:36 +0700 Subject: [PATCH 03/12] Fix in asyncio mode as well --- lib/core/fuzzer.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py index 85cc1304b..a4688ebd7 100755 --- a/lib/core/fuzzer.py +++ b/lib/core/fuzzer.py @@ -53,7 +53,6 @@ def __init__( self._requester = requester self._dictionary = dictionary self._base_path: str = "" - self.exc: Exception | None = None self.match_callbacks = match_callbacks self.not_found_callbacks = not_found_callbacks self.error_callbacks = error_callbacks @@ -148,6 +147,7 @@ def __init__( not_found_callbacks=not_found_callbacks, error_callbacks=error_callbacks, ) + self._exc: Exception | None = None self._threads = [] self._play_event = threading.Event() self._quit_event = threading.Event() @@ -207,8 +207,8 @@ def start(self) -> None: thread.start() def is_finished(self) -> bool: - if self.exc: - raise self.exc + if self._exc: + raise self._exc for thread in self._threads: if thread.is_alive(): @@ -266,7 +266,7 @@ def thread_proc(self) -> None: break except Exception as e: - self.exc = e + self._exc = e finally: time.sleep(options["delay"]) @@ -358,12 +358,6 @@ async def start(self) -> None: await asyncio.gather(*self._background_tasks) - def is_finished(self) -> bool: - if self.exc: - raise self.exc - - return len(self._background_tasks) == 0 - def play(self) -> None: self._play_event.set() @@ -376,7 +370,12 @@ def quit(self) -> None: async def scan(self, path: str) -> None: scanners = self.get_scanners_for(path) - response = await self._requester.request(path) + try: + response = await self._requester.request(path) + except RequestException as e: + for callback in self.error_callbacks: + callback(e) + return if self.is_excluded(response): for callback in self.not_found_callbacks: @@ -390,11 +389,8 @@ async def scan(self, path: str) -> None: callback(response) return - try: - for callback in self.match_callbacks: - callback(response) - except Exception as e: - self.exc = e + for callback in self.match_callbacks: + callback(response) async def task_proc(self) -> None: async with self.sem: @@ -405,8 +401,5 @@ async def task_proc(self) -> None: await self.scan(self._base_path + path) except StopIteration: pass - except RequestException as e: - for callback in self.error_callbacks: - callback(e) finally: await asyncio.sleep(options["delay"]) From 669888bafd9675728c9110f21cede5344bdbe448 Mon Sep 17 00:00:00 2001 From: Pham Sy Minh <59408894+shelld3v@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:53:03 +0700 Subject: [PATCH 04/12] Fix #1285 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 01bd4e608..393dfd6ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Jinja2>=3.0.0 defusedxml>=0.7.0 pyopenssl>=21.0.0 requests>=2.27.0 -requests_ntlm>=1.1.0 +requests_ntlm>=1.3.0 colorama>=0.4.4 ntlm_auth>=1.5.0 beautifulsoup4>=4.8.0 From 9137c863ab4ae0780ea61d542db8ee3e7c748c08 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Sat, 9 Nov 2024 16:39:19 +0700 Subject: [PATCH 05/12] Fix a bug with async mode --- lib/connection/requester.py | 18 ++++++------------ lib/connection/response.py | 8 ++++---- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/connection/requester.py b/lib/connection/requester.py index c67428346..4fb91b3a2 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -188,9 +188,7 @@ def request(self, path: str, proxy: str | None = None) -> Response: self.increase_rate() err_msg = None - - # Safe quote all special characters to prevent them from being encoded - url = safequote(self._url + path if self._url else path) + url = self._url + safequote(path) # Why using a loop instead of max_retries argument? Check issue #1009 for _ in range(options["max_retries"] + 1): @@ -221,7 +219,7 @@ def request(self, path: str, proxy: str | None = None) -> Response: timeout=options["timeout"], stream=True, ) - response = Response(origin_response) + response = Response(url, origin_response) log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B' @@ -371,12 +369,9 @@ async def request( self.increase_rate() err_msg = None - - # Safe quote all special characters to prevent them from being encoded - url = safequote(self._url + path if self._url else path) - parsed_url = urlparse(url) - + url = self._url + safequote(path) session = session or self.session + for _ in range(options["max_retries"] + 1): try: if self.agents: @@ -388,16 +383,15 @@ async def request( url, headers=self.headers, data=options["data"], + extensions={"target": f"/{safequote(path)}".encode()}, ) - if p := parsed_url.path: - request.extensions = {"target": p.encode()} xresponse = await session.send( request, stream=True, follow_redirects=options["follow_redirects"], ) - response = await AsyncResponse.create(xresponse) + response = await AsyncResponse.create(url, xresponse) await xresponse.aclose() log_msg = f'"{options["http_method"]} {response.url}" {response.status} - {response.length}B' diff --git a/lib/connection/response.py b/lib/connection/response.py index b344f7620..cbf491751 100755 --- a/lib/connection/response.py +++ b/lib/connection/response.py @@ -35,9 +35,9 @@ class BaseResponse: - def __init__(self, response: requests.Response | httpx.Response) -> None: + def __init__(self, url, response: requests.Response | httpx.Response) -> None: self.datetime = time.strftime("%Y-%m-%d %H:%M:%S") - self.url = str(response.url) + self.url = url self.full_path = parse_path(self.url) self.path = clean_path(self.full_path) self.status = response.status_code @@ -99,8 +99,8 @@ def __init__(self, response: requests.Response) -> None: class AsyncResponse(BaseResponse): @classmethod - async def create(cls, response: httpx.Response) -> AsyncResponse: - self = cls(response) + async def create(cls, url, response: httpx.Response) -> AsyncResponse: + self = cls(url, response) async for chunk in response.aiter_bytes(chunk_size=ITER_CHUNK_SIZE): self.body += chunk From 4719e298e590889251da86e9e347cce7a74de40c Mon Sep 17 00:00:00 2001 From: 4shen0ne <4shen.01@gmail.com> Date: Wed, 13 Nov 2024 21:09:55 +0800 Subject: [PATCH 06/12] fix bug in #1430 --- lib/controller/controller.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 88be4dc57..3517b7895 100755 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -205,12 +205,9 @@ def run(self) -> None: self.requester = Requester() if options["async_mode"]: self.loop = asyncio.new_event_loop() - try: - self.loop.add_signal_handler(signal.SIGINT, self.handle_pause) - except NotImplementedError: - # Windows - signal.signal(signal.SIGINT, self.handle_pause) - signal.signal(signal.SIGTERM, self.handle_pause) + + signal.signal(signal.SIGINT, lambda *_: self.handle_pause()) + signal.signal(signal.SIGTERM, lambda *_: self.handle_pause()) while options["urls"]: url = options["urls"][0] @@ -514,18 +511,14 @@ def is_timed_out(self) -> bool: def process(self) -> None: while True: - try: - while not self.fuzzer.is_finished(): - if self.is_timed_out(): - raise SkipTargetInterrupt( - "Runtime exceeded the maximum set by the user" - ) - time.sleep(0.5) - - break + while not self.fuzzer.is_finished(): + if self.is_timed_out(): + raise SkipTargetInterrupt( + "Runtime exceeded the maximum set by the user" + ) + time.sleep(0.5) - except KeyboardInterrupt: - self.handle_pause() + break def add_directory(self, path: str) -> None: """Add directory to the recursion queue""" From c4f15e7e75ad6392e9227731a9b3b612e1da5a99 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Wed, 13 Nov 2024 21:55:40 +0700 Subject: [PATCH 07/12] Move warning to dirsearch.py --- dirsearch.py | 5 +++++ lib/controller/controller.py | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dirsearch.py b/dirsearch.py index dcd539b69..97635d60f 100755 --- a/dirsearch.py +++ b/dirsearch.py @@ -66,6 +66,11 @@ def main(): options.update(parse_options()) + if options["session_file"]: + print("WARNING: Running an untrusted session file might lead to unwanted code execution!") + if input("[c]ontinue / [q]uit: ") != "c": + exit(1) + from lib.controller.controller import Controller Controller() diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 88be4dc57..bef0c7446 100755 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -72,11 +72,6 @@ class Controller: def __init__(self) -> None: if options["session_file"]: - print("WARNING: Running an untrusted session file might lead to unwanted code execution!") - interface.in_line("[c]continue / [q]uit: ") - if input() != "c": - exit(1) - self._import(options["session_file"]) self.old_session = True else: From 82eeb841f88d2e9edd355956d8ae65aae068f7ae Mon Sep 17 00:00:00 2001 From: 4shen0ne <4shen.01@gmail.com> Date: Fri, 15 Nov 2024 15:49:48 +0800 Subject: [PATCH 08/12] clear quit event when starting fuzzer --- lib/core/fuzzer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/core/fuzzer.py b/lib/core/fuzzer.py index 4eec7821d..e5d4122e9 100755 --- a/lib/core/fuzzer.py +++ b/lib/core/fuzzer.py @@ -202,6 +202,7 @@ def start(self) -> None: self.setup_scanners() self.setup_threads() self.play() + self._quit_event.clear() for thread in self._threads: thread.start() From 365e4ba25d5c51dba047f74bf37efc3fb90a2a20 Mon Sep 17 00:00:00 2001 From: shelld3v <59408894+shelld3v@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:06:54 +0700 Subject: [PATCH 09/12] Bug fixes --- lib/connection/requester.py | 42 +++++++++++++++++-------------------- lib/connection/response.py | 4 ++-- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/lib/connection/requester.py b/lib/connection/requester.py index 4fb91b3a2..c72465f04 100755 --- a/lib/connection/requester.py +++ b/lib/connection/requester.py @@ -101,21 +101,6 @@ def set_url(self, url: str) -> None: def set_header(self, key: str, value: str) -> None: self.headers[key] = value.lstrip() - def set_proxy(self, proxy: str) -> None: - if not proxy: - return - - if not proxy.startswith(PROXY_SCHEMES): - proxy = f"http://{proxy}" - - if self.proxy_cred and "@" not in proxy: - # socks5://localhost:9050 => socks5://[credential]@localhost:9050 - proxy = proxy.replace("://", f"://{self.proxy_cred}@", 1) - - self.session.proxies = {"https": proxy} - if not proxy.startswith("https://"): - self.session.proxies["http"] = proxy - def is_rate_exceeded(self) -> bool: return self._rate >= options["max_rate"] > 0 @@ -193,9 +178,19 @@ def request(self, path: str, proxy: str | None = None) -> Response: # Why using a loop instead of max_retries argument? Check issue #1009 for _ in range(options["max_retries"] + 1): try: + proxies = {} try: - proxy = proxy or random.choice(options["proxies"]) - self.set_proxy(proxy) + proxy_url = proxy or random.choice(options["proxies"]) + if not proxy_url.startswith(PROXY_SCHEMES): + proxy_url = f"http://{proxy_url}" + + if self.proxy_cred and "@" not in proxy_url: + # socks5://localhost:9050 => socks5://[credential]@localhost:9050 + proxy_url = proxy_url.replace("://", f"://{self.proxy_cred}@", 1) + + proxies["https"] = proxy_url + if not proxy_url.startswith("https://"): + proxies["http"] = proxy_url except IndexError: pass @@ -210,13 +205,14 @@ def request(self, path: str, proxy: str | None = None) -> Response: headers=self.headers, data=options["data"], ) - prepped = self.session.prepare_request(request) - prepped.url = url + prep = self.session.prepare_request(request) + prep.url = url origin_response = self.session.send( - prepped, + prep, allow_redirects=options["follow_redirects"], timeout=options["timeout"], + proxies=proxies, stream=True, ) response = Response(url, origin_response) @@ -357,11 +353,11 @@ async def replay_request(self, path: str, proxy: str) -> AsyncResponse: mounts={"all://": transport}, timeout=httpx.Timeout(options["timeout"]), ) - return await self.request(path, self.replay_session) + return await self.request(path, self.replay_session, replay=True) # :path: is expected not to start with "/" async def request( - self, path: str, session: httpx.AsyncClient | None = None + self, path: str, session: httpx.AsyncClient | None = None, replay: bool = False ) -> AsyncResponse: while self.is_rate_exceeded(): await asyncio.sleep(0.1) @@ -383,7 +379,7 @@ async def request( url, headers=self.headers, data=options["data"], - extensions={"target": f"/{safequote(path)}".encode()}, + extensions={"target": (url if replay else f"/{safequote(path)}").encode()}, ) xresponse = await session.send( diff --git a/lib/connection/response.py b/lib/connection/response.py index cbf491751..108fe6db0 100755 --- a/lib/connection/response.py +++ b/lib/connection/response.py @@ -77,8 +77,8 @@ def __eq__(self, other: Any) -> bool: class Response(BaseResponse): - def __init__(self, response: requests.Response) -> None: - super().__init__(response) + def __init__(self, url, response: requests.Response) -> None: + super().__init__(url, response) for chunk in response.iter_content(chunk_size=ITER_CHUNK_SIZE): self.body += chunk From 0ba737dde4161efb29ebfce9125aa0550a61a592 Mon Sep 17 00:00:00 2001 From: partoneplay Date: Mon, 16 Dec 2024 11:03:02 +0800 Subject: [PATCH 10/12] add OpenAI and Ollama REST API Endpoint --- db/dicc.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/db/dicc.txt b/db/dicc.txt index 73a333c9d..88c248c0e 100644 --- a/db/dicc.txt +++ b/db/dicc.txt @@ -3047,14 +3047,21 @@ api/apidocs/swagger.json api/application.wadl api/batch api/cask/graphql +api/chat api/config api/config.json +api/copy +api/create api/credential.json api/credentials.json api/database.json +api/delete api/docs api/docs/ +api/embed +api/embeddings api/error_log +api/generate api/index.html api/jsonws api/jsonws/invoke @@ -3062,6 +3069,10 @@ api/login.json api/package_search/v4/documentation api/profile api/proxy +api/ps +api/pull +api/push +api/show api/snapshots api/spec/swagger.json api/swagger @@ -3073,6 +3084,7 @@ api/swagger/index.html api/swagger/static/index.html api/swagger/swagger api/swagger/ui/index +api/tags api/timelion/run api/user.json api/users.json @@ -8928,9 +8940,19 @@ v1.0/ v1.1 v1/ v1/api-docs +v1/audio/speech +v1/batches +v1/chat/completions +v1/embeddings +v1/files +v1/fine_tuning/jobs +v1/images/generations +v1/models +v1/moderations v1/public/yql v1/test/js/console.html v1/test/js/console_ajax.js +v1/uploads v2 v2.0 v2/ From b3ccc28fb7c45ca360a42cbdcef511f72f1f27c2 Mon Sep 17 00:00:00 2001 From: partoneplay Date: Mon, 16 Dec 2024 11:06:36 +0800 Subject: [PATCH 11/12] Contributors --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ab2c275a3..66903333d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -83,6 +83,7 @@ - [jxdv](https://github.com/jxdv) - [Xeonacid](https://github.com/Xeonacid) - [Valentijn Scholten](https://www.github.com/valentijnscholten) +- [partoneplay](https://github.com/partoneplay) Special thanks to all the people who are named here! From 47926b13f8786f13ec202cbbff9247d796e81592 Mon Sep 17 00:00:00 2001 From: partoneplay Date: Mon, 16 Dec 2024 11:12:07 +0800 Subject: [PATCH 12/12] CHNAGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89627f41a..bbc02bda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Load targets from a Nmap XML report - Added --async option to enable asynchronous mode (use coroutines instead of threads) - Added option to disable CLI output entirely +- Added OpenAI and Ollama REST API endpoints to the dictionary ## [0.4.3] - October 2nd, 2022 - Automatically detect the URI scheme (`http` or `https`) if no scheme is provided