From 9742da0a7253ed59cbe287e83d85b0238ec0dfff Mon Sep 17 00:00:00 2001 From: panyi Date: Sun, 22 Sep 2024 01:24:09 +0800 Subject: [PATCH] =?UTF-8?q?Feature=20v2.5.2=20=E6=B7=BB=E5=8A=A0csv?= =?UTF-8?q?=E5=90=8E=E7=BC=80=E7=9A=84=E8=BE=93=E5=87=BA,=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0debug=E8=8E=AB=E4=BA=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 -- core/fofaMain.py | 9 +++++- docs/CHANGELOG.md | 46 ++++++++++++++++++++++++------ docs/TODO.md | 19 ------------- fofa.py | 8 ++++-- tookit/config.py | 3 +- tookit/outputData.py | 67 ++++++++++++++++++++++++++++++-------------- 7 files changed, 100 insertions(+), 55 deletions(-) delete mode 100644 docs/TODO.md diff --git a/README.md b/README.md index 431df73..c827535 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,6 @@ if __name__ == '__main__': 详情请见[CHANGELOG](docs/CHANGELOG.md) -### TODO List - -详情请见[TODO](docs/TODO.md) ### 贡献者 diff --git a/core/fofaMain.py b/core/fofaMain.py index 6d9a360..2772d21 100644 --- a/core/fofaMain.py +++ b/core/fofaMain.py @@ -331,15 +331,20 @@ def setIndexTimestamp(self, searchbs64, timestamp_index): exit(0) request_url = getUrl(searchbs64) + if config.DEBUG: + print("[+] 当前请求网址: "+request_url) rep = requests.get(request_url, headers=fofaUseragent.getFofaPageNumHeaders(), timeout=self.timeout, proxies=self.get_proxy()) # request should be success rep.raise_for_status() + if config.DEBUG: + print("[+] 当前响应: " + rep.text) # request should not be limited # '{"code":820006,"message":"[820006] 资源访问每天限制","data":""}' if len(rep.text) <= 55 and '820006' in rep.text: - raise RuntimeError("API call limit reached for today,call at next day or use proxy") + raise RuntimeError("\033[1;31m[-] error:{}\033[0m".format( + "API call limit reached for today,call at next day or use proxy")) # print(rep.text) timelist = self.getTimeList(rep.text) # print(timelist) @@ -603,6 +608,8 @@ def removeDuplicate(self): f.close() final.close() return int(len(text_list)) + except FileNotFoundError: + print("\033[1;31m[-] 未保存文件,去重失败\033[0m") except Exception as e: print("\033[1;31m[-] error:{}\033[0m".format(e)) return 0 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 945858c..72ccaac 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,65 +1,86 @@ # CHANGELOG 代码变更记录 +### 2.5.2 + ++ 添加csv后缀的输出 ++ 添加debug模式 + ### 2.5.1 + + 添加 --fofa-key 参数 (优化中) + 修复BUG [issue 86](https://github.com/Cl0udG0d/Fofa-hack/issues/86) ### 2.4.18 + + 合入(pull request)(https://github.com/Cl0udG0d/Fofa-hack/pull/85) ### 2.4.17 + + 删除文档里面的存在问题的部分 + 修改setup.py，更新fofa-hack包为0.02版本 [pip fofa-hack](https://pypi.org/manage/project/fofa-hack/release/0.0.2/) ### 2.4.16 + + 新增 --authorization-file 参数 ### 2.4.15 + + 添加批量从文件使用代理的功能 --proxy-file + 添加使用网址代理的功能 --proxy-url + 在使用AUTHORIZATION的情况下,一次获取50条数据 ### 2.4.14 + + 修改并测试了代理的问题 ### 2.4.13 + + 新增icon文件和网址功能 [issue 64](https://github.com/Cl0udG0d/Fofa-hack/issues/64) + 修复BUG [issue 65](https://github.com/Cl0udG0d/Fofa-hack/issues/65) ### 2.4.12 + + 修复[issue 63](https://github.com/Cl0udG0d/Fofa-hack/issues/63) + 完善Authorization逻辑 ### 2.4.11 + + 新增SECURITY文件 + 新增用户Authorization查询 ### 2.4.10 + + 修复删除文件问题 + 修复[issue 59](https://github.com/Cl0udG0d/Fofa-hack/issues/59) ### 2.4.9 + + 修复语言问题 + 修改代码冗余部分 ### 2.4.8 + + 修复setup.py问题 + 合并[新增可设置保存文件名代码](https://github.com/Cl0udG0d/Fofa-hack/pull/58) + 新增贡献者[10cks](https://github.com/10cks) ### 2.4.7 + + 去除没有获取到语言情况下为空的问题 + 生成fofa-hack pip包 ### 2.4.6 + + 修复 [issues 54](https://github.com/Cl0udG0d/Fofa-hack/issues/54) + 终端颜色输出问题解决 ### 2.4.5 + + 修复 [issues 51](https://github.com/Cl0udG0d/Fofa-hack/issues/51) + 修复 [issues 53](https://github.com/Cl0udG0d/Fofa-hack/issues/53) ### 2.4.4 + + unit.py 修剪关键字功能扩展 ### 2.4.3 @@ -69,6 +90,7 @@ ### 2.4.2 + 修复打包问题 + ### 2.4.1 + 根据环境切换语言 @@ -78,6 +100,7 @@ + 删除多余逻辑文件 + 更新获取数据逻辑,页面爬虫 -> api接口获取 + 默认步长设置为 50,防止等待过久 + ### 2.3.9 + 修复exit(0)退出报错问题 @@ -105,11 +128,13 @@ ### 2.3.4 + 新增英文readme + ### 2.3.3 -+ 修复部分已知BUG - + 修复搜索结果为0时出现的列表索引越界问题 - + 修复搜索结果时间戳列表为空时的报错问题 ++ 修复部分已知BUG + + 修复搜索结果为0时出现的列表索引越界问题 + + 修复搜索结果时间戳列表为空时的报错问题 + ### 2.3.2 + 修改运行方式readme @@ -227,21 +252,26 @@ + 合并代码 - 优化时间戳逻辑 ### 2.0.5 + + 新增贡献者 tastypear + 尝试优化时间戳逻辑 ### 2.0.4 + + 删除.github文件夹 + 新增TODO List + 删除csv导出 ### 2.0.3 + + 修复fofa单关键字下第二页错误情况 ### 2.0.2 + + 修复fofa多个查询条件下出现的错误 ### 2.0.1 + + 修改filename,timeSleep等为类内部变量 + 新增logoutInitMsg方法输出初始化信息 + 新增fofa_fuzz_spider方法 @@ -254,14 +284,15 @@ + 支持多种导出方式 txt,json,csv + 因FOFA对普通用户的限制停止该项目的更新 + ### 1.3.0 + `README`添加示例运行参考 + 新增工具文件夹`tookit`,创建`levelData`等级模板文件 + 新增 `--level`参数,内容明细如下 - + level=1 [ url ] - + level=2 [ url , port , title , ip ] - + level=3 [ url , port , title , ip , city , asn , organization , server , rep ] + + level=1 [ url ] + + level=2 [ url , port , title , ip ] + + level=3 [ url , port , title , ip , city , asn , organization , server , rep ] ### 1.2.8 @@ -297,8 +328,8 @@ ### 1.2.2 + BUG修复 -> 存储结果到`md5(搜索关键字)_运行时间戳.txt`文件中 +> 存储结果到`md5(搜索关键字)_运行时间戳.txt`文件中 ### 1.2.1 @@ -316,7 +347,6 @@ + 添加部分代码注释 + 自动生成文件名 搜索关键字_运行时间戳.txt - ### 此前的一些提交 + 2023-1-27，合入[wanswu](https://github.com/wanswu)师傅提交的代码，通过账号密码进行登录 diff --git a/docs/TODO.md b/docs/TODO.md deleted file mode 100644 index 4dcfa6e..0000000 --- a/docs/TODO.md +++ /dev/null @@ -1,19 +0,0 @@ -# TODO - -+ 功能 - + 2.2.x版本添加代理 - + git action维护代理池 - + https://github.com/fate0/getproxy - + 添加csv文件支持 - + 挑战fofa-hack极限 - + 从文件批量读取语法 - + 集成fofa-script - + 集成fofa 公安版 -+ BUG - + 高级语法混合使用情况下的报错处理，增加程序稳定性 - + 修复level = 2,3 情况下的BUG - + -o json 出来还是txt - + level为3的时候结果不变 - + python fofa.py -k "47.75.*.*/24" -l 2 错误 - + 1w多条数据时，爬两三页直接报错 - diff --git a/fofa.py b/fofa.py index 1ffa88a..f39ef2e 100644 --- a/fofa.py +++ b/fofa.py @@ -50,7 +50,7 @@ def main(): parser.add_argument('--timeout', '-to', help=_('爬取每一页的超时时间,默认为180秒'), default=180) parser.add_argument('--endcount', '-e', help=_('爬取结束数量')) parser.add_argument('--level', '-l', help=_('爬取等级: 1-3 ,数字越大内容越详细,默认为 1')) - parser.add_argument('--output', '-o', help=_('输出格式:txt、json,默认为txt')) + parser.add_argument('--output', '-o', help=_('输出格式:txt、json、csv,默认为txt')) parser.add_argument('--outputname','-on', help=_("指定输出文件名，默认文件名为 fofaHack")) parser.add_argument('--fuzz', '-f', help=_('关键字fuzz参数,增加内容获取粒度'), action='store_true') @@ -60,7 +60,8 @@ def main(): parser.add_argument('--fofa-key', type=str, help="fofa api key值(配合fofa终身会员使用)") - + parser.add_argument('--debug', + help="fofa-hack调试模式,运行过程中输出更多运行日志", action='store_true') proxy_group = parser.add_mutually_exclusive_group() proxy_group.add_argument('--proxy', help=_("指定代理,代理格式 --proxy '127.0.0.1:7890'")) proxy_group.add_argument('--proxy-url', help=_("指定代理url，即访问URL响应为proxy,代理格式 --proxy-url http://127.0.0.1/proxy_pool/get")) @@ -69,6 +70,9 @@ def main(): # help="运行类型,默认为普通方式") args = parser.parse_args() + if args.debug: + config.DEBUG = args.debug + if args.fofa_key: config.FOFA_KEY = args.fofa_key diff --git a/tookit/config.py b/tookit/config.py index 3254499..59247de 100644 --- a/tookit/config.py +++ b/tookit/config.py @@ -5,7 +5,7 @@ # @File : config.py # @Github: https://github.com/Cl0udG0d -VERSION_NUM = "2.5.1" +VERSION_NUM = "2.5.2" ROOT_PATH = "" AUTHORIZATION = "" AUTHORIZATION_LIST = [ @@ -15,6 +15,7 @@ FOFA_KEY = "" +DEBUG = False ### ============================================================================ ### 代理相关的配置参数 diff --git a/tookit/outputData.py b/tookit/outputData.py index a883bfd..bd2137b 100644 --- a/tookit/outputData.py +++ b/tookit/outputData.py @@ -1,9 +1,9 @@ +import csv import json import os class OutputData: - ''' 常量 ''' @@ -12,18 +12,16 @@ class OutputData: CONST_CSV = "csv" STANDARD_LIST = [CONST_TXT, CONST_JSON, CONST_CSV] - ENCODING_TYPE="utf-8" + ENCODING_TYPE = "utf-8" - def __init__(self,filename, level="1",pattern="txt"): - self.filename=filename + def __init__(self, filename, level="1", pattern="txt"): + self.filename = filename # self.outputname=outputname self.pattern = pattern if self.checkPatternStandard(pattern) else "txt" - self.level=level + self.level = level + self.header_written = False # self.path=os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))),self.filename) - def initFile(self): - - return def checkPatternStandard(self, pattern): """ @@ -33,24 +31,25 @@ def checkPatternStandard(self, pattern): """ return pattern in self.STANDARD_LIST - def output(self,data): + def output(self, data): # self.filename="{}.{}".format(self.filename,self.pattern) - if self.pattern==self.CONST_TXT: + if self.pattern == self.CONST_TXT: self.outputTxt(data) - elif self.pattern==self.CONST_JSON: + elif self.pattern == self.CONST_JSON: self.outputJson(data) + elif self.pattern == self.CONST_CSV: + self.outputCsv(data) else: pass - def outputTxt(self,data): + def outputTxt(self, data): with open(self.filename, 'a+', encoding=self.ENCODING_TYPE) as f: for i in data: f.write(str(i) + "\n") - def readAllJsonData(self): # print(os.path.exists(self.filename)) - if os.path.exists(self.filename)==False or os.path.getsize(self.filename)==0: + if os.path.exists(self.filename) == False or os.path.getsize(self.filename) == 0: return {} else: with open(self.filename, 'r+', encoding=self.ENCODING_TYPE) as load_f: @@ -59,16 +58,16 @@ def readAllJsonData(self): return load_dict return {} - def outputJson(self,newdata): - data=self.readAllJsonData() + def outputJson(self, newdata): + data = self.readAllJsonData() with open(self.filename, 'w+', encoding=self.ENCODING_TYPE) as load_f: for i in newdata: - if self.level=="1": + if self.level == "1": data[i] = i - elif self.level=="2": - keyword =i["url"] + elif self.level == "2": + keyword = i["url"] data[keyword] = {} data[keyword]["url"] = i["url"] data[keyword]["port"] = i["port"] @@ -85,8 +84,34 @@ def outputJson(self,newdata): # # r.close() - def outputCsv(self): - return + def getCsvHeaders(self, data): + headers = [] + if len(data)>0: + if type(data[0]) is dict: + + headers = list(data[0].keys()) + else: + headers = ["link"] + return headers + + def outputCsv(self, data): + headers = self.getCsvHeaders(data) + with open('score.csv', mode='a+', newline='', encoding='utf-8-sig') as file: + writer = csv.writer(file) + + # 如果尚未写入表头，则写入表头 + if not self.header_written: + writer.writerow(headers) + self.header_written = True + + for i in data: + if self.level == "1": + writer.writerow([i]) + elif self.level == "2" or self.level == "3": + writer.writerow(list(i.values())) + + + if __name__ == '__main__': output = OutputData("123", "json")