update v2.0

.gitignore

@@ -1,4 +1,5 @@
 .idea/
 .venv/
 __pycache__/
-todo.txt
+tests/
+todo*

README.md

@@ -39,10 +39,10 @@
 - [x] 奇安信漏洞通告
 
 
-### TODO
+### ~~TODO~~ DONE
 
-- [ ] 每日所有漏洞信息邮件推送（模板doing，V2.0实现）
-- [ ] flask+datatables+echarts实现完整前后端（V2.0实现）
+- [x] 每日所有漏洞信息邮件推送  ~~（模板doing，V2.0实现）~~
+- [x] flask+datatables+echarts实现完整前后端  ~~（V2.0实现）~~
 
 
 ## WHY
@@ -115,9 +115,9 @@ notify:
     enable: false
     access_token: 
     secret: 
-
+
+  #填写相关配置信息，在每天6点会推送前一天的漏洞汇总
   email:
-    enable: false
     smtp_server:
     smtp_port:
     username:
@@ -166,16 +166,92 @@ while True:
 
 安装完需要的库，配置好config.yaml后，即可开始运行。
 
-![image-20240730174829661](README/image-20240730174829661.png)
+![image-20240801234222860](README/image-20240801234222860.png)
+
+注意，需要一直在后台保持运行，可配合screen等工具实现。
+
+#### 高危漏洞预警
+
+**注意：** 只会推送severity为high及以上的漏洞。
 
 推送内容如下：
 
 ![image-20240730174957206](README/image-20240730174957206.png)
 
-**注意：** 只会推送severity为high及以上的漏洞，但是所有漏洞均会存入数据库：
+#### 数据库信息
+
+所有漏洞均会存入数据库：
 
 ![image-20240731172901942](README/image-20240731172901942.png)
 
+#### 每日漏洞汇总
+
+每天六点会推送前一天的漏洞汇总，邮件内容如下：
+
+![image-20240801233643812](README/image-20240801233643812.png)
+
+由于漏洞描述可能较长，影响观感，因此邮件中隐藏了描述字段，每个漏洞均可通过点击link跳转至poc或详情页。
+
+#### 数据可视化
+
+在V2.0中，实现了数据的可视化，默认端口为5000。
+
+**重要！！！**  在该项目中，后端由flask实现，在main.py中，通过run_flask_app来运行应用：
+
+```python
+def run_flask_app():
+    app.run(debug=False)
+```
+
+默认配置下，仅能本机访问，即`host = '127.0.0.1'`，若想进行远程访问，有如下两种方法：
+
+- 推荐通过nps等隧道工具实现，较为安全，但是需要配置隧道工具
+- 修改`app.run(debug=False)`为`app.run(debug=False,host='0.0.0.0')`，修改后直接对所有主机开放，简单粗暴，但是存在一定安全风险，如有意外作者概不负责！
+
+
+
+前端展示共有3处路由：首页('/'或'/index')，漏洞总览('/vuls')和每日漏洞详情('/daily/[date]')
+
+
+
+##### /index
+
+首页有两部分，第一部分是漏洞的统计结果，有四张图表，分别统计了所有漏洞中，CVE的占比，各种severity漏洞的占比，各漏洞源漏洞的占比以及近7天来的漏洞数量变化趋势。
+
+![image-20240801235519773](README/image-20240801235519773.png)
+
+第二部分为最新的十条漏洞数据：
+
+![image-20240801235832583](README/image-20240801235832583.png)
+
+为了更加美观，在图表中没有采取换行的方式，如有超长字段，会隐藏一部分，鼠标悬停即可查看完整内容。
+
+同样，单击右侧link即可跳转至漏洞POC或详情页。
+
+
+
+##### /vuls
+
+该路由展示了全量的漏洞数据，并且添加了搜索功能：
+
+![image-20240802000129644](README/image-20240802000129644.png)
+
+右上方可选每页展示的漏洞条数，通过最下方页码即可跳转。
+
+搜索框中为模糊搜索，支持全字段，例如搜索name，cve，source，severity，date等等：
+
+![image-20240802000310469](README/image-20240802000310469.png)
+
+##### /daily/date
+
+通过输入%Y-%m-%d形式的日期，可查询指定日期的漏洞情况，以2024-07-31为例：
+
+`/daily/2024-07-31`
+
+![image-20240802000456751](README/image-20240802000456751.png)
+
+展示了当前日期CVE漏洞占比，各severity和source的漏洞占比，以及当天更新的所有漏洞信息。
+
 ### 扩展
 
 如需扩展其他漏洞源数据，实现base_collector.py中的VulnerabilityCollector类即可，漏洞信息的字段如下：

collectors/collector_github.py

@@ -42,7 +42,7 @@ def parse_data(self, raw_data):
                 continue
 
             vulnerability = {
-                'name': '',
+                'name': cves,
                 'cve': cves,
                 'severity': severity,
                 'description': desc,

config.py

@@ -1,7 +1,13 @@
 import yaml
+import os
 
+config_dir = os.path.dirname(os.path.abspath(__file__))
+
+
+def load_config(config_path=None):
+    if config_path is None:
+        config_path = os.path.join(config_dir, 'config.yaml')
 
-def load_config(config_path='config.yaml'):
     with open(config_path, 'r') as f:
         config = yaml.safe_load(f)
     return config

config.yaml

@@ -21,7 +21,6 @@ notify:
     secret:
 
   email:
-    enable: false
     smtp_server:
     smtp_port:
     username:

database/db_class.py

@@ -47,7 +47,7 @@ def fetch_results(self, query, params=None):
             print("Failed to connect to the database")
             return []
 
-        cursor = self.connection.cursor()
+        cursor = self.connection.cursor(dictionary=True)
         results = []
         try:
             cursor.execute(query, params)
@@ -57,7 +57,7 @@ def fetch_results(self, query, params=None):
         cursor.close()
         return results
 
-    def insert_vulnerability(self,  data):
+    def insert_vulnerability(self, data):
         query = f"""
         INSERT INTO vulnerabilities (name, cve, severity, description, source, date, link)
         VALUES (%s, %s, %s, %s, %s, %s, %s)
@@ -75,6 +75,6 @@ def insert_vulnerability(self,  data):
         ))
 
     def check_vulnerability_exists(self, key, value):
-        query = f"SELECT EXISTS(SELECT 1 FROM vulnerabilities WHERE {key}=%s)"
+        query = f"SELECT EXISTS(SELECT 1 FROM vulnerabilities WHERE {key}=%s) as exists_flag"
         result = self.fetch_results(query, (value,))
-        return result[0][0] if result else False
+        return result[0]['exists_flag'] if result else False

flaskr/app.py

@@ -0,0 +1,124 @@
+from flask import Flask, render_template, request, jsonify
+from config import cfg
+from database.db_class import MySQLDatabase
+
+app = Flask(__name__)
+mysql_db = MySQLDatabase()
+
+
+@app.route('/')
+@app.route('/index')
+def index():
+    try:
+
+        latest_vulnerabilities_query = "SELECT * FROM vulnerabilities ORDER BY date DESC LIMIT 10"
+        latest_vulnerabilities = mysql_db.fetch_results(latest_vulnerabilities_query)
+
+
+        severity_data_query = """
+            SELECT COUNT(*) as total,
+                   SUM(CASE WHEN severity = 'high' THEN 1 ELSE 0 END) as high_count,
+                   SUM(CASE WHEN severity = 'medium' THEN 1 ELSE 0 END) as medium_count,
+                   SUM(CASE WHEN severity = 'low' THEN 1 ELSE 0 END) as low_count,
+                   SUM(CASE WHEN severity = 'critical' THEN 1 ELSE 0 END) as critical_count,
+                   SUM(CASE WHEN severity = '' THEN 1 ELSE 0 END) as none_count
+            FROM vulnerabilities
+        """
+        severity_data = mysql_db.fetch_results(severity_data_query)[0]
+
+        source_data_query = "SELECT source, COUNT(*) as count FROM vulnerabilities GROUP BY source"
+        source_data = mysql_db.fetch_results(source_data_query)
+
+        cve_data_query = "SELECT COUNT(*) as total, SUM(CASE WHEN cve <> '' THEN 1 ELSE 0 END) as cve_count FROM vulnerabilities"
+        cve_data = mysql_db.fetch_results(cve_data_query)[0]
+
+        trend_data_query = """
+            SELECT date, COUNT(*) as count
+            FROM vulnerabilities
+            WHERE date >= CURDATE() - INTERVAL 7 DAY
+            GROUP BY date
+            ORDER BY date
+        """
+        trend_data = mysql_db.fetch_results(trend_data_query)
+
+        return render_template('index.html', latest_vulnerabilities=latest_vulnerabilities,
+                               severity_data=severity_data, source_data=source_data, cve_data=cve_data,
+                               trend_data=trend_data)
+    except Exception as err:
+        return f"Error: {err}"
+
+@app.route('/vuls', methods=['GET'])
+def vuls():
+    try:
+        search = request.args.get('search', '')
+        per_page = int(request.args.get('per_page', 10))
+        page = int(request.args.get('page', 1))
+        offset = (page - 1) * per_page
+
+        if search:
+            query = """
+                SELECT * FROM vulnerabilities
+                WHERE name LIKE %s OR cve LIKE %s OR severity LIKE %s OR description LIKE %s OR source LIKE %s
+                ORDER BY date DESC
+                LIMIT %s, %s
+            """
+            params = (f'%{search}%', f'%{search}%', f'%{search}%', f'%{search}%', f'%{search}%', offset, per_page)
+            vulnerabilities = mysql_db.fetch_results(query, params)
+
+            count_query = """
+                SELECT COUNT(*) as total
+                FROM vulnerabilities
+                WHERE name LIKE %s OR cve LIKE %s OR severity LIKE %s OR description LIKE %s OR source LIKE %s
+            """
+            count_params = (f'%{search}%', f'%{search}%', f'%{search}%', f'%{search}%', f'%{search}%')
+        else:
+            query = "SELECT * FROM vulnerabilities ORDER BY date DESC LIMIT %s, %s"
+            params = (offset, per_page)
+            vulnerabilities = mysql_db.fetch_results(query, params)
+
+            count_query = "SELECT COUNT(*) as total FROM vulnerabilities"
+            count_params = ()
+
+        total_result = mysql_db.fetch_results(count_query, count_params)[0]
+        total = total_result['total']
+
+        return render_template('vuls.html', vulnerabilities=vulnerabilities, total=total, per_page=per_page, page=page, search=search)
+    except Exception as err:
+        return f"Error: {err}"
+
+@app.route('/daily/<date>')
+def daily(date):
+    try:
+        vulnerabilities_query = "SELECT * FROM vulnerabilities WHERE date = %s ORDER BY date DESC"
+        vulnerabilities = mysql_db.fetch_results(vulnerabilities_query, (date,))
+
+        severity_data_query = """
+            SELECT COUNT(*) as total,
+                   SUM(CASE WHEN severity = 'high' THEN 1 ELSE 0 END) as high_count,
+                   SUM(CASE WHEN severity = 'medium' THEN 1 ELSE 0 END) as medium_count,
+                   SUM(CASE WHEN severity = 'low' THEN 1 ELSE 0 END) as low_count,
+                   SUM(CASE WHEN severity = 'critical' THEN 1 ELSE 0 END) as critical_count,
+                   SUM(CASE WHEN severity = '' THEN 1 ELSE 0 END) as none_count
+            FROM vulnerabilities
+            WHERE date = %s
+        """
+        severity_data = mysql_db.fetch_results(severity_data_query, (date,))[0]
+
+        source_data_query = "SELECT source, COUNT(*) as count FROM vulnerabilities WHERE date = %s GROUP BY source"
+        source_data = mysql_db.fetch_results(source_data_query, (date,))
+
+
+        cve_data_query = "SELECT COUNT(*) as total, SUM(CASE WHEN cve <> '' THEN 1 ELSE 0 END) as cve_count FROM vulnerabilities WHERE date = %s"
+        cve_data = mysql_db.fetch_results(cve_data_query, (date,))[0]
+
+        vuln_count = len(vulnerabilities)
+
+        return render_template('daily.html', date=date, vulnerabilities=vulnerabilities,
+                               severity_data=severity_data, source_data=source_data, cve_data=cve_data,
+                               vuln_count=vuln_count)
+    except Exception as err:
+        return f"Error: {err}"
+
+@app.errorhandler(404)
+def page_not_found(error):
+    return render_template('404.html'), 404