forked from testerSunshine/12306
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
05f1486
commit 9d70f83
Showing
11 changed files
with
200 additions
and
219 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,88 +1,81 @@ | ||
| # encoding=utf8 | ||
| import collections | ||
| import json | ||
| import datetime | ||
| import os | ||
| import re | ||
| import sys | ||
| import csv | ||
| import requests | ||
| from config import urlConf | ||
| import threading | ||
| from config.urlConf import urls | ||
|
|
||
| try: | ||
| reload(sys) | ||
| sys.setdefaultencoding('utf-8') | ||
| except NameError: | ||
| pass | ||
| from myUrllib.httpUtils import HTTPClient | ||
|
|
||
| cdn_list = [] | ||
|
|
||
| class CDNProxy: | ||
| def __init__(self, host=None): | ||
| self.host = host | ||
|
|
||
| class CDNProxy(threading.Thread): | ||
| def __init__(self, cdns): | ||
| super().__init__() | ||
| self.cdns = cdns | ||
| self.urlConf = urlConf.urls | ||
| self.httpClint = requests | ||
| self.city_list = [] | ||
| self.timeout = 5 | ||
|
|
||
| def _set_header(self): | ||
| """设置header""" | ||
| return { | ||
| "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", | ||
| "X-Requested-With": "xmlHttpRequest", | ||
| "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36", | ||
| "Referer": "https://kyfw.12306.cn/otn/login/init", | ||
| "Accept": "*/*", | ||
| } | ||
| def run(self): | ||
| for cdn in self.cdns: | ||
| http = HTTPClient(0) | ||
| url = urls["loginInitCdn"] | ||
| http._cdn = cdn.replace("\n", "") | ||
| start_time = datetime.datetime.now() | ||
| rep = http.send(url) | ||
| if rep and "message" not in rep and (datetime.datetime.now() - start_time).microseconds / 1000 < 1000: | ||
| if cdn.replace("\n", "") not in cdn_list: # 如果有重复的cdn，则放弃加入 | ||
| print(f"加入cdn: {cdn}") | ||
| cdn_list.append(cdn.replace("\n", "")) | ||
|
|
||
|
|
||
| def open_cdn_file(cdnFile): | ||
| cdn = [] | ||
| path = os.path.join(os.path.dirname(__file__), f'../{cdnFile}') | ||
| try: | ||
| with open(path, "r", encoding="utf-8") as f: | ||
| for i in f.readlines(): | ||
| if i and "kyfw.12306.cn:443" not in i: | ||
| cdn.append(i.replace("\n", "")) | ||
| return cdn | ||
| except Exception: | ||
| with open(path, "r") as f: | ||
| for i in f.readlines(): | ||
| if i and "kyfw.12306.cn:443" not in i: | ||
| cdn.append(i.replace("\n", "")) | ||
| return cdn | ||
|
|
||
|
|
||
| def filterCdn(): | ||
| """ | ||
| 过滤cdn, 过滤逻辑为当前cdn响应值小于1000毫秒 | ||
| 过滤日志: | ||
| 加入cdn: 116.77.75.146 | ||
| :return: | ||
| """ | ||
| cdns = open_cdn_file("cdn_list") | ||
| cdnss = [cdns[i:i + 50] for i in range(0, len(cdns), 50)] | ||
| cdnThread = [] | ||
| for cdn in cdnss: | ||
| t = CDNProxy(cdn) | ||
| cdnThread.append(t) | ||
| for cdn_t in cdnThread: | ||
| cdn_t.start() | ||
|
|
||
| def get_city_id(self): | ||
| """ | ||
| 获取所有城市md5参数 | ||
| :return: | ||
| """ | ||
| try: | ||
| if self.host: | ||
| while True: | ||
| url = self.urlConf["cdn_host"]["req_url"] | ||
| data = {"host": self.host, "lintType": "电信,多线,联通,移动"} | ||
| rep = self.httpClint.post(url, data, headers=self._set_header(), timeout=self.timeout) | ||
| city_re = re.compile(r"<li id=\"(\S+)\" class=\"PingListCent PingRLlist") | ||
| self.city_list = re.findall(city_re, rep.content) | ||
| if self.city_list: | ||
| print(self.city_list) | ||
| break | ||
| else: | ||
| pass | ||
| except: | ||
| pass | ||
| for cdn_j in cdnThread: | ||
| cdn_j.join() | ||
|
|
||
| def open_cdn_file(self): | ||
| cdn = [] | ||
| # cdn_re = re.compile("CONNECT (\S+) HTTP/1.1") | ||
| # path = os.path.join(os.path.dirname(__file__), '../cdn_list') | ||
| # with open(path, "r") as f: | ||
| # for i in f.readlines(): | ||
| # # print(i.replace("\n", "")) | ||
| # cdn_list = re.findall(cdn_re, i) | ||
| # if cdn_list and "kyfw.12306.cn:443" not in cdn_list: | ||
| # print(cdn_list[0].split(":")[0]) | ||
| # cdn.append(cdn_list[0].split(":")[0]) | ||
| # return cdn | ||
| path = os.path.join(os.path.dirname(__file__), '../cdn_list') | ||
| try: | ||
| with open(path, "r", encoding="utf-8") as f: | ||
| for i in f.readlines(): | ||
| # print(i.replace("\n", "")) | ||
| if i and "kyfw.12306.cn:443" not in i: | ||
| cdn.append(i.replace("\n", "")) | ||
| return cdn | ||
| except Exception: | ||
| with open(path, "r") as f: | ||
| for i in f.readlines(): | ||
| # print(i.replace("\n", "")) | ||
| if i and "kyfw.12306.cn:443" not in i: | ||
| cdn.append(i.replace("\n", "")) | ||
| return cdn | ||
| print(f"当前有效cdn个数为: {len(cdn_list)}") | ||
| if cdn_list: | ||
| f = open(r"../filter_cdn_list", "a+") | ||
| for c in cdn_list: | ||
| f.writelines(f"{c}\n") | ||
| f.close() | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| cdn = CDNProxy() | ||
| print(cdn.open_cdn_file()) | ||
| filterCdn() |
Oops, something went wrong.