diff --git a/HISTORY.md b/HISTORY.md index 29733e3..747c354 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,6 @@ ### Version History: +- 1.5.4: Print instructions on how to fix 'multiprocess' errors - 1.5.3: Change to streaming upload - 1.5.2: Defuse the circular import bomb brought in the previous version... - 1.5.1: Improve multiprocess (and fix filter() for Python3) diff --git a/HISTORY.rst b/HISTORY.rst index 272a2cb..6300f89 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,7 @@ Version History: ~~~~~~~~~~~~~~~~ +- 1.5.4: Print instructions on how to fix 'multiprocess' errors - 1.5.3: Change to streaming upload - 1.5.2: Defuse the circular import bomb brought in the previous version... diff --git a/bypy/bypy.py b/bypy/bypy.py index ed0e817..3726bec 100755 --- a/bypy/bypy.py +++ b/bypy/bypy.py @@ -537,19 +537,39 @@ def __filter_multi_results(self, results): errors = list(filter(lambda x: x != const.ENoError, results)) return const.ENoError if len(errors) == 0 else errors[-1] - def __multi_process(self, worker, iter, process = "dl / ul"): + def __multi_process(self, worker, iterator, process = "dl / ul"): if not self.__check_prompt_multiprocess(): return const.EArgument self.__warn_multi_processes(process) # patch the printer restoremp = set_mp_print() with UPool(self.processes) as pool: - # http://stackoverflow.com/a/35134329/404271 - # On Python 2.x, Ctrl-C won't work if we use pool.map(), wait() or get() - ar = pool.map_async(worker, iter) - results = ar.get(const.TenYearInSeconds) - restoremp() - return self.__filter_multi_results(results) + try: + # http://stackoverflow.com/a/35134329/404271 + # On Python 2.x, Ctrl-C won't work if we use pool.map(), wait() or get() + ar = pool.map_async(worker, iterator) + results = ar.get(const.TenYearInSeconds) + restoremp() + return self.__filter_multi_results(results) + except pickle.PicklingError as pe: + errmsg = ("Exception:\n{}\n" + "--------------------------------\n" + "Error: Your Python 'multiprocess' library is probably " + "not properly installed (missing C extensions). " + "You need to install a C compiler first before " + "installing the Python 'multiprocess' library. " + "(All these hassles can be saved if Python's builtin " + "'multiprocessing' works properly, sigh)\n" + "Fix for debian derivatives:\n" + "- Install gcc: # apt-get install gcc\n" + "- Reinstall Python 'multiprocess' library:\n" + " # pip uninstall -y multiprocess\n" + " # pip install -v multiprocess\n" + "- If there's no errors/warnings in the above actions, " + "then this error should be gone when you run 'bypy' with '{}' again.\n" + ).format(formatex(pe), const.MultiprocessOption) + perr(errmsg) + self.quit(const.EFatal) def __print_error_json(self, r): try: @@ -3389,7 +3409,7 @@ def getparser(): type=int, metavar='RCOUNT', help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]") if Pool: - parser.add_argument("--processes", + parser.add_argument(const.MultiprocessOption, dest="processes", default=const.DefaultProcessCount, type=int, help="Number of parallel processes. (Only applies to dir sync/dl/ul). [default: %(default)s]") diff --git a/bypy/const.py b/bypy/const.py index cbc047b..608db24 100755 --- a/bypy/const.py +++ b/bypy/const.py @@ -12,7 +12,7 @@ # https://packaging.python.org/single_source_version/ __title__ = 'bypy' -__version__ = '1.5.3' +__version__ = '1.5.4' __author__ = 'Hou Tianze' __license__ = 'MIT' __desc__ = 'Python client for Baidu Yun (Personal Cloud Storage) 百度云/百度网盘 Python 客户端' @@ -198,5 +198,6 @@ CleanOptionLong= '--clean' DisableSslCheckOption = '--disable-ssl-check' CaCertsOption = '--cacerts' +MultiprocessOption= '--processes' # vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 ff=unix fileencoding=utf-8 diff --git a/bypy/requester.py b/bypy/requester.py index 6785845..dc31870 100644 --- a/bypy/requester.py +++ b/bypy/requester.py @@ -106,6 +106,8 @@ def request(cls, method, url, **kwargs): @classmethod def disable_warnings(cls, debug): + failures = 0 + exmsg = '' try: import requests.packages.urllib3 as ul3 if debug: @@ -113,19 +115,26 @@ def disable_warnings(cls, debug): #ul3.disable_warnings(ul3.exceptions.InsecureRequestWarning) #ul3.disable_warnings(ul3.exceptions.InsecurePlatformWarning) ul3.disable_warnings() - except: - try: - import urllib3 as ul3 - if debug: - pdbg("Using urllib3 to disable warnings") - ul3.disable_warnings() - except Exception as ex: - perr("Failed to disable warnings for Urllib3.\n" - "Possibly the requests library is out of date?\n" - "You can upgrade it by running '{}'.\n{}".format( - const.PipUpgradeCommand, formatex(ex))) - # i don't know why under Ubuntu, 'pip install requests' doesn't install the requests.packages.* packages - pass + except Exception as ex: + failures += 1 + exmsg += formatex(ex) + '-' * 64 + '\n' + + # i don't know why under Ubuntu, 'pip install requests' + # doesn't install the requests.packages.* packages + try: + import urllib3 as ul3 + if debug: + pdbg("Using urllib3 to disable warnings") + ul3.disable_warnings() + except Exception as ex: + failures += 1 + exmsg += formatex(ex) + + if failures >= 2: + perr("Failed to disable warnings for Urllib3.\n" + "Possibly the requests library is out of date?\n" + "You can upgrade it by running '{}'.\nExceptions:\n{}".format( + const.PipUpgradeCommand, exmsg)) # only if user specifies '-ddd' or more 'd's, the following # debugging information will be shown, as it's very talkative. diff --git a/update/update.json b/update/update.json index 23ca3f4..781df7e 100644 --- a/update/update.json +++ b/update/update.json @@ -1,5 +1,5 @@ { "comment": "Update info", - "recommendedVersion": "1.5.3", + "recommendedVersion": "1.5.4", "minimumRequiredVersion": "1.4.3" }