Print instructions fixing 'multiprocess' erros; Properly disable urll…

…ib3 warnings; Close #345 , Close #371

HISTORY.md

@@ -1,5 +1,6 @@
 ### Version History:
 
+- 1.5.4: Print instructions on how to fix 'multiprocess' errors
 - 1.5.3: Change to streaming upload
 - 1.5.2: Defuse the circular import bomb brought in the previous version...
 - 1.5.1: Improve multiprocess (and fix filter() for Python3)

HISTORY.rst

@@ -1,6 +1,7 @@
 Version History:
 ~~~~~~~~~~~~~~~~
 
+-  1.5.4: Print instructions on how to fix 'multiprocess' errors
 -  1.5.3: Change to streaming upload
 -  1.5.2: Defuse the circular import bomb brought in the previous
    version...

bypy/bypy.py

@@ -537,19 +537,39 @@ def __filter_multi_results(self, results):
 		errors = list(filter(lambda x: x != const.ENoError, results))
 		return const.ENoError if len(errors) == 0 else errors[-1]
 
-	def __multi_process(self, worker, iter, process = "dl / ul"):
+	def __multi_process(self, worker, iterator, process = "dl / ul"):
 		if not self.__check_prompt_multiprocess():
 			return const.EArgument
 		self.__warn_multi_processes(process)
 		# patch the printer
 		restoremp = set_mp_print()
 		with UPool(self.processes) as pool:
-			# http://stackoverflow.com/a/35134329/404271
-			# On Python 2.x, Ctrl-C won't work if we use pool.map(), wait() or get()
-			ar = pool.map_async(worker, iter)
-			results = ar.get(const.TenYearInSeconds)
-			restoremp()
-			return self.__filter_multi_results(results)
+			try:
+				# http://stackoverflow.com/a/35134329/404271
+				# On Python 2.x, Ctrl-C won't work if we use pool.map(), wait() or get()
+				ar = pool.map_async(worker, iterator)
+				results = ar.get(const.TenYearInSeconds)
+				restoremp()
+				return self.__filter_multi_results(results)
+			except pickle.PicklingError as pe:
+				errmsg = ("Exception:\n{}\n"
+						"--------------------------------\n"
+						"Error: Your Python 'multiprocess' library is probably "
+						"not properly installed (missing C extensions). "
+						"You need to install a C compiler first before "
+						"installing the Python 'multiprocess' library. "
+						"(All these hassles can be saved if Python's builtin "
+						"'multiprocessing' works properly, sigh)\n"
+						"Fix for debian derivatives:\n"
+						"- Install gcc: # apt-get install gcc\n"
+						"- Reinstall Python 'multiprocess' library:\n"
+						"  # pip uninstall -y multiprocess\n"
+						"  # pip install -v multiprocess\n"
+						"- If there's no errors/warnings in the above actions, "
+						"then this error should be gone when you run 'bypy' with '{}' again.\n"
+						).format(formatex(pe), const.MultiprocessOption)
+				perr(errmsg)
+				self.quit(const.EFatal)
 
 	def __print_error_json(self, r):
 		try:
@@ -3389,7 +3409,7 @@ def getparser():
 		type=int, metavar='RCOUNT',
 		help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]")
 	if Pool:
-		parser.add_argument("--processes",
+		parser.add_argument(const.MultiprocessOption,
 			dest="processes", default=const.DefaultProcessCount, type=int,
 			help="Number of parallel processes. (Only applies to dir sync/dl/ul). [default: %(default)s]")
 

bypy/const.py

@@ -12,7 +12,7 @@
 
 # https://packaging.python.org/single_source_version/
 __title__ = 'bypy'
-__version__ = '1.5.3'
+__version__ = '1.5.4'
 __author__ = 'Hou Tianze'
 __license__ = 'MIT'
 __desc__ = 'Python client for Baidu Yun (Personal Cloud Storage) 百度云/百度网盘 Python 客户端'
@@ -198,5 +198,6 @@
 CleanOptionLong= '--clean'
 DisableSslCheckOption = '--disable-ssl-check'
 CaCertsOption = '--cacerts'
+MultiprocessOption= '--processes'
 
 # vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 ff=unix fileencoding=utf-8

bypy/requester.py

@@ -106,26 +106,35 @@ def request(cls, method, url, **kwargs):
 
 	@classmethod
 	def disable_warnings(cls, debug):
+		failures = 0
+		exmsg = ''
 		try:
 			import requests.packages.urllib3 as ul3
 			if debug:
 				pdbg("Using requests.packages.urllib3 to disable warnings")
 			#ul3.disable_warnings(ul3.exceptions.InsecureRequestWarning)
 			#ul3.disable_warnings(ul3.exceptions.InsecurePlatformWarning)
 			ul3.disable_warnings()
-		except:
-			try:
-				import urllib3 as ul3
-				if debug:
-					pdbg("Using urllib3 to disable warnings")
-				ul3.disable_warnings()
-			except Exception as ex:
-				perr("Failed to disable warnings for Urllib3.\n"
-					"Possibly the requests library is out of date?\n"
-					"You can upgrade it by running '{}'.\n{}".format(
-						const.PipUpgradeCommand, formatex(ex)))
-			# i don't know why under Ubuntu, 'pip install requests' doesn't install the requests.packages.* packages
-				pass
+		except Exception as ex:
+			failures += 1
+			exmsg += formatex(ex) + '-' * 64 + '\n'
+
+		# i don't know why under Ubuntu, 'pip install requests'
+		# doesn't install the requests.packages.* packages
+		try:
+			import urllib3 as ul3
+			if debug:
+				pdbg("Using urllib3 to disable warnings")
+			ul3.disable_warnings()
+		except Exception as ex:
+			failures += 1
+			exmsg += formatex(ex)
+
+		if failures >= 2:
+			perr("Failed to disable warnings for Urllib3.\n"
+				"Possibly the requests library is out of date?\n"
+				"You can upgrade it by running '{}'.\nExceptions:\n{}".format(
+					const.PipUpgradeCommand, exmsg))
 
 	# only if user specifies '-ddd' or more 'd's, the following
 	# debugging information will be shown, as it's very talkative.

update/update.json

@@ -1,5 +1,5 @@
 {
 	"comment": "Update info",
-	"recommendedVersion": "1.5.3",
+	"recommendedVersion": "1.5.4",
 	"minimumRequiredVersion": "1.4.3"
 }