From aa388ca24da050ea0da09fe23b28923d398bcb8f Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 16:56:33 +0300 Subject: [PATCH 01/22] Drop support for Python versions below 3.5 This is a first commit in the series of removal lower Python versions. It includes: - clean-up of tox config - clean-up of travis config - fix yaml warning when it is used without Loader - fix isinstance calls - remove PY2 conditional from py3compat --- .travis.yml | 4 - circus/client.py | 6 +- circus/commands/util.py | 5 +- circus/consumer.py | 3 +- circus/controller.py | 3 +- circus/papa_process_proxy.py | 8 +- circus/process.py | 6 +- circus/py3compat.py | 151 +++++++++--------------------- circus/stream/file_stream.py | 12 +-- circus/tests/test_config.py | 6 +- circus/tests/test_logging.py | 12 +-- circus/tests/test_process.py | 12 +-- circus/util.py | 37 +++----- circus/watcher.py | 9 +- examples/uwsgi_lossless_reload.py | 4 +- tox.ini | 19 +--- 16 files changed, 88 insertions(+), 209 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2477719fe..848409301 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,10 +31,6 @@ jobs: env: TOX_ENV=py36 - python: 3.5 env: TOX_ENV=py35 - - python: 3.4 - env: TOX_ENV=py34 - - python: 2.7 - env: TOX_ENV=py27 script: - tox -v -e $TOX_ENV diff --git a/circus/client.py b/circus/client.py index 9a7b95b0b..eb360103f 100644 --- a/circus/client.py +++ b/circus/client.py @@ -9,7 +9,7 @@ import tornado from circus.exc import CallError -from circus.py3compat import string_types, b +from circus.py3compat import b from circus.util import DEFAULT_ENDPOINT_DEALER, get_connection @@ -57,7 +57,7 @@ def send_message(self, command, **props): @tornado.gen.coroutine def call(self, cmd): - if isinstance(cmd, string_types): + if isinstance(cmd, str): raise DeprecationWarning('call() takes a mapping') call_id = uuid.uuid4().hex @@ -116,7 +116,7 @@ def send_message(self, command, **props): return self.call(make_message(command, **props)) def call(self, cmd): - if isinstance(cmd, string_types): + if isinstance(cmd, str): raise DeprecationWarning('call() takes a mapping') call_id = uuid.uuid4().hex diff --git a/circus/commands/util.py b/circus/commands/util.py index d73380e3d..6c3953327 100644 --- a/circus/commands/util.py +++ b/circus/commands/util.py @@ -1,5 +1,4 @@ from circus.exc import ArgumentError, MessageError -from circus.py3compat import string_types from circus import util import warnings try: @@ -124,7 +123,7 @@ def _valid_prefix(): raise MessageError("%r isn't a number" % key) elif key in ('uid', 'gid',): - if not isinstance(val, int) and not isinstance(val, string_types): + if not isinstance(val, int) and not isinstance(val, str): raise MessageError("%r isn't an integer or string" % key) elif key in ('send_hup', 'shell', 'copy_env', 'respawn', 'stop_children', @@ -138,7 +137,7 @@ def _valid_prefix(): raise MessageError("%r isn't a valid object" % key) for k, v in val.items(): - if not isinstance(v, string_types): + if not isinstance(v, str): raise MessageError("%r isn't a string" % k) elif key == 'hooks': diff --git a/circus/consumer.py b/circus/consumer.py index b80c6a903..0d06afea0 100644 --- a/circus/consumer.py +++ b/circus/consumer.py @@ -2,7 +2,6 @@ import zmq from circus.util import DEFAULT_ENDPOINT_SUB, get_connection -from circus.py3compat import b class CircusConsumer(object): @@ -15,7 +14,7 @@ def __init__(self, topics, context=None, endpoint=DEFAULT_ENDPOINT_SUB, self.pubsub_socket = self.context.socket(zmq.SUB) get_connection(self.pubsub_socket, self.endpoint, ssh_server) for topic in self.topics: - self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, b(topic)) + self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, topic.encode('utf8')) self._init_poller() self.timeout = timeout diff --git a/circus/controller.py b/circus/controller.py index c227c8a81..1aae173f8 100644 --- a/circus/controller.py +++ b/circus/controller.py @@ -22,7 +22,6 @@ from circus.commands import get_commands, ok, error, errors from circus import logger from circus.exc import MessageError, ConflictError -from circus.py3compat import string_types from circus.sighandler import SysHandler @@ -254,7 +253,7 @@ def send_response(self, mid, cid, msg, resp, cast=False): if cid is None: return - if isinstance(resp, string_types): + if isinstance(resp, str): raise DeprecationWarning('Takes only a mapping') resp['id'] = mid diff --git a/circus/papa_process_proxy.py b/circus/papa_process_proxy.py index ca8bf0508..02e9070d1 100644 --- a/circus/papa_process_proxy.py +++ b/circus/papa_process_proxy.py @@ -4,16 +4,10 @@ import psutil import select import time -from circus.py3compat import PY2, string_types __author__ = 'Scott Maxwell' -if PY2: - class TimeoutError(Exception): - """The operation timed out.""" - - def _bools_to_papa_out(pipe, close): return papa.PIPE if pipe else papa.DEVNULL if close else None @@ -61,7 +55,7 @@ def spawn(self): socket_names = set(socket_name.lower() for socket_name in self.watcher._get_sockets_fds()) self.cmd = self._fix_socket_name(self.cmd, socket_names) - if isinstance(self.args, string_types): + if isinstance(self.args, str): self.args = self._fix_socket_name(self.args, socket_names) else: self.args = [self._fix_socket_name(arg, socket_names) diff --git a/circus/process.py b/circus/process.py index 0b995511d..cefe47fdc 100644 --- a/circus/process.py +++ b/circus/process.py @@ -23,7 +23,7 @@ from psutil import (Popen, STATUS_ZOMBIE, STATUS_DEAD, NoSuchProcess, AccessDenied) -from circus.py3compat import bytestring, string_types, quote +from circus.py3compat import bytestring, quote from circus.sockets import CircusSocket from circus.util import (get_info, to_uid, to_gid, debuglog, get_working_dir, ObjectDict, replace_gnu_args, get_default_gid, @@ -401,7 +401,7 @@ def format_args(self, sockets_fds=None): self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: - if isinstance(self.args, string_types): + if isinstance(self.args, str): args = shlex.split(bytestring(replace_gnu_args( self.args, **format_kwargs))) else: @@ -419,7 +419,7 @@ def format_args(self, sockets_fds=None): if shell_args and IS_WINDOWS: logger.warn("shell_args won't apply for " "windows platforms: %s", shell_args) - elif isinstance(shell_args, string_types): + elif isinstance(shell_args, str): args += shlex.split(bytestring(replace_gnu_args( shell_args, **format_kwargs))) elif shell_args: diff --git a/circus/py3compat.py b/circus/py3compat.py index 9a203a293..86d372864 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -1,126 +1,61 @@ +import io import sys +import collections +from shlex import quote # NOQA -PY2 = sys.version_info[0] < 3 -if PY2: - string_types = basestring # NOQA - integer_types = (int, long) # NOQA - text_type = unicode # NOQA - long = long # NOQA - bytes = str +text_type = str +long = int +unicode = str + + +def sort_by_field(obj, field='name'): # NOQA + def _by_field(item): + return item[field] + + obj.sort(key=_by_field) - def bytestring(s): # NOQA - if isinstance(s, unicode): # NOQA - return s.encode('utf-8') - return s - def cast_bytes(s, encoding='utf8'): - """cast unicode or bytes to bytes""" - if isinstance(s, unicode): - return s.encode(encoding) - return str(s) - - def cast_unicode(s, encoding='utf8', errors='replace'): - """cast bytes or unicode to unicode. - errors options are strict, ignore or replace""" - if isinstance(s, unicode): - return s - return str(s).decode(encoding) - - def cast_string(s, errors='replace'): - return s if isinstance(s, basestring) else str(s) # NOQA - - try: - import cStringIO - StringIO = cStringIO.StringIO # NOQA - except ImportError: - import StringIO - StringIO = StringIO.StringIO # NOQA - - BytesIO = StringIO - - eval(compile('def raise_with_tb(E): raise E, None, sys.exc_info()[2]', - 'py3compat.py', 'exec')) - - def is_callable(c): # NOQA - return callable(c) - - def get_next(c): # NOQA - return c.next - - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) # NOQA - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) # NOQA - del X - - def sort_by_field(obj, field='name'): # NOQA - def _by_field(item1, item2): - return cmp(item1[field], item2[field]) # NOQA - - obj.sort(_by_field) - -else: - import collections - string_types = str - integer_types = int - text_type = str - long = int - unicode = str - - def sort_by_field(obj, field='name'): # NOQA - def _by_field(item): - return item[field] - - obj.sort(key=_by_field) - - def bytestring(s): # NOQA +def bytestring(s): # NOQA + return s + + +def cast_bytes(s, encoding='utf8'): # NOQA + """cast unicode or bytes to bytes""" + if isinstance(s, bytes): return s + return str(s).encode(encoding) + - def cast_bytes(s, encoding='utf8'): # NOQA - """cast unicode or bytes to bytes""" - if isinstance(s, bytes): - return s - return str(s).encode(encoding) +def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA + """cast bytes or unicode to unicode. + errors options are strict, ignore or replace""" + if isinstance(s, bytes): + return s.decode(encoding, errors=errors) + return str(s) - def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA - """cast bytes or unicode to unicode. - errors options are strict, ignore or replace""" - if isinstance(s, bytes): - return s.decode(encoding, errors=errors) - return str(s) - cast_string = cast_unicode +cast_string = cast_unicode - import io - StringIO = io.StringIO # NOQA - BytesIO = io.BytesIO # NOQA - def raise_with_tb(E): # NOQA - raise E.with_traceback(sys.exc_info()[2]) +StringIO = io.StringIO # NOQA +BytesIO = io.BytesIO # NOQA - def is_callable(c): # NOQA - return isinstance(c, collections.Callable) - def get_next(c): # NOQA - return c.__next__ +def raise_with_tb(E): # NOQA + raise E.with_traceback(sys.exc_info()[2]) - MAXSIZE = sys.maxsize # NOQA + +def is_callable(c): # NOQA + return isinstance(c, collections.Callable) + + +def get_next(c): # NOQA + return c.__next__ + + +MAXSIZE = sys.maxsize # NOQA b = cast_bytes s = cast_string u = cast_unicode - -try: - # PY >= 3.3 - from shlex import quote # NOQA -except ImportError: - from pipes import quote # NOQA diff --git a/circus/stream/file_stream.py b/circus/stream/file_stream.py index 22fb6c74d..bc8379cc1 100644 --- a/circus/stream/file_stream.py +++ b/circus/stream/file_stream.py @@ -6,7 +6,7 @@ import re from stat import ST_DEV, ST_INO, ST_MTIME from circus import logger -from circus.py3compat import s, PY2 +from circus.py3compat import s class _FileStreamBase(object): @@ -55,13 +55,9 @@ def write_data(self, data): try: self._file.write(file_data) except Exception: - # we can strip the string down on Py3 but not on Py2 - if not PY2: - file_data = file_data.encode('latin-1', errors='replace') - file_data = file_data.decode('latin-1') - self._file.write(file_data) - else: - raise + file_data = file_data.encode('latin-1', errors='replace') + file_data = file_data.decode('latin-1') + self._file.write(file_data) self._file.flush() diff --git a/circus/tests/test_config.py b/circus/tests/test_config.py index 150783ad6..be9a2b6cb 100644 --- a/circus/tests/test_config.py +++ b/circus/tests/test_config.py @@ -10,7 +10,6 @@ from circus.sockets import CircusSocket from circus.tests.support import TestCase, EasyTestSuite, IS_WINDOWS from circus.util import replace_gnu_args -from circus.py3compat import PY2 HERE = os.path.join(os.path.dirname(__file__)) @@ -277,10 +276,7 @@ def test_override(self): watchers = conf['watchers'] self.assertEqual(len(watchers), 3) watchers = conf['watchers'] - if PY2: - watchers.sort() - else: - watchers = sorted(watchers, key=lambda a: a['__name__']) + watchers = sorted(watchers, key=lambda a: a['__name__']) self.assertEqual(watchers[2]['env']['INI'], 'private.ini') self.assertEqual(conf['check_delay'], 555) diff --git a/circus/tests/test_logging.py b/circus/tests/test_logging.py index d73ca512a..02bdcaf2b 100644 --- a/circus/tests/test_logging.py +++ b/circus/tests/test_logging.py @@ -207,7 +207,7 @@ def test_loggerconfig_default_opt(self): @skipIf(not hasDictConfig(), "Needs logging.config.dictConfig()") def test_loggerconfig_yaml_ini(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_yaml_ini.txt" run_circusd( [], {"circus.loggerconfig": "logging.yaml"}, @@ -217,7 +217,7 @@ def test_loggerconfig_yaml_ini(self): @skipIf(not hasDictConfig(), "Needs logging.config.dictConfig()") def test_loggerconfig_yaml_opt(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_yaml_opt.txt" run_circusd( ["--logger-config", "logging.yaml"], {}, @@ -227,7 +227,7 @@ def test_loggerconfig_yaml_opt(self): @skipIf(not hasDictConfig(), "Needs logging.config.dictConfig()") def test_loggerconfig_json_ini(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_json_ini.txt" run_circusd( [], {"circus.loggerconfig": "logging.json"}, @@ -237,7 +237,7 @@ def test_loggerconfig_json_ini(self): @skipIf(not hasDictConfig(), "Needs logging.config.dictConfig()") def test_loggerconfig_json_opt(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_json_opt.txt" run_circusd( ["--logger-config", "logging.json"], {}, @@ -246,7 +246,7 @@ def test_loggerconfig_json_opt(self): ) def test_loggerconfig_ini_ini(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_ini_ini.txt" run_circusd( [], {"circus.loggerconfig": "logging.ini"}, @@ -255,7 +255,7 @@ def test_loggerconfig_ini_ini(self): ) def test_loggerconfig_ini_opt(self): - config = yaml.load(EXAMPLE_YAML) + config = yaml.load(EXAMPLE_YAML, Loader=yaml.FullLoader) config["handlers"]["logfile"]["filename"] = "log_ini_opt.txt" run_circusd( ["--logger-config", "logging.ini"], {}, diff --git a/circus/tests/test_process.py b/circus/tests/test_process.py index b2d9ae389..981140940 100644 --- a/circus/tests/test_process.py +++ b/circus/tests/test_process.py @@ -5,8 +5,7 @@ from circus.process import Process from circus.tests.support import (TestCircus, skipIf, EasyTestSuite, DEBUG, poll_for, IS_WINDOWS, PYTHON, SLEEP) -import circus.py3compat -from circus.py3compat import StringIO, PY2 +from circus.py3compat import StringIO RLIMIT = """\ @@ -44,10 +43,7 @@ def _nose_no_s(): - if PY2: - return not hasattr(sys.stdout, 'fileno') - else: - return isinstance(sys.stdout, StringIO) + return isinstance(sys.stdout, StringIO) class TestProcess(TestCircus): @@ -95,9 +91,9 @@ def test_rlimits(self): output[limit] = value def srt2ints(val): - return [circus.py3compat.long(key) for key in val[1:-1].split(',')] + return [int(key) for key in val[1:-1].split(',')] - wanted = [circus.py3compat.long(20), circus.py3compat.long(20)] + wanted = [int(20), int(20)] self.assertEqual(srt2ints(output['NOFILE']), wanted) self.assertEqual(srt2ints(output['NPROC']), wanted) diff --git a/circus/util.py b/circus/util.py index e7324ce48..2f67cfdeb 100644 --- a/circus/util.py +++ b/circus/util.py @@ -29,21 +29,13 @@ from tornado.ioloop import IOLoop from tornado import gen from tornado import concurrent -from circus.py3compat import ( - integer_types, bytestring, raise_with_tb, text_type +from circus.py3compat import raise_with_tb + +from configparser import ( + ConfigParser, MissingSectionHeaderError, ParsingError, DEFAULTSECT ) -try: - from configparser import ( - ConfigParser, MissingSectionHeaderError, ParsingError, DEFAULTSECT - ) -except ImportError: - from ConfigParser import ( # NOQA - ConfigParser, MissingSectionHeaderError, ParsingError, DEFAULTSECT - ) -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse # NOQA + +from urllib.parse import urlparse from datetime import timedelta from functools import wraps @@ -63,7 +55,6 @@ from circus.exc import ConflictError from circus import logger -from circus.py3compat import string_types # default endpoints @@ -135,7 +126,7 @@ def get_working_dir(): def bytes2human(n): """Translates bytes into a human repr. """ - if not isinstance(n, integer_types): + if not isinstance(n, int): raise TypeError(n) prefix = {} @@ -369,7 +360,7 @@ def to_uid(name): # NOQA except KeyError: raise ValueError("%r isn't a valid user id" % name) - if not isinstance(name, string_types): + if not isinstance(name, str): raise TypeError(name) try: @@ -404,7 +395,7 @@ def to_gid(name): # NOQA except (KeyError, OverflowError): raise ValueError("No such group: %r" % name) - if not isinstance(name, string_types): + if not isinstance(name, str): raise TypeError(name) try: @@ -573,8 +564,6 @@ def resolve_name(import_name, silent=False, reload=False): reloaded :return: imported object """ - # force the import name to automatically convert to strings - import_name = bytestring(import_name) try: if ':' in import_name: module, obj = import_name.split(':', 1) @@ -738,7 +727,9 @@ def configure_logger(logger, level='INFO', output="-", loggerconfig=None, "Try: pip install PyYAML" % (shell_escape_arg(loggerconfig),)) with open(loggerconfig, "r") as fh: - logging.config.dictConfig(yaml.load(fh.read())) + logging.config.dictConfig( + yaml.load(fh.read(), Loader=yaml.FullLoader) + ) else: raise Exception("Logger configuration file %s is not in one " "of the recognized formats. The file name " @@ -794,7 +785,7 @@ def _read(self, fp, fpname): mo = self._optcre.match(line) # 2.7 if mo: optname, vi, optval = mo.group('option', 'vi', 'value') - self.optionxform = text_type + self.optionxform = str optname = self.optionxform(optname.rstrip()) # We don't want to override. if optname in cursect: @@ -876,7 +867,7 @@ def process_pth(sitedir, name): packages = set() fullname = os.path.join(sitedir, name) try: - f = open(fullname, "rU") + f = open(fullname, "r") except IOError: return with f: diff --git a/circus/watcher.py b/circus/watcher.py index a536ef80f..12294e7cc 100644 --- a/circus/watcher.py +++ b/circus/watcher.py @@ -26,7 +26,7 @@ from circus.stream.papa_redirector import PapaRedirector from circus.util import parse_env_dict, resolve_name, tornado_sleep, IS_WINDOWS from circus.util import papa -from circus.py3compat import bytestring, is_callable, b, PY2 +from circus.py3compat import bytestring, is_callable, b class Watcher(object): @@ -1123,12 +1123,7 @@ def set_opt(self, key, val): self.shell = val action = 1 elif key == "env": - if PY2 and IS_WINDOWS: - # Windows on Python 2 does not accept Unicode values - # in env dictionary - self.env = dict((b(k), b(v)) for k, v in val.iteritems()) - else: - self.env = val + self.env = val action = 1 elif key == "cmd": self.cmd = val diff --git a/examples/uwsgi_lossless_reload.py b/examples/uwsgi_lossless_reload.py index ea10633e9..a2fa83bd7 100644 --- a/examples/uwsgi_lossless_reload.py +++ b/examples/uwsgi_lossless_reload.py @@ -28,7 +28,6 @@ from json import loads import signal from circus import logger -from circus.py3compat import PY2 import re worker_states = { @@ -54,8 +53,7 @@ def get_uwsgi_stats(name, wid, base_port): "Error: No stats seem available for WID %d of %s", wid, name) return # recent versions of uWSGI had some garbage in the JSON so strip it out - if not PY2: - data = data.decode('latin', 'replace') + data = data.decode('latin', 'replace') data = NON_JSON_CHARACTERS.sub('', data) return loads(data) diff --git a/tox.ini b/tox.ini index 18f8ea4da..13ea4c6da 100644 --- a/tox.ini +++ b/tox.ini @@ -1,27 +1,12 @@ [tox] -envlist = py27,py34,py35,py36,py37,py38,flake8,docs - -[testenv:py27] -passenv = PWD TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH - -deps = - {[testenv]deps} - nose-cov - coverage - gevent - circus-web - -commands = - nosetests -vs --with-coverage --cover-package=circus circus/tests +envlist = py35,py36,py37,py38,flake8,docs [testenv] passenv = PWD deps = nose mock - !py34: PyYAML - py34: PyYAML<5.3 - six + PyYAML tornado>=3.0,<5.0 pyzmq>=17.0 From 818a2c0b90ce737df8ebfb4b3a4d7c5f4c126486 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 17:30:59 +0300 Subject: [PATCH 02/22] Remove bytestring, replace shlex.quote import, is_callable --- circus/process.py | 20 ++++++++++---------- circus/py3compat.py | 15 --------------- circus/watcher.py | 6 +++--- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/circus/process.py b/circus/process.py index cefe47fdc..fc0e1a897 100644 --- a/circus/process.py +++ b/circus/process.py @@ -23,7 +23,7 @@ from psutil import (Popen, STATUS_ZOMBIE, STATUS_DEAD, NoSuchProcess, AccessDenied) -from circus.py3compat import bytestring, quote +from shlex import quote from circus.sockets import CircusSocket from circus.util import (get_info, to_uid, to_gid, debuglog, get_working_dir, ObjectDict, replace_gnu_args, get_default_gid, @@ -370,7 +370,7 @@ def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ - logger.debug('cmd: ' + bytestring(self.cmd)) + logger.debug('cmd: ' + self.cmd) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) @@ -402,14 +402,14 @@ def format_args(self, sockets_fds=None): if self.args is not None: if isinstance(self.args, str): - args = shlex.split(bytestring(replace_gnu_args( - self.args, **format_kwargs))) + args = shlex.split(replace_gnu_args( + self.args, **format_kwargs)) else: - args = [bytestring(replace_gnu_args(arg, **format_kwargs)) + args = [replace_gnu_args(arg, **format_kwargs) for arg in self.args] - args = shlex.split(bytestring(cmd), posix=not IS_WINDOWS) + args + args = shlex.split(cmd, posix=not IS_WINDOWS) + args else: - args = shlex.split(bytestring(cmd), posix=not IS_WINDOWS) + args = shlex.split(cmd, posix=not IS_WINDOWS) if self.shell: # subprocess.Popen(shell=True) implies that 1st arg is the @@ -420,10 +420,10 @@ def format_args(self, sockets_fds=None): logger.warn("shell_args won't apply for " "windows platforms: %s", shell_args) elif isinstance(shell_args, str): - args += shlex.split(bytestring(replace_gnu_args( - shell_args, **format_kwargs))) + args += shlex.split(replace_gnu_args( + shell_args, **format_kwargs)) elif shell_args: - args += [bytestring(replace_gnu_args(arg, **format_kwargs)) + args += [replace_gnu_args(arg, **format_kwargs) for arg in shell_args] elif format_kwargs.get('shell_args', False): diff --git a/circus/py3compat.py b/circus/py3compat.py index 86d372864..71a72fc9e 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -1,12 +1,5 @@ import io import sys -import collections -from shlex import quote # NOQA - - -text_type = str -long = int -unicode = str def sort_by_field(obj, field='name'): # NOQA @@ -16,10 +9,6 @@ def _by_field(item): obj.sort(key=_by_field) -def bytestring(s): # NOQA - return s - - def cast_bytes(s, encoding='utf8'): # NOQA """cast unicode or bytes to bytes""" if isinstance(s, bytes): @@ -46,10 +35,6 @@ def raise_with_tb(E): # NOQA raise E.with_traceback(sys.exc_info()[2]) -def is_callable(c): # NOQA - return isinstance(c, collections.Callable) - - def get_next(c): # NOQA return c.__next__ diff --git a/circus/watcher.py b/circus/watcher.py index 12294e7cc..380c5f9d7 100644 --- a/circus/watcher.py +++ b/circus/watcher.py @@ -26,7 +26,7 @@ from circus.stream.papa_redirector import PapaRedirector from circus.util import parse_env_dict, resolve_name, tornado_sleep, IS_WINDOWS from circus.util import papa -from circus.py3compat import bytestring, is_callable, b +from circus.py3compat import b class Watcher(object): @@ -382,7 +382,7 @@ def _create_redirectors(self): def _resolve_hook(self, name, callable_or_name, ignore_failure, reload_module=False): - if is_callable(callable_or_name): + if callable(callable_or_name): self.hooks[name] = callable_or_name else: # will raise ImportError on failure @@ -431,7 +431,7 @@ def __repr__(self): def notify_event(self, topic, msg): """Publish a message on the event publisher channel""" - name = bytestring(self.res_name) + name = self.res_name multipart_msg = [b("watcher.%s.%s" % (name, topic)), json.dumps(msg)] From 6f1221afcf8979ea1030255262787830f048a783 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 22:35:32 +0300 Subject: [PATCH 03/22] Replace StringIO, remove get_next, replace sort calls --- circus/config.py | 7 +++---- circus/py3compat.py | 18 ------------------ circus/tests/test_process.py | 4 ++-- circus/tests/test_stats_client.py | 5 ++--- circus/tests/test_stream.py | 6 +++--- 5 files changed, 10 insertions(+), 30 deletions(-) diff --git a/circus/config.py b/circus/config.py index af274c5c2..0e7adc872 100644 --- a/circus/config.py +++ b/circus/config.py @@ -11,7 +11,6 @@ import six from circus import logger -from circus.py3compat import sort_by_field from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, DEFAULT_ENDPOINT_MULTICAST, DEFAULT_ENDPOINT_STATS, StrictConfigParser, replace_gnu_args, to_signum, @@ -281,9 +280,9 @@ def get_config(config_file): watchers.append(watcher) # making sure we return consistent lists - sort_by_field(watchers) - sort_by_field(plugins) - sort_by_field(sockets) + watchers.sort(key=lambda x: x['name']) + plugins.sort(key=lambda x: x['name']) + sockets.sort(key=lambda x: x['name']) # Second pass to make sure env sections apply to all watchers. diff --git a/circus/py3compat.py b/circus/py3compat.py index 71a72fc9e..f79d557cd 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -1,14 +1,6 @@ -import io import sys -def sort_by_field(obj, field='name'): # NOQA - def _by_field(item): - return item[field] - - obj.sort(key=_by_field) - - def cast_bytes(s, encoding='utf8'): # NOQA """cast unicode or bytes to bytes""" if isinstance(s, bytes): @@ -27,20 +19,10 @@ def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA cast_string = cast_unicode -StringIO = io.StringIO # NOQA -BytesIO = io.BytesIO # NOQA - - def raise_with_tb(E): # NOQA raise E.with_traceback(sys.exc_info()[2]) -def get_next(c): # NOQA - return c.__next__ - - -MAXSIZE = sys.maxsize # NOQA - b = cast_bytes s = cast_string u = cast_unicode diff --git a/circus/tests/test_process.py b/circus/tests/test_process.py index 981140940..80a7e506d 100644 --- a/circus/tests/test_process.py +++ b/circus/tests/test_process.py @@ -1,3 +1,4 @@ +import io import os import sys import time @@ -5,7 +6,6 @@ from circus.process import Process from circus.tests.support import (TestCircus, skipIf, EasyTestSuite, DEBUG, poll_for, IS_WINDOWS, PYTHON, SLEEP) -from circus.py3compat import StringIO RLIMIT = """\ @@ -43,7 +43,7 @@ def _nose_no_s(): - return isinstance(sys.stdout, StringIO) + return isinstance(sys.stdout, io.StringIO) class TestProcess(TestCircus): diff --git a/circus/tests/test_stats_client.py b/circus/tests/test_stats_client.py index d3f4335c3..43dfc807c 100644 --- a/circus/tests/test_stats_client.py +++ b/circus/tests/test_stats_client.py @@ -7,7 +7,6 @@ from circus.tests.support import TestCircus, EasyTestSuite, skipIf, IS_WINDOWS from circus.client import AsyncCircusClient from circus.stream import FileStream -from circus.py3compat import get_next from circus.util import tornado_sleep @@ -81,10 +80,10 @@ def test_handler(self): from circus.stats.client import StatsClient client = StatsClient(endpoint=self.arbiter.stats_endpoint) - next = get_next(client.iter_messages()) + message_iterator = client.iter_messages() for i in range(10): - watcher, pid, stat = next() + watcher, pid, stat = next(message_iterator) self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'), watcher) yield self.stop_arbiter() diff --git a/circus/tests/test_stream.py b/circus/tests/test_stream.py index 69314a2cd..ecd9605b8 100644 --- a/circus/tests/test_stream.py +++ b/circus/tests/test_stream.py @@ -1,3 +1,4 @@ +import io import time import sys import os @@ -5,7 +6,6 @@ import tornado from datetime import datetime -from circus.py3compat import StringIO from circus.client import make_message from circus.tests.support import TestCircus, async_poll_for, truncate_file @@ -181,7 +181,7 @@ def get_stream(self, *args, **kw): stream = FancyStdoutStream(*args, **kw) # patch some details that will be used - stream.out = StringIO() + stream.out = io.StringIO() stream.now = lambda: now return stream @@ -261,7 +261,7 @@ def get_stream(self, *args, **kw): # patch some details that will be used stream._file.close() - stream._file = StringIO() + stream._file = io.StringIO() stream._open = lambda: stream._file stream.now = lambda: now From da91a4e5410aefedb545aee4c19acd527b47d345 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 22:41:10 +0300 Subject: [PATCH 04/22] Remove raise_with_tb --- circus/py3compat.py | 4 ---- circus/util.py | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/circus/py3compat.py b/circus/py3compat.py index f79d557cd..da07085c5 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -19,10 +19,6 @@ def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA cast_string = cast_unicode -def raise_with_tb(E): # NOQA - raise E.with_traceback(sys.exc_info()[2]) - - b = cast_bytes s = cast_string u = cast_unicode diff --git a/circus/util.py b/circus/util.py index 2f67cfdeb..f7a7aa7ab 100644 --- a/circus/util.py +++ b/circus/util.py @@ -29,7 +29,6 @@ from tornado.ioloop import IOLoop from tornado import gen from tornado import concurrent -from circus.py3compat import raise_with_tb from configparser import ( ConfigParser, MissingSectionHeaderError, ParsingError, DEFAULTSECT @@ -605,7 +604,7 @@ def resolve_name(import_name, silent=False, reload=False): return __import__(import_name) except ImportError as e: if not silent: - raise_with_tb(ImportStringError(import_name, e)) + raise ImportStringError(import_name, e).with_traceback(sys.exc_info()[2]) _SECTION_NAME = r'\w\.\-' From 14a44ffe88d61fe9a646d79acb8985b57ecf01a6 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 22:41:37 +0300 Subject: [PATCH 05/22] Remove unused sys import --- circus/py3compat.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/circus/py3compat.py b/circus/py3compat.py index da07085c5..10e16f7ac 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -1,6 +1,3 @@ -import sys - - def cast_bytes(s, encoding='utf8'): # NOQA """cast unicode or bytes to bytes""" if isinstance(s, bytes): From 2319f59be2a69332d3226cf02b4050421f018d70 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 22 Mar 2020 22:47:41 +0300 Subject: [PATCH 06/22] Remove cast_bytes --- circus/client.py | 5 ++--- circus/plugins/__init__.py | 4 ++-- circus/py3compat.py | 1 - circus/stats/publisher.py | 3 +-- circus/tests/test_circusctl.py | 4 ++-- circus/watcher.py | 5 +++-- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/circus/client.py b/circus/client.py index eb360103f..8bccfbf26 100644 --- a/circus/client.py +++ b/circus/client.py @@ -9,7 +9,6 @@ import tornado from circus.exc import CallError -from circus.py3compat import b from circus.util import DEFAULT_ENDPOINT_DEALER, get_connection @@ -31,7 +30,7 @@ def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint - self._id = b(uuid.uuid4().hex) + self._id = uuid.uuid4().hex.encode('utf8') self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) @@ -90,7 +89,7 @@ def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint - self._id = b(uuid.uuid4().hex) + self._id = uuid.uuid4().hex.encode('utf8') self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) diff --git a/circus/plugins/__init__.py b/circus/plugins/__init__.py index 0fa74d28d..cbd83e0e9 100644 --- a/circus/plugins/__init__.py +++ b/circus/plugins/__init__.py @@ -12,7 +12,7 @@ from circus import logger, __version__ from circus.client import make_message, cast_message -from circus.py3compat import b, s +from circus.py3compat import s from circus.util import (debuglog, to_bool, resolve_name, configure_logger, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, get_connection) @@ -40,7 +40,7 @@ def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server=None, self.endpoint = endpoint self.check_delay = check_delay self.ssh_server = ssh_server - self._id = b(uuid.uuid4().hex) + self._id = uuid.uuid4().hex.encode('utf8') self.running = False self.loop = ioloop.IOLoop() diff --git a/circus/py3compat.py b/circus/py3compat.py index 10e16f7ac..13d5cd7a8 100644 --- a/circus/py3compat.py +++ b/circus/py3compat.py @@ -16,6 +16,5 @@ def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA cast_string = cast_unicode -b = cast_bytes s = cast_string u = cast_unicode diff --git a/circus/stats/publisher.py b/circus/stats/publisher.py index 5062e3c3a..a6727a745 100644 --- a/circus/stats/publisher.py +++ b/circus/stats/publisher.py @@ -1,6 +1,5 @@ import zmq import zmq.utils.jsonapi as json -from circus.py3compat import b from circus import logger @@ -22,7 +21,7 @@ def publish(self, name, stat): stat = json.dumps(stat) logger.debug('Sending %s' % stat) - self.socket.send_multipart([b(topic), stat]) + self.socket.send_multipart([topic.encode('utf8'), stat]) except zmq.ZMQError: if self.socket.closed: diff --git a/circus/tests/test_circusctl.py b/circus/tests/test_circusctl.py index 9106ba39a..661c638c2 100644 --- a/circus/tests/test_circusctl.py +++ b/circus/tests/test_circusctl.py @@ -10,7 +10,7 @@ from circus.tests.support import (TestCircus, async_poll_for, EasyTestSuite, skipIf, DEBUG, PYTHON, SLEEP) from circus.util import tornado_sleep, DEFAULT_ENDPOINT_DEALER -from circus.py3compat import b, s +from circus.py3compat import s def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): @@ -22,7 +22,7 @@ def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): stdin=subprocess.PIPE if stdin else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate(b(stdin) if stdin else None) + stdout, stderr = proc.communicate(stdin.encode('utf8') if stdin else None) stdout = s(stdout) stderr = s(stderr) if queue: diff --git a/circus/watcher.py b/circus/watcher.py index 380c5f9d7..293be133c 100644 --- a/circus/watcher.py +++ b/circus/watcher.py @@ -26,7 +26,6 @@ from circus.stream.papa_redirector import PapaRedirector from circus.util import parse_env_dict, resolve_name, tornado_sleep, IS_WINDOWS from circus.util import papa -from circus.py3compat import b class Watcher(object): @@ -433,7 +432,9 @@ def notify_event(self, topic, msg): name = self.res_name - multipart_msg = [b("watcher.%s.%s" % (name, topic)), json.dumps(msg)] + multipart_msg = [ + ("watcher.%s.%s" % (name, topic)).encode('utf8'), json.dumps(msg) + ] if self.evpub_socket is not None and not self.evpub_socket.closed: self.evpub_socket.send_multipart(multipart_msg) From 07ad8a46ab78803a578896748b81d8a1a70dffac Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Mon, 23 Mar 2020 21:08:18 +0300 Subject: [PATCH 07/22] Remove cast_unicode and all its references --- circus/plugins/__init__.py | 3 +-- circus/py3compat.py | 20 -------------------- circus/stats/client.py | 3 +-- circus/stats/streamer.py | 2 -- circus/stream/__init__.py | 5 ++--- circus/stream/file_stream.py | 3 +-- circus/tests/test_arbiter.py | 3 +-- circus/tests/test_circusctl.py | 5 ++--- circus/tests/test_command_kill.py | 4 ++-- circus/tests/test_command_signal.py | 3 +-- circus/tests/test_watcher.py | 3 +-- 11 files changed, 12 insertions(+), 42 deletions(-) delete mode 100644 circus/py3compat.py diff --git a/circus/plugins/__init__.py b/circus/plugins/__init__.py index cbd83e0e9..fb0bcb3b4 100644 --- a/circus/plugins/__init__.py +++ b/circus/plugins/__init__.py @@ -12,7 +12,6 @@ from circus import logger, __version__ from circus.client import make_message, cast_message -from circus.py3compat import s from circus.util import (debuglog, to_bool, resolve_name, configure_logger, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, get_connection) @@ -151,7 +150,7 @@ def handle_init(self): @staticmethod def split_data(data): topic, msg = data - topic_parts = s(topic).split(".") + topic_parts = str(topic).split(".") return topic_parts[1], topic_parts[2], msg @staticmethod diff --git a/circus/py3compat.py b/circus/py3compat.py deleted file mode 100644 index 13d5cd7a8..000000000 --- a/circus/py3compat.py +++ /dev/null @@ -1,20 +0,0 @@ -def cast_bytes(s, encoding='utf8'): # NOQA - """cast unicode or bytes to bytes""" - if isinstance(s, bytes): - return s - return str(s).encode(encoding) - - -def cast_unicode(s, encoding='utf8', errors='replace'): # NOQA - """cast bytes or unicode to unicode. - errors options are strict, ignore or replace""" - if isinstance(s, bytes): - return s.decode(encoding, errors=errors) - return str(s) - - -cast_string = cast_unicode - - -s = cast_string -u = cast_unicode diff --git a/circus/stats/client.py b/circus/stats/client.py index b755043bf..2ddfee80e 100644 --- a/circus/stats/client.py +++ b/circus/stats/client.py @@ -13,7 +13,6 @@ from circus.consumer import CircusConsumer from circus import __version__ from circus.util import DEFAULT_ENDPOINT_STATS -from circus.py3compat import s class StatsClient(CircusConsumer): @@ -48,7 +47,7 @@ def iter_messages(self): pass continue - topic = s(topic).split('.') + topic = topic.decode('utf8', errors='replace').split('.') if len(topic) == 3: __, watcher, subtopic = topic yield watcher, subtopic, json.loads(stat) diff --git a/circus/stats/streamer.py b/circus/stats/streamer.py index 3ab7aebde..9983423df 100644 --- a/circus/stats/streamer.py +++ b/circus/stats/streamer.py @@ -14,7 +14,6 @@ from circus.stats.collector import WatcherStatsCollector, SocketStatsCollector from circus.stats.publisher import StatsPublisher from circus import logger -from circus.py3compat import s class StatsStreamer(object): @@ -184,7 +183,6 @@ def handle_recv(self, data): logger.debug('Received an event from circusd: %s' % str(data)) topic, msg = data try: - topic = s(topic) watcher = topic.split('.')[1:-1][0] action = topic.split('.')[-1] msg = json.loads(msg) diff --git a/circus/stream/__init__.py b/circus/stream/__init__.py index 21693189e..7f4cb1341 100644 --- a/circus/stream/__init__.py +++ b/circus/stream/__init__.py @@ -12,7 +12,6 @@ from circus.stream.file_stream import WatchedFileStream # noqa: F401 from circus.stream.file_stream import TimedRotatingFileStream # noqa: F401 from circus.stream.redirector import Redirector # noqa: F401 -from circus.py3compat import s class QueueStream(Queue): @@ -32,7 +31,7 @@ def __init__(self, **kwargs): pass def __call__(self, data): - sys.stdout.write(s(data['data'])) + sys.stdout.write(data['data'].decode('utf8', errors='replace')) sys.stdout.flush() def close(self): @@ -111,7 +110,7 @@ def prefix(self, data): return color + prefix def __call__(self, data): - for line in s(data['data']).split('\n'): + for line in data['data'].decode('utf8', errors='replace').split('\n'): if line: self.out.write(self.prefix(data)) self.out.write(line) diff --git a/circus/stream/file_stream.py b/circus/stream/file_stream.py index bc8379cc1..42eb4bcfa 100644 --- a/circus/stream/file_stream.py +++ b/circus/stream/file_stream.py @@ -6,7 +6,6 @@ import re from stat import ST_DEV, ST_INO, ST_MTIME from circus import logger -from circus.py3compat import s class _FileStreamBase(object): @@ -37,7 +36,7 @@ def close(self): def write_data(self, data): # data to write on file - file_data = s(data['data']) + file_data = data['data'] # If we want to prefix the stream with the current datetime if self._time_format is not None: diff --git a/circus/tests/test_arbiter.py b/circus/tests/test_arbiter.py index 662431570..18d25ac4b 100644 --- a/circus/tests/test_arbiter.py +++ b/circus/tests/test_arbiter.py @@ -23,7 +23,6 @@ from circus.tests.support import (MockWatcher, has_circusweb, poll_for_callable, get_available_port) from circus import watcher as watcher_mod -from circus.py3compat import s _GENERIC = os.path.join(os.path.dirname(__file__), 'generic.py') @@ -39,7 +38,7 @@ def __init__(self, *args, **kwargs): def handle_recv(self, data): topic, msg = data - topic_parts = s(topic).split(".") + topic_parts = topic.decode('utf8', errors='replace').split(".") watcher = topic_parts[1] action = topic_parts[2] with open(self.config['file'], 'a+') as f: diff --git a/circus/tests/test_circusctl.py b/circus/tests/test_circusctl.py index 661c638c2..6c911c471 100644 --- a/circus/tests/test_circusctl.py +++ b/circus/tests/test_circusctl.py @@ -10,7 +10,6 @@ from circus.tests.support import (TestCircus, async_poll_for, EasyTestSuite, skipIf, DEBUG, PYTHON, SLEEP) from circus.util import tornado_sleep, DEFAULT_ENDPOINT_DEALER -from circus.py3compat import s def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): @@ -23,8 +22,8 @@ def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate(stdin.encode('utf8') if stdin else None) - stdout = s(stdout) - stderr = s(stderr) + stdout = stdout.decode('utf8', errors='replace') + stderr = stderr.decode('utf8', errors='replace') if queue: queue.put(stderr) queue.put(stdout) diff --git a/circus/tests/test_command_kill.py b/circus/tests/test_command_kill.py index df194abc2..648adfff2 100644 --- a/circus/tests/test_command_kill.py +++ b/circus/tests/test_command_kill.py @@ -9,7 +9,6 @@ from circus.client import AsyncCircusClient from circus.stream import QueueStream, Empty from circus.util import tornado_sleep -from circus.py3compat import s def send(msg): @@ -64,7 +63,8 @@ def read(self, timeout=None): while time.time() - start < timeout: try: msg = self._stream.get_nowait() - lines = [l for l in s(msg['data']).split('\n') if l] + raw_lines = msg['data'].decode('utf8', errors='replace').split('\n') + lines = [l for l in raw_lines if l] self._buffer.extend(lines) raise tornado.gen.Return(self._buffer.pop(0)) except Empty: diff --git a/circus/tests/test_command_signal.py b/circus/tests/test_command_signal.py index c5809e125..324f9d3bb 100644 --- a/circus/tests/test_command_signal.py +++ b/circus/tests/test_command_signal.py @@ -9,7 +9,6 @@ from circus.client import AsyncCircusClient from circus.stream import QueueStream, Empty from circus.util import tornado_sleep -from circus.py3compat import s exiting = False @@ -72,7 +71,7 @@ def read_from_stream(stream, desired_channel, timeout=10): while not channels[desired_channel] and time.time() - start < timeout: try: data = stream.get_nowait() - data = s(data['data']).split('\n') + data = data['data'].decode('utf8', errors='replace').split('\n') accumulator += data.pop(0) if data: data.insert(0, accumulator) diff --git a/circus/tests/test_watcher.py b/circus/tests/test_watcher.py index 828fce28d..5c7c67845 100644 --- a/circus/tests/test_watcher.py +++ b/circus/tests/test_watcher.py @@ -28,7 +28,6 @@ from circus.tests.support import PYTHON from circus.util import get_python_version, tornado_sleep from circus.watcher import Watcher -from circus.py3compat import s if hasattr(signal, 'SIGKILL'): SIGKILL = signal.SIGKILL @@ -245,7 +244,7 @@ def test_copy_path(self): messages.append(m) except Queue.Empty: pass - data = ''.join(s(m['data']) for m in messages) + data = ''.join(m['data'].decode('utf8', errors='replace') for m in messages) if 'XYZ' in data: resp = True break From 12dc6cbb3e7f10ba38c7bbd5cd3537c278eb3776 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Mon, 23 Mar 2020 21:11:04 +0300 Subject: [PATCH 08/22] flake8 fixes --- circus/tests/test_command_kill.py | 4 +++- circus/tests/test_watcher.py | 5 ++++- circus/util.py | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/circus/tests/test_command_kill.py b/circus/tests/test_command_kill.py index 648adfff2..50c66cc98 100644 --- a/circus/tests/test_command_kill.py +++ b/circus/tests/test_command_kill.py @@ -63,7 +63,9 @@ def read(self, timeout=None): while time.time() - start < timeout: try: msg = self._stream.get_nowait() - raw_lines = msg['data'].decode('utf8', errors='replace').split('\n') + raw_lines = msg['data'].decode( + 'utf8', errors='replace' + ).split('\n') lines = [l for l in raw_lines if l] self._buffer.extend(lines) raise tornado.gen.Return(self._buffer.pop(0)) diff --git a/circus/tests/test_watcher.py b/circus/tests/test_watcher.py index 5c7c67845..637a0b3c8 100644 --- a/circus/tests/test_watcher.py +++ b/circus/tests/test_watcher.py @@ -244,7 +244,10 @@ def test_copy_path(self): messages.append(m) except Queue.Empty: pass - data = ''.join(m['data'].decode('utf8', errors='replace') for m in messages) + data = ''.join( + m['data'].decode('utf8', errors='replace') + for m in messages + ) if 'XYZ' in data: resp = True break diff --git a/circus/util.py b/circus/util.py index f7a7aa7ab..1c1ca4ade 100644 --- a/circus/util.py +++ b/circus/util.py @@ -604,7 +604,9 @@ def resolve_name(import_name, silent=False, reload=False): return __import__(import_name) except ImportError as e: if not silent: - raise ImportStringError(import_name, e).with_traceback(sys.exc_info()[2]) + raise ImportStringError(import_name, e).with_traceback( + sys.exc_info()[2] + ) _SECTION_NAME = r'\w\.\-' From 942070a2eac38f27d672c085131ead09768279a4 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Mon, 23 Mar 2020 21:23:50 +0300 Subject: [PATCH 09/22] Fix str conversion in stream --- circus/stream/__init__.py | 2 +- circus/stream/file_stream.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/circus/stream/__init__.py b/circus/stream/__init__.py index 7f4cb1341..5af024012 100644 --- a/circus/stream/__init__.py +++ b/circus/stream/__init__.py @@ -110,7 +110,7 @@ def prefix(self, data): return color + prefix def __call__(self, data): - for line in data['data'].decode('utf8', errors='replace').split('\n'): + for line in data['data'].split('\n'): if line: self.out.write(self.prefix(data)) self.out.write(line) diff --git a/circus/stream/file_stream.py b/circus/stream/file_stream.py index 42eb4bcfa..a7d4840a0 100644 --- a/circus/stream/file_stream.py +++ b/circus/stream/file_stream.py @@ -37,6 +37,8 @@ def close(self): def write_data(self, data): # data to write on file file_data = data['data'] + if isinstance(data['data'], bytes): + file_data = data['data'].decode('utf8', errors='replace') # If we want to prefix the stream with the current datetime if self._time_format is not None: From 6e6066658c2f87da06205ebcdc15bb8923325c02 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Mon, 23 Mar 2020 22:21:43 +0300 Subject: [PATCH 10/22] Revert b and s methods as util.to_bytes and util.to_str A lot of real world examples failed (even example1.ini) because of the types misuse between bytes and str in streamer and other parts of the package. I reverted b and s methods as to_bytes and to_str respectively, keeping the same signature. --- circus/client.py | 6 +++--- circus/consumer.py | 4 ++-- circus/plugins/__init__.py | 6 +++--- circus/stats/client.py | 4 ++-- circus/stats/publisher.py | 3 ++- circus/stats/streamer.py | 2 ++ circus/stream/__init__.py | 6 +++--- circus/stream/file_stream.py | 5 ++--- circus/tests/test_arbiter.py | 4 ++-- circus/tests/test_circusctl.py | 9 +++++---- circus/tests/test_command_kill.py | 7 ++----- circus/tests/test_command_signal.py | 5 ++--- circus/tests/test_watcher.py | 7 ++----- circus/util.py | 15 +++++++++++++++ 14 files changed, 47 insertions(+), 36 deletions(-) diff --git a/circus/client.py b/circus/client.py index 8bccfbf26..91edc279b 100644 --- a/circus/client.py +++ b/circus/client.py @@ -9,7 +9,7 @@ import tornado from circus.exc import CallError -from circus.util import DEFAULT_ENDPOINT_DEALER, get_connection +from circus.util import DEFAULT_ENDPOINT_DEALER, get_connection, to_bytes def make_message(command, **props): @@ -30,7 +30,7 @@ def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint - self._id = uuid.uuid4().hex.encode('utf8') + self._id = to_bytes(uuid.uuid4().hex) self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) @@ -89,7 +89,7 @@ def __init__(self, context=None, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0, ssh_server=None, ssh_keyfile=None): self._init_context(context) self.endpoint = endpoint - self._id = uuid.uuid4().hex.encode('utf8') + self._id = to_bytes(uuid.uuid4().hex) self.socket = self.context.socket(zmq.DEALER) self.socket.setsockopt(zmq.IDENTITY, self._id) self.socket.setsockopt(zmq.LINGER, 0) diff --git a/circus/consumer.py b/circus/consumer.py index 0d06afea0..daf18674f 100644 --- a/circus/consumer.py +++ b/circus/consumer.py @@ -1,7 +1,7 @@ import errno import zmq -from circus.util import DEFAULT_ENDPOINT_SUB, get_connection +from circus.util import DEFAULT_ENDPOINT_SUB, get_connection, to_bytes class CircusConsumer(object): @@ -14,7 +14,7 @@ def __init__(self, topics, context=None, endpoint=DEFAULT_ENDPOINT_SUB, self.pubsub_socket = self.context.socket(zmq.SUB) get_connection(self.pubsub_socket, self.endpoint, ssh_server) for topic in self.topics: - self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, topic.encode('utf8')) + self.pubsub_socket.setsockopt(zmq.SUBSCRIBE, to_bytes(topic)) self._init_poller() self.timeout = timeout diff --git a/circus/plugins/__init__.py b/circus/plugins/__init__.py index fb0bcb3b4..e5c7d2467 100644 --- a/circus/plugins/__init__.py +++ b/circus/plugins/__init__.py @@ -14,7 +14,7 @@ from circus.client import make_message, cast_message from circus.util import (debuglog, to_bool, resolve_name, configure_logger, DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, - get_connection) + get_connection, to_str, to_bytes) class CircusPlugin(object): @@ -39,7 +39,7 @@ def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server=None, self.endpoint = endpoint self.check_delay = check_delay self.ssh_server = ssh_server - self._id = uuid.uuid4().hex.encode('utf8') + self._id = to_bytes(uuid.uuid4().hex) self.running = False self.loop = ioloop.IOLoop() @@ -150,7 +150,7 @@ def handle_init(self): @staticmethod def split_data(data): topic, msg = data - topic_parts = str(topic).split(".") + topic_parts = to_str(topic).split(".") return topic_parts[1], topic_parts[2], msg @staticmethod diff --git a/circus/stats/client.py b/circus/stats/client.py index 2ddfee80e..b8dbb5163 100644 --- a/circus/stats/client.py +++ b/circus/stats/client.py @@ -12,7 +12,7 @@ from circus.consumer import CircusConsumer from circus import __version__ -from circus.util import DEFAULT_ENDPOINT_STATS +from circus.util import DEFAULT_ENDPOINT_STATS, to_str class StatsClient(CircusConsumer): @@ -47,7 +47,7 @@ def iter_messages(self): pass continue - topic = topic.decode('utf8', errors='replace').split('.') + topic = to_str(topic).split('.') if len(topic) == 3: __, watcher, subtopic = topic yield watcher, subtopic, json.loads(stat) diff --git a/circus/stats/publisher.py b/circus/stats/publisher.py index a6727a745..671b7105b 100644 --- a/circus/stats/publisher.py +++ b/circus/stats/publisher.py @@ -2,6 +2,7 @@ import zmq.utils.jsonapi as json from circus import logger +from circus.util import to_bytes class StatsPublisher(object): @@ -21,7 +22,7 @@ def publish(self, name, stat): stat = json.dumps(stat) logger.debug('Sending %s' % stat) - self.socket.send_multipart([topic.encode('utf8'), stat]) + self.socket.send_multipart([to_bytes(topic), stat]) except zmq.ZMQError: if self.socket.closed: diff --git a/circus/stats/streamer.py b/circus/stats/streamer.py index 9983423df..569a6c7ed 100644 --- a/circus/stats/streamer.py +++ b/circus/stats/streamer.py @@ -14,6 +14,7 @@ from circus.stats.collector import WatcherStatsCollector, SocketStatsCollector from circus.stats.publisher import StatsPublisher from circus import logger +from circus.util import to_str class StatsStreamer(object): @@ -183,6 +184,7 @@ def handle_recv(self, data): logger.debug('Received an event from circusd: %s' % str(data)) topic, msg = data try: + topic = to_str(topic) watcher = topic.split('.')[1:-1][0] action = topic.split('.')[-1] msg = json.loads(msg) diff --git a/circus/stream/__init__.py b/circus/stream/__init__.py index 5af024012..8537767a2 100644 --- a/circus/stream/__init__.py +++ b/circus/stream/__init__.py @@ -7,7 +7,7 @@ except ImportError: from Queue import Queue, Empty # NOQA -from circus.util import resolve_name +from circus.util import resolve_name, to_str from circus.stream.file_stream import FileStream from circus.stream.file_stream import WatchedFileStream # noqa: F401 from circus.stream.file_stream import TimedRotatingFileStream # noqa: F401 @@ -31,7 +31,7 @@ def __init__(self, **kwargs): pass def __call__(self, data): - sys.stdout.write(data['data'].decode('utf8', errors='replace')) + sys.stdout.write(to_str(data['data'])) sys.stdout.flush() def close(self): @@ -110,7 +110,7 @@ def prefix(self, data): return color + prefix def __call__(self, data): - for line in data['data'].split('\n'): + for line in to_str(data['data']).split('\n'): if line: self.out.write(self.prefix(data)) self.out.write(line) diff --git a/circus/stream/file_stream.py b/circus/stream/file_stream.py index a7d4840a0..eb1f8ac3c 100644 --- a/circus/stream/file_stream.py +++ b/circus/stream/file_stream.py @@ -6,6 +6,7 @@ import re from stat import ST_DEV, ST_INO, ST_MTIME from circus import logger +from circus.util import to_str class _FileStreamBase(object): @@ -36,9 +37,7 @@ def close(self): def write_data(self, data): # data to write on file - file_data = data['data'] - if isinstance(data['data'], bytes): - file_data = data['data'].decode('utf8', errors='replace') + file_data = to_str(data['data']) # If we want to prefix the stream with the current datetime if self._time_format is not None: diff --git a/circus/tests/test_arbiter.py b/circus/tests/test_arbiter.py index 18d25ac4b..c9f23c8e2 100644 --- a/circus/tests/test_arbiter.py +++ b/circus/tests/test_arbiter.py @@ -19,7 +19,7 @@ EasyTestSuite, skipIf, get_ioloop, SLEEP, PYTHON) from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_MULTICAST, - DEFAULT_ENDPOINT_SUB) + DEFAULT_ENDPOINT_SUB, to_str) from circus.tests.support import (MockWatcher, has_circusweb, poll_for_callable, get_available_port) from circus import watcher as watcher_mod @@ -38,7 +38,7 @@ def __init__(self, *args, **kwargs): def handle_recv(self, data): topic, msg = data - topic_parts = topic.decode('utf8', errors='replace').split(".") + topic_parts = to_str(topic).split(".") watcher = topic_parts[1] action = topic_parts[2] with open(self.config['file'], 'a+') as f: diff --git a/circus/tests/test_circusctl.py b/circus/tests/test_circusctl.py index 6c911c471..90289e23d 100644 --- a/circus/tests/test_circusctl.py +++ b/circus/tests/test_circusctl.py @@ -9,7 +9,8 @@ from circus.circusctl import USAGE, VERSION, CircusCtl from circus.tests.support import (TestCircus, async_poll_for, EasyTestSuite, skipIf, DEBUG, PYTHON, SLEEP) -from circus.util import tornado_sleep, DEFAULT_ENDPOINT_DEALER +from circus.util import (tornado_sleep, DEFAULT_ENDPOINT_DEALER, to_str, + to_bytes) def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): @@ -21,9 +22,9 @@ def run_ctl(args, queue=None, stdin='', endpoint=DEFAULT_ENDPOINT_DEALER): stdin=subprocess.PIPE if stdin else None, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate(stdin.encode('utf8') if stdin else None) - stdout = stdout.decode('utf8', errors='replace') - stderr = stderr.decode('utf8', errors='replace') + stdout, stderr = proc.communicate(to_bytes(stdin) if stdin else None) + stdout = to_str(stdout) + stderr = to_str(stderr) if queue: queue.put(stderr) queue.put(stdout) diff --git a/circus/tests/test_command_kill.py b/circus/tests/test_command_kill.py index 50c66cc98..5dcb899b7 100644 --- a/circus/tests/test_command_kill.py +++ b/circus/tests/test_command_kill.py @@ -8,7 +8,7 @@ from circus.tests.support import skipIf, IS_WINDOWS from circus.client import AsyncCircusClient from circus.stream import QueueStream, Empty -from circus.util import tornado_sleep +from circus.util import tornado_sleep, to_str def send(msg): @@ -63,10 +63,7 @@ def read(self, timeout=None): while time.time() - start < timeout: try: msg = self._stream.get_nowait() - raw_lines = msg['data'].decode( - 'utf8', errors='replace' - ).split('\n') - lines = [l for l in raw_lines if l] + lines = [l for l in to_str(msg['data']).split('\n') if l] self._buffer.extend(lines) raise tornado.gen.Return(self._buffer.pop(0)) except Empty: diff --git a/circus/tests/test_command_signal.py b/circus/tests/test_command_signal.py index 324f9d3bb..50eb3faa8 100644 --- a/circus/tests/test_command_signal.py +++ b/circus/tests/test_command_signal.py @@ -8,8 +8,7 @@ from circus.tests.support import skipIf, IS_WINDOWS from circus.client import AsyncCircusClient from circus.stream import QueueStream, Empty -from circus.util import tornado_sleep - +from circus.util import tornado_sleep, to_str exiting = False channels = {0: [], 1: [], 2: [], 3: []} @@ -71,7 +70,7 @@ def read_from_stream(stream, desired_channel, timeout=10): while not channels[desired_channel] and time.time() - start < timeout: try: data = stream.get_nowait() - data = data['data'].decode('utf8', errors='replace').split('\n') + data = to_str(data['data']).split('\n') accumulator += data.pop(0) if data: data.insert(0, accumulator) diff --git a/circus/tests/test_watcher.py b/circus/tests/test_watcher.py index 637a0b3c8..6c9222c78 100644 --- a/circus/tests/test_watcher.py +++ b/circus/tests/test_watcher.py @@ -26,7 +26,7 @@ from circus.tests.support import async_poll_for, EasyTestSuite from circus.tests.support import MagicMockFuture, skipIf, IS_WINDOWS from circus.tests.support import PYTHON -from circus.util import get_python_version, tornado_sleep +from circus.util import get_python_version, tornado_sleep, to_str from circus.watcher import Watcher if hasattr(signal, 'SIGKILL'): @@ -244,10 +244,7 @@ def test_copy_path(self): messages.append(m) except Queue.Empty: pass - data = ''.join( - m['data'].decode('utf8', errors='replace') - for m in messages - ) + data = ''.join(to_str(m['data']) for m in messages) if 'XYZ' in data: resp = True break diff --git a/circus/util.py b/circus/util.py index 1c1ca4ade..29205ff7d 100644 --- a/circus/util.py +++ b/circus/util.py @@ -334,6 +334,21 @@ def to_signum(signum): raise ValueError('signal invalid: {}'.format(signum)) +def to_str(s, encoding='utf8', errors='replace'): + """cast bytes or string to string. + errors options are strict, ignore or replace""" + if isinstance(s, bytes): + return s.decode(encoding, errors=errors) + return str(s) + + +def to_bytes(s, encoding='utf8'): # NOQA + """cast str or bytes to bytes""" + if isinstance(s, bytes): + return s + return str(s).encode(encoding) + + if pwd is None: def to_uid(name): From 10b4364d66ed8edf27b1e6297c106697c7a52a44 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:00:40 +0300 Subject: [PATCH 11/22] Remove unnecessary imports - remove try/except imports - remove leftover six imports (with relevatn changes in actual code) - remove unused itertools imports --- bootstrap.py | 5 +---- circus/config.py | 4 +--- circus/controller.py | 8 ++------ circus/stream/__init__.py | 5 +---- circus/tests/support.py | 6 +----- circus/tests/test_arbiter.py | 5 +---- circus/tests/test_circusd.py | 3 +-- circus/tests/test_logging.py | 12 +++--------- circus/watcher.py | 5 ----- 9 files changed, 11 insertions(+), 42 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index ec3757a96..b2b9af466 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -68,10 +68,7 @@ except ImportError: ez = {} - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen + from urllib.request import urlopen exec(urlopen('http://python-distribute.org/distribute_setup.py').read(), ez) setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True) diff --git a/circus/config.py b/circus/config.py index 0e7adc872..2b58085a9 100644 --- a/circus/config.py +++ b/circus/config.py @@ -8,8 +8,6 @@ except ImportError: resource = None # NOQA -import six - from circus import logger from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB, DEFAULT_ENDPOINT_MULTICAST, DEFAULT_ENDPOINT_STATS, @@ -293,7 +291,7 @@ def _extend(target, source): target[name] = value def _expand_vars(target, key, env): - if isinstance(target[key], six.string_types): + if isinstance(target[key], str): target[key] = replace_gnu_args(target[key], env=env) elif isinstance(target[key], dict): for k in target[key].keys(): diff --git a/circus/controller.py b/circus/controller.py index 1aae173f8..9abe399c1 100644 --- a/circus/controller.py +++ b/circus/controller.py @@ -2,12 +2,8 @@ import sys import traceback import functools -try: - from queue import Queue, Empty - from urllib.parse import urlparse -except ImportError: - from Queue import Queue, Empty # NOQA - from urlparse import urlparse # NOQA +from queue import Queue, Empty +from urllib.parse import urlparse import zmq diff --git a/circus/stream/__init__.py b/circus/stream/__init__.py index 8537767a2..57d1a2e60 100644 --- a/circus/stream/__init__.py +++ b/circus/stream/__init__.py @@ -2,10 +2,7 @@ import random from datetime import datetime -try: - from queue import Queue, Empty -except ImportError: - from Queue import Queue, Empty # NOQA +from queue import Queue, Empty from circus.util import resolve_name, to_str from circus.stream.file_stream import FileStream diff --git a/circus/tests/support.py b/circus/tests/support.py index 0eac5e3e1..ac248a7c6 100644 --- a/circus/tests/support.py +++ b/circus/tests/support.py @@ -12,11 +12,7 @@ import socket import sysconfig -try: - from unittest import skip, skipIf, TestCase, TestSuite, findTestCases -except ImportError: - from unittest2 import skip, skipIf, TestCase, TestSuite # NOQA - from unittest2 import findTestCases # NOQA +from unittest import skip, skipIf, TestCase, TestSuite, findTestCases from tornado.testing import AsyncTestCase import mock diff --git a/circus/tests/test_arbiter.py b/circus/tests/test_arbiter.py index c9f23c8e2..63a5ce1f1 100644 --- a/circus/tests/test_arbiter.py +++ b/circus/tests/test_arbiter.py @@ -6,10 +6,7 @@ from time import time import zmq.utils.jsonapi as json import mock -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse # NOQA +from urllib.parse import urlparse from circus.arbiter import Arbiter from circus.client import AsyncCircusClient diff --git a/circus/tests/test_circusd.py b/circus/tests/test_circusd.py index 02fbff095..775bf192f 100644 --- a/circus/tests/test_circusd.py +++ b/circus/tests/test_circusd.py @@ -1,7 +1,6 @@ import sys import os import tempfile -import six from copy import copy from circus.circusd import get_maxfd, daemonize, main @@ -94,7 +93,7 @@ def test_daemon(self): def test_maxfd(self): max = get_maxfd() - self.assertTrue(isinstance(max, six.integer_types)) + self.assertTrue(isinstance(max, int)) @skipIf(has_gevent(), "Gevent is loaded") @skipIf(IS_WINDOWS, "Daemonizing not supported on Windows") diff --git a/circus/tests/test_logging.py b/circus/tests/test_logging.py index 02bdcaf2b..abe619359 100644 --- a/circus/tests/test_logging.py +++ b/circus/tests/test_logging.py @@ -1,12 +1,6 @@ -try: - from io import StringIO - from io import BytesIO -except ImportError: - from cStringIO import StringIO # NOQA -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser # NOQA +from io import StringIO +from io import BytesIO +from configparser import ConfigParser from circus.tests.support import TestCase from circus.tests.support import EasyTestSuite from circus.tests.support import skipIf, PYTHON, IS_WINDOWS diff --git a/circus/watcher.py b/circus/watcher.py index 293be133c..3a843cac2 100644 --- a/circus/watcher.py +++ b/circus/watcher.py @@ -6,11 +6,6 @@ import sys from random import randint -try: - from itertools import zip_longest as izip_longest -except ImportError: - # noinspection PyUnresolvedReferences - from itertools import izip_longest # NOQA import site from tornado import gen From c927185543b36a2c8a4e6c2413698ebdb4a795e0 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:07:42 +0300 Subject: [PATCH 12/22] Remove unnecessary coding line It is not required as of Python 3 - https://www.python.org/dev/peps/pep-3120/ --- circus/circusctl.py | 1 - circus/client.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/circus/circusctl.py b/circus/circusctl.py index 93acf580b..d9132ac46 100644 --- a/circus/circusctl.py +++ b/circus/circusctl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 - import argparse import cmd import getopt diff --git a/circus/client.py b/circus/client.py index 91edc279b..e95d39243 100644 --- a/circus/client.py +++ b/circus/client.py @@ -1,5 +1,3 @@ - -# -*- coding: utf-8 - import errno import uuid From 791149f8541b0dfa65a3e1fc575b0a9e7ebf8619 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:09:11 +0300 Subject: [PATCH 13/22] Add test skip for windows --- circus/tests/test_arbiter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circus/tests/test_arbiter.py b/circus/tests/test_arbiter.py index 63a5ce1f1..d4c73a5c6 100644 --- a/circus/tests/test_arbiter.py +++ b/circus/tests/test_arbiter.py @@ -16,7 +16,7 @@ EasyTestSuite, skipIf, get_ioloop, SLEEP, PYTHON) from circus.util import (DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_MULTICAST, - DEFAULT_ENDPOINT_SUB, to_str) + DEFAULT_ENDPOINT_SUB, to_str, IS_WINDOWS) from circus.tests.support import (MockWatcher, has_circusweb, poll_for_callable, get_available_port) from circus import watcher as watcher_mod @@ -486,6 +486,7 @@ def test_stop_watchers3(self): self.assertEqual(resp.get('status'), "active") yield self.stop_arbiter() + @skipIf(IS_WINDOWS, "Streams not supported on Windows") @tornado.testing.gen_test def test_plugins(self): fd, datafile = mkstemp() From d64b384d9fdf959301122db6e0fb982174c9bddf Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:09:42 +0300 Subject: [PATCH 14/22] Add removal of pycache folders in make clean --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6e13cc168..9aa6c6309 100644 --- a/Makefile +++ b/Makefile @@ -36,3 +36,4 @@ clean: rm -rf bin .tox include/ lib/ man/ circus.egg-info/ build/ find . -name "*.pyc" | xargs rm -f find . -name "*.un~" | xargs rm -f + find . -name "__pycache__" | xargs rm -rf From d07d1f86f48fb239aa6016f26c49effb9f367607 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:10:20 +0300 Subject: [PATCH 15/22] Drop python versions below 3.5 and remove six from requirements --- setup.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 4348690d1..9a784defb 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,11 @@ from setuptools import setup, find_packages from circus import __version__ -if not hasattr(sys, 'version_info') or sys.version_info < (2, 7, 0, 'final'): - raise SystemExit("Circus requires Python 2.7 or higher.") +if not hasattr(sys, 'version_info') or sys.version_info < (3, 5, 0, 'final'): + raise SystemExit("Circus requires Python 3.5 or higher.") -install_requires = ['psutil', 'pyzmq>=17.0', 'tornado>=3.0,<5.0', 'six'] +install_requires = ['psutil', 'pyzmq>=17.0', 'tornado>=3.0,<5.0'] try: import argparse # NOQA @@ -29,10 +29,6 @@ zip_safe=False, classifiers=[ "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", From 93841a0a0c80914b2e8753785f48e3811d7cf362 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:32:25 +0300 Subject: [PATCH 16/22] Remove venv/lib for older python versions --- .../lib/python2.7/no-global-site-packages.txt | 0 .../tests/venv/lib/python2.7/orig-prefix.txt | 1 - .../python2.7/site-packages/easy-install.pth | 3 - .../lib/python3.2/no-global-site-packages.txt | 0 .../tests/venv/lib/python3.2/orig-prefix.txt | 1 - .../python3.2/site-packages/easy-install.pth | 3 - .../pip-7.7-py3.2.egg/EGG-INFO/PKG-INFO | 630 ------------------ .../pip-7.7-py3.2.egg/EGG-INFO/SOURCES.txt | 50 -- .../EGG-INFO/dependency_links.txt | 1 - .../EGG-INFO/entry_points.txt | 4 - .../pip-7.7-py3.2.egg/EGG-INFO/not-zip-safe | 1 - .../pip-7.7-py3.2.egg/EGG-INFO/top_level.txt | 1 - .../lib/python3.4/no-global-site-packages.txt | 0 .../tests/venv/lib/python3.4/orig-prefix.txt | 1 - .../python3.4/site-packages/easy-install.pth | 3 - 15 files changed, 699 deletions(-) delete mode 100644 circus/tests/venv/lib/python2.7/no-global-site-packages.txt delete mode 100644 circus/tests/venv/lib/python2.7/orig-prefix.txt delete mode 100644 circus/tests/venv/lib/python2.7/site-packages/easy-install.pth delete mode 100644 circus/tests/venv/lib/python3.2/no-global-site-packages.txt delete mode 100644 circus/tests/venv/lib/python3.2/orig-prefix.txt delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/easy-install.pth delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/PKG-INFO delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/SOURCES.txt delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/dependency_links.txt delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/entry_points.txt delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/not-zip-safe delete mode 100644 circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/top_level.txt delete mode 100644 circus/tests/venv/lib/python3.4/no-global-site-packages.txt delete mode 100644 circus/tests/venv/lib/python3.4/orig-prefix.txt delete mode 100644 circus/tests/venv/lib/python3.4/site-packages/easy-install.pth diff --git a/circus/tests/venv/lib/python2.7/no-global-site-packages.txt b/circus/tests/venv/lib/python2.7/no-global-site-packages.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/circus/tests/venv/lib/python2.7/orig-prefix.txt b/circus/tests/venv/lib/python2.7/orig-prefix.txt deleted file mode 100644 index 2a4512010..000000000 --- a/circus/tests/venv/lib/python2.7/orig-prefix.txt +++ /dev/null @@ -1 +0,0 @@ -/System/Library/Frameworks/Python.framework/Versions/2.7 \ No newline at end of file diff --git a/circus/tests/venv/lib/python2.7/site-packages/easy-install.pth b/circus/tests/venv/lib/python2.7/site-packages/easy-install.pth deleted file mode 100644 index a3a085a15..000000000 --- a/circus/tests/venv/lib/python2.7/site-packages/easy-install.pth +++ /dev/null @@ -1,3 +0,0 @@ -import sys; sys.__plen = len(sys.path) -./pip-7.7-py2.7.egg -import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/circus/tests/venv/lib/python3.2/no-global-site-packages.txt b/circus/tests/venv/lib/python3.2/no-global-site-packages.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/circus/tests/venv/lib/python3.2/orig-prefix.txt b/circus/tests/venv/lib/python3.2/orig-prefix.txt deleted file mode 100644 index 6eece0977..000000000 --- a/circus/tests/venv/lib/python3.2/orig-prefix.txt +++ /dev/null @@ -1 +0,0 @@ -/Library/Frameworks/Python.framework/Versions/3.2 \ No newline at end of file diff --git a/circus/tests/venv/lib/python3.2/site-packages/easy-install.pth b/circus/tests/venv/lib/python3.2/site-packages/easy-install.pth deleted file mode 100644 index d60d24779..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/easy-install.pth +++ /dev/null @@ -1,3 +0,0 @@ -import sys; sys.__plen = len(sys.path) -./pip-7.7-py3.2.egg -import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/PKG-INFO b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/PKG-INFO deleted file mode 100644 index 5ab3eff95..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/PKG-INFO +++ /dev/null @@ -1,630 +0,0 @@ -Metadata-Version: 1.0 -Name: pip -Version: 1.1 -Summary: pip installs packages. Python packages. An easy_install replacement -Home-page: http://www.pip-installer.org -Author: The pip developers -Author-email: python-virtualenv@groups.google.com -License: MIT -Description: pip - === - - `pip` is a tool for installing and managing Python packages, such as - those found in the `Python Package Index`_. It's a replacement for - easy_install_. - :: - - $ pip install simplejson - [... progress report ...] - Successfully installed simplejson - - .. _`Python Package Index`: http://pypi.python.org/pypi - .. _easy_install: http://peak.telecommunity.com/DevCenter/EasyInstall - - Upgrading a package:: - - $ pip install --upgrade simplejson - [... progress report ...] - Successfully installed simplejson - - Removing a package:: - - $ pip uninstall simplejson - Uninstalling simplejson: - /home/me/env/lib/python2.7/site-packages/simplejson - /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info - Proceed (y/n)? y - Successfully uninstalled simplejson - - .. comment: - - The main website for pip is `www.pip-installer.org - `_. You can also install - the `in-development version `_ - of pip with ``easy_install pip==dev``. - - - - Community - --------- - - The homepage for pip is at `pip-installer.org `_. - Bugs can be filed in the `pip issue tracker - `_. Discussion happens on the - `virtualenv email group - `_. - - - ==== - News - ==== - - Changelog - ========= - - Next release (1.2) schedule - --------------------------- - - Beta and final releases planned for the second half of 2012. - - 1.1 (2012-02-16) - ---------------- - - * Fixed issue #326 - don't crash when a package's setup.py emits UTF-8 and - then fails. Thanks Marc Abramowitz. - - * Added ``--target`` option for installing directly to arbitrary directory. - Thanks Stavros Korokithakis. - - * Added support for authentication with Subversion repositories. Thanks - Qiangning Hong. - - * Fixed issue #315 - ``--download`` now downloads dependencies as well. - Thanks Qiangning Hong. - - * Errors from subprocesses will display the current working directory. - Thanks Antti Kaihola. - - * Fixed issue #369 - compatibility with Subversion 1.7. Thanks Qiangning - Hong. Note that setuptools remains incompatible with Subversion 1.7; to - get the benefits of pip's support you must use Distribute rather than - setuptools. - - * Fixed issue #57 - ignore py2app-generated OS X mpkg zip files in finder. - Thanks Rene Dudfield. - - * Fixed issue #182 - log to ~/Library/Logs/ by default on OS X framework - installs. Thanks Dan Callahan for report and patch. - - * Fixed issue #310 - understand version tags without minor version ("py3") - in sdist filenames. Thanks Stuart Andrews for report and Olivier Girardot for - patch. - - * Fixed issue #7 - Pip now supports optionally installing setuptools - "extras" dependencies; e.g. "pip install Paste[openid]". Thanks Matt Maker - and Olivier Girardot. - - * Fixed issue #391 - freeze no longer borks on requirements files with - --index-url or --find-links. Thanks Herbert Pfennig. - - * Fixed issue #288 - handle symlinks properly. Thanks lebedov for the patch. - - * Fixed issue #49 - pip install -U no longer reinstalls the same versions of - packages. Thanks iguananaut for the pull request. - - * Removed ``-E`` option and ``PIP_RESPECT_VIRTUALENV``; both use a - restart-in-venv mechanism that's broken, and neither one is useful since - every virtualenv now has pip inside it. - - * Fixed issue #366 - pip throws IndexError when it calls `scraped_rel_links` - - * Fixed issue #22 - pip search should set and return a userful shell status code - - * Fixed issue #351 and #365 - added global ``--exists-action`` command line - option to easier script file exists conflicts, e.g. from editable - requirements from VCS that have a changed repo URL. - - - 1.0.2 (2011-07-16) - ------------------ - - * Fixed docs issues. - * Fixed issue #295 - Reinstall a package when using the ``install -I`` option - * Fixed issue #283 - Finds a Git tag pointing to same commit as origin/master - * Fixed issue #279 - Use absolute path for path to docs in setup.py - * Fixed issue #314 - Correctly handle exceptions on Python3. - * Fixed issue #320 - Correctly parse ``--editable`` lines in requirements files - - 1.0.1 (2011-04-30) - ------------------ - - * Start to use git-flow. - * Fixed issue #274 - `find_command` should not raise AttributeError - * Fixed issue #273 - respect Content-Disposition header. Thanks Bradley Ayers. - * Fixed issue #233 - pathext handling on Windows. - * Fixed issue #252 - svn+svn protocol. - * Fixed issue #44 - multiple CLI searches. - * Fixed issue #266 - current working directory when running setup.py clean. - - 1.0 (2011-04-04) - ---------------- - - * Added Python 3 support! Huge thanks to Vinay Sajip, Vitaly Babiy, Kelsey - Hightower, and Alex Gronholm, among others. - - * Download progress only shown on a real TTY. Thanks Alex Morega. - - * Fixed finding of VCS binaries to not be fooled by same-named directories. - Thanks Alex Morega. - - * Fixed uninstall of packages from system Python for users of Debian/Ubuntu - python-setuptools package (workaround until fixed in Debian and Ubuntu). - - * Added `get-pip.py `_ - installer. Simply download and execute it, using the Python interpreter of - your choice:: - - $ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py - $ python get-pip.py - - This may have to be run as root. - - .. note:: - - Make sure you have `distribute `_ - installed before using the installer! - - 0.8.3 - ----- - - * Moved main repository to Github: https://github.com/pypa/pip - - * Transferred primary maintenance from Ian to Jannis Leidel, Carl Meyer, Brian Rosner - - * Fixed issue #14 - No uninstall-on-upgrade with URL package. Thanks Oliver Tonnhofer - - * Fixed issue #163 - Egg name not properly resolved. Thanks Igor Sobreira - - * Fixed issue #178 - Non-alphabetical installation of requirements. Thanks Igor Sobreira - - * Fixed issue #199 - Documentation mentions --index instead of --index-url. Thanks Kelsey Hightower - - * Fixed issue #204 - rmtree undefined in mercurial.py. Thanks Kelsey Hightower - - * Fixed bug in Git vcs backend that would break during reinstallation. - - * Fixed bug in Mercurial vcs backend related to pip freeze and branch/tag resolution. - - * Fixed bug in version string parsing related to the suffix "-dev". - - 0.8.2 - ----- - - * Avoid redundant unpacking of bundles (from pwaller) - - * Fixed issue #32, #150, #161 - Fixed checking out the correct - tag/branch/commit when updating an editable Git requirement. - - * Fixed issue #49 - Added ability to install version control requirements - without making them editable, e.g.:: - - pip install git+https://github.com/pypa/pip/ - - * Fixed issue #175 - Correctly locate build and source directory on Mac OS X. - - * Added ``git+https://`` scheme to Git VCS backend. - - 0.8.1 - ----- - - * Added global --user flag as shortcut for --install-option="--user". From - Ronny Pfannschmidt. - - * Added support for `PyPI mirrors `_ as - defined in `PEP 381 `_, from - Jannis Leidel. - - * Fixed issue #138 - Git revisions ignored. Thanks John-Scott Atlakson. - - * Fixed issue #95 - Initial editable install of github package from a tag fails. Thanks John-Scott Atlakson. - - * Fixed issue #107 - Can't install if a directory in cwd has the same name as the package you're installing. - - * Fixed issue #39 - --install-option="--prefix=~/.local" ignored with -e. - Thanks Ronny Pfannschmidt and Wil Tan. - - - - 0.8 - --- - - * Track which ``build/`` directories pip creates, never remove directories - it doesn't create. From Hugo Lopes Tavares. - - * Pip now accepts file:// index URLs. Thanks Dave Abrahams. - - * Various cleanup to make test-running more consistent and less fragile. - Thanks Dave Abrahams. - - * Real Windows support (with passing tests). Thanks Dave Abrahams. - - * ``pip-2.7`` etc. scripts are created (Python-version specific scripts) - - * ``contrib/build-standalone`` script creates a runnable ``.zip`` form of - pip, from Jannis Leidel - - * Editable git repos are updated when reinstalled - - * Fix problem with ``--editable`` when multiple ``.egg-info/`` directories - are found. - - * A number of VCS-related fixes for ``pip freeze``, from Hugo Lopes Tavares. - - * Significant test framework changes, from Hugo Lopes Tavares. - - 0.7.2 - ----- - - * Set zip_safe=False to avoid problems some people are encountering where - pip is installed as a zip file. - - 0.7.1 - ----- - - * Fixed opening of logfile with no directory name. Thanks Alexandre Conrad. - - * Temporary files are consistently cleaned up, especially after - installing bundles, also from Alex Conrad. - - * Tests now require at least ScriptTest 1.0.3. - - 0.7 - --- - - * Fixed uninstallation on Windows - * Added ``pip search`` command. - * Tab-complete names of installed distributions for ``pip uninstall``. - * Support tab-completion when there is a global-option before the - subcommand. - * Install header files in standard (scheme-default) location when installing - outside a virtualenv. Install them to a slightly more consistent - non-standard location inside a virtualenv (since the standard location is - a non-writable symlink to the global location). - * pip now logs to a central location by default (instead of creating - ``pip-log.txt`` all over the place) and constantly overwrites the - file in question. On Unix and Mac OS X this is ``'$HOME/.pip/pip.log'`` - and on Windows it's ``'%HOME%\\pip\\pip.log'``. You are still able to - override this location with the ``$PIP_LOG_FILE`` environment variable. - For a complete (appended) logfile use the separate ``'--log'`` command line - option. - * Fixed an issue with Git that left an editable packge as a checkout of a - remote branch, even if the default behaviour would have been fine, too. - * Fixed installing from a Git tag with older versions of Git. - * Expand "~" in logfile and download cache paths. - * Speed up installing from Mercurial repositories by cloning without - updating the working copy multiple times. - * Fixed installing directly from directories (e.g. - ``pip install path/to/dir/``). - * Fixed installing editable packages with ``svn+ssh`` URLs. - * Don't print unwanted debug information when running the freeze command. - * Create log file directory automatically. Thanks Alexandre Conrad. - * Make test suite easier to run successfully. Thanks Dave Abrahams. - * Fixed "pip install ." and "pip install .."; better error for directory - without setup.py. Thanks Alexandre Conrad. - * Support Debian/Ubuntu "dist-packages" in zip command. Thanks duckx. - * Fix relative --src folder. Thanks Simon Cross. - * Handle missing VCS with an error message. Thanks Alexandre Conrad. - * Added --no-download option to install; pairs with --no-install to separate - download and installation into two steps. Thanks Simon Cross. - * Fix uninstalling from requirements file containing -f, -i, or - --extra-index-url. - * Leftover build directories are now removed. Thanks Alexandre Conrad. - - 0.6.3 - ----- - - * Fixed import error on Windows with regard to the backwards compatibility - package - - 0.6.2 - ----- - - * Fixed uninstall when /tmp is on a different filesystem. - - * Fixed uninstallation of distributions with namespace packages. - - 0.6.1 - ----- - - * Added support for the ``https`` and ``http-static`` schemes to the - Mercurial and ``ftp`` scheme to the Bazaar backend. - - * Fixed uninstallation of scripts installed with easy_install. - - * Fixed an issue in the package finder that could result in an - infinite loop while looking for links. - - * Fixed issue with ``pip bundle`` and local files (which weren't being - copied into the bundle), from Whit Morriss. - - 0.6 - --- - - * Add ``pip uninstall`` and uninstall-before upgrade (from Carl - Meyer). - - * Extended configurability with config files and environment variables. - - * Allow packages to be upgraded, e.g., ``pip install Package==0.1`` - then ``pip install Package==0.2``. - - * Allow installing/upgrading to Package==dev (fix "Source version does not - match target version" errors). - - * Added command and option completion for bash and zsh. - - * Extended integration with virtualenv by providing an option to - automatically use an active virtualenv and an option to warn if no active - virtualenv is found. - - * Fixed a bug with pip install --download and editable packages, where - directories were being set with 0000 permissions, now defaults to 755. - - * Fixed uninstallation of easy_installed console_scripts. - - * Fixed uninstallation on Mac OS X Framework layout installs - - * Fixed bug preventing uninstall of editables with source outside venv. - - * Creates download cache directory if not existing. - - 0.5.1 - ----- - - * Fixed a couple little bugs, with git and with extensions. - - 0.5 - --- - - * Added ability to override the default log file name (``pip-log.txt``) - with the environmental variable ``$PIP_LOG_FILE``. - - * Made the freeze command print installed packages to stdout instead of - writing them to a file. Use simple redirection (e.g. - ``pip freeze > stable-req.txt``) to get a file with requirements. - - * Fixed problem with freezing editable packages from a Git repository. - - * Added support for base URLs using ```` when parsing - HTML pages. - - * Fixed installing of non-editable packages from version control systems. - - * Fixed issue with Bazaar's bzr+ssh scheme. - - * Added --download-dir option to the install command to retrieve package - archives. If given an editable package it will create an archive of it. - - * Added ability to pass local file and directory paths to ``--find-links``, - e.g. ``--find-links=file:///path/to/my/private/archive`` - - * Reduced the amount of console log messages when fetching a page to find a - distribution was problematic. The full messages can be found in pip-log.txt. - - * Added ``--no-deps`` option to install ignore package dependencies - - * Added ``--no-index`` option to ignore the package index (PyPI) temporarily - - * Fixed installing editable packages from Git branches. - - * Fixes freezing of editable packages from Mercurial repositories. - - * Fixed handling read-only attributes of build files, e.g. of Subversion and - Bazaar on Windows. - - * When downloading a file from a redirect, use the redirected - location's extension to guess the compression (happens specifically - when redirecting to a bitbucket.org tip.gz file). - - * Editable freeze URLs now always use revision hash/id rather than tip or - branch names which could move. - - * Fixed comparison of repo URLs so incidental differences such as - presence/absence of final slashes or quoted/unquoted special - characters don't trigger "ignore/switch/wipe/backup" choice. - - * Fixed handling of attempt to checkout editable install to a - non-empty, non-repo directory. - - 0.4 - --- - - * Make ``-e`` work better with local hg repositories - - * Construct PyPI URLs the exact way easy_install constructs URLs (you - might notice this if you use a custom index that is - slash-sensitive). - - * Improvements on Windows (from `Ionel Maries Cristian - `_). - - * Fixed problem with not being able to install private git repositories. - - * Make ``pip zip`` zip all its arguments, not just the first. - - * Fix some filename issues on Windows. - - * Allow the ``-i`` and ``--extra-index-url`` options in requirements - files. - - * Fix the way bundle components are unpacked and moved around, to make - bundles work. - - * Adds ``-s`` option to allow the access to the global site-packages if a - virtualenv is to be created. - - * Fixed support for Subversion 1.6. - - 0.3.1 - ----- - - * Improved virtualenv restart and various path/cleanup problems on win32. - - * Fixed a regression with installing from svn repositories (when not - using ``-e``). - - * Fixes when installing editable packages that put their source in a - subdirectory (like ``src/``). - - * Improve ``pip -h`` - - 0.3 - --- - - * Added support for editable packages created from Git, Mercurial and Bazaar - repositories and ability to freeze them. Refactored support for version - control systems. - - * Do not use ``sys.exit()`` from inside the code, instead use a - return. This will make it easier to invoke programmatically. - - * Put the install record in ``Package.egg-info/installed-files.txt`` - (previously they went in - ``site-packages/install-record-Package.txt``). - - * Fix a problem with ``pip freeze`` not including ``-e svn+`` when an - svn structure is peculiar. - - * Allow ``pip -E`` to work with a virtualenv that uses a different - version of Python than the parent environment. - - * Fixed Win32 virtualenv (``-E``) option. - - * Search the links passed in with ``-f`` for packages. - - * Detect zip files, even when the file doesn't have a ``.zip`` - extension and it is served with the wrong Content-Type. - - * Installing editable from existing source now works, like ``pip - install -e some/path/`` will install the package in ``some/path/``. - Most importantly, anything that package requires will also be - installed by pip. - - * Add a ``--path`` option to ``pip un/zip``, so you can avoid zipping - files that are outside of where you expect. - - * Add ``--simulate`` option to ``pip zip``. - - 0.2.1 - ----- - - * Fixed small problem that prevented using ``pip.py`` without actually - installing pip. - - * Fixed ``--upgrade``, which would download and appear to install - upgraded packages, but actually just reinstall the existing package. - - * Fixed Windows problem with putting the install record in the right - place, and generating the ``pip`` script with Setuptools. - - * Download links that include embedded spaces or other unsafe - characters (those characters get %-encoded). - - * Fixed use of URLs in requirement files, and problems with some blank - lines. - - * Turn some tar file errors into warnings. - - 0.2 - --- - - * Renamed to ``pip``, and to install you now do ``pip install - PACKAGE`` - - * Added command ``pip zip PACKAGE`` and ``pip unzip PACKAGE``. This - is particularly intended for Google App Engine to manage libraries - to stay under the 1000-file limit. - - * Some fixes to bundles, especially editable packages and when - creating a bundle using unnamed packages (like just an svn - repository without ``#egg=Package``). - - 0.1.4 - ----- - - * Added an option ``--install-option`` to pass options to pass - arguments to ``setup.py install`` - - * ``.svn/`` directories are no longer included in bundles, as these - directories are specific to a version of svn -- if you build a - bundle on a system with svn 1.5, you can't use the checkout on a - system with svn 1.4. Instead a file ``svn-checkout.txt`` is - included that notes the original location and revision, and the - command you can use to turn it back into an svn checkout. (Probably - unpacking the bundle should, maybe optionally, recreate this - information -- but that is not currently implemented, and it would - require network access.) - - * Avoid ambiguities over project name case, where for instance - MyPackage and mypackage would be considered different packages. - This in particular caused problems on Macs, where ``MyPackage/`` and - ``mypackage/`` are the same directory. - - * Added support for an environmental variable - ``$PIP_DOWNLOAD_CACHE`` which will cache package downloads, so - future installations won't require large downloads. Network access - is still required, but just some downloads will be avoided when - using this. - - 0.1.3 - ----- - - * Always use ``svn checkout`` (not ``export``) so that - ``tag_svn_revision`` settings give the revision of the package. - - * Don't update checkouts that came from ``.pybundle`` files. - - 0.1.2 - ----- - - * Improve error text when there are errors fetching HTML pages when - seeking packages. - - * Improve bundles: include empty directories, make them work with - editable packages. - - * If you use ``-E env`` and the environment ``env/`` doesn't exist, a - new virtual environment will be created. - - * Fix ``dependency_links`` for finding packages. - - 0.1.1 - ----- - - * Fixed a NameError exception when running pip outside of a - virtualenv environment. - - * Added HTTP proxy support (from Prabhu Ramachandran) - - * Fixed use of ``hashlib.md5`` on python2.5+ (also from Prabhu - Ramachandran) - - 0.1 - --- - - * Initial release - -Keywords: easy_install distutils setuptools egg virtualenv -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Topic :: Software Development :: Build Tools -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.4 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.1 -Classifier: Programming Language :: Python :: 3.2 diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/SOURCES.txt b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/SOURCES.txt deleted file mode 100644 index 5695276f8..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/SOURCES.txt +++ /dev/null @@ -1,50 +0,0 @@ -AUTHORS.txt -LICENSE.txt -MANIFEST.in -setup.cfg -setup.py -docs/ci-server-step-by-step.txt -docs/configuration.txt -docs/contributing.txt -docs/glossary.txt -docs/index.txt -docs/installing.txt -docs/news.txt -docs/other-tools.txt -docs/requirements.txt -docs/usage.txt -pip/__init__.py -pip/_pkgutil.py -pip/backwardcompat.py -pip/basecommand.py -pip/baseparser.py -pip/download.py -pip/exceptions.py -pip/index.py -pip/locations.py -pip/log.py -pip/req.py -pip/runner.py -pip/status_codes.py -pip/util.py -pip.egg-info/PKG-INFO -pip.egg-info/SOURCES.txt -pip.egg-info/dependency_links.txt -pip.egg-info/entry_points.txt -pip.egg-info/not-zip-safe -pip.egg-info/top_level.txt -pip/commands/__init__.py -pip/commands/bundle.py -pip/commands/completion.py -pip/commands/freeze.py -pip/commands/help.py -pip/commands/install.py -pip/commands/search.py -pip/commands/uninstall.py -pip/commands/unzip.py -pip/commands/zip.py -pip/vcs/__init__.py -pip/vcs/bazaar.py -pip/vcs/git.py -pip/vcs/mercurial.py -pip/vcs/subversion.py \ No newline at end of file diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/dependency_links.txt b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/entry_points.txt b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/entry_points.txt deleted file mode 100644 index 5f7b7cf99..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] -pip = pip:main -pip-2.7 = pip:main - diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/not-zip-safe b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/not-zip-safe deleted file mode 100644 index 8b1378917..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/top_level.txt b/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/top_level.txt deleted file mode 100644 index a1b589e38..000000000 --- a/circus/tests/venv/lib/python3.2/site-packages/pip-7.7-py3.2.egg/EGG-INFO/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/circus/tests/venv/lib/python3.4/no-global-site-packages.txt b/circus/tests/venv/lib/python3.4/no-global-site-packages.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/circus/tests/venv/lib/python3.4/orig-prefix.txt b/circus/tests/venv/lib/python3.4/orig-prefix.txt deleted file mode 100644 index 95343c9f7..000000000 --- a/circus/tests/venv/lib/python3.4/orig-prefix.txt +++ /dev/null @@ -1 +0,0 @@ -/Library/Frameworks/Python.framework/Versions/3.4 diff --git a/circus/tests/venv/lib/python3.4/site-packages/easy-install.pth b/circus/tests/venv/lib/python3.4/site-packages/easy-install.pth deleted file mode 100644 index 1cf37515f..000000000 --- a/circus/tests/venv/lib/python3.4/site-packages/easy-install.pth +++ /dev/null @@ -1,3 +0,0 @@ -import sys; sys.__plen = len(sys.path) -./pip-7.7-py3.4.egg -import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new) From 7e507fdc99867cf4352b6f30fec1d4c63fdd4c15 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 14:44:25 +0300 Subject: [PATCH 17/22] flake8 fixes (ignore F401 unused imports) --- circus/controller.py | 2 +- circus/stream/__init__.py | 2 +- circus/tests/support.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/circus/controller.py b/circus/controller.py index 9abe399c1..ad7b5e7b9 100644 --- a/circus/controller.py +++ b/circus/controller.py @@ -2,7 +2,7 @@ import sys import traceback import functools -from queue import Queue, Empty +from queue import Queue, Empty # noqa: F401 from urllib.parse import urlparse diff --git a/circus/stream/__init__.py b/circus/stream/__init__.py index 57d1a2e60..6aa0c56d3 100644 --- a/circus/stream/__init__.py +++ b/circus/stream/__init__.py @@ -2,7 +2,7 @@ import random from datetime import datetime -from queue import Queue, Empty +from queue import Queue, Empty # noqa: F401 from circus.util import resolve_name, to_str from circus.stream.file_stream import FileStream diff --git a/circus/tests/support.py b/circus/tests/support.py index ac248a7c6..bfd22eef5 100644 --- a/circus/tests/support.py +++ b/circus/tests/support.py @@ -12,7 +12,7 @@ import socket import sysconfig -from unittest import skip, skipIf, TestCase, TestSuite, findTestCases +from unittest import skip, skipIf, TestCase, TestSuite, findTestCases # noqa: F401 from tornado.testing import AsyncTestCase import mock From a998cfe1e70be16325b86ce44fffe511d0b78bc8 Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 15:00:16 +0300 Subject: [PATCH 18/22] Remove Python 2.7 related condition for threads --- circus/_patch.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/circus/_patch.py b/circus/_patch.py index b4256c8b1..c3ba53083 100644 --- a/circus/_patch.py +++ b/circus/_patch.py @@ -33,15 +33,3 @@ def _delete(self): # NOQA raise threading.Thread._delete = _delete - - # see http://bugs.python.org/issue14308 - if get_python_version() < (2, 7, 0): - def _stop(self): - # DummyThreads delete self.__block, but they have no waiters to - # notify anyway (join() is forbidden on them). - if not hasattr(self, '_Thread__block'): - return - self._Thread__stop_old() - - threading.Thread._Thread__stop_old = threading.Thread._Thread__stop - threading.Thread._Thread__stop = _stop From 29a25d9d5154073ecbb261602462987bd0354c1b Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sun, 29 Mar 2020 15:07:07 +0300 Subject: [PATCH 19/22] flake8 fix unused import --- circus/_patch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/circus/_patch.py b/circus/_patch.py index c3ba53083..1673de91f 100644 --- a/circus/_patch.py +++ b/circus/_patch.py @@ -1,6 +1,5 @@ import threading from threading import _active_limbo_lock, _active, _sys -from .util import get_python_version debugger = False From 22c98e0f161989ef9d1dd0624517a3257d7e2f9b Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Fri, 3 Apr 2020 11:33:57 +0300 Subject: [PATCH 20/22] Change from lambda to itemgetter in sort calls --- circus/config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/circus/config.py b/circus/config.py index 2b58085a9..8939b5dbb 100644 --- a/circus/config.py +++ b/circus/config.py @@ -1,4 +1,5 @@ import glob +import operator import os import signal import warnings @@ -278,9 +279,10 @@ def get_config(config_file): watchers.append(watcher) # making sure we return consistent lists - watchers.sort(key=lambda x: x['name']) - plugins.sort(key=lambda x: x['name']) - sockets.sort(key=lambda x: x['name']) + name = operator.itemgetter('name') + watchers.sort(key=name) + plugins.sort(key=name) + sockets.sort(key=name) # Second pass to make sure env sections apply to all watchers. From 12ca560cec4dee9c70f59b866a8efa0bb865377c Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Fri, 3 Apr 2020 21:59:46 +0300 Subject: [PATCH 21/22] Add tox env 'extra' with gevent, papa and circus-web Also fix gevent related test --- .travis.yml | 2 ++ circus/tests/test_circusd.py | 2 +- tox.ini | 9 ++++++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 848409301..62a28158a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ jobs: include: - python: 3.8 env: TOX_ENV=py38 + - python: 3.8 + env: TOX_ENV=extra - python: 3.7 env: TOX_ENV=py37 - python: 3.6 diff --git a/circus/tests/test_circusd.py b/circus/tests/test_circusd.py index 775bf192f..d3fca2d1d 100644 --- a/circus/tests/test_circusd.py +++ b/circus/tests/test_circusd.py @@ -83,7 +83,7 @@ def test_daemon(self): # daemonize() to work self.assertRaises(ValueError, daemonize) - for module in sys.modules.keys(): + for module in list(sys.modules): if module.startswith('gevent'): del sys.modules[module] diff --git a/tox.ini b/tox.ini index 13ea4c6da..31f6897f4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,12 @@ [tox] -envlist = py35,py36,py37,py38,flake8,docs +envlist = py35,py36,py37,py38,extra,flake8,docs + +[testenv:extra] +deps = + {[testenv]deps} + circus-web + gevent + papa [testenv] passenv = PWD From 48dadba3deb78778806565bb52b30501032fc58d Mon Sep 17 00:00:00 2001 From: Ivan Elfimov Date: Sat, 4 Apr 2020 18:41:31 +0300 Subject: [PATCH 22/22] Merge extra and main test envs, update tox command - coverage was not picking up by coveralls, because I removed the coverage package from deps while removing py27 config - remove PYTHONHASHSEED variable, because it is not necessary in Python 3 - remove extra env from travis config --- .travis.yml | 2 -- tox.ini | 17 +++++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62a28158a..848409301 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,6 @@ jobs: include: - python: 3.8 env: TOX_ENV=py38 - - python: 3.8 - env: TOX_ENV=extra - python: 3.7 env: TOX_ENV=py37 - python: 3.6 diff --git a/tox.ini b/tox.ini index 31f6897f4..900c9c3fb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,29 +1,26 @@ [tox] -envlist = py35,py36,py37,py38,extra,flake8,docs - -[testenv:extra] -deps = - {[testenv]deps} - circus-web - gevent - papa +envlist = py35,py36,py37,py38,flake8,docs [testenv] passenv = PWD deps = nose + nose-cov + coverage mock + circus-web + gevent + papa PyYAML tornado>=3.0,<5.0 pyzmq>=17.0 setenv = TESTING=1 - PYTHONHASHSEED=random PYTHONUNBUFFERED=1 commands = - nosetests -vs circus/tests + nosetests -vs --with-coverage --cover-package=circus circus/tests [testenv:docs]