diff --git a/.travis.yml b/.travis.yml index ac74982333..fc29780c4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,7 @@ env: - TRAVIS_FLAVOR=rabbitmq - TRAVIS_FLAVOR=etcd - TRAVIS_FLAVOR=pgbouncer + - TRAVIS_FLAVOR=supervisord # Override travis defaults with empty jobs before_install: echo "OVERRIDING TRAVIS STEPS" diff --git a/Rakefile b/Rakefile index b3609b4b37..a33623d0fd 100755 --- a/Rakefile +++ b/Rakefile @@ -25,6 +25,7 @@ require './ci/redis' require './ci/snmpd' require './ci/sysstat' require './ci/ssh' +require './ci/supervisord' require './ci/tomcat' CLOBBER.include '**/*.pyc' diff --git a/checks.d/supervisord.py b/checks.d/supervisord.py new file mode 100644 index 0000000000..293304be86 --- /dev/null +++ b/checks.d/supervisord.py @@ -0,0 +1,157 @@ +from collections import defaultdict +import errno +import socket +import time +import xmlrpclib + +from checks import AgentCheck + +import supervisor.xmlrpc + +DEFAULT_HOST = 'localhost' +DEFAULT_PORT = '9001' +DEFAULT_SOCKET_IP = 'http://127.0.0.1' + +DD_STATUS = { + 'STOPPED': AgentCheck.CRITICAL, + 'STARTING': AgentCheck.UNKNOWN, + 'RUNNING': AgentCheck.OK, + 'BACKOFF': AgentCheck.CRITICAL, + 'STOPPING': AgentCheck.CRITICAL, + 'EXITED': AgentCheck.CRITICAL, + 'FATAL': AgentCheck.CRITICAL, + 'UNKNOWN': AgentCheck.UNKNOWN +} + +PROCESS_STATUS = { + AgentCheck.CRITICAL: 'down', + AgentCheck.OK: 'up', + AgentCheck.UNKNOWN: 'unknown' +} + +SERVER_TAG = 'supervisord_server' + +PROCESS_TAG = 'supervisord_process' + +FORMAT_TIME = lambda x: time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(x)) + +class SupervisordCheck(AgentCheck): + + def check(self, instance): + server_name = instance.get('name') + + if not server_name or not server_name.strip(): + raise Exception("Supervisord server name not specified in yaml configuration.") + + supe = self._connect(instance) + count_by_status = defaultdict(int) + + # Grab process information + try: + proc_names = instance.get('proc_names') + if proc_names: + if not isinstance(proc_names, list) or not len(proc_names): + raise Exception("Empty or invalid proc_names.") + processes = [] + for proc_name in proc_names: + try: + processes.append(supe.getProcessInfo(proc_name)) + except xmlrpclib.Fault, e: + if e.faultCode == 10: # bad process name + self.warning('Process not found: %s' % proc_name) + else: + raise Exception('An error occurred while reading' + 'process %s information: %s %s' + % (proc_name, e.faultCode, e.faultString)) + else: + processes = supe.getAllProcessInfo() + except socket.error, e: + host = instance.get('host', DEFAULT_HOST) + port = instance.get('port', DEFAULT_PORT) + sock = instance.get('socket') + if sock is None: + msg = 'Cannot connect to http://%s:%s. ' \ + 'Make sure supervisor is running and XML-RPC ' \ + 'inet interface is enabled.' % (host, port) + else: + msg = 'Cannot connect to %s. Make sure sure supervisor ' \ + 'is running and socket is enabled and socket file' \ + ' has the right permissions.' % sock + + if e.errno not in [errno.EACCES, errno.ENOENT]: # permissions denied, no such file + self.service_check('supervisord.server.check', AgentCheck.CRITICAL, + tags=['%s:%s' % (SERVER_TAG, server_name)], + message='Supervisord server %s is down.' % server_name) + + raise Exception(msg) + except xmlrpclib.ProtocolError, e: + if e.errcode == 401: # authorization error + raise Exception('Username or password to %s are incorrect.' % + server_name) + else: + raise Exception('An error occurred while connecting to %s: ' + '%s %s ' % (server_name, e.errcode, e.errmsg)) + + # Report service checks and uptime for each process + for proc in processes: + proc_name = proc['name'] + tags = ['%s:%s' % (SERVER_TAG, server_name), + '%s:%s' % (PROCESS_TAG, proc_name)] + + # Report Service Check + status = DD_STATUS[proc['statename']] + msg = self._build_message(proc) + count_by_status[status] += 1 + self.service_check('supervisord.process.check', + status, tags=tags, message=msg) + # Report Uptime + uptime = self._extract_uptime(proc) + self.gauge('supervisord.process.uptime', uptime, tags=tags) + + # Report counts by status + tags = ['%s:%s' % (SERVER_TAG, server_name)] + for status in PROCESS_STATUS: + self.gauge('supervisord.process.count', count_by_status[status], + tags=tags + ['status:%s' % PROCESS_STATUS[status]]) + + @staticmethod + def _connect(instance): + sock = instance.get('socket') + if sock is not None: + host = instance.get('host', DEFAULT_SOCKET_IP) + transport = supervisor.xmlrpc.SupervisorTransport(None, None, sock) + server = xmlrpclib.ServerProxy(host, transport=transport) + else: + host = instance.get('host', DEFAULT_HOST) + port = instance.get('port', DEFAULT_PORT) + user = instance.get('user') + password = instance.get('pass') + auth = '%s:%s@' % (user, password) if user and password else '' + server = xmlrpclib.Server('http://%s%s:%s/RPC2' % (auth, host, port)) + return server.supervisor + + @staticmethod + def _extract_uptime(proc): + start, now = int(proc['start']), int(proc['now']) + status = proc['statename'] + active_state = status in ['BACKOFF', 'RUNNING', 'STOPPING'] + return now - start if active_state else 0 + + @staticmethod + def _build_message(proc): + start, stop, now = int(proc['start']), int(proc['stop']), int(proc['now']) + proc['now_str'] = FORMAT_TIME(now) + proc['start_str'] = FORMAT_TIME(start) + proc['stop_str'] = '' if stop == 0 else FORMAT_TIME(stop) + + return """Current time: %(now_str)s +Process name: %(name)s +Process group: %(group)s +Description: %(description)s +Error log file: %(stderr_logfile)s +Stdout log file: %(stdout_logfile)s +Log file: %(logfile)s +State: %(statename)s +Start time: %(start_str)s +Stop time: %(stop_str)s +Exit Status: %(exitstatus)s""" % proc diff --git a/ci/resources/supervisord/program_1.sh b/ci/resources/supervisord/program_1.sh new file mode 100755 index 0000000000..f27c85ec09 --- /dev/null +++ b/ci/resources/supervisord/program_1.sh @@ -0,0 +1,3 @@ +# dummy program that runs for 30 seconds and dies + +sleep 30 \ No newline at end of file diff --git a/ci/resources/supervisord/program_2.sh b/ci/resources/supervisord/program_2.sh new file mode 100755 index 0000000000..c6285ba59b --- /dev/null +++ b/ci/resources/supervisord/program_2.sh @@ -0,0 +1,3 @@ +# dummy program that runs for 60 seconds and dies + +sleep 60 \ No newline at end of file diff --git a/ci/resources/supervisord/program_3.sh b/ci/resources/supervisord/program_3.sh new file mode 100755 index 0000000000..6d2e7a75b1 --- /dev/null +++ b/ci/resources/supervisord/program_3.sh @@ -0,0 +1,3 @@ +# dummy program that runs for 90 seconds and dies + +sleep 90 \ No newline at end of file diff --git a/ci/resources/supervisord/supervisord.conf b/ci/resources/supervisord/supervisord.conf new file mode 100644 index 0000000000..5f9474174d --- /dev/null +++ b/ci/resources/supervisord/supervisord.conf @@ -0,0 +1,31 @@ +[unix_http_server] +file=VOLATILE_DIR/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix://VOLATILE_DIR//supervisor.sock + +[supervisord] +logfile=VOLATILE_DIR/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=VOLATILE_DIR/supervisord.pid +childlogdir=VOLATILE_DIR + +[program:program_1] +command=/bin/sh VOLATILE_DIR/program_1.sh +autostart=true +autorestart=false + +[program:program_2] +command=/bin/sh VOLATILE_DIR/program_2.sh +autostart=true +autorestart=false + +[program:program_3] +command=/bin/sh VOLATILE_DIR/program_3.sh +autostart=true +autorestart=false diff --git a/ci/resources/supervisord/supervisord.yaml b/ci/resources/supervisord/supervisord.yaml new file mode 100755 index 0000000000..5bc09be5db --- /dev/null +++ b/ci/resources/supervisord/supervisord.yaml @@ -0,0 +1,6 @@ +init_config: + +instances: + - name: travis + socket: unix://VOLATILE_DIR//supervisor.sock + host: http://127.0.0.1 diff --git a/ci/supervisord.rb b/ci/supervisord.rb new file mode 100644 index 0000000000..e2514386a9 --- /dev/null +++ b/ci/supervisord.rb @@ -0,0 +1,65 @@ +require './ci/common' + +namespace :ci do + namespace :supervisord do |flavor| + task :before_install => ['ci:common:before_install'] + + task :install => ['ci:common:install'] do + sh %(pip install supervisor) + end + + task :before_script => ['ci:common:before_script'] do + sh %(cp $TRAVIS_BUILD_DIR/ci/resources/supervisord/supervisord.conf $VOLATILE_DIR/) + sh %(sed -i -- 's/VOLATILE_DIR/#{ENV['VOLATILE_DIR'].gsub '/','\/'}/g' $VOLATILE_DIR/supervisord.conf) + + sh %(cp $TRAVIS_BUILD_DIR/ci/resources/supervisord/supervisord.yaml $VOLATILE_DIR/) + sh %(sed -i -- 's/VOLATILE_DIR/#{ENV['VOLATILE_DIR'].gsub '/','\/'}/g' $VOLATILE_DIR/supervisord.yaml) + + sh %(cp $TRAVIS_BUILD_DIR/ci/resources/supervisord/program_1.sh $VOLATILE_DIR/) + sh %(cp $TRAVIS_BUILD_DIR/ci/resources/supervisord/program_2.sh $VOLATILE_DIR/) + sh %(cp $TRAVIS_BUILD_DIR/ci/resources/supervisord/program_3.sh $VOLATILE_DIR/) + sh %(chmod a+x $VOLATILE_DIR/program_*.sh) + + sh %(supervisord -c $VOLATILE_DIR/supervisord.conf) + sh %(sed -i -- 's/VOLATILE_DIR/#{ENV['VOLATILE_DIR'].gsub '/','\/'}/g' $VOLATILE_DIR/supervisord.conf) + + sleep_for 10 + end + + task :script => ['ci:common:script'] do + Rake::Task['ci:common:run_tests'].invoke(['supervisord']) + end + + task :cleanup => ['ci:common:cleanup'] do + sh %(rm -f $VOLATILE_DIR/supervisord.conf) + sh %(rm -f $VOLATILE_DIR/supervisord.yaml) + sh %(rm -f $VOLATILE_DIR/program*.sh) + sh %(rm -f $VOLATILE_DIR/supervisord.conf) + sh %(rm -f $VOLATILE_DIR/supervisord.log) + sh %(rm -f $VOLATILE_DIR/supervisord.pid) + sh %(rm -f $VOLATILE_DIR/program*.log) + sh %(rm -f $VOLATILE_DIR/ci.log) + + sh %(unlink $VOLATILE_DIR/supervisor.sock) + end + + task :execute do + exception = nil + begin + %w(before_install install before_script script).each do |t| + Rake::Task["#{flavor.scope.path}:#{t}"].invoke + end + rescue => e + exception = e + puts "Failed task: #{e.class} #{e.message}".red + end + if ENV['SKIP_CLEANUP'] + puts 'Skipping cleanup, disposable environments are great'.yellow + else + puts 'Cleaning up' + Rake::Task["#{flavor.scope.path}:cleanup"].invoke + end + fail exception if exception + end + end +end diff --git a/conf.d/supervisord.yaml.example b/conf.d/supervisord.yaml.example new file mode 100755 index 0000000000..d54c88bd8c --- /dev/null +++ b/conf.d/supervisord.yaml.example @@ -0,0 +1,49 @@ +# +# There are two ways to get started with the supervisord check. +# +# You can configure inet_http_server in /etc/supervisord.conf. Below is an +# example inet_http_server configuration: +# +# [inet_http_server] +# port:localhost:9001 +# username:user # optional +# password:pass # optional +# +# OR, you can use supervisorctl socket to communicate with supervisor. +# If supervisor is running as root, make sure chmod property is set +# to a permission accessible to non-root users. See the example below: +# +# [supervisorctl] +# serverurl=unix:///var/run//supervisor.sock +# +# [unix_http_server] +# file=/var/run/supervisor.sock +# chmod=777 +# +# Reload supervsior, specify the inet or unix socket server information +# in this yaml file along with an optional list of the processes you want +# to monitor per instance, and you're good to go! +# +# See http://supervisord.org/configuration.html for more information on +# configuring supervisord sockets and inet http servers. +# + +init_config: + +instances: +# - name: server0 # Required. An arbitrary name to identify the supervisord server +# host: localhost # Optional. Defaults to localhost. The host where supervisord server is running +# port: 9001 # Optional. Defaults to 9001. The port number. +# user: user # Optional. Required only if a username is configured. +# pass: pass # Optional. Required only if a password is configured. +# proc_names: # Optional. The process to monitor within this supervisord instance. +# - apache2 # If not specified, the check will monitor all processes. +# - webapp +# - java +# server_check: false # Optional. Defaults to true. Service check for connection to supervisord server. +# - name: server1 +# host: localhost +# port: 9002 +# - name: server2 +# socket: unix:///var/run//supervisor.sock +# host: http://127.0.0.1 # Optional. Defaults to http://127.0.0.1 diff --git a/requirements.txt b/requirements.txt index 3014816b51..d512a182d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ httplib2 kafka-python==0.9.0-9bed11db98387c0d9e456528130b330631dc50af requests paramiko +supervisor==3.1.3 diff --git a/tests/test_supervisord.py b/tests/test_supervisord.py new file mode 100644 index 0000000000..5727139a07 --- /dev/null +++ b/tests/test_supervisord.py @@ -0,0 +1,460 @@ +import os +from socket import socket +from time import sleep + +from mock import patch +from nose.plugins.attrib import attr +from nose.tools import ok_, eq_ +import unittest +import xmlrpclib + +from checks import AgentCheck +from tests.common import get_check + +class TestSupervisordCheck(unittest.TestCase): + + TEST_CASES = [{ + 'yaml': """ +init_config: +instances: + - name: server1 + host: localhost + port: 9001""", + 'expected_instances': [{ + 'host': 'localhost', + 'name': 'server1', + 'port': 9001 + }], + 'expected_metrics': { + 'server1': [ + ('supervisord.process.count', 1, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:up']}), + ('supervisord.process.count', 1, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:down']}), + ('supervisord.process.count', 1, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:unknown']}), + ('supervisord.process.uptime', 0, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'supervisord_process:python']}), + ('supervisord.process.uptime', 125, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'supervisord_process:mysql']}), + ('supervisord.process.uptime', 0, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'supervisord_process:java']}) + ] + }, + 'expected_service_checks': { + 'server1': [{ + 'status': AgentCheck.OK, + 'tags': ['supervisord_server:server1', 'supervisord_process:mysql'], + 'check': 'supervisord.process.check' + }, { + 'status': AgentCheck.CRITICAL, + 'tags': ['supervisord_server:server1', 'supervisord_process:java'], + 'check': 'supervisord.process.check' + }, { + 'status': AgentCheck.UNKNOWN, + 'tags': ['supervisord_server:server1', 'supervisord_process:python'], + 'check': 'supervisord.process.check' + }] + } + }, { + 'yaml': """ +init_config: + +instances: + - name: server0 + host: localhost + port: 9001 + user: user + pass: pass + proc_names: + - apache2 + - webapp + - name: server1 + host: 10.60.130.82""", + 'expected_instances': [{ + 'name': 'server0', + 'host': 'localhost', + 'port': 9001, + 'user': 'user', + 'pass': 'pass', + 'proc_names': ['apache2', 'webapp'], + }, { + 'host': '10.60.130.82', + 'name': 'server1' + }], + 'expected_metrics': { + 'server0': [ + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:up']}), + ('supervisord.process.count', 2, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:down']}), + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:unknown']}), + ('supervisord.process.uptime', 0, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'supervisord_process:apache2']}), + ('supervisord.process.uptime', 2, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'supervisord_process:webapp']}), + ], + 'server1': [ + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:up']}), + ('supervisord.process.count', 1, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:down']}), + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'status:unknown']}), + ('supervisord.process.uptime', 0, {'type': 'gauge', 'tags': ['supervisord_server:server1', 'supervisord_process:ruby']}) + ] + }, + 'expected_service_checks': { + 'server0': [{ + 'status': AgentCheck.CRITICAL, + 'tags': ['supervisord_server:server0', 'supervisord_process:apache2'], + 'check': 'supervisord.process.check' + }, { + 'status': AgentCheck.CRITICAL, + 'tags': ['supervisord_server:server0', 'supervisord_process:webapp'], + 'check': 'supervisord.process.check' + }], + 'server1': [{ + 'status': AgentCheck.CRITICAL, + 'tags': ['supervisord_server:server1', 'supervisord_process:ruby'], + 'check': 'supervisord.process.check' + }] + } + }, { + 'yaml': """ +init_config: + +instances: + - name: server0 + host: invalid_host + port: 9009""", + 'expected_instances': [{ + 'name': 'server0', + 'host': 'invalid_host', + 'port': 9009 + }], + 'error_message': """Cannot connect to http://invalid_host:9009. Make sure supervisor is running and XML-RPC inet interface is enabled.""" + }, { + 'yaml': """ +init_config: + +instances: + - name: server0 + host: localhost + port: 9010 + user: invalid_user + pass: invalid_pass""", + 'expected_instances': [{ + 'name': 'server0', + 'host': 'localhost', + 'port': 9010, + 'user': 'invalid_user', + 'pass': 'invalid_pass' + }], + 'error_message': """Username or password to server0 are incorrect.""" + }, { + 'yaml': """ +init_config: + +instances: + - name: server0 + host: localhost + port: 9001 + proc_names: + - mysql + - invalid_process""", + 'expected_instances': [{ + 'name': 'server0', + 'host': 'localhost', + 'port': 9001, + 'proc_names': ['mysql', 'invalid_process'] + }], + 'expected_metrics': { + 'server0': [ + ('supervisord.process.count', 1, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:up']}), + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:down']}), + ('supervisord.process.count', 0, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'status:unknown']}), + ('supervisord.process.uptime', 125, {'type': 'gauge', 'tags': ['supervisord_server:server0', 'supervisord_process:mysql']}) + ] + }, + 'expected_service_checks': { + 'server0': [{ + 'status': AgentCheck.OK, + 'tags': ['supervisord_server:server0', 'supervisord_process:mysql'], + 'check': 'supervisord.process.check' + }] + } + }] + + def setUp(self): + self.patcher = patch('xmlrpclib.Server', self.mock_server) + self.patcher.start() + + def tearDown(self): + self.patcher.stop() + + ### Integration Test ##################################################### + + def test_check(self): + """Integration test for supervisord check. Using a mocked supervisord.""" + for tc in self.TEST_CASES: + check, instances = get_check('supervisord', tc['yaml']) + ok_(check is not None, msg=check) + eq_(tc['expected_instances'], instances) + for instance in instances: + name = instance['name'] + + try: + # Run the check + check.check(instance) + except Exception, e: + if 'error_message' in tc: # excepted error + eq_(str(e), tc['error_message']) + else: + ok_(False, msg=str(e)) + else: + # Assert that the check collected the right metrics + expected_metrics = tc['expected_metrics'][name] + self.assert_metrics(expected_metrics, check.get_metrics()) + + # Assert that the check generated the right service checks + expected_service_checks = tc['expected_service_checks'][name] + self.assert_service_checks(expected_service_checks, + check.get_service_checks()) + + @attr(requires='supervisord') + def test_travis_supervisord(self): + """Integration test for supervisord check. Using a supervisord on Travis.""" + + # Load yaml config + config_str = open(os.environ['VOLATILE_DIR'] + '/supervisord.yaml', 'r').read() + ok_(config_str is not None and len(config_str) > 0, msg=config_str) + + # init the check and get the instances + check, instances = get_check('supervisord', config_str) + ok_(check is not None, msg=check) + eq_(len(instances), 1) + + + # Supervisord should run 3 programs for 30, 60 and 90 seconds + # respectively. The tests below will ensure that the process count + # metric is reported correctly after (roughly) 10, 40, 70 and 100 seconds + for i in range(4): + try: + # Run the check + check.check(instances[0]) + except Exception, e: + # Make sure that it ran successfully + ok_(False, msg=str(e)) + else: + up, down = 0, 0 + for name, timestamp, value, meta in check.get_metrics(): + if name == 'supervisord.process.count': + if 'status:up' in meta['tags']: + up = value + elif 'status:down' in meta['tags']: + down = value + eq_(up, 3 - i) + eq_(down, i) + sleep(30) + + + ### Unit Tests ########################################################### + + def test_build_message(self): + """Unit test supervisord build service check message.""" + process = { + 'now': 1414815513, + 'group': 'mysql', + 'description': 'pid 787, uptime 0:02:05', + 'pid': 787, + 'stderr_logfile': '/var/log/supervisor/mysql-stderr---supervisor-3ATI82.log', + 'stop': 0, + 'statename': 'RUNNING', + 'start': 1414815388, + 'state': 20, + 'stdout_logfile': '/var/log/mysql/mysql.log', + 'logfile': '/var/log/mysql/mysql.log', + 'exitstatus': 0, + 'spawnerr': '', + 'name': 'mysql' + } + + expected_message = """Current time: 2014-11-01 04:18:33 +Process name: mysql +Process group: mysql +Description: pid 787, uptime 0:02:05 +Error log file: /var/log/supervisor/mysql-stderr---supervisor-3ATI82.log +Stdout log file: /var/log/mysql/mysql.log +Log file: /var/log/mysql/mysql.log +State: RUNNING +Start time: 2014-11-01 04:16:28 +Stop time: \nExit Status: 0""" + + check, _ = get_check('supervisord', self.TEST_CASES[0]['yaml']) + eq_(expected_message, check._build_message(process)) + + ### Helper Methods ####################################################### + + @staticmethod + def mock_server(url): + return MockXmlRcpServer(url) + + @staticmethod + def assert_metrics(expected, actual): + actual = [TestSupervisordCheck.norm_metric(metric) for metric in actual] + eq_(len(actual), len(expected), msg='Invalid # metrics reported.\n' + 'Expected: {0}. Found: {1}'.format(len(expected), len(actual))) + ok_(all([expected_metric in actual for expected_metric in expected]), + msg='Reported metrics are incorrect.\nExpected: {0}.\n' + 'Found: {1}'.format(expected, actual)) + + @staticmethod + def assert_service_checks(expected, actual): + actual = [TestSupervisordCheck.norm_service_check(service_check) + for service_check in actual] + eq_(len(actual), len(expected), msg='Invalid # service checks reported.' + '\nExpected: {0}. Found: {1}.'.format(len(expected), len(actual))) + ok_(all([expected_service_check in actual + for expected_service_check in expected]), + msg='Reported service checks are incorrect.\nExpected:{0}\n' + 'Found:{1}'.format(expected, actual)) + + @staticmethod + def norm_metric(metric): + '''Removes hostname and timestamp''' + metric[3].pop('hostname') + return (metric[0], metric[2], metric[3]) + + @staticmethod + def norm_service_check(service_check): + '''Removes timestamp, host_name, message and id''' + for field in ['timestamp', 'host_name', 'message', 'id']: + service_check.pop(field) + return service_check + + +class MockXmlRcpServer: + """Class that mocks an XML RPC server. Initialized using a mocked + supervisord server url, which is used to initialize the supervisord + server. + """ + def __init__(self, url): + self.supervisor = MockSupervisor(url) + + +class MockSupervisor: + """Class that mocks a supervisord sever. Initialized using the server url + and mocks process methods providing mocked process information for testing + purposes. + """ + MOCK_PROCESSES = { + 'http://localhost:9001/RPC2': [{ + 'now': 1414815513, + 'group': 'mysql', + 'description': 'pid 787, uptime 0:02:05', + 'pid': 787, + 'stderr_logfile': '/var/log/supervisor/mysql-stderr---supervisor-3ATI82.log', + 'stop': 0, + 'statename': 'RUNNING', + 'start': 1414815388, + 'state': 20, + 'stdout_logfile': '/var/log/mysql/mysql.log', + 'logfile': '/var/log/mysql/mysql.log', + 'exitstatus': 0, + 'spawnerr': '', + 'name': 'mysql' + }, { + 'now': 1414815738, + 'group': 'java', + 'description': 'Nov 01 04:22 AM', + 'pid': 0, + 'stderr_logfile': '/var/log/supervisor/java-stderr---supervisor-lSdcKZ.log', + 'stop': 1414815722, + 'statename': 'STOPPED', + 'start': 1414815388, + 'state': 0, + 'stdout_logfile': '/var/log/java/java.log', + 'logfile': '/var/log/java/java.log', + 'exitstatus': 21, + 'spawnerr': '', + 'name': 'java' + }, { + 'now': 1414815738, + 'group': 'python', + 'description': '', + 'pid': 2765, + 'stderr_logfile': '/var/log/supervisor/python-stderr---supervisor-vFzxIg.log', + 'stop': 1414815737, + 'statename': 'STARTING', + 'start': 1414815737, + 'state': 10, + 'stdout_logfile': '/var/log/python/python.log', + 'logfile': '/var/log/python/python.log', + 'exitstatus': 0, + 'spawnerr': '', + 'name': 'python' + }], + 'http://user:pass@localhost:9001/RPC2': [{ + 'now': 1414869824, + 'group': 'apache2', + 'description': 'Exited too quickly (process log may have details)', + 'pid': 0, + 'stderr_logfile': '/var/log/supervisor/apache2-stderr---supervisor-0PkXWd.log', + 'stop': 1414867047, + 'statename': 'FATAL', + 'start': 1414867047, + 'state': 200, + 'stdout_logfile': '/var/log/apache2/apache2.log', + 'logfile': '/var/log/apache2/apache2.log', + 'exitstatus': 0, + 'spawnerr': 'Exited too quickly (process log may have details)', + 'name': 'apache2' + }, { + 'now': 1414871104, + 'group': 'webapp', + 'description': '', + 'pid': 17600, + 'stderr_logfile': '/var/log/supervisor/webapp-stderr---supervisor-onZK__.log', + 'stop': 1414871101, + 'statename': 'STOPPING', + 'start': 1414871102, + 'state': 40, + 'stdout_logfile': '/var/log/company/webapp.log', + 'logfile': '/var/log/company/webapp.log', + 'exitstatus': 1, + 'spawnerr': '', + 'name': 'webapp' + }], + 'http://10.60.130.82:9001/RPC2': [{ + 'now': 1414871588, + 'group': 'ruby', + 'description': 'Exited too quickly (process log may have details)', + 'pid': 0, + 'stderr_logfile': '/var/log/supervisor/ruby-stderr---supervisor-BU7Wat.log', + 'stop': 1414871588, + 'statename': 'BACKOFF', + 'start': 1414871588, + 'state': 30, + 'stdout_logfile': '/var/log/ruby/ruby.log', + 'logfile': '/var/log/ruby/ruby.log', + 'exitstatus': 0, + 'spawnerr': 'Exited too quickly (process log may have details)', + 'name': 'ruby' + }] + } + + def __init__(self, url): + self.url = url + + def getAllProcessInfo(self): + self._validate_request() + return self.MOCK_PROCESSES[self.url] + + def getProcessInfo(self, proc_name): + self._validate_request(proc=proc_name) + for proc in self.MOCK_PROCESSES[self.url]: + if proc['name'] == proc_name: + return proc + raise Exception('Process not found: %s' % proc_name) + + def _validate_request(self, proc=None): + '''Validates request and simulates errors when not valid''' + if 'invalid_host' in self.url: + # Simulate connecting to an invalid host/port in order to + # raise `socket.error: [Errno 111] Connection refused` + socket().connect(('localhost', 38837)) + elif 'invalid_pass' in self.url: + # Simulate xmlrpc exception for invalid credentials + raise xmlrpclib.ProtocolError(self.url[7:], 401, + 'Unauthorized', None) + elif proc is not None and 'invalid' in proc: + # Simulate xmlrpc exception for process not found + raise xmlrpclib.Fault(10, 'BAD_NAME') \ No newline at end of file