From 902c4c9b3f9ffcee71f6dadcc3125da576e9ef93 Mon Sep 17 00:00:00 2001 From: Jorge Bastida Date: Fri, 26 Jun 2015 17:06:27 +0200 Subject: [PATCH] First python 3.4 working version --- .travis.yml | 3 +- CHANGELOG | 8 + README.rst | 5 +- awslogs/__init__.py | 4 +- awslogs/bin.py | 28 +-- awslogs/core.py | 57 +---- setup.py | 4 +- tests.py | 601 +++++++++++++------------------------------- 8 files changed, 205 insertions(+), 505 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c76f08..8c14c79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: python - python: + - "3.4" + - "3.3" - "2.7" - "2.6" diff --git a/CHANGELOG b/CHANGELOG index ea350d6..7bddbb4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,11 @@ +0.0.3 +===== +* awslogs now use ``boto3`` instead of ``boto2``. +* awslogs don't longer require ``gevent``. +* massive refactoring of the internals now that ``aws`` provides ``filter_log_events`` out the box. +* awslogs don't longer support retrieving logs from several groups at the same time. +* awslogs now support python python: 2.7, 3.3 and 3.4 + 0.0.3 ===== * Filter streams by ``start`` and ``end`` in order to reduce the initial volume of queries #9. diff --git a/README.rst b/README.rst index 83fbc8e..cccf5c2 100644 --- a/README.rst +++ b/README.rst @@ -24,11 +24,10 @@ One of the most powerful features is to query events from several streams and co Features -------- -* Aggregate logs from accross streams and groups. +* Aggregate logs from accross streams. - Aggregate all streams in a group. - Aggregate streams matching a regular expression. - - Filter both groups and streams using regular expressions. * Colored output. * List existing groups @@ -72,7 +71,7 @@ Options * ``awslogs groups``: List existing groups * ``awslogs streams GROUP``: List existing streams withing ``GROUP`` -* ``awslogs get [GROUP_EXPRESSION [STREAM_EXPRESSION]]``: Get logs matching ``GROUP_EXPRESSION`` and ``STREAM_EXPRESSION`` +* ``awslogs get [GROUP [STREAM_EXPRESSION]]``: Get logs matching ``STREAM_EXPRESSION`` in ``GROUP``. - Expressions can be regular expressions or the wildcard ``ALL`` if you want any and don't want to type ``.*``. diff --git a/awslogs/__init__.py b/awslogs/__init__.py index 2bb309c..cf034c0 100644 --- a/awslogs/__init__.py +++ b/awslogs/__init__.py @@ -1,2 +1,2 @@ -from core import __version__ -from core import AWSLogs +from .core import __version__ +from .core import AWSLogs diff --git a/awslogs/bin.py b/awslogs/bin.py index 716a40a..55f1ab0 100644 --- a/awslogs/bin.py +++ b/awslogs/bin.py @@ -6,15 +6,15 @@ import boto3 from termcolor import colored -import exceptions -from core import AWSLogs +from . import exceptions +from .core import AWSLogs __version__ = "0.0.3" def keyboard_signal_handler(signal, frame): - print 'You pressed Ctrl+C!' + print('You pressed Ctrl+C!') sys.exit(0) signal.signal(signal.SIGINT, keyboard_signal_handler) @@ -29,37 +29,37 @@ def main(argv=None): def add_common_arguments(parser): parser.add_argument("--aws-access-key-id", dest="aws_access_key_id", - type=unicode, + type=str, default=None, help="aws access key id") parser.add_argument("--aws-secret-access-key", dest="aws_secret_access_key", - type=unicode, + type=str, default=None, help="aws secret access key") parser.add_argument("--aws-session-token", dest="aws_session_token", - type=unicode, + type=str, default=None, help="aws session token") parser.add_argument("--aws-region", dest="aws_region", - type=unicode, + type=str, default=os.environ.get('AWS_REGION', None), help="aws region") def add_date_range_arguments(parser): parser.add_argument("-s", "--start", - type=unicode, + type=str, dest='start', default='24h', help="Start time") parser.add_argument("-e", "--end", - type=unicode, + type=str, dest='end', help="End time") @@ -72,13 +72,13 @@ def add_date_range_arguments(parser): get_parser.add_argument("log_group_name", - type=unicode, + type=str, default="ALL", nargs='?', help="log group name") get_parser.add_argument("log_stream_name", - type=unicode, + type=str, default="ALL", nargs='?', help="log stream name") @@ -117,7 +117,7 @@ def add_date_range_arguments(parser): add_date_range_arguments(streams_parser) streams_parser.add_argument("log_group_name", - type=unicode, + type=str, help="log group name") # Parse input @@ -126,7 +126,7 @@ def add_date_range_arguments(parser): try: logs = AWSLogs(**vars(options)) getattr(logs, options.func)() - except exceptions.BaseAWSLogsException, exc: + except exceptions.BaseAWSLogsException as exc: sys.stderr.write(colored("{0}\n".format(exc.hint()), "red")) return exc.code except Exception: @@ -135,7 +135,7 @@ def add_date_range_arguments(parser): options = vars(options) options['aws_access_key_id'] = 'SENSITIVE' options['aws_secret_access_key'] = 'SENSITIVE' - options['aws_secret_access_key'] = 'SENSITIVE' + options['aws_session_token'] = 'SENSITIVE' sys.stderr.write("\n") sys.stderr.write("=" * 80) sys.stderr.write("\nYou've found a bug! Please, raise an issue attaching the following traceback\n") diff --git a/awslogs/core.py b/awslogs/core.py index f88d62f..4f8d63a 100644 --- a/awslogs/core.py +++ b/awslogs/core.py @@ -6,55 +6,12 @@ import boto3 from termcolor import colored -from boto import logs as botologs from dateutil.parser import parse -import exceptions +from . import exceptions __version__ = '0.1.0' -#NO_MORE_EVENTS = object() - -# -# class AWSConnection(object): -# """Wrapper on top of boto's ``connect_to_region`` which retry api -# calls if some well-known errors occur.""" -# -# def __init__(self, aws_region, *args, **kwargs): -# -# if aws_region not in (r.name for r in botologs.regions()): -# raise exceptions.InvalidRegionError(aws_region) -# -# try: -# self.connection = botologs.connect_to_region(aws_region, *args, **kwargs) -# except boto.exception.NoAuthHandlerFound, exc: -# raise exceptions.NoAuthHandlerFoundError(*exc.args) -# -# if not self.connection: -# raise exceptions.ConnectionError() -# -# def __bool__(self): -# return bool(self.connection) -# -# def __getattr__(self, name): -# -# def aws_connection_wrap(*args, **kwargs): -# while True: -# try: -# return getattr(self.connection, name)(*args, **kwargs) -# except boto.exception.JSONResponseError, exc: -# if exc.error_code == u'ThrottlingException': -# gevent.sleep(1) -# continue -# elif exc.error_code == u'AccessDeniedException': -# hint = exc.body.get('Message', 'AccessDeniedException') -# raise exceptions.AccessDeniedError(hint) -# raise -# except Exception, exc: -# raise -# -# return aws_connection_wrap - class AWSLogs(object): @@ -134,17 +91,17 @@ def get_logs(self): def list_logs(self): for event in self.get_logs(): - print event + print(event) def list_groups(self): """Lists available CloudWatch logs groups""" for group in self.get_groups(): - print group + print(group) - def list_streams(self, *args, **kwargs): + def list_streams(self): """Lists available CloudWatch logs streams in ``log_group_name``.""" - for stream in self.get_streams(*args, **kwargs): - print stream + for stream in self.get_streams(): + print(stream) def get_groups(self): """Returns available CloudWatch logs groups""" @@ -164,7 +121,7 @@ def get_streams(self, log_group_name=None): """Returns available CloudWatch logs streams in ``log_group_name``.""" kwargs = {'logGroupName': log_group_name or self.log_group_name} window_start = self.start or 0 - window_end = self.end or sys.maxint + window_end = self.end or sys.maxsize while True: response = self.client.describe_log_streams(**kwargs) diff --git a/setup.py b/setup.py index fb6a79c..88259fb 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,8 @@ install_requires = [ 'boto3>=1.0.0', 'termcolor>=1.1', - 'python-dateutil>=2.4.0' + 'python-dateutil>=2.4.0', + 'future' ] tests_require = [] @@ -18,7 +19,6 @@ if sys.version_info < (3, 3): tests_require.append('mock>=1.0') - setup( name='awslogs', version='0.1.0', diff --git a/tests.py b/tests.py index 0d79744..5fabc10 100644 --- a/tests.py +++ b/tests.py @@ -1,104 +1,112 @@ import sys import unittest from datetime import datetime -from StringIO import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO -import boto -import gevent -from gevent.pool import Pool from termcolor import colored -from mock import Mock, patch, call + +try: + from mock import patch, Mock, call +except ImportError: + from unittest.mock import patch, Mock, call from awslogs import AWSLogs from awslogs.exceptions import UnknownDateError, ConnectionError -from awslogs.core import NO_MORE_EVENTS from awslogs.bin import main class TestAWSLogs(unittest.TestCase): - def setUp(self): - super(TestAWSLogs, self).setUp() - self.aws = AWSLogs(connection_cls=Mock) + # def setUp(self): + # super(TestAWSLogs, self).setUp() + # self.aws = AWSLogs() - def _stream(self, name, start=0, end=sys.maxint): + def _stream(self, name, start=0, end=sys.maxsize): return {'logStreamName': name, 'firstEventTimestamp': start, 'lastEventTimestamp': end} @patch('awslogs.core.datetime') def test_parse_datetime(self, datetime_mock): + + awslogs = AWSLogs() datetime_mock.now.return_value = datetime(2015, 1, 1, 3, 0, 0, 0) def epoch(dt): return int(dt.strftime("%s")) * 1000 - self.assertEqual(self.aws.parse_datetime(''), None) - self.assertEqual(self.aws.parse_datetime(None), None) + self.assertEqual(awslogs.parse_datetime(''), None) + self.assertEqual(awslogs.parse_datetime(None), None) - self.assertEqual(self.aws.parse_datetime('1m'), + self.assertEqual(awslogs.parse_datetime('1m'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1m ago'), + self.assertEqual(awslogs.parse_datetime('1m ago'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1minute'), + self.assertEqual(awslogs.parse_datetime('1minute'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1minute ago'), + self.assertEqual(awslogs.parse_datetime('1minute ago'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1minutes'), + self.assertEqual(awslogs.parse_datetime('1minutes'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1minutes ago'), + self.assertEqual(awslogs.parse_datetime('1minutes ago'), epoch(datetime(2015, 1, 1, 2, 59, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1h'), + self.assertEqual(awslogs.parse_datetime('1h'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1h ago'), + self.assertEqual(awslogs.parse_datetime('1h ago'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1hour'), + self.assertEqual(awslogs.parse_datetime('1hour'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1hour ago'), + self.assertEqual(awslogs.parse_datetime('1hour ago'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1hours'), + self.assertEqual(awslogs.parse_datetime('1hours'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1hours ago'), + self.assertEqual(awslogs.parse_datetime('1hours ago'), epoch(datetime(2015, 1, 1, 2, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1d'), + self.assertEqual(awslogs.parse_datetime('1d'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1d ago'), + self.assertEqual(awslogs.parse_datetime('1d ago'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1day'), + self.assertEqual(awslogs.parse_datetime('1day'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1day ago'), + self.assertEqual(awslogs.parse_datetime('1day ago'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1days'), + self.assertEqual(awslogs.parse_datetime('1days'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1days ago'), + self.assertEqual(awslogs.parse_datetime('1days ago'), epoch(datetime(2014, 12, 31, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1w'), + self.assertEqual(awslogs.parse_datetime('1w'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1w ago'), + self.assertEqual(awslogs.parse_datetime('1w ago'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1week'), + self.assertEqual(awslogs.parse_datetime('1week'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1week ago'), + self.assertEqual(awslogs.parse_datetime('1week ago'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1weeks'), + self.assertEqual(awslogs.parse_datetime('1weeks'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1weeks ago'), + self.assertEqual(awslogs.parse_datetime('1weeks ago'), epoch(datetime(2014, 12, 25, 3, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1/1/2013'), + self.assertEqual(awslogs.parse_datetime('1/1/2013'), epoch(datetime(2013, 1, 1, 0, 0, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1/1/2012 12:34'), + self.assertEqual(awslogs.parse_datetime('1/1/2012 12:34'), epoch(datetime(2012, 1, 1, 12, 34, 0, 0))) - self.assertEqual(self.aws.parse_datetime('1/1/2011 12:34:56'), + self.assertEqual(awslogs.parse_datetime('1/1/2011 12:34:56'), epoch(datetime(2011, 1, 1, 12, 34, 56, 0))) - self.assertRaises(UnknownDateError, self.aws.parse_datetime, '???') + self.assertRaises(UnknownDateError, awslogs.parse_datetime, '???') - def test_get_groups(self): - self.aws.connection.describe_log_groups.side_effect = [ + @patch('boto3.client') + def test_get_groups(self, botoclient): + client = Mock() + botoclient.return_value = client + client.describe_log_groups.side_effect = [ {'logGroups': [{'logGroupName': 'A'}, {'logGroupName': 'B'}, {'logGroupName': 'C'}], @@ -110,18 +118,19 @@ def test_get_groups(self): {'logGroups': [{'logGroupName': 'G'}]}, ] - expected = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] - self.assertEqual([g for g in self.aws.get_groups()], expected) + awslogs = AWSLogs() + self.assertEqual([g for g in awslogs.get_groups()], ['A', 'B', 'C', 'D', 'E', 'F', 'G']) - expected = [call(next_token=None), - call(next_token=1), - call(next_token=2)] + expected = [call(), + call(nextToken=1), + call(nextToken=2)] + self.assertEqual(client.describe_log_groups.call_args_list, expected) - self.assertEqual(self.aws.connection.describe_log_groups.call_args_list, - expected) - - def test_get_streams(self): - self.aws.connection.describe_log_streams.side_effect = [ + @patch('boto3.client') + def test_get_streams(self, botoclient): + client = Mock() + botoclient.return_value = client + client.describe_log_streams.side_effect = [ {'logStreams': [self._stream('A'), self._stream('B'), self._stream('C')], @@ -133,36 +142,37 @@ def test_get_streams(self): {'logStreams': [self._stream('G')]}, ] - expected = ['A', 'B', 'C', 'D', 'E', 'F', 'G'] - self.assertEqual([g for g in self.aws.get_streams('group')], expected) + awslogs = AWSLogs(log_group_name='group') + self.assertEqual([g for g in awslogs.get_streams()], ['A', 'B', 'C', 'D', 'E', 'F', 'G']) - expected = [call(log_group_name="group", next_token=None), - call(log_group_name="group", next_token=1), - call(log_group_name="group", next_token=2)] + expected = [call(logGroupName="group"), + call(logGroupName="group", nextToken=1), + call(logGroupName="group", nextToken=2)] - self.assertEqual(self.aws.connection.describe_log_streams.call_args_list, - expected) + self.assertEqual(client.describe_log_streams.call_args_list, expected) - def test_get_streams_filtered_by_date(self): - self.aws.connection.describe_log_streams.side_effect = [ + @patch('boto3.client') + @patch('awslogs.core.AWSLogs.parse_datetime') + def test_get_streams_filtered_by_date(self, parse_datetime, botoclient): + client = Mock() + botoclient.return_value = client + client.describe_log_streams.side_effect = [ {'logStreams': [self._stream('A', 0, 1), self._stream('B', 0, 6), self._stream('C'), - self._stream('D', sys.maxint - 1, sys.maxint)], + self._stream('D', sys.maxsize - 1, sys.maxsize)], } ] + parse_datetime.side_effect = [5, 7] + awslogs = AWSLogs(log_group_name='group', start='5', end='7') + self.assertEqual([g for g in awslogs.get_streams()], ['B', 'C']) + self.assertEqual(client.describe_log_streams.call_args_list, [call(logGroupName="group")]) - self.aws.start, self.aws.end = 5, 7 - expected = ['B', 'C'] - - self.assertEqual([g for g in self.aws.get_streams('group')], expected) + @patch('boto3.client') + def test_get_streams_from_pattern(self, botoclient): + client = Mock() + botoclient.return_value = client - expected = [call(log_group_name="group", next_token=None)] - - self.assertEqual(self.aws.connection.describe_log_streams.call_args_list, - expected) - - def test_get_streams_from_pattern(self): side_effect = [ {'logStreams': [self._stream('AAA'), self._stream('ABA'), @@ -175,291 +185,39 @@ def test_get_streams_from_pattern(self): {'logStreams': [self._stream('CAC')]}, ] - self.aws.connection.describe_log_streams.side_effect = side_effect - expected = ['AAA', 'ABA', 'ACA', 'BAA', 'BBA', 'BBB', 'CAC'] - actual = [s for s in self.aws._get_streams_from_pattern('X', 'ALL')] - self.assertEqual(actual, expected) - - self.aws.connection.describe_log_streams.side_effect = side_effect - expected = ['AAA', 'ABA', 'ACA'] - actual = [s for s in self.aws._get_streams_from_pattern('X', 'A')] - self.assertEqual(actual, expected) + awslogs = AWSLogs() - self.aws.connection.describe_log_streams.side_effect = side_effect - expected = ['AAA', 'ACA'] - actual = [s for s in self.aws._get_streams_from_pattern('X', 'A[AC]A')] - self.assertEqual(actual, expected) - - def test_get_groups_from_pattern(self): - side_effect = [ - {'logGroups': [{'logGroupName': 'AAA'}, - {'logGroupName': 'ABA'}, - {'logGroupName': 'ACA'}], - 'nextToken': 1}, - {'logGroups': [{'logGroupName': 'BAA'}, - {'logGroupName': 'BBA'}, - {'logGroupName': 'BBB'}], - 'nextToken': 2}, - {'logGroups': [{'logGroupName': 'CAC'}]}, - ] - - self.aws.connection.describe_log_groups.side_effect = side_effect + client.describe_log_streams.side_effect = side_effect expected = ['AAA', 'ABA', 'ACA', 'BAA', 'BBA', 'BBB', 'CAC'] - actual = [s for s in self.aws._get_groups_from_pattern('ALL')] + actual = [s for s in awslogs._get_streams_from_pattern('X', 'ALL')] self.assertEqual(actual, expected) - self.aws.connection.describe_log_groups.side_effect = side_effect + client.describe_log_streams.side_effect = side_effect expected = ['AAA', 'ABA', 'ACA'] - actual = [s for s in self.aws._get_groups_from_pattern('A')] + actual = [s for s in awslogs._get_streams_from_pattern('X', 'A')] self.assertEqual(actual, expected) - self.aws.connection.describe_log_groups.side_effect = side_effect + client.describe_log_streams.side_effect = side_effect expected = ['AAA', 'ACA'] - actual = [s for s in self.aws._get_groups_from_pattern('A[AC]A')] + actual = [s for s in awslogs._get_streams_from_pattern('X', 'A[AC]A')] self.assertEqual(actual, expected) - def test_get_streams_from_patterns(self): - groups = [ - {'logGroups': [{'logGroupName': 'AAA'}, - {'logGroupName': 'BAB'}, - {'logGroupName': 'CCC'}]}, - ] - - streams = [ - {'logStreams': [self._stream('ABB'), - self._stream('ABC'), - self._stream('ACD')]}, - {'logStreams': [self._stream('BBB'), - self._stream('BBD'), - self._stream('BBE')]}, - {'logStreams': [self._stream('CCC')]}, - ] - - self.aws.connection.describe_log_groups.side_effect = groups - self.aws.connection.describe_log_streams.side_effect = streams - expected = [('AAA', 'ABB'), ('AAA', 'ABC')] - actual = [s for s in self.aws._get_streams_from_patterns('A', 'AB')] - self.assertEqual(actual, expected) - - self.aws.connection.describe_log_groups.side_effect = groups - self.aws.connection.describe_log_streams.side_effect = streams - expected = [('AAA', 'ABB'), ('AAA', 'ABC'), ('BAB', 'BBB'), - ('BAB', 'BBD'), ('BAB', 'BBE')] - actual = [s for s in self.aws._get_streams_from_patterns('[AB]A.*', '.*B.*')] - self.assertEqual(actual, expected) - - def test_raw_events_queue_consumer_exit_if_exhausted(self): - self.aws.stream_status = {('A', 'B'): self.aws.EXHAUSTED} - pool = Pool(size=1) - pool.spawn(self.aws._raw_events_queue_consumer) - pool.join() - self.assertEqual(self.aws.events_queue.get(), NO_MORE_EVENTS) - self.assertTrue(self.aws.events_queue.empty()) - - def test_raw_events_queue_consumer_exit_when_exhausted(self): - self.aws.stream_status = {('A', 'B'): self.aws.EXHAUSTED} - self.aws.raw_events_queue.put((0, {'message': 'Hello'})) - pool = Pool(size=1) - pool.spawn(self.aws._raw_events_queue_consumer) - pool.join() - self.assertEqual(self.aws.events_queue.get(), 'Hello\n') - self.assertEqual(self.aws.events_queue.get(), NO_MORE_EVENTS) - self.assertTrue(self.aws.events_queue.empty()) - - @patch('awslogs.core.gevent.sleep') - @patch('awslogs.core.AWSLogs._get_min_timestamp') - @patch('awslogs.core.AWSLogs._get_all_streams_exhausted') - def test_raw_events_queue_consumer_waits_streams(self, _get_all_streams_exhausted, _get_min_timestamp, sleep): - _get_min_timestamp.side_effect = [5, 5, 6, 7, 8, 9, 10] - _get_all_streams_exhausted.side_effect = [ - False, - False, - False, - False, - False, - True, - True - ] - self.aws.stream_status = {('A', 'B'): self.aws.ACTIVE, - ('A', 'C'): self.aws.EXHAUSTED} - self.aws.raw_events_queue.put((8, {'message': 'Hello 8'})) - self.aws.raw_events_queue.put((7, {'message': 'Hello 7'})) - self.aws.raw_events_queue.put((9, {'message': 'Hello 9'})) - self.aws.raw_events_queue.put((6, {'message': 'Hello 6'})) - - pool = Pool(size=1) - pool.spawn(self.aws._raw_events_queue_consumer) - pool.join() - self.assertEqual(self.aws.events_queue.get(), 'Hello 6\n') - self.assertEqual(self.aws.events_queue.get(), 'Hello 7\n') - self.assertEqual(self.aws.events_queue.get(), 'Hello 8\n') - self.assertEqual(self.aws.events_queue.get(), 'Hello 9\n') - self.assertEqual(self.aws.events_queue.get(), NO_MORE_EVENTS) - self.assertTrue(self.aws.events_queue.empty()) - - self.assertEqual(sleep.call_args_list, [call(0.3), call(0.3)]) - - def test_publisher_queue_consumer_with_empty_queue(self): - self.aws.connection = Mock() - pool = Pool(size=1) - pool.spawn(self.aws._publisher_queue_consumer) - pool.join() - self.assertEqual(self.aws.connection.call_count, 0) - - def test_publisher_queue_consumer(self): - self.aws.publishers_queue.put((0, ('group', 'stream', None))) - self.aws.connection = Mock() - self.aws.connection.get_log_events.side_effect = [ - {'events': [{'timestamp': 1, 'message': 'Hello 1'}, - {'timestamp': 2, 'message': 'Hello 2'}, - {'timestamp': 3, 'message': 'Hello 3'}]} - ] - pool = Pool(size=1) - pool.spawn(self.aws._publisher_queue_consumer) - pool.join() - - self.assertEqual( - self.aws.raw_events_queue.get(), - (1, {'timestamp': 1, - 'message': 'Hello 1', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (2, {'timestamp': 2, - 'message': 'Hello 2', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (3, {'timestamp': 3, - 'message': 'Hello 3', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertTrue(self.aws.raw_events_queue.empty()) - self.assertTrue(self.aws.publishers_queue.empty()) - - def test_publisher_queue_consumer_paginated(self): - self.aws.publishers_queue.put((0, ('group', 'stream', None))) - self.aws.connection = Mock() - self.aws.connection.get_log_events.side_effect = [ - {'events': [{'timestamp': 1, 'message': 'Hello 1'}, - {'timestamp': 2, 'message': 'Hello 2'}, - {'timestamp': 3, 'message': 'Hello 3'}], - 'nextForwardToken': 'token'}, - {'events': [{'timestamp': 4, 'message': 'Hello 4'}, - {'timestamp': 5, 'message': 'Hello 5'}, - {'timestamp': 6, 'message': 'Hello 6'}]} - ] - pool = Pool(size=1) - pool.spawn(self.aws._publisher_queue_consumer) - pool.join() - - self.assertEqual( - self.aws.raw_events_queue.get(), - (1, {'timestamp': 1, - 'message': 'Hello 1', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (2, {'timestamp': 2, - 'message': 'Hello 2', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (3, {'timestamp': 3, - 'message': 'Hello 3', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (4, {'timestamp': 4, - 'message': 'Hello 4', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (5, {'timestamp': 5, - 'message': 'Hello 5', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertEqual( - self.aws.raw_events_queue.get(), - (6, {'timestamp': 6, - 'message': 'Hello 6', - 'stream': 'stream', - 'group': 'group'}) - ) - - self.assertTrue(self.aws.raw_events_queue.empty()) - self.assertTrue(self.aws.publishers_queue.empty()) - - def test_get_min_timestamp(self): - self.assertEqual(self.aws._get_min_timestamp(), None) - - self.aws.stream_status = {('A', 'A'): AWSLogs.ACTIVE, - ('B', 'B'): AWSLogs.ACTIVE, - ('C', 'C'): AWSLogs.EXHAUSTED} - self.aws.stream_max_timestamp = { - ('A', 'A'): datetime(2015, 1, 1, 13, 30), - ('B', 'B'): datetime(2015, 1, 1, 14, 30), - ('C', 'C'): datetime(2015, 1, 1, 15, 30) - } - - self.assertEqual(self.aws._get_min_timestamp(), - datetime(2015, 1, 1, 13, 30)) - - self.aws.stream_status[('A', 'A')] = AWSLogs.EXHAUSTED - self.assertEqual(self.aws._get_min_timestamp(), - datetime(2015, 1, 1, 14, 30)) - - self.aws.stream_status[('B', 'B')] = AWSLogs.EXHAUSTED - self.assertEqual(self.aws._get_min_timestamp(), None) - - def test_get_all_streams_exhausted(self): - self.aws.stream_status = {} - self.assertTrue(self.aws._get_all_streams_exhausted()) - - self.aws.stream_status = {('A', 'A'): AWSLogs.ACTIVE, - ('B', 'B'): AWSLogs.EXHAUSTED} - self.assertFalse(self.aws._get_all_streams_exhausted()) - - self.aws.stream_status = {('A', 'A'): AWSLogs.EXHAUSTED, - ('B', 'B'): AWSLogs.EXHAUSTED} - self.assertTrue(self.aws._get_all_streams_exhausted()) - - @patch('awslogs.core.AWSConnection') + @patch('boto3.client') @patch('sys.stdout', new_callable=StringIO) - def test_main_get(self, mock_stdout, AWSConnection): - instance = Mock() - AWSConnection.return_value = instance + def test_main_get(self, mock_stdout, botoclient): + client = Mock() + botoclient.return_value = client + awslogs = AWSLogs() + logs = [ - {'events': [{'timestamp': 1, 'message': 'Hello 1'}, - {'timestamp': 2, 'message': 'Hello 2'}, - {'timestamp': 3, 'message': 'Hello 3'}], - 'nextForwardToken': 'token'}, - {'events': [{'timestamp': 4, 'message': 'Hello 4'}, - {'timestamp': 5, 'message': 'Hello 5'}, - {'timestamp': 6, 'message': 'Hello 6'}], - 'nextForwardToken': 'token'}, + {'events': [{'timestamp': 1, 'message': 'Hello 1', 'logStreamName': 'DDD'}, + {'timestamp': 2, 'message': 'Hello 2', 'logStreamName': 'EEE'}, + {'timestamp': 3, 'message': 'Hello 3', 'logStreamName': 'DDD'}], + 'nextToken': 'token'}, + {'events': [{'timestamp': 4, 'message': 'Hello 4', 'logStreamName': 'EEE'}, + {'timestamp': 5, 'message': 'Hello 5', 'logStreamName': 'DDD'}, + {'timestamp': 6, 'message': 'Hello 6', 'logStreamName': 'EEE'}], + 'nextToken': 'token'}, {'events': []} ] @@ -474,26 +232,28 @@ def test_main_get(self, mock_stdout, AWSConnection): self._stream('EEE')]} ] - instance.get_log_events.side_effect = logs - instance.describe_log_groups.side_effect = groups - instance.describe_log_streams.side_effect = streams + client.filter_log_events.side_effect = logs + client.describe_log_groups.side_effect = groups + client.describe_log_streams.side_effect = streams main("awslogs get AAA DDD --no-color".split()) + self.assertEqual( mock_stdout.getvalue(), ("AAA DDD Hello 1\n" - "AAA DDD Hello 2\n" + "AAA EEE Hello 2\n" "AAA DDD Hello 3\n" - "AAA DDD Hello 4\n" + "AAA EEE Hello 4\n" "AAA DDD Hello 5\n" - "AAA DDD Hello 6\n") + "AAA EEE Hello 6\n") ) - @patch('awslogs.core.AWSConnection') + @patch('boto3.client') @patch('sys.stdout', new_callable=StringIO) - def test_main_groups(self, mock_stdout, AWSConnection): - instance = Mock() - AWSConnection.return_value = instance + def test_main_groups(self, mock_stdout, botoclient): + client = Mock() + botoclient.return_value = client + awslogs = AWSLogs() groups = [ {'logGroups': [{'logGroupName': 'AAA'}, @@ -501,7 +261,7 @@ def test_main_groups(self, mock_stdout, AWSConnection): {'logGroupName': 'CCC'}]}, ] - instance.describe_log_groups.side_effect = groups + client.describe_log_groups.side_effect = groups main("awslogs groups".split()) self.assertEqual( @@ -511,11 +271,12 @@ def test_main_groups(self, mock_stdout, AWSConnection): "CCC\n") ) - @patch('awslogs.core.AWSConnection') + @patch('boto3.client') @patch('sys.stdout', new_callable=StringIO) - def test_main_streams(self, mock_stdout, AWSConnection): - instance = Mock() - AWSConnection.return_value = instance + def test_main_streams(self, mock_stdout, botoclient): + client = Mock() + botoclient.return_value = client + awslogs = AWSLogs() groups = [ {'logGroups': [{'logGroupName': 'AAA'}, @@ -528,8 +289,8 @@ def test_main_streams(self, mock_stdout, AWSConnection): self._stream('EEE')]} ] - instance.describe_log_groups.side_effect = groups - instance.describe_log_streams.side_effect = streams + client.describe_log_groups.side_effect = groups + client.describe_log_streams.side_effect = streams main("awslogs streams AAA".split()) self.assertEqual( @@ -545,7 +306,6 @@ def test_unknown_date_error(self, mock_stderr): self.assertEqual(mock_stderr.getvalue(), colored("awslogs doesn't understand 'X' as a date.\n", "red")) - @patch('awslogs.bin.AWSLogs') @patch('sys.stderr', new_callable=StringIO) def test_unknown_error(self, mock_stderr, mock_awslogs): @@ -566,73 +326,48 @@ def test_connection_error(self, mock_stderr, mock_awslogs): self.assertEqual(mock_stderr.getvalue(), colored("awslogs can't connecto to AWS.\n", "red")) - @patch('awslogs.core.botologs.connect_to_region') - @patch('sys.stderr', new_callable=StringIO) - def test_access_denied_error(self, mock_stderr, connect_to_region): - instance = Mock() - connect_to_region.return_value = instance - exc = boto.exception.JSONResponseError( - status=400, - reason='Bad Request', - body={u'Message': u'User XXX...', '__type': 'AccessDeniedException'} - ) - instance.describe_log_groups.side_effect = exc - - code = main("awslogs groups --aws-region=eu-west-1".split()) - self.assertEqual(code, 4) - self.assertEqual(mock_stderr.getvalue(), colored("User XXX...\n", "red")) - - @patch('awslogs.core.botologs.connect_to_region') - @patch('sys.stderr', new_callable=StringIO) - def test_no_handler_was_ready_to_authenticate(self, mock_stderr, connect_to_region): - instance = Mock() - connect_to_region.side_effect = boto.exception.NoAuthHandlerFound( - "No handler was ready to authenticate" - ) - - code = main("awslogs groups --aws-region=eu-west-1".split()) - self.assertEqual(code, 5) - self.assertTrue("No handler was ready to authenticate" in mock_stderr.getvalue()) - - @patch('sys.stderr', new_callable=StringIO) - def test_invalid_aws_region(self, mock_stderr): - code = main("awslogs groups --aws-region=xxx".split()) - self.assertEqual(code, 6) - self.assertEqual(mock_stderr.getvalue(), - colored("xxx is not a valid AWS region name\n", "red")) - - @patch('sys.stderr', new_callable=StringIO) - def test_empty_aws_region(self, mock_stderr): - code = main("awslogs groups".split()) - self.assertEqual(code, 6) - self.assertEqual(mock_stderr.getvalue(), - colored("You need to provide a valid AWS region name using --aws-region\n", "red")) - - def test_regression_next_token_unbound(self): - """Test that the loop continues without raising unbound exception. - - `next_token` was unbound because the loop didn't continue after an - Empty exception with `self.watch` set to True.""" - - class MockPublishersQueue(object): - def get(self, block): - raise gevent.queue.Empty - - class MockAWSLogs(AWSLogs): - WATCH_SLEEP = 0 - continue_reached = False - - def __init__(self): - self.publishers_queue = MockPublishersQueue() - - @property - def watch(self): - if self.continue_reached: - return False - self.continue_reached = True - return True - - logs = MockAWSLogs() - logs._publisher_queue_consumer() - - self.assertTrue(logs.continue_reached) + # @patch('awslogs.core.boto3.client') + # @patch('sys.stderr', new_callable=StringIO) + # def test_access_denied_error(self, mock_stderr, botoclient): + # client = Mock() + # botoclient.return_value = client + # + # exc = boto.exception.JSONResponseError( + # status=400, + # reason='Bad Request', + # body={u'Message': u'User XXX...', '__type': 'AccessDeniedException'} + # ) + # client.describe_log_groups.side_effect = exc + # + # code = main("awslogs groups --aws-region=eu-west-1".split()) + # self.assertEqual(code, 4) + # self.assertEqual(mock_stderr.getvalue(), colored("User XXX...\n", "red")) + + # @patch('awslogs.core.botologs.connect_to_region') + # @patch('sys.stderr', new_callable=StringIO) + # def test_no_handler_was_ready_to_authenticate(self, mock_stderr, connect_to_region): + # instance = Mock() + # connect_to_region.side_effect = boto.exception.NoAuthHandlerFound( + # "No handler was ready to authenticate" + # ) + # + # code = main("awslogs groups --aws-region=eu-west-1".split()) + # self.assertEqual(code, 5) + # self.assertTrue("No handler was ready to authenticate" in mock_stderr.getvalue()) + # + # @patch('sys.stderr', new_callable=StringIO) + # def test_invalid_aws_region(self, mock_stderr): + # code = main("awslogs groups --aws-region=xxx".split()) + # self.assertEqual(code, 6) + # self.assertEqual(mock_stderr.getvalue(), + # colored("xxx is not a valid AWS region name\n", "red")) + # + # @patch('boto3.client') + # @patch('sys.stderr', new_callable=StringIO) + # def test_empty_aws_region(self, mock_stderr, botoclient): + # client = Mock() + # botoclient.return_value = client + # code = main("awslogs groups".split()) + # self.assertEqual(code, 6) + # self.assertEqual(mock_stderr.getvalue(), + # colored("You need to provide a valid AWS region name using --aws-region\n", "red"))