From e270af61349b481526a2f55d4f80e642ff763d1f Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Mon, 2 Mar 2015 16:19:09 +0200 Subject: [PATCH 01/11] Add framework for handling credit cutoff messages --- go/vumitools/billing_worker.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/go/vumitools/billing_worker.py b/go/vumitools/billing_worker.py index 4de108ae1..6ba2709da 100644 --- a/go/vumitools/billing_worker.py +++ b/go/vumitools/billing_worker.py @@ -165,7 +165,7 @@ def determine_session_length(cls, session_metadata_field, msg): is returned """ metadata = msg['helper_metadata'].get(session_metadata_field, {}) - + if 'session_start' not in metadata: return None @@ -183,7 +183,7 @@ def create_transaction_for_inbound(self, msg): self.validate_metadata(msg) msg_mdh = self.get_metadata_helper(msg) session_created = msg['session_event'] == 'new' - yield self.billing_api.create_transaction( + transaction = yield self.billing_api.create_transaction( account_number=msg_mdh.get_account_key(), message_id=msg['message_id'], tag_pool_name=msg_mdh.tag[0], tag_name=msg_mdh.tag[1], @@ -192,6 +192,7 @@ def create_transaction_for_inbound(self, msg): session_created=session_created, transaction_type=self.TRANSACTION_TYPE_MESSAGE, session_length=self._determine_session_length(msg)) + returnValue(transaction) @inlineCallbacks def create_transaction_for_outbound(self, msg): @@ -199,7 +200,7 @@ def create_transaction_for_outbound(self, msg): self.validate_metadata(msg) msg_mdh = self.get_metadata_helper(msg) session_created = msg['session_event'] == 'new' - yield self.billing_api.create_transaction( + transaction = yield self.billing_api.create_transaction( account_number=msg_mdh.get_account_key(), message_id=msg['message_id'], tag_pool_name=msg_mdh.tag[0], tag_name=msg_mdh.tag[1], @@ -208,6 +209,7 @@ def create_transaction_for_outbound(self, msg): session_created=session_created, transaction_type=self.TRANSACTION_TYPE_MESSAGE, session_length=self._determine_session_length(msg)) + returnValue(transaction) @inlineCallbacks def process_inbound(self, config, msg, connector_name): @@ -223,8 +225,11 @@ def process_inbound(self, config, msg, connector_name): log.info( "Not billing for inbound message: %r" % msg.to_json()) else: - yield self.create_transaction_for_inbound(msg) + transaction = yield self.create_transaction_for_inbound(msg) msg_mdh.set_paid() + if transaction.get('credit_cutoff_reached', False): + self._handle_credit_cutoff_inbound(msg) + except BillingError: log.warning( "BillingError for inbound message, sending without billing:" @@ -251,8 +256,10 @@ def process_outbound(self, config, msg, connector_name): log.info( "Not billing for outbound message: %r" % msg.to_json()) else: - yield self.create_transaction_for_outbound(msg) + transaction = yield self.create_transaction_for_outbound(msg) msg_mdh.set_paid() + if transaction.get('credit_cutoff_reached', False): + self._handle_credit_cutoff_outbound(msg) except BillingError: log.warning( "BillingError for outbound message, sending without billing:" @@ -265,6 +272,12 @@ def process_outbound(self, config, msg, connector_name): log.err() yield self.publish_outbound(msg, self.receive_inbound_connector, None) + def _handle_credit_cutoff_inbound(self, msg): + pass + + def _handle_credit_cutoff_outbound(self, msg): + pass + @inlineCallbacks def process_event(self, config, event, connector_name): """Process an event message. From 5d8bfb132453b1799008b54f29f44d516789e993 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Tue, 3 Mar 2015 12:12:16 +0200 Subject: [PATCH 02/11] Have the billing dispatcher handle credit cutoffs --- go/vumitools/billing_worker.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/go/vumitools/billing_worker.py b/go/vumitools/billing_worker.py index 6ba2709da..076cb90f5 100644 --- a/go/vumitools/billing_worker.py +++ b/go/vumitools/billing_worker.py @@ -9,12 +9,15 @@ from vumi import log from vumi.dispatchers.endpoint_dispatchers import Dispatcher from vumi.config import ConfigText, ConfigFloat, ConfigBool +from vumi.message import TransportUserMessage from vumi.utils import http_request_full from go.vumitools.app_worker import GoWorkerMixin, GoWorkerConfigMixin from go.billing.utils import JSONEncoder, JSONDecoder, BillingError +SESSION_CLOSE = TransportUserMessage.SESSION_CLOSE + class BillingApi(object): """Proxy to the billing REST API""" @@ -95,6 +98,9 @@ class BillingDispatcherConfig(Dispatcher.CONFIG_CLASS, GoWorkerConfigMixin): "Name of the session metadata field to look for in each message to " "calculate session length", static=True, default='session_metadata') + credit_limit_message = ConfigText( + "The message to send when terminating session based transports.", + static=True, default='Vumi Go account has run out of credits.') def post_validate(self): if len(self.receive_inbound_connectors) != 1: @@ -133,6 +139,7 @@ def setup_dispatcher(self): self.billing_api = BillingApi(self.api_url, config.retry_delay) self.disable_billing = config.disable_billing self.session_metadata_field = config.session_metadata_field + self.credit_limit_message = config.credit_limit_message @inlineCallbacks def teardown_dispatcher(self): @@ -225,10 +232,8 @@ def process_inbound(self, config, msg, connector_name): log.info( "Not billing for inbound message: %r" % msg.to_json()) else: - transaction = yield self.create_transaction_for_inbound(msg) + yield self.create_transaction_for_inbound(msg) msg_mdh.set_paid() - if transaction.get('credit_cutoff_reached', False): - self._handle_credit_cutoff_inbound(msg) except BillingError: log.warning( @@ -259,7 +264,7 @@ def process_outbound(self, config, msg, connector_name): transaction = yield self.create_transaction_for_outbound(msg) msg_mdh.set_paid() if transaction.get('credit_cutoff_reached', False): - self._handle_credit_cutoff_outbound(msg) + self._handle_credit_cutoff(msg) except BillingError: log.warning( "BillingError for outbound message, sending without billing:" @@ -270,13 +275,16 @@ def process_outbound(self, config, msg, connector_name): "Error processing outbound message, sending without billing:" " %r" % (msg,)) log.err() - yield self.publish_outbound(msg, self.receive_inbound_connector, None) - - def _handle_credit_cutoff_inbound(self, msg): - pass - - def _handle_credit_cutoff_outbound(self, msg): - pass + if msg is not None: + yield self.publish_outbound( + msg, self.receive_inbound_connector, None) + + def _handle_credit_cutoff(self, msg): + if msg.session_event is not None: + msg.session_event = SESSION_CLOSE + msg.content = self.credit_limit_message + else: + msg = None @inlineCallbacks def process_event(self, config, event, connector_name): From 0b665d3755f8c0467727786e282aab6462f37731 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Wed, 4 Mar 2015 12:39:15 +0200 Subject: [PATCH 03/11] Added tests for outbound credit cutoff messages --- go/vumitools/billing_worker.py | 11 ++++--- go/vumitools/tests/test_billing_worker.py | 40 ++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/go/vumitools/billing_worker.py b/go/vumitools/billing_worker.py index 076cb90f5..8bd9b6885 100644 --- a/go/vumitools/billing_worker.py +++ b/go/vumitools/billing_worker.py @@ -264,7 +264,7 @@ def process_outbound(self, config, msg, connector_name): transaction = yield self.create_transaction_for_outbound(msg) msg_mdh.set_paid() if transaction.get('credit_cutoff_reached', False): - self._handle_credit_cutoff(msg) + msg = self._handle_credit_cutoff(msg) except BillingError: log.warning( "BillingError for outbound message, sending without billing:" @@ -280,11 +280,12 @@ def process_outbound(self, config, msg, connector_name): msg, self.receive_inbound_connector, None) def _handle_credit_cutoff(self, msg): - if msg.session_event is not None: - msg.session_event = SESSION_CLOSE - msg.content = self.credit_limit_message + if msg.get('session_event') is not None: + msg['session_event'] = SESSION_CLOSE + msg['content'] = self.credit_limit_message + return msg else: - msg = None + return None @inlineCallbacks def process_event(self, config, event, connector_name): diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index 0bc12cb30..35187525f 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -5,6 +5,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.web.client import Agent, Request, Response +from vumi.message import TransportUserMessage from vumi.tests.helpers import VumiTestCase from vumi.tests.utils import LogCatcher from vumi.utils import mkheaders, StringProducer @@ -17,11 +18,14 @@ from go.billing.api import BillingError from go.billing.utils import JSONEncoder +SESSION_CLOSE = TransportUserMessage.SESSION_CLOSE + class BillingApiMock(object): - def __init__(self): + def __init__(self, credit_cutoff=False): self.transactions = [] + self.credit_cutoff = credit_cutoff def _record(self, items, vars): del vars["self"] @@ -50,6 +54,7 @@ def create_transaction(self, account_number, message_id, tag_pool_name, "status": "Completed", "transaction_type": transaction_type, "session_length": session_length, + "credit_cutoff_reached": self.credit_cutoff } @@ -739,3 +744,36 @@ def test_outbound_message_session_length_custom_field(self): "outbound", session_created=False, session_metadata_field='foo') + + @inlineCallbacks + def test_outbound_message_credit_cutoff_session(self): + self.billing_api = BillingApiMock(credit_cutoff=True) + dispatcher = yield self.get_dispatcher() + + yield self.make_dispatch_outbound( + "outbound", + user_account="12345", + tag=("pool1", "1234"), + helper_metadata={}, + session_event='new' + ) + + published_msg = self.ri_helper.get_dispatched_outbound()[0] + self.assertEqual(published_msg['session_event'], SESSION_CLOSE) + self.assertEqual( + published_msg['content'], dispatcher.credit_limit_message) + + @inlineCallbacks + def test_outbound_message_credit_cutoff_message(self): + self.billing_api = BillingApiMock(credit_cutoff=True) + yield self.get_dispatcher() + + yield self.make_dispatch_outbound( + "outbound", + user_account="12345", + tag=("pool1", "1234"), + helper_metadata={}, + ) + + published_msgs = self.ri_helper.get_dispatched_outbound() + self.assertEqual(len(published_msgs), 0) From 01771085df1578a7cdc5501535839f868e7b4687 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Wed, 4 Mar 2015 12:48:00 +0200 Subject: [PATCH 04/11] Added tests for inbound message credit cutoff --- go/vumitools/tests/test_billing_worker.py | 33 ++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index 35187525f..ccb942446 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -772,8 +772,39 @@ def test_outbound_message_credit_cutoff_message(self): "outbound", user_account="12345", tag=("pool1", "1234"), - helper_metadata={}, + helper_metadata={} ) published_msgs = self.ri_helper.get_dispatched_outbound() self.assertEqual(len(published_msgs), 0) + + @inlineCallbacks + def test_inbound_message_credit_cutoff_session(self): + self.billing_api = BillingApiMock(credit_cutoff=True) + yield self.get_dispatcher + + msg = yield self.make_dispatch_inbound( + "inbound", + user_account="12345", + tag=("pool1", "1234"), + helper_metadata={}, + session_event='new' + ) + + published_msg = self.ri_helper.get_dispatched_inbound()[0] + self.assertEqual(msg, published_msg) + + @inlineCallbacks + def test_inbound_message_credit_cutoff_message(self): + self.billing_api = BillingApiMock(credit_cutoff=True) + yield self.get_dispatcher + + msg = yield self.make_dispatch_inbound( + "inbound", + user_account="12345", + tag=("pool1", "1234"), + helper_metadata={} + ) + + published_msg = self.ri_helper.get_dispatched_inbound()[0] + self.assertEqual(msg, published_msg) From 63ef90f975a0a84fda5fce470d786fe8258f74f8 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 13:31:12 +0200 Subject: [PATCH 05/11] Only set paid if transaction exists. --- go/vumitools/billing_worker.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/go/vumitools/billing_worker.py b/go/vumitools/billing_worker.py index 8bd9b6885..17f232100 100644 --- a/go/vumitools/billing_worker.py +++ b/go/vumitools/billing_worker.py @@ -232,8 +232,9 @@ def process_inbound(self, config, msg, connector_name): log.info( "Not billing for inbound message: %r" % msg.to_json()) else: - yield self.create_transaction_for_inbound(msg) - msg_mdh.set_paid() + result = yield self.create_transaction_for_inbound(msg) + if result.get('transaction'): + msg_mdh.set_paid() except BillingError: log.warning( @@ -261,9 +262,10 @@ def process_outbound(self, config, msg, connector_name): log.info( "Not billing for outbound message: %r" % msg.to_json()) else: - transaction = yield self.create_transaction_for_outbound(msg) - msg_mdh.set_paid() - if transaction.get('credit_cutoff_reached', False): + result = yield self.create_transaction_for_outbound(msg) + if result.get('transaction'): + msg_mdh.set_paid() + if result.get('credit_cutoff_reached', False): msg = self._handle_credit_cutoff(msg) except BillingError: log.warning( From 11f5977cf12a0d299377752d4846d939b0411a56 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 13:35:49 +0200 Subject: [PATCH 06/11] Stop sending of new sessions if credit cutoff --- go/vumitools/billing_worker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/vumitools/billing_worker.py b/go/vumitools/billing_worker.py index 17f232100..20b8b5580 100644 --- a/go/vumitools/billing_worker.py +++ b/go/vumitools/billing_worker.py @@ -17,6 +17,7 @@ from go.billing.utils import JSONEncoder, JSONDecoder, BillingError SESSION_CLOSE = TransportUserMessage.SESSION_CLOSE +SESSION_NEW = TransportUserMessage.SESSION_NEW class BillingApi(object): @@ -282,7 +283,8 @@ def process_outbound(self, config, msg, connector_name): msg, self.receive_inbound_connector, None) def _handle_credit_cutoff(self, msg): - if msg.get('session_event') is not None: + session_event = msg.get('session_event') + if session_event is not None and session_event != SESSION_NEW: msg['session_event'] = SESSION_CLOSE msg['content'] = self.credit_limit_message return msg From a0afe0692c1bdb9b6c72dd228c84069f29e81b25 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 13:38:13 +0200 Subject: [PATCH 07/11] Check that the number of dispatched outbound messages is correct --- go/vumitools/tests/test_billing_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index ccb942446..edb6f1c7a 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -758,7 +758,7 @@ def test_outbound_message_credit_cutoff_session(self): session_event='new' ) - published_msg = self.ri_helper.get_dispatched_outbound()[0] + [published_msg] = self.ri_helper.get_dispatched_outbound() self.assertEqual(published_msg['session_event'], SESSION_CLOSE) self.assertEqual( published_msg['content'], dispatcher.credit_limit_message) From 7e20f66375c5fe45f5ea62596956825e27e50db5 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 13:53:03 +0200 Subject: [PATCH 08/11] Add test to check that new sessions are not started when credit cutoff is reached. --- go/vumitools/tests/test_billing_worker.py | 56 +++++++++++++++-------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index edb6f1c7a..ae8e8d49c 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -36,24 +36,26 @@ def create_transaction(self, account_number, message_id, tag_pool_name, session_created, transaction_type, session_length): self._record(self.transactions, locals()) return { - "id": 1, - "account_number": account_number, - "message_id": message_id, - "tag_pool_name": tag_pool_name, - "tag_name": tag_name, - "provider": provider, - "message_direction": message_direction, - "message_cost": 80, - "session_created": session_created, - "session_cost": 30, - "markup_percent": decimal.Decimal('10.0'), - "credit_amount": -35, - "credit_factor": decimal.Decimal('0.4'), - "created": "2013-10-30T10:42:51.144745+02:00", - "last_modified": "2013-10-30T10:42:51.144745+02:00", - "status": "Completed", - "transaction_type": transaction_type, - "session_length": session_length, + "transaction": { + "id": 1, + "account_number": account_number, + "message_id": message_id, + "tag_pool_name": tag_pool_name, + "tag_name": tag_name, + "provider": provider, + "message_direction": message_direction, + "message_cost": 80, + "session_created": session_created, + "session_cost": 30, + "markup_percent": decimal.Decimal('10.0'), + "credit_amount": -35, + "credit_factor": decimal.Decimal('0.4'), + "created": "2013-10-30T10:42:51.144745+02:00", + "last_modified": "2013-10-30T10:42:51.144745+02:00", + "status": "Completed", + "transaction_type": transaction_type, + "session_length": session_length, + }, "credit_cutoff_reached": self.credit_cutoff } @@ -755,7 +757,7 @@ def test_outbound_message_credit_cutoff_session(self): user_account="12345", tag=("pool1", "1234"), helper_metadata={}, - session_event='new' + session_event='resume' ) [published_msg] = self.ri_helper.get_dispatched_outbound() @@ -763,6 +765,22 @@ def test_outbound_message_credit_cutoff_session(self): self.assertEqual( published_msg['content'], dispatcher.credit_limit_message) + @inlineCallbacks + def test_outbound_message_credit_cutoff_session_start(self): + self.billing_api = BillingApiMock(credit_cutoff=True) + dispatcher = yield self.get_dispatcher() + + yield self.make_dispatch_outbound( + "outbound", + user_account="12345", + tag=("pool1", "1234"), + helper_metadata={}, + session_event='new' + ) + + self.assertEqual(len(self.ri_helper.get_dispatched_outbound()), 0) + + @inlineCallbacks def test_outbound_message_credit_cutoff_message(self): self.billing_api = BillingApiMock(credit_cutoff=True) From 900aa5d49a0260da8ce4b134a5a9bc2461786943 Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 13:59:53 +0200 Subject: [PATCH 09/11] Syntax fix --- go/vumitools/tests/test_billing_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index ae8e8d49c..276470bb4 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -799,7 +799,7 @@ def test_outbound_message_credit_cutoff_message(self): @inlineCallbacks def test_inbound_message_credit_cutoff_session(self): self.billing_api = BillingApiMock(credit_cutoff=True) - yield self.get_dispatcher + yield self.get_dispatcher() msg = yield self.make_dispatch_inbound( "inbound", From 28c282fe5179edbe8ac65cc41d80516abbb5296d Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 14:21:41 +0200 Subject: [PATCH 10/11] Assert on correct message for inbound credit cutoff tests --- go/vumitools/tests/test_billing_worker.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index 276470bb4..ece1d43b8 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -806,16 +806,17 @@ def test_inbound_message_credit_cutoff_session(self): user_account="12345", tag=("pool1", "1234"), helper_metadata={}, - session_event='new' + session_event='resume' ) - published_msg = self.ri_helper.get_dispatched_inbound()[0] + self.add_md(msg, is_paid=True) + [published_msg] = self.ro_helper.get_dispatched_inbound() self.assertEqual(msg, published_msg) @inlineCallbacks def test_inbound_message_credit_cutoff_message(self): self.billing_api = BillingApiMock(credit_cutoff=True) - yield self.get_dispatcher + yield self.get_dispatcher() msg = yield self.make_dispatch_inbound( "inbound", @@ -824,5 +825,6 @@ def test_inbound_message_credit_cutoff_message(self): helper_metadata={} ) - published_msg = self.ri_helper.get_dispatched_inbound()[0] + self.add_md(msg, is_paid=True) + [published_msg] = self.ro_helper.get_dispatched_inbound() self.assertEqual(msg, published_msg) From d08d94338b15e24d7da0d03890d50cddcfa85fec Mon Sep 17 00:00:00 2001 From: Rudi Giesler Date: Fri, 6 Mar 2015 15:58:30 +0200 Subject: [PATCH 11/11] Remove unused variable and extra line --- go/vumitools/tests/test_billing_worker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/vumitools/tests/test_billing_worker.py b/go/vumitools/tests/test_billing_worker.py index ece1d43b8..322e51ddd 100644 --- a/go/vumitools/tests/test_billing_worker.py +++ b/go/vumitools/tests/test_billing_worker.py @@ -768,7 +768,7 @@ def test_outbound_message_credit_cutoff_session(self): @inlineCallbacks def test_outbound_message_credit_cutoff_session_start(self): self.billing_api = BillingApiMock(credit_cutoff=True) - dispatcher = yield self.get_dispatcher() + yield self.get_dispatcher() yield self.make_dispatch_outbound( "outbound", @@ -780,7 +780,6 @@ def test_outbound_message_credit_cutoff_session_start(self): self.assertEqual(len(self.ri_helper.get_dispatched_outbound()), 0) - @inlineCallbacks def test_outbound_message_credit_cutoff_message(self): self.billing_api = BillingApiMock(credit_cutoff=True)