diff --git a/corehq/apps/data_interfaces/tasks.py b/corehq/apps/data_interfaces/tasks.py index 5c1ebbf9ca19..31a9b5e043d1 100644 --- a/corehq/apps/data_interfaces/tasks.py +++ b/corehq/apps/data_interfaces/tasks.py @@ -20,9 +20,11 @@ ) from corehq.form_processor.utils.general import should_use_sql_backend from corehq.motech.repeaters.dbaccessors import ( - get_repeat_records_by_payload_id, - iter_repeat_records_by_repeater, + get_couch_repeat_record_ids_by_payload_id, + get_sql_repeat_records_by_payload_id, + iter_repeat_record_ids_by_repeater, ) +from corehq.motech.repeaters.models import SQLRepeatRecord from corehq.sql_db.util import get_db_aliases_for_partitioned_query from corehq.toggles import DISABLE_CASE_UPDATE_RULE_SCHEDULED_TASK from corehq.util.decorators import serial_task @@ -224,8 +226,13 @@ def delete_old_rule_submission_logs(): @task(serializer='pickle') -def task_operate_on_payloads(record_ids, domain, action=''): - return operate_on_payloads(record_ids, domain, action, +def task_operate_on_payloads( + record_ids: List[str], + domain: str, + action, # type: Literal['resend', 'cancel', 'requeue'] # 3.8+ + use_sql: bool, +): + return operate_on_payloads(record_ids, domain, action, use_sql, task=task_operate_on_payloads) @@ -234,10 +241,12 @@ def task_generate_ids_and_operate_on_payloads( payload_id: Optional[str], repeater_id: Optional[str], domain: str, - action: str = '', + action, # type: Literal['resend', 'cancel', 'requeue'] # 3.8+ + use_sql: bool, ) -> dict: - repeat_record_ids = _get_repeat_record_ids(payload_id, repeater_id, domain) - return operate_on_payloads(repeat_record_ids, domain, action, + repeat_record_ids = _get_repeat_record_ids(payload_id, repeater_id, domain, + use_sql) + return operate_on_payloads(repeat_record_ids, domain, action, use_sql, task=task_generate_ids_and_operate_on_payloads) @@ -245,13 +254,22 @@ def _get_repeat_record_ids( payload_id: Optional[str], repeater_id: Optional[str], domain: str, + use_sql: bool, ) -> List[str]: if not payload_id and not repeater_id: return [] if payload_id: - results = get_repeat_records_by_payload_id(domain, payload_id) + if use_sql: + records = get_sql_repeat_records_by_payload_id(domain, payload_id) + return [r.id for r in records] + else: + return get_couch_repeat_record_ids_by_payload_id(domain, payload_id) else: - results = iter_repeat_records_by_repeater(domain, repeater_id) - ids = [x['id'] for x in results] - - return ids + if use_sql: + queryset = SQLRepeatRecord.objects.filter( + domain=domain, + repeater_stub__repeater_id=repeater_id, + ) + return [r['id'] for r in queryset.values('id')] + else: + return list(iter_repeat_record_ids_by_repeater(domain, repeater_id)) diff --git a/corehq/apps/data_interfaces/tests/test_tasks.py b/corehq/apps/data_interfaces/tests/test_tasks.py index 49cb56de191f..198d27587a8f 100644 --- a/corehq/apps/data_interfaces/tests/test_tasks.py +++ b/corehq/apps/data_interfaces/tests/test_tasks.py @@ -9,45 +9,77 @@ class TestTasks(TestCase): - def test_task_operate_on_payloads_no_action(self): + @patch('corehq.apps.data_interfaces.utils.DownloadBase') + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_task_operate_on_payloads_no_action( + self, + unused_1, + unused_2, + ): response = task_operate_on_payloads( record_ids=['payload_id'], domain='test_domain', - action='' + action='', + use_sql=False, ) - self.assertEqual(response, - {'messages': {'errors': ['No action specified']}}) + self.assertEqual(response, { + 'messages': { + 'errors': [ + "Could not perform action for repeat record (id=payload_id): " + "Unknown action ''", + ], + 'success': [], + 'success_count_msg': '', + } + }) def test_task_operate_on_payloads_no_payload_ids(self): response = task_operate_on_payloads( record_ids=[], domain='test_domain', - action='test_action' + action='test_action', + use_sql=False, ) self.assertEqual(response, {'messages': {'errors': ['No payloads specified']}}) + @patch('corehq.apps.data_interfaces.utils.DownloadBase') + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') @patch('corehq.apps.data_interfaces.tasks._get_repeat_record_ids') def test_task_generate_ids_and_operate_on_payloads_no_action( self, get_repeat_record_ids_mock, + unused_1, + unused_2, ): get_repeat_record_ids_mock.return_value = ['c0ffee', 'deadbeef'] response = task_generate_ids_and_operate_on_payloads( payload_id='c0ffee', repeater_id=None, domain='test_domain', - action='' + action='', + use_sql=False, ) - self.assertEqual(response, - {'messages': {'errors': ['No action specified']}}) + self.assertEqual(response, { + 'messages': { + 'errors': [ + "Could not perform action for repeat record (id=c0ffee): " + "Unknown action ''", + "Could not perform action for repeat record (id=deadbeef): " + "Unknown action ''", + ], + 'success': [], + 'success_count_msg': '', + } + }) def test_task_generate_ids_and_operate_on_payloads_no_data(self): response = task_generate_ids_and_operate_on_payloads( payload_id=None, repeater_id=None, domain='test_domain', - action='' + action='', + use_sql=False, ) self.assertEqual(response, {'messages': {'errors': ['No payloads specified']}}) diff --git a/corehq/apps/data_interfaces/tests/test_utils.py b/corehq/apps/data_interfaces/tests/test_utils.py index e4ad5aa6db93..574bc385fb6c 100644 --- a/corehq/apps/data_interfaces/tests/test_utils.py +++ b/corehq/apps/data_interfaces/tests/test_utils.py @@ -1,5 +1,8 @@ -from unittest.case import TestCase +from datetime import datetime from unittest.mock import Mock, patch +from uuid import uuid4 + +from django.test import SimpleTestCase, TestCase from couchdbkit import ResourceNotFound @@ -8,46 +11,60 @@ task_generate_ids_and_operate_on_payloads, ) from corehq.apps.data_interfaces.utils import ( - _validate_record, + _get_couch_repeat_record, operate_on_payloads, ) +from corehq.motech.models import ConnectionSettings +from corehq.motech.repeaters.models import ( + FormRepeater, + RepeaterStub, + RepeatRecord, + SQLRepeatRecord, +) +DOMAIN = 'test-domain' -class TestUtils(TestCase): + +class TestUtils(SimpleTestCase): def test__get_ids_no_data(self): - response = _get_repeat_record_ids(None, None, 'test_domain') + response = _get_repeat_record_ids(None, None, 'test_domain', False) self.assertEqual(response, []) - @patch('corehq.apps.data_interfaces.tasks.get_repeat_records_by_payload_id') - @patch('corehq.apps.data_interfaces.tasks.iter_repeat_records_by_repeater') - def test__get_ids_payload_id_in_data(self, mock_iter_repeat_records_by_repeater, - mock_get_repeat_records_by_payload_id): + @patch('corehq.apps.data_interfaces.tasks.get_couch_repeat_record_ids_by_payload_id') + @patch('corehq.apps.data_interfaces.tasks.iter_repeat_record_ids_by_repeater') + def test__get_ids_payload_id_in_data( + self, + iter_by_repeater, + get_by_payload_id, + ): payload_id = Mock() - _get_repeat_record_ids(payload_id, None, 'test_domain') + _get_repeat_record_ids(payload_id, None, 'test_domain', False) - self.assertEqual(mock_get_repeat_records_by_payload_id.call_count, 1) - mock_get_repeat_records_by_payload_id.assert_called_with('test_domain', payload_id) - self.assertEqual(mock_iter_repeat_records_by_repeater.call_count, 0) + self.assertEqual(get_by_payload_id.call_count, 1) + get_by_payload_id.assert_called_with( + 'test_domain', payload_id) + self.assertEqual(iter_by_repeater.call_count, 0) - @patch('corehq.apps.data_interfaces.tasks.get_repeat_records_by_payload_id') - @patch('corehq.apps.data_interfaces.tasks.iter_repeat_records_by_repeater') + @patch('corehq.apps.data_interfaces.tasks.get_couch_repeat_record_ids_by_payload_id') + @patch('corehq.apps.data_interfaces.tasks.iter_repeat_record_ids_by_repeater') def test__get_ids_payload_id_not_in_data( self, - mock_iter_repeat_records_by_repeater, - mock_get_repeat_records_by_payload_id, + iter_by_repeater, + get_by_payload_id, ): REPEATER_ID = 'c0ffee' - _get_repeat_record_ids(None, REPEATER_ID, 'test_domain') + _get_repeat_record_ids(None, REPEATER_ID, 'test_domain', False) - mock_get_repeat_records_by_payload_id.assert_not_called() - mock_iter_repeat_records_by_repeater.assert_called_with('test_domain', REPEATER_ID) - self.assertEqual(mock_iter_repeat_records_by_repeater.call_count, 1) + get_by_payload_id.assert_not_called() + iter_by_repeater.assert_called_with( + 'test_domain', REPEATER_ID) + self.assertEqual(iter_by_repeater.call_count, 1) @patch('corehq.motech.repeaters.models.RepeatRecord') def test__validate_record_record_does_not_exist(self, mock_RepeatRecord): mock_RepeatRecord.get.side_effect = [ResourceNotFound] - response = _validate_record('id_1', 'test_domain') + response = _get_couch_repeat_record('test_domain', 'id_1') mock_RepeatRecord.get.assert_called_once() self.assertIsNone(response) @@ -57,7 +74,7 @@ def test__validate_record_invalid_domain(self, mock_RepeatRecord): mock_payload = Mock() mock_payload.domain = 'domain' mock_RepeatRecord.get.return_value = mock_payload - response = _validate_record('id_1', 'test_domain') + response = _get_couch_repeat_record('test_domain', 'id_1') mock_RepeatRecord.get.assert_called_once() self.assertIsNone(response) @@ -67,7 +84,7 @@ def test__validate_record_success(self, mock_RepeatRecord): mock_payload = Mock() mock_payload.domain = 'test_domain' mock_RepeatRecord.get.return_value = mock_payload - response = _validate_record('id_1', 'test_domain') + response = _get_couch_repeat_record('test_domain', 'id_1') mock_RepeatRecord.get.assert_called_once() self.assertEqual(response, mock_payload) @@ -76,235 +93,333 @@ def test__validate_record_success(self, mock_RepeatRecord): class TestTasks(TestCase): def setUp(self): - self.mock_payload_one, self.mock_payload_two = Mock(id='id_1'), Mock(id='id_2') - self.mock_payload_ids = [self.mock_payload_one.id, self.mock_payload_two.id] + self.mock_payload_one = Mock(id='id_1') + self.mock_payload_two = Mock(id='id_2') + self.mock_payload_ids = [self.mock_payload_one.id, + self.mock_payload_two.id] @patch('corehq.apps.data_interfaces.tasks._get_repeat_record_ids') @patch('corehq.apps.data_interfaces.tasks.operate_on_payloads') - def test_generate_ids_and_operate_on_payloads_success(self, mock_operate_on_payloads, mock__get_ids): + def test_generate_ids_and_operate_on_payloads_success( + self, + mock_operate_on_payloads, + mock__get_ids, + ): payload_id = 'c0ffee' repeater_id = 'deadbeef' task_generate_ids_and_operate_on_payloads( - payload_id, repeater_id, 'test_domain', 'test_action') + payload_id, repeater_id, 'test_domain', 'test_action', False) mock__get_ids.assert_called_once() - mock__get_ids.assert_called_with('c0ffee', 'deadbeef', 'test_domain') - mock_record_ids = mock__get_ids('c0ffee', 'deadbeef', 'test_domain') + mock__get_ids.assert_called_with( + 'c0ffee', 'deadbeef', 'test_domain', False) + + mock_record_ids = mock__get_ids( + 'c0ffee', 'deadbeef', 'test_domain', False) mock_operate_on_payloads.assert_called_once() - mock_operate_on_payloads.assert_called_with(mock_record_ids, 'test_domain', 'test_action', - task=task_generate_ids_and_operate_on_payloads) + mock_operate_on_payloads.assert_called_with( + mock_record_ids, 'test_domain', 'test_action', False, + task=task_generate_ids_and_operate_on_payloads) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_false_resend(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_false_resend( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'resend') - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully resend payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully resend 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'resend', False) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully resent repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': "Successfully performed resend action on " + "1 form(s)", } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_resend(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_resend(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_true_resend(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_true_resend( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'resend', from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully resend payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'resend', False, from_excel=True) + expected_response = { + 'errors': [], + 'success': ['Successfully resent repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_resend(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_resend(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_false_resend(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_false_resend( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'resend', task=Mock()) - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully resend payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully resend 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'resend', False, task=Mock()) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully resent repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': 'Successfully performed resend action on ' + '1 form(s)', } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_resend(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_resend(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_true_resend(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_true_resend( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'resend', task=Mock(), from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully resend payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'resend', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': [], + 'success': ['Successfully resent repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_resend(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_resend(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_false_cancel(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_false_cancel( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'cancel') - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully cancelled payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully cancel 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'cancel', False) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully cancelled repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': 'Successfully performed cancel action on ' + '1 form(s)', } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_cancel(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_cancel(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_true_cancel(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_true_cancel( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'cancel', from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully cancelled payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'cancel', False, from_excel=True) + expected_response = { + 'errors': [], + 'success': ['Successfully cancelled repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_cancel(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_cancel(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_false_cancel(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_false_cancel( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'cancel', task=Mock()) - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully cancelled payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully cancel 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'cancel', False, task=Mock()) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully cancelled repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': 'Successfully performed cancel action on ' + '1 form(s)', } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_cancel(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_cancel(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_true_cancel(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_true_cancel( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'cancel', task=Mock(), from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully cancelled payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'cancel', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': [], + 'success': ['Successfully cancelled repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_cancel(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_cancel(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_false_requeue(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_false_requeue( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'requeue') - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully requeue 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'requeue', False) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully requeued repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': 'Successfully performed requeue action ' + 'on 1 form(s)', } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_requeue(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_requeue(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_no_task_from_excel_true_requeue(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_no_task_from_excel_true_requeue( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'requeue', from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'requeue', False, from_excel=True) + expected_response = { + 'errors': [], + 'success': [f'Successfully requeued repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 0) - self._check_requeue(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_requeue(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_false_requeue(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_false_requeue( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'requeue', task=Mock()) - expected_response = { - 'messages': { - 'errors': [], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - 'success_count_msg': _("Successfully requeue 1 form(s)") - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'requeue', False, task=Mock()) + expected_response = { + 'messages': { + 'errors': [], + 'success': ['Successfully requeued repeat record ' + f'(id={self.mock_payload_one.id})'], + 'success_count_msg': 'Successfully performed requeue action ' + 'on 1 form(s)', } + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_requeue(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_requeue(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_with_task_from_excel_true_requeue(self, mock__validate_record, mock_DownloadBase): + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_with_task_from_excel_true_requeue( + self, + mock__validate_record, + mock_DownloadBase, + ): mock__validate_record.side_effect = [self.mock_payload_one, None] - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'requeue', task=Mock(), from_excel=True) - expected_response = { - 'errors': [], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - } + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'requeue', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': [], + 'success': ['Successfully requeued repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 2) - self._check_requeue(self.mock_payload_one, self.mock_payload_two, response, expected_response) + self._check_requeue(self.mock_payload_one, self.mock_payload_two, + response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_throws_exception_resend(self, mock__validate_record, mock_DownloadBase): - mock__validate_record.side_effect = [self.mock_payload_one, self.mock_payload_two] - self.mock_payload_two.fire.side_effect = [Exception] - - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'resend', task=Mock(), from_excel=True) - expected_response = { - 'errors': [_("Could not perform action for payload (id={}): {}").format(self.mock_payload_two.id, - Exception)], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - } + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_throws_exception_resend( + self, + mock__validate_record, + mock_DownloadBase, + ): + mock__validate_record.side_effect = [self.mock_payload_one, + self.mock_payload_two] + self.mock_payload_two.fire.side_effect = [Exception('Boom!')] + + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'resend', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': ['Could not perform action for repeat record ' + f'(id={self.mock_payload_two.id}): Boom!'], + 'success': ['Successfully resent repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 3) self.assertEqual(self.mock_payload_one.fire.call_count, 1) @@ -312,18 +427,25 @@ def test_operate_on_payloads_throws_exception_resend(self, mock__validate_record self.assertEqual(response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_throws_exception_cancel(self, mock__validate_record, mock_DownloadBase): - mock__validate_record.side_effect = [self.mock_payload_one, self.mock_payload_two] - self.mock_payload_two.cancel.side_effect = [Exception] - - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'cancel', task=Mock(), from_excel=True) - expected_response = { - 'errors': [_("Could not perform action for payload (id={}): {}").format(self.mock_payload_two.id, - Exception)], - 'success': [_('Successfully cancelled payload (id={})').format(self.mock_payload_one.id)], - } + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_throws_exception_cancel( + self, + mock__validate_record, + mock_DownloadBase, + ): + mock__validate_record.side_effect = [self.mock_payload_one, + self.mock_payload_two] + self.mock_payload_two.cancel.side_effect = [Exception('Boom!')] + + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'cancel', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': ['Could not perform action for repeat record ' + f'(id={self.mock_payload_two.id}): Boom!'], + 'success': ['Successfully cancelled repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 3) self.assertEqual(self.mock_payload_one.cancel.call_count, 1) @@ -333,18 +455,25 @@ def test_operate_on_payloads_throws_exception_cancel(self, mock__validate_record self.assertEqual(response, expected_response) @patch('corehq.apps.data_interfaces.utils.DownloadBase') - @patch('corehq.apps.data_interfaces.utils._validate_record') - def test_operate_on_payloads_throws_exception_requeue(self, mock__validate_record, mock_DownloadBase): - mock__validate_record.side_effect = [self.mock_payload_one, self.mock_payload_two] - self.mock_payload_two.requeue.side_effect = [Exception] - - with patch('corehq.apps.data_interfaces.utils._') as _: - response = operate_on_payloads(self.mock_payload_ids, 'test_domain', 'requeue', task=Mock(), from_excel=True) - expected_response = { - 'errors': [_("Could not perform action for payload (id={}): {}").format(self.mock_payload_two.id, - Exception)], - 'success': [_('Successfully requeue payload (id={})').format(self.mock_payload_one.id)], - } + @patch('corehq.apps.data_interfaces.utils._get_couch_repeat_record') + def test_operate_on_payloads_throws_exception_requeue( + self, + mock__validate_record, + mock_DownloadBase, + ): + mock__validate_record.side_effect = [self.mock_payload_one, + self.mock_payload_two] + self.mock_payload_two.requeue.side_effect = [Exception('Boom!')] + + response = operate_on_payloads(self.mock_payload_ids, 'test_domain', + 'requeue', False, task=Mock(), + from_excel=True) + expected_response = { + 'errors': ['Could not perform action for repeat record ' + f'(id={self.mock_payload_two.id}): Boom!'], + 'success': ['Successfully requeued repeat record ' + f'(id={self.mock_payload_one.id})'], + } self.assertEqual(mock_DownloadBase.set_progress.call_count, 3) self.assertEqual(self.mock_payload_one.requeue.call_count, 1) @@ -353,21 +482,111 @@ def test_operate_on_payloads_throws_exception_requeue(self, mock__validate_recor self.assertEqual(self.mock_payload_two.save.call_count, 0) self.assertEqual(response, expected_response) - def _check_resend(self, mock_payload_one, mock_payload_two, response, expected_response): + def _check_resend(self, mock_payload_one, mock_payload_two, + response, expected_response): self.assertEqual(mock_payload_one.fire.call_count, 1) self.assertEqual(mock_payload_two.fire.call_count, 0) self.assertEqual(response, expected_response) - def _check_cancel(self, mock_payload_one, mock_payload_two, response, expected_response): + def _check_cancel(self, mock_payload_one, mock_payload_two, + response, expected_response): self.assertEqual(mock_payload_one.cancel.call_count, 1) self.assertEqual(mock_payload_one.save.call_count, 1) self.assertEqual(mock_payload_two.cancel.call_count, 0) self.assertEqual(mock_payload_two.save.call_count, 0) self.assertEqual(response, expected_response) - def _check_requeue(self, mock_payload_one, mock_payload_two, response, expected_response): + def _check_requeue(self, mock_payload_one, mock_payload_two, + response, expected_response): self.assertEqual(mock_payload_one.requeue.call_count, 1) self.assertEqual(mock_payload_one.save.call_count, 1) self.assertEqual(mock_payload_two.requeue.call_count, 0) self.assertEqual(mock_payload_two.save.call_count, 0) self.assertEqual(response, expected_response) + + +class TestGetRepeatRecordIDs(TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.instance_id = str(uuid4()) + url = 'https://www.example.com/api/' + conn = ConnectionSettings.objects.create(domain=DOMAIN, name=url, url=url) + cls.repeater = FormRepeater( + domain=DOMAIN, + connection_settings_id=conn.id, + include_app_id_param=False, + ) + cls.repeater.save() + cls.repeater_stub = RepeaterStub.objects.create( + domain=DOMAIN, + repeater_id=cls.repeater.get_id, + ) + cls.create_repeat_records() + + @classmethod + def tearDownClass(cls): + for record in cls.couch_records + cls.sql_records: + record.delete() + cls.repeater_stub.delete() + cls.repeater.delete() + super().tearDownClass() + + @classmethod + def create_repeat_records(cls): + now = datetime.now() + cls.couch_records = [] + cls.sql_records = [] + for __ in range(3): + couch_record = RepeatRecord( + domain=DOMAIN, + repeater_id=cls.repeater._id, + repeater_type='FormRepeater', + payload_id=cls.instance_id, + registered_on=now, + ) + couch_record.save() + cls.couch_records.append(couch_record) + + cls.sql_records.append(SQLRepeatRecord.objects.create( + domain=DOMAIN, + couch_id=couch_record._id, + payload_id=cls.instance_id, + repeater_stub=cls.repeater_stub, + registered_at=now, + )) + + def test_no_payload_id_no_repeater_id_sql(self): + result = _get_repeat_record_ids(payload_id=None, repeater_id=None, + domain=DOMAIN, use_sql=True) + self.assertEqual(result, []) + + def test_no_payload_id_no_repeater_id_couch(self): + result = _get_repeat_record_ids(payload_id=None, repeater_id=None, + domain=DOMAIN, use_sql=False) + self.assertEqual(result, []) + + def test_payload_id_sql(self): + result = _get_repeat_record_ids(payload_id=self.instance_id, + repeater_id=None, + domain=DOMAIN, use_sql=True) + self.assertEqual(set(result), {r.pk for r in self.sql_records}) + + def test_payload_id_couch(self): + result = _get_repeat_record_ids(payload_id=self.instance_id, + repeater_id=None, + domain=DOMAIN, use_sql=False) + self.assertEqual(set(result), {r._id for r in self.couch_records}) + + def test_repeater_id_sql(self): + result = _get_repeat_record_ids(payload_id=None, + repeater_id=self.repeater._id, + domain=DOMAIN, use_sql=True) + self.assertEqual(set(result), {r.pk for r in self.sql_records}) + + def test_repeater_id_couch(self): + result = _get_repeat_record_ids(payload_id=None, + repeater_id=self.repeater._id, + domain=DOMAIN, use_sql=False) + self.assertEqual(set(result), {r._id for r in self.couch_records}) diff --git a/corehq/apps/data_interfaces/utils.py b/corehq/apps/data_interfaces/utils.py index 63404773ad9e..2fc482b29b5e 100644 --- a/corehq/apps/data_interfaces/utils.py +++ b/corehq/apps/data_interfaces/utils.py @@ -1,3 +1,5 @@ +from typing import List, Optional + from django.utils.translation import ugettext as _ from couchdbkit import ResourceNotFound @@ -7,6 +9,7 @@ from corehq.apps.casegroups.models import CommCareCaseGroup from corehq.apps.hqcase.utils import get_case_by_identifier from corehq.form_processor.interfaces.dbaccessors import FormAccessors +from corehq.motech.repeaters.const import RECORD_CANCELLED_STATE def add_cases_to_case_group(domain, case_group_id, uploaded_data, progress_tracker): @@ -116,11 +119,16 @@ def property_references_parent(case_property): ) -def operate_on_payloads(repeat_record_ids, domain, action, task=None, from_excel=False): +def operate_on_payloads( + repeat_record_ids: List[str], + domain: str, + action, # type: Literal['resend', 'cancel', 'requeue'] # 3.8+ + use_sql: bool, + task: Optional = None, + from_excel: bool = False, +): if not repeat_record_ids: return {'messages': {'errors': [_('No payloads specified')]}} - if not action: - return {'messages': {'errors': [_('No action specified')]}} response = { 'errors': [], @@ -133,22 +141,30 @@ def operate_on_payloads(repeat_record_ids, domain, action, task=None, from_excel DownloadBase.set_progress(task, 0, len(repeat_record_ids)) for record_id in repeat_record_ids: - valid_record = _validate_record(record_id, domain) + if use_sql: + record = _get_sql_repeat_record(domain, record_id) + else: + record = _get_couch_repeat_record(domain, record_id) - if valid_record: + if record: try: - message = '' if action == 'resend': - valid_record.fire(force_send=True) + record.fire(force_send=True) message = _("Successfully resent repeat record (id={})").format(record_id) elif action == 'cancel': - valid_record.cancel() - valid_record.save() + if use_sql: + record.state = RECORD_CANCELLED_STATE + else: + record.cancel() + record.save() message = _("Successfully cancelled repeat record (id={})").format(record_id) elif action == 'requeue': - valid_record.requeue() - valid_record.save() + record.requeue() + if not use_sql: + record.save() message = _("Successfully requeued repeat record (id={})").format(record_id) + else: + raise ValueError(f'Unknown action {action!r}') response['success'].append(message) success_count = success_count + 1 except Exception as e: @@ -161,18 +177,32 @@ def operate_on_payloads(repeat_record_ids, domain, action, task=None, from_excel if from_excel: return response - response["success_count_msg"] = \ - _("Successfully {action} {count} form(s)".format(action=action, count=success_count)) + if success_count: + response["success_count_msg"] = _( + "Successfully performed {action} action on {count} form(s)" + ).format(action=action, count=success_count) + else: + response["success_count_msg"] = '' return {"messages": response} -def _validate_record(r, domain): +def _get_couch_repeat_record(domain, record_id): from corehq.motech.repeaters.models import RepeatRecord + try: - payload = RepeatRecord.get(r) + couch_record = RepeatRecord.get(record_id) except ResourceNotFound: return None - if payload.domain != domain: + if couch_record.domain != domain: + return None + return couch_record + + +def _get_sql_repeat_record(domain, record_id): + from corehq.motech.repeaters.models import SQLRepeatRecord + + try: + return SQLRepeatRecord.objects.get(domain=domain, pk=record_id) + except SQLRepeatRecord.DoesNotExist: return None - return payload diff --git a/corehq/ex-submodules/casexml/apps/case/templates/case/partials/repeat_records.html b/corehq/ex-submodules/casexml/apps/case/templates/case/partials/repeat_records.html index b0fcc87996b2..12b638333740 100644 --- a/corehq/ex-submodules/casexml/apps/case/templates/case/partials/repeat_records.html +++ b/corehq/ex-submodules/casexml/apps/case/templates/case/partials/repeat_records.html @@ -23,7 +23,7 @@ {{ repeat_record.state|lower }} {% for attempt in repeat_record.attempts %} -
{{ attempt.datetime }}: {{ attempt.message }}
+
{{ attempt.created_at }}: {{ attempt.message }}
{% endfor %} diff --git a/corehq/motech/repeaters/dbaccessors.py b/corehq/motech/repeaters/dbaccessors.py index 2afd4dcf089f..ce61e3de7315 100644 --- a/corehq/motech/repeaters/dbaccessors.py +++ b/corehq/motech/repeaters/dbaccessors.py @@ -2,6 +2,7 @@ from dimagi.utils.parsing import json_format_datetime +from corehq.sql_db.util import estimate_row_count from corehq.util.couch_helpers import paginate_view from corehq.util.test_utils import unit_testing_only @@ -30,6 +31,14 @@ def get_cancelled_repeat_record_count(domain, repeater_id): def get_repeat_record_count(domain, repeater_id=None, state=None): + from .models import are_repeat_records_migrated + + if are_repeat_records_migrated(domain): + return get_sql_repeat_record_count(domain, repeater_id, state) + return get_couch_repeat_record_count(domain, repeater_id, state) + + +def get_couch_repeat_record_count(domain, repeater_id=None, state=None): from .models import RepeatRecord kwargs = dict( include_docs=False, @@ -37,12 +46,21 @@ def get_repeat_record_count(domain, repeater_id=None, state=None): descending=True, ) kwargs.update(_get_startkey_endkey_all_records(domain, repeater_id, state)) - result = RepeatRecord.get_db().view('repeaters/repeat_records', **kwargs).one() - return result['value'] if result else 0 +def get_sql_repeat_record_count(domain, repeater_id=None, state=None): + from .models import SQLRepeatRecord + + queryset = SQLRepeatRecord.objects.filter(domain=domain) + if repeater_id: + queryset = queryset.filter(repeater_stub__repeater_id=repeater_id) + if state: + queryset = queryset.filter(state=state) + return estimate_row_count(queryset) + + def get_overdue_repeat_record_count(overdue_threshold=datetime.timedelta(minutes=10)): from .models import RepeatRecord overdue_datetime = datetime.datetime.utcnow() - overdue_threshold @@ -75,6 +93,14 @@ def _get_startkey_endkey_all_records(domain, repeater_id=None, state=None): def get_paged_repeat_records(domain, skip, limit, repeater_id=None, state=None): + from .models import are_repeat_records_migrated + + if are_repeat_records_migrated(domain): + return get_paged_sql_repeat_records(domain, skip, limit, repeater_id, state) + return get_paged_couch_repeat_records(domain, skip, limit, repeater_id, state) + + +def get_paged_couch_repeat_records(domain, skip, limit, repeater_id=None, state=None): from .models import RepeatRecord kwargs = { 'include_docs': True, @@ -90,6 +116,19 @@ def get_paged_repeat_records(domain, skip, limit, repeater_id=None, state=None): return [RepeatRecord.wrap(result['doc']) for result in results] +def get_paged_sql_repeat_records(domain, skip, limit, repeater_id=None, state=None): + from .models import SQLRepeatRecord + + queryset = SQLRepeatRecord.objects.filter(domain=domain) + if repeater_id: + queryset = queryset.filter(repeater_stub__repeater_id=repeater_id) + if state: + queryset = queryset.filter(state=state) + return (queryset.order_by('-registered_at')[skip:skip + limit] + .select_related('repeater_stub') + .prefetch_related('sqlrepeatrecordattempt_set')) + + def iter_repeat_records_by_domain(domain, repeater_id=None, state=None, chunk_size=1000): from .models import RepeatRecord kwargs = { @@ -108,9 +147,20 @@ def iter_repeat_records_by_domain(domain, repeater_id=None, state=None, chunk_si def iter_repeat_records_by_repeater(domain, repeater_id, chunk_size=1000): + return _iter_repeat_records_by_repeater(domain, repeater_id, chunk_size, + include_docs=True) + + +def iter_repeat_record_ids_by_repeater(domain, repeater_id, chunk_size=1000): + return _iter_repeat_records_by_repeater(domain, repeater_id, chunk_size, + include_docs=False) + + +def _iter_repeat_records_by_repeater(domain, repeater_id, chunk_size, + include_docs): from corehq.motech.repeaters.models import RepeatRecord kwargs = { - 'include_docs': True, + 'include_docs': include_docs, 'reduce': False, 'descending': True, } @@ -120,20 +170,51 @@ def iter_repeat_records_by_repeater(domain, repeater_id, chunk_size=1000): 'repeaters/repeat_records', chunk_size, **kwargs): - yield RepeatRecord.wrap(doc['doc']) + if include_docs: + yield RepeatRecord.wrap(doc['doc']) + else: + yield doc['id'] def get_repeat_records_by_payload_id(domain, payload_id): + repeat_records = get_sql_repeat_records_by_payload_id(domain, payload_id) + if repeat_records: + return repeat_records + return get_couch_repeat_records_by_payload_id(domain, payload_id) + + +def get_couch_repeat_records_by_payload_id(domain, payload_id): + return _get_couch_repeat_records_by_payload_id(domain, payload_id, + include_docs=True) + + +def get_couch_repeat_record_ids_by_payload_id(domain, payload_id): + return _get_couch_repeat_records_by_payload_id(domain, payload_id, + include_docs=False) + + +def _get_couch_repeat_records_by_payload_id(domain, payload_id, include_docs): from .models import RepeatRecord results = RepeatRecord.get_db().view( 'repeaters/repeat_records_by_payload_id', startkey=[domain, payload_id], endkey=[domain, payload_id], - include_docs=True, + include_docs=include_docs, reduce=False, descending=True ).all() - return [RepeatRecord.wrap(result['doc']) for result in results] + if include_docs: + return [RepeatRecord.wrap(result['doc']) for result in results] + return [result['id'] for result in results] + + +def get_sql_repeat_records_by_payload_id(domain, payload_id): + from corehq.motech.repeaters.models import SQLRepeatRecord + + return (SQLRepeatRecord.objects + .filter(domain=domain, payload_id=payload_id) + .order_by('-registered_at') + .all()) def get_repeaters_by_domain(domain): diff --git a/corehq/motech/repeaters/migrations/0004_attempt_strings.py b/corehq/motech/repeaters/migrations/0004_attempt_strings.py new file mode 100644 index 000000000000..fa3f5463140c --- /dev/null +++ b/corehq/motech/repeaters/migrations/0004_attempt_strings.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.19 on 2021-04-10 14:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('repeaters', '0003_migrate_connectionsettings'), + ] + + operations = [ + migrations.AlterField( + model_name='sqlrepeatrecordattempt', + name='message', + field=models.TextField(blank=True, default=''), + ), + migrations.AlterField( + model_name='sqlrepeatrecordattempt', + name='traceback', + field=models.TextField(blank=True, default=''), + ), + ] diff --git a/corehq/motech/repeaters/models.py b/corehq/motech/repeaters/models.py index 665a8d0403a5..ccbfb8445509 100644 --- a/corehq/motech/repeaters/models.py +++ b/corehq/motech/repeaters/models.py @@ -70,7 +70,6 @@ from typing import Any, Optional from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse -from django.utils.functional import cached_property from django.conf import settings from django.db import models from django.utils import timezone @@ -282,10 +281,16 @@ def __str__(self): def __repr__(self): return f"<{self.__class__.__name__} {self._id} {self.name!r}>" - @cached_property + @property def connection_settings(self): if not self.connection_settings_id: return self.create_connection_settings() + return self._get_connection_settings() + + # Cache across instances to avoid N+1 query problem when calling + # Repeater.get_url() for each row in repeat record report + @quickcache(['self.connection_settings_id']) + def _get_connection_settings(self): return ConnectionSettings.objects.get(pk=self.connection_settings_id) @property @@ -771,6 +776,22 @@ class RepeatRecordAttempt(DocumentSchema): def message(self): return self.success_response if self.succeeded else self.failure_reason + @property + def state(self): + state = RECORD_PENDING_STATE + if self.succeeded: + state = RECORD_SUCCESS_STATE + elif self.cancelled: + state = RECORD_CANCELLED_STATE + elif self.failure_reason: + state = RECORD_FAILURE_STATE + return state + + @property + def created_at(self): + # Used by .../case/partials/repeat_records.html + return self.datetime + class RepeatRecord(Document): """ @@ -1062,6 +1083,16 @@ class Meta: ] ordering = ['registered_at'] + def requeue(self): + # Changing "success" to "pending" and "cancelled" to "failed" + # preserves the value of `self.failure_reason`. + if self.state == RECORD_SUCCESS_STATE: + self.state = RECORD_PENDING_STATE + self.save() + elif self.state == RECORD_CANCELLED_STATE: + self.state = RECORD_FAILURE_STATE + self.save() + def add_success_attempt(self, response): """ ``response`` can be a Requests response instance, or True if the @@ -1070,7 +1101,7 @@ def add_success_attempt(self, response): self.repeater_stub.reset_next_attempt() self.sqlrepeatrecordattempt_set.create( state=RECORD_SUCCESS_STATE, - message=format_response(response), + message=format_response(response) or '', ) self.state = RECORD_SUCCESS_STATE self.save() @@ -1125,15 +1156,50 @@ def attempts(self): @property def num_attempts(self): - return self.sqlrepeatrecordattempt_set.count() + # Uses `len(queryset)` instead of `queryset.count()` to use + # prefetched attempts, if available. + return len(self.attempts) + + def get_numbered_attempts(self): + for i, attempt in enumerate(self.attempts, start=1): + yield i, attempt + + @property + def record_id(self): + # Used by Repeater.get_url() ... by SQLRepeatRecordReport._make_row() + return self.pk + + @property + def last_checked(self): + # Used by .../case/partials/repeat_records.html + return self.repeater.last_attempt_at + + @property + def url(self): + # Used by .../case/partials/repeat_records.html + return self.repeater.couch_repeater.get_url(self) + + @property + def failure_reason(self): + if has_failed(self): + return self.last_message + else: + return '' + + @property + def last_message(self): + # Uses `list(queryset)[-1]` instead of `queryset.last()` to use + # prefetched attempts, if available. + attempts = list(self.attempts) + return attempts[-1].message if attempts else '' class SQLRepeatRecordAttempt(models.Model): repeat_record = models.ForeignKey(SQLRepeatRecord, on_delete=models.CASCADE) state = models.TextField(choices=RECORD_STATES) - message = models.TextField(null=True, blank=True) - traceback = models.TextField(null=True, blank=True) + message = models.TextField(blank=True, default='') + traceback = models.TextField(blank=True, default='') created_at = models.DateTimeField(default=timezone.now) class Meta: @@ -1244,6 +1310,14 @@ def later_might_be_better(resp): RECORD_CANCELLED_STATE) # Don't retry +def is_queued(record): + return record.state in (RECORD_PENDING_STATE, RECORD_FAILURE_STATE) + + +def has_failed(record): + return record.state in (RECORD_FAILURE_STATE, RECORD_CANCELLED_STATE) + + def format_response(response) -> Optional[str]: if not is_response(response): return None @@ -1261,6 +1335,17 @@ def is_response(duck): return hasattr(duck, 'status_code') and hasattr(duck, 'reason') +@quickcache(['domain'], timeout=5 * 60) +def are_repeat_records_migrated(domain) -> bool: + """ + Returns True if ``domain`` has SQLRepeatRecords. + + .. note:: Succeeded and cancelled RepeatRecords may not have been + migrated to SQLRepeatRecords. + """ + return SQLRepeatRecord.objects.filter(domain=domain).exists() + + def domain_can_forward(domain): return domain and ( domain_has_privilege(domain, ZAPIER_INTEGRATION) diff --git a/corehq/motech/repeaters/templates/repeaters/partials/attempt_history.html b/corehq/motech/repeaters/templates/repeaters/partials/attempt_history.html index f61dcc5b719a..1fed5721d6e4 100644 --- a/corehq/motech/repeaters/templates/repeaters/partials/attempt_history.html +++ b/corehq/motech/repeaters/templates/repeaters/partials/attempt_history.html @@ -3,20 +3,20 @@