diff --git a/api/conf/settings.py b/api/conf/settings.py index bc0cdfa5e8..45988f0227 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -342,7 +342,7 @@ def _build_redis_url(base_url, db_number, **query_args): } ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) - ELASTICSEARCH_DSL_AUTOSYNC = True + ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") diff --git a/api/external_data/migrations/0024_denials_data_migration.py b/api/external_data/migrations/0024_denials_data_migration.py index c579082dbe..5f18ea5f22 100644 --- a/api/external_data/migrations/0024_denials_data_migration.py +++ b/api/external_data/migrations/0024_denials_data_migration.py @@ -1,65 +1,64 @@ # Generated by Django 4.2.10 on 2024-04-02 14:59 import logging -from django.db import migrations -from django.forms.models import model_to_dict + +from django.db import IntegrityError, migrations, transaction from django.db.models import Q -from django.db import IntegrityError -from django.db import transaction -from django.conf import settings +from django.forms.models import model_to_dict + + log = logging.getLogger(__name__) + required_fields = [ - "reference", - "regime_reg_ref", - "notifying_government", - "item_list_codes", - "item_description", - "end_use", - "is_revoked", - "is_revoked_comment", - "reason_for_refusal", + "reference", + "regime_reg_ref", + "notifying_government", + "item_list_codes", + "item_description", + "end_use", + "is_revoked", + "is_revoked_comment", + "reason_for_refusal", ] -def denials_data_migration(apps, schema_editor): - settings.ELASTICSEARCH_DSL_AUTOSYNC = False +def denials_data_migration(apps, schema_editor): DenialEntity = apps.get_model("external_data", "DenialEntity") Denial = apps.get_model("external_data", "Denial") # There are a handfull (10) of regime_reg_refs that are null which was in the initial load - # Assumption here is that they can be deleted since it's erroneous data as we now know - # regime_reg_ref is considered a unique DN record. + # Assumption here is that they can be deleted since it's erroneous data as we now know + # regime_reg_ref is considered a unique DN record. - total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref='')) + total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref="")) log.info( - "Delete null regime_reg_ref total -> %s", - total_null_regime_reg_ref.count(), + "Delete null regime_reg_ref total -> %s", + total_null_regime_reg_ref.count(), ) total_null_regime_reg_ref.delete() - duplicate_denial_errors = [] - with transaction.atomic(): - sid = transaction.savepoint() - for denial_entity in DenialEntity.objects.all(): - try: - denial_entity_dict = {key:value for (key,value) in model_to_dict(denial_entity).items() if key in required_fields} - denial , _ = Denial.objects.get_or_create(**denial_entity_dict) - denial_entity.denial = denial - denial_entity.save() - except IntegrityError as e: - duplicate_denial_errors.append(denial_entity.regime_reg_ref) - - if duplicate_denial_errors: - log.info( - "There are the following duplicate denials in the database rolling back this migration: -> %s", - duplicate_denial_errors, - ) - transaction.savepoint_rollback(sid) - else: - transaction.savepoint_commit(sid) - - settings.ELASTICSEARCH_DSL_AUTOSYNC = True + + denials_to_create = [] + denials_entitiy_to_update = [] + denials_dict_map = {} + for denial_entity in DenialEntity.objects.all(): + denial_entity_dict = { + key: value for (key, value) in model_to_dict(denial_entity).items() if key in required_fields + } + denials_dict_map[denial_entity_dict["regime_reg_ref"]] = denial_entity_dict + + for denials_dict_map_item in denials_dict_map.values(): + denial = Denial(**denials_dict_map_item) + denials_to_create.append(denial) + + Denial.objects.bulk_create(denials_to_create) + + for denial_entity in DenialEntity.objects.all(): + denial_entity.denial = Denial.objects.get(regime_reg_ref=denial_entity.regime_reg_ref) + denials_entitiy_to_update.append(denial_entity) + DenialEntity.objects.bulk_update(denials_entitiy_to_update, ["denial"]) + class Migration(migrations.Migration): dependencies = [ ("external_data", "0023_set_denial_entity_type"), diff --git a/api/external_data/migrations/tests/test_0024_denials_data_migration.py b/api/external_data/migrations/tests/test_0024_denials_data_migration.py index a17be6567c..e1bfe8e0ba 100644 --- a/api/external_data/migrations/tests/test_0024_denials_data_migration.py +++ b/api/external_data/migrations/tests/test_0024_denials_data_migration.py @@ -4,11 +4,65 @@ test_data = [ -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.123","name":"name 1","address":"address 1","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 2","address":"address 2","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 3","address":"address 3","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 4","address":"address 4","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","name":"bad record","address":"bad record","notifying_government":"UK","country":"bad","item_list_codes":"all","item_description":"bad","end_use":"bad","reason_for_refusal":"bad "} + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.123", + "name": "name 1", + "address": "address 1", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 2", + "address": "address 2", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 3", + "address": "address 3", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 4", + "address": "address 4", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "name": "bad record", + "address": "bad record", + "notifying_government": "UK", + "country": "bad", + "item_list_codes": "all", + "item_description": "bad", + "end_use": "bad", + "reason_for_refusal": "bad ", + }, ] @@ -18,14 +72,10 @@ class TestDenialDataMigration(MigratorTestCase): migrate_from = ("external_data", "0023_set_denial_entity_type") migrate_to = ("external_data", "0024_denials_data_migration") - def prepare(self): DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") for row in test_data: - DenialEntity.objects.create(**row) - - - + DenialEntity.objects.create(**row) def test_0023_denials_data_migration(self): DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") @@ -33,27 +83,4 @@ def test_0023_denials_data_migration(self): self.assertEqual(DenialEntity.objects.all().count(), 4) self.assertEqual(Denial.objects.all().count(), 2) - self.assertEqual(Denial.objects.get(regime_reg_ref='reg.123.1234').denial_entity.count(), 3) - - -@pytest.mark.django_db() -class TestDenialDataDuplicatesMigration(MigratorTestCase): - - migrate_from = ("external_data", "0023_set_denial_entity_type") - migrate_to = ("external_data", "0024_denials_data_migration") - - - def prepare(self): - DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") - for row in test_data: - DenialEntity.objects.create(**row) - test_data[0]["end_use"] = "end_use b" - DenialEntity.objects.create(**test_data[0]) - - - - def test_0024_denials_data_migration_duplicates(self): - DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") - Denial = self.new_state.apps.get_model("external_data", "Denial") - self.assertEqual(DenialEntity.objects.all().count(), 5) - self.assertEqual(Denial.objects.all().count(), 0) + self.assertEqual(Denial.objects.get(regime_reg_ref="reg.123.1234").denial_entity.count(), 3)