Skip to content

Commit

Permalink
Merge pull request #1965 from uktrade/uat
Browse files Browse the repository at this point in the history 
PROD Hotfix
  • Loading branch information
@depsiatwal
depsiatwal authored May 1, 2024
2 parents e179267 + 88e4cfb commit f90d6cb
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 78 deletions.
2 changes: 1 addition & 1 deletion api/conf/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ def _build_redis_url(base_url, db_number, **query_args):
}

ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False)
ELASTICSEARCH_DSL_AUTOSYNC = True

ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS}
ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS}
SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias")
Expand Down
85 changes: 42 additions & 43 deletions api/external_data/migrations/0024_denials_data_migration.py
View file
Original file line number Diff line number Diff line change
@@ -1,65 +1,64 @@
# Generated by Django 4.2.10 on 2024-04-02 14:59

import logging
from django.db import migrations
from django.forms.models import model_to_dict

from django.db import IntegrityError, migrations, transaction
from django.db.models import Q
from django.db import IntegrityError
from django.db import transaction
from django.conf import settings
from django.forms.models import model_to_dict


log = logging.getLogger(__name__)


required_fields = [
"reference",
"regime_reg_ref",
"notifying_government",
"item_list_codes",
"item_description",
"end_use",
"is_revoked",
"is_revoked_comment",
"reason_for_refusal",
"reference",
"regime_reg_ref",
"notifying_government",
"item_list_codes",
"item_description",
"end_use",
"is_revoked",
"is_revoked_comment",
"reason_for_refusal",
]

def denials_data_migration(apps, schema_editor):
settings.ELASTICSEARCH_DSL_AUTOSYNC = False

def denials_data_migration(apps, schema_editor):
DenialEntity = apps.get_model("external_data", "DenialEntity")
Denial = apps.get_model("external_data", "Denial")

# There are a handfull (10) of regime_reg_refs that are null which was in the initial load
# Assumption here is that they can be deleted since it's erroneous data as we now know
# regime_reg_ref is considered a unique DN record.
# Assumption here is that they can be deleted since it's erroneous data as we now know
# regime_reg_ref is considered a unique DN record.

total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref=''))
total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref=""))
log.info(
"Delete null regime_reg_ref total -> %s",
total_null_regime_reg_ref.count(),
"Delete null regime_reg_ref total -> %s",
total_null_regime_reg_ref.count(),
)
total_null_regime_reg_ref.delete()
duplicate_denial_errors = []
with transaction.atomic():
sid = transaction.savepoint()
for denial_entity in DenialEntity.objects.all():
try:
denial_entity_dict = {key:value for (key,value) in model_to_dict(denial_entity).items() if key in required_fields}
denial , _ = Denial.objects.get_or_create(**denial_entity_dict)
denial_entity.denial = denial
denial_entity.save()
except IntegrityError as e:
duplicate_denial_errors.append(denial_entity.regime_reg_ref)

if duplicate_denial_errors:
log.info(
"There are the following duplicate denials in the database rolling back this migration: -> %s",
duplicate_denial_errors,
)
transaction.savepoint_rollback(sid)
else:
transaction.savepoint_commit(sid)

settings.ELASTICSEARCH_DSL_AUTOSYNC = True

denials_to_create = []
denials_entitiy_to_update = []
denials_dict_map = {}
for denial_entity in DenialEntity.objects.all():
denial_entity_dict = {
key: value for (key, value) in model_to_dict(denial_entity).items() if key in required_fields
}
denials_dict_map[denial_entity_dict["regime_reg_ref"]] = denial_entity_dict

for denials_dict_map_item in denials_dict_map.values():
denial = Denial(**denials_dict_map_item)
denials_to_create.append(denial)

Denial.objects.bulk_create(denials_to_create)

for denial_entity in DenialEntity.objects.all():
denial_entity.denial = Denial.objects.get(regime_reg_ref=denial_entity.regime_reg_ref)
denials_entitiy_to_update.append(denial_entity)

DenialEntity.objects.bulk_update(denials_entitiy_to_update, ["denial"])

class Migration(migrations.Migration):
dependencies = [
("external_data", "0023_set_denial_entity_type"),
Expand Down
95 changes: 61 additions & 34 deletions api/external_data/migrations/tests/test_0024_denials_data_migration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,65 @@


test_data = [
{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.123","name":"name 1","address":"address 1","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"},
{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 2","address":"address 2","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"},
{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 3","address":"address 3","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"},
{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 4","address":"address 4","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"},
{"reference":"DN2010\/0057","name":"bad record","address":"bad record","notifying_government":"UK","country":"bad","item_list_codes":"all","item_description":"bad","end_use":"bad","reason_for_refusal":"bad "}
{
"reference": "DN2010\/0057",
"regime_reg_ref": "reg.123.123",
"name": "name 1",
"address": "address 1",
"notifying_government": "UK",
"country": "UK",
"item_list_codes": "all",
"item_description": "desc a",
"end_use": "use 1",
"reason_for_refusal": "a",
},
{
"reference": "DN2010\/0057",
"regime_reg_ref": "reg.123.1234",
"name": "name 2",
"address": "address 2",
"notifying_government": "UK",
"country": "UK",
"item_list_codes": "all",
"item_description": "desc a",
"end_use": "use 1",
"reason_for_refusal": "a",
},
{
"reference": "DN2010\/0057",
"regime_reg_ref": "reg.123.1234",
"name": "name 3",
"address": "address 3",
"notifying_government": "UK",
"country": "UK",
"item_list_codes": "all",
"item_description": "desc a",
"end_use": "use 1",
"reason_for_refusal": "a",
},
{
"reference": "DN2010\/0057",
"regime_reg_ref": "reg.123.1234",
"name": "name 4",
"address": "address 4",
"notifying_government": "UK",
"country": "UK",
"item_list_codes": "all",
"item_description": "desc a",
"end_use": "use 1",
"reason_for_refusal": "a",
},
{
"reference": "DN2010\/0057",
"name": "bad record",
"address": "bad record",
"notifying_government": "UK",
"country": "bad",
"item_list_codes": "all",
"item_description": "bad",
"end_use": "bad",
"reason_for_refusal": "bad ",
},
]


Expand All @@ -18,42 +72,15 @@ class TestDenialDataMigration(MigratorTestCase):
migrate_from = ("external_data", "0023_set_denial_entity_type")
migrate_to = ("external_data", "0024_denials_data_migration")


def prepare(self):
DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity")
for row in test_data:
DenialEntity.objects.create(**row)



DenialEntity.objects.create(**row)

def test_0023_denials_data_migration(self):
DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity")
Denial = self.new_state.apps.get_model("external_data", "Denial")

self.assertEqual(DenialEntity.objects.all().count(), 4)
self.assertEqual(Denial.objects.all().count(), 2)
self.assertEqual(Denial.objects.get(regime_reg_ref='reg.123.1234').denial_entity.count(), 3)


@pytest.mark.django_db()
class TestDenialDataDuplicatesMigration(MigratorTestCase):

migrate_from = ("external_data", "0023_set_denial_entity_type")
migrate_to = ("external_data", "0024_denials_data_migration")


def prepare(self):
DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity")
for row in test_data:
DenialEntity.objects.create(**row)
test_data[0]["end_use"] = "end_use b"
DenialEntity.objects.create(**test_data[0])



def test_0024_denials_data_migration_duplicates(self):
DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity")
Denial = self.new_state.apps.get_model("external_data", "Denial")
self.assertEqual(DenialEntity.objects.all().count(), 5)
self.assertEqual(Denial.objects.all().count(), 0)
self.assertEqual(Denial.objects.get(regime_reg_ref="reg.123.1234").denial_entity.count(), 3)

0 comments on commit f90d6cb

Please sign in to comment.