diff --git a/api/applications/tests/test_matching_denials.py b/api/applications/tests/test_matching_denials.py index f292ff6825..d5a3c6323b 100644 --- a/api/applications/tests/test_matching_denials.py +++ b/api/applications/tests/test_matching_denials.py @@ -1,40 +1,25 @@ -import csv -import io import pytest +import os +from django.conf import settings from django.urls import reverse -from faker import Faker from rest_framework import status from api.applications.tests.factories import DenialMatchFactory -from api.external_data import models, serializers +from api.external_data import models from test_helpers.clients import DataTestClient class ApplicationDenialMatchesOnApplicationTests(DataTestClient): def setUp(self): super().setUp() - self.faker = Faker() self.application = self.create_standard_application_case(self.organisation) - denials = [ - {name: self.faker.word() for name in serializers.DenialFromCSVFileSerializer.required_headers} - for _ in range(5) - ] - - content = io.StringIO() - writer = csv.DictWriter( - content, - fieldnames=[*serializers.DenialFromCSVFileSerializer.required_headers, "field_n"], - delimiter=",", - quoting=csv.QUOTE_MINIMAL, - ) - writer.writeheader() - writer.writerows(denials) - response = self.client.post( - reverse("external_data:denial-list"), {"csv_file": content.getvalue()}, **self.gov_headers - ) + file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") + with open(file_path, "rb") as f: + content = f.read() + response = self.client.post(reverse("external_data:denial-list"), {"csv_file": content}, **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(models.DenialEntity.objects.count(), 5) + self.assertEqual(models.DenialEntity.objects.count(), 4) @pytest.mark.xfail(reason="This test is flaky and should be rewritten") # Occasionally causes this error: @@ -60,7 +45,7 @@ def test_adding_denials_to_application(self): def test_revoke_denial_without_comment_failure(self): response = self.client.get(reverse("external_data:denial-list"), **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.json()["count"], 5) + self.assertEqual(response.json()["count"], 4) denials = response.json()["results"] @@ -77,7 +62,7 @@ def test_revoke_denial_without_comment_failure(self): def test_revoke_denial_success(self): response = self.client.get(reverse("external_data:denial-list"), **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.json()["count"], 5) + self.assertEqual(response.json()["count"], 4) denials = response.json()["results"] diff --git a/api/external_data/migrations/0023_set_denial_entity_type.py b/api/external_data/migrations/0023_set_denial_entity_type.py new file mode 100644 index 0000000000..253ad64833 --- /dev/null +++ b/api/external_data/migrations/0023_set_denial_entity_type.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.11 on 2024-04-18 12:00 + +from django.db import migrations +from api.external_data.enums import DenialEntityType + + +def get_denial_entity_type(data): + + if isinstance(data, dict): + entity_type = "" + normalised_entity_type_dict = {keys.lower(): values.lower() for keys, values in data.items()} + + is_end_user_flag = normalised_entity_type_dict.get("end_user_flag", "false") == "true" + is_consignee_flag = normalised_entity_type_dict.get("consignee_flag", "false") == "true" + is_other_role = len(normalised_entity_type_dict.get("other_role", "")) > 0 + + if is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.CONSIGNEE + elif is_end_user_flag and not is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and not is_consignee_flag and is_other_role: + entity_type = DenialEntityType.THIRD_PARTY + + return entity_type + + +def set_denial_entity_type(apps, schema_editor): + + DenialEntity = apps.get_model("external_data", "DenialEntity") + + for denial_entity in DenialEntity.objects.filter(entity_type__isnull=True): + + denial_entity_type = get_denial_entity_type(denial_entity.data) + + if denial_entity_type in ["end_user", "consignee", "third_party"]: + denial_entity.entity_type = denial_entity_type + denial_entity.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("external_data", "0022_denialentity_entity_type"), + ] + + operations = [migrations.RunPython(set_denial_entity_type, migrations.RunPython.noop)] diff --git a/api/external_data/migrations/0024_denials_data_migration.py b/api/external_data/migrations/0024_denials_data_migration.py new file mode 100644 index 0000000000..4a7bfafec6 --- /dev/null +++ b/api/external_data/migrations/0024_denials_data_migration.py @@ -0,0 +1,64 @@ +# Generated by Django 4.2.10 on 2024-04-02 14:59 + +import logging +from django.db import migrations +from django.forms.models import model_to_dict +from django.db.models import Q +from django.db import IntegrityError +from django.db import transaction + +log = logging.getLogger(__name__) + +required_fields = [ + "reference", + "regime_reg_ref", + "notifying_government", + "item_list_codes", + "item_description", + "end_use", + "is_revoked", + "is_revoked_comment", + "reason_for_refusal", +] + +def denials_data_migration(apps, schema_editor): + DenialEntity = apps.get_model("external_data", "DenialEntity") + Denial = apps.get_model("external_data", "Denial") + + # There are a handfull (10) of regime_reg_refs that are null which was in the initial load + # Assumption here is that they can be deleted since it's erroneous data as we now know + # regime_reg_ref is considered a unique DN record. + + total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref='')) + log.info( + "Delete null regime_reg_ref total -> %s", + total_null_regime_reg_ref.count(), + ) + total_null_regime_reg_ref.delete() + duplicate_denial_errors = [] + with transaction.atomic(): + sid = transaction.savepoint() + for denial_entity in DenialEntity.objects.all(): + try: + denial_entity_dict = {key:value for (key,value) in model_to_dict(denial_entity).items() if key in required_fields} + denial , _ = Denial.objects.get_or_create(**denial_entity_dict) + denial_entity.denial = denial + denial_entity.save() + except IntegrityError as e: + duplicate_denial_errors.append(denial_entity.regime_reg_ref) + + if duplicate_denial_errors: + log.info( + "There are the following duplicate denials in the database rolling back this migration: -> %s", + duplicate_denial_errors, + ) + transaction.savepoint_rollback(sid) + else: + transaction.savepoint_commit(sid) + +class Migration(migrations.Migration): + dependencies = [ + ("external_data", "0023_set_denial_entity_type"), + ] + + operations = [migrations.RunPython(denials_data_migration, migrations.RunPython.noop, atomic=False)] diff --git a/api/external_data/migrations/tests/test_0023_set_denial_entity_type.py b/api/external_data/migrations/tests/test_0023_set_denial_entity_type.py new file mode 100644 index 0000000000..8510c63e80 --- /dev/null +++ b/api/external_data/migrations/tests/test_0023_set_denial_entity_type.py @@ -0,0 +1,90 @@ +import pytest + +from django_test_migrations.contrib.unittest_case import MigratorTestCase +from api.external_data.enums import DenialEntityType + +test_data = [ + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "name": "Organisation Name", + "address": "1000 Street Name, City Name", + "notifying_government": "Country Name", + "country": "Country Name", + "item_list_codes": "0A00100", + "item_description": "Medium Size Widget", + "consignee_name": "Example Name", + "end_use": "Used in industry", + "reason_for_refusal": "Risk of outcome", + "spire_entity_id": 123, + "data": {"END_USER_FLAG": "true", "CONSIGNEE_FLAG": "false", "OTHER_ROLE": ""}, + "entity_type": None, + }, + { + "reference": "DN2000/0010", + "regime_reg_ref": "AB-CD-EF-300", + "name": "Organisation Name 3", + "address": "2001 Street Name, City Name 3", + "notifying_government": "Country Name 3", + "country": "Country Name 3", + "item_list_codes": "0A00201", + "item_description": "Unspecified Size Widget", + "consignee_name": "Example Name 3", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 3", + "spire_entity_id": 125, + "data": {"END_USER_FLAG": "false", "CONSIGNEE_FLAG": "true", "OTHER_ROLE": ""}, + "entity_type": None, + }, + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "name": "Organisation Name", + "address": "1000 Street Name, City Name", + "notifying_government": "Country Name", + "country": "Country Name", + "item_list_codes": "0A00100", + "item_description": "Medium Size Widget", + "consignee_name": "Example Name", + "end_use": "Used in industry", + "reason_for_refusal": "Risk of outcome", + "spire_entity_id": 123, + "data": {"END_USER_FLAG": "true", "CONSIGNEE_FLAG": "true", "OTHER_ROLE": ""}, + "entity_type": None, + }, + { + "reference": "DN3000/0000", + "regime_reg_ref": "AB-CD-EF-100", + "name": "Organisation Name XYZ", + "address": "2000 Street Name, City Name 2", + "notifying_government": "Country Name 2", + "country": "Country Name 2", + "item_list_codes": "0A00200", + "item_description": "Large Size Widget", + "consignee_name": "Example Name 2", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 2", + "spire_entity_id": 124, + "data": {"END_USER_FLAG": "false", "CONSIGNEE_FLAG": "false", "OTHER_ROLE": "other"}, + "entity_type": None, + }, +] + + +@pytest.mark.django_db() +class TestDenialEntityTypeSet(MigratorTestCase): + migrate_from = ("external_data", "0022_denialentity_entity_type") + migrate_to = ("external_data", "0023_set_denial_entity_type") + + def prepare(self): + DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") + for row in test_data: + DenialEntity.objects.create(**row) + + def test_0023_set_denial_entity_type(self): + DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") + + self.assertEqual(DenialEntity.objects.all().count(),4) + self.assertEqual(DenialEntity.objects.filter(entity_type=DenialEntityType.END_USER).count(),2) + self.assertEqual(DenialEntity.objects.filter(entity_type=DenialEntityType.CONSIGNEE).count(),1) + self.assertEqual(DenialEntity.objects.filter(entity_type=DenialEntityType.THIRD_PARTY).count(),1) diff --git a/api/external_data/migrations/tests/test_0024_denials_data_migration.py b/api/external_data/migrations/tests/test_0024_denials_data_migration.py new file mode 100644 index 0000000000..a17be6567c --- /dev/null +++ b/api/external_data/migrations/tests/test_0024_denials_data_migration.py @@ -0,0 +1,59 @@ +import pytest + +from django_test_migrations.contrib.unittest_case import MigratorTestCase + + +test_data = [ +{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.123","name":"name 1","address":"address 1","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, +{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 2","address":"address 2","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, +{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 3","address":"address 3","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, +{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 4","address":"address 4","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, +{"reference":"DN2010\/0057","name":"bad record","address":"bad record","notifying_government":"UK","country":"bad","item_list_codes":"all","item_description":"bad","end_use":"bad","reason_for_refusal":"bad "} +] + + +@pytest.mark.django_db() +class TestDenialDataMigration(MigratorTestCase): + + migrate_from = ("external_data", "0023_set_denial_entity_type") + migrate_to = ("external_data", "0024_denials_data_migration") + + + def prepare(self): + DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") + for row in test_data: + DenialEntity.objects.create(**row) + + + + + def test_0023_denials_data_migration(self): + DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") + Denial = self.new_state.apps.get_model("external_data", "Denial") + + self.assertEqual(DenialEntity.objects.all().count(), 4) + self.assertEqual(Denial.objects.all().count(), 2) + self.assertEqual(Denial.objects.get(regime_reg_ref='reg.123.1234').denial_entity.count(), 3) + + +@pytest.mark.django_db() +class TestDenialDataDuplicatesMigration(MigratorTestCase): + + migrate_from = ("external_data", "0023_set_denial_entity_type") + migrate_to = ("external_data", "0024_denials_data_migration") + + + def prepare(self): + DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") + for row in test_data: + DenialEntity.objects.create(**row) + test_data[0]["end_use"] = "end_use b" + DenialEntity.objects.create(**test_data[0]) + + + + def test_0024_denials_data_migration_duplicates(self): + DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") + Denial = self.new_state.apps.get_model("external_data", "Denial") + self.assertEqual(DenialEntity.objects.all().count(), 5) + self.assertEqual(Denial.objects.all().count(), 0) diff --git a/api/external_data/serializers.py b/api/external_data/serializers.py index 9d87629d5b..5b1ef5ac09 100644 --- a/api/external_data/serializers.py +++ b/api/external_data/serializers.py @@ -1,8 +1,8 @@ import csv import io +import logging from django.db import transaction - from django_elasticsearch_dsl_drf.serializers import DocumentSerializer from rest_framework import serializers @@ -55,47 +55,116 @@ def get_entity_type(self, obj): class DenialFromCSVFileSerializer(serializers.Serializer): csv_file = serializers.CharField() + required_headers = [ "reference", + "regime_reg_ref", "name", "address", "notifying_government", "country", "item_list_codes", "item_description", - "consignee_name", "end_use", + "reason_for_refusal", + "spire_entity_id", ] @transaction.atomic def validate_csv_file(self, value): csv_file = io.StringIO(value) - dialect = csv.Sniffer().sniff(csv_file.read(1024)) - csv_file.seek(0) - reader = csv.reader(csv_file, dialect=dialect) - headers = next(reader, None) + reader = csv.DictReader(csv_file) + + # Check if required headers are present + if not (set(self.required_headers)).issubset(set(reader.fieldnames)): # type: ignore + raise serializers.ValidationError("Missing required headers in CSV file") + + logging_counts = {"denial": {"created": 0, "updated": 0}, "denial_entity": {"created": 0, "updated": 0}} + logging_regime_reg_ref_values = { + "denial": {"created": [], "updated": []}, + "denial_entity": {"created": [], "updated": []}, + } errors = [] - valid_serializers = [] for i, row in enumerate(reader, start=1): - data = dict(zip(headers, row)) - serializer = DenialEntitySerializer( - data={ - "data": data, - "created_by": self.context["request"].user, - **{field: data.pop(field, None) for field in self.required_headers}, + denial_entity_data = { + **{field: row[field] for field in self.required_headers}, + "created_by": self.context["request"].user, + } + # Create a serializer instance to validate data + serializer = DenialEntitySerializer(data=denial_entity_data) + if serializer.is_valid() and isinstance(serializer.validated_data, dict): + # Try to update an existing Denial record or create a new one + regime_reg_ref = serializer.validated_data["regime_reg_ref"] + denial_data = { + "reference": serializer.validated_data["reference"], + "notifying_government": serializer.validated_data["notifying_government"], + "item_list_codes": serializer.validated_data["item_list_codes"], + "item_description": serializer.validated_data["item_description"], + "end_use": serializer.validated_data["end_use"], + "reason_for_refusal": serializer.validated_data["reason_for_refusal"], } - ) - - if serializer.is_valid(): - valid_serializers.append(serializer) + denial, is_denial_created = models.Denial.objects.update_or_create( + regime_reg_ref=regime_reg_ref, defaults=denial_data + ) + + if is_denial_created: + logging_counts["denial"]["created"] += 1 + logging_regime_reg_ref_values["denial"]["created"].append(denial.regime_reg_ref) + else: + logging_counts["denial"]["updated"] += 1 + logging_regime_reg_ref_values["denial"]["updated"].append(denial.regime_reg_ref) + + # We assume that a DenialEntity object already exists if we can + # match on all of the following fields + denial_entity_lookup_fields = { + "reference": serializer.validated_data["reference"], + "regime_reg_ref": regime_reg_ref, + "name": serializer.validated_data["name"], + "address": serializer.validated_data["address"], + } + # Link the validated DenialEntity data with the Denial + denial_entity, is_denial_entity_created = models.DenialEntity.objects.update_or_create( + defaults=serializer.validated_data, denial=denial, **denial_entity_lookup_fields + ) + + if is_denial_entity_created: + logging_counts["denial_entity"]["created"] += 1 + logging_regime_reg_ref_values["denial_entity"]["created"].append(denial_entity.regime_reg_ref) + else: + logging_counts["denial_entity"]["updated"] += 1 + logging_regime_reg_ref_values["denial_entity"]["updated"].append(denial_entity.regime_reg_ref) else: - self.add_bulk_errors(errors=errors, row_number=i + 1, line_errors=serializer.errors) + self.add_bulk_errors(errors, i, serializer.errors) + if errors: raise serializers.ValidationError(errors) - else: - # only save if no errors - for serializer in valid_serializers: - serializer.save() + + if logging_counts["denial"]["created"]: + logging.info( + "Created %s Denial records with regime_reg_ref values:\n%s", + logging_counts["denial"]["created"], + "\n".join(logging_regime_reg_ref_values["denial"]["created"]), + ) + if logging_counts["denial"]["updated"]: + logging.info( + "Updated %s Denial records with regime_reg_ref values:\n%s", + logging_counts["denial"]["updated"], + "\n".join(logging_regime_reg_ref_values["denial"]["updated"]), + ) + + if logging_counts["denial_entity"]["created"]: + logging.info( + "Created %s DenialEntity records with regime_reg_ref values:\n%s", + logging_counts["denial_entity"]["created"], + "\n".join(logging_regime_reg_ref_values["denial_entity"]["created"]), + ) + if logging_counts["denial_entity"]["updated"]: + logging.info( + "Updated %s DenialEntity records with regime_reg_ref values:\n%s", + logging_counts["denial_entity"]["updated"], + "\n".join(logging_regime_reg_ref_values["denial_entity"]["updated"]), + ) + return csv_file @staticmethod diff --git a/api/external_data/tests/denial_valid.csv b/api/external_data/tests/denial_valid.csv index 4fab6ec7bc..c021b828f3 100644 --- a/api/external_data/tests/denial_valid.csv +++ b/api/external_data/tests/denial_valid.csv @@ -1,5 +1,5 @@ -reference,name,address,notifying_government,country,item_list_codes,item_description,consignee_name,end_use,end_user_flag,consignee_flag,other_role -FOO123,Jim Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,true,true,false -BAR123,Jak Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,false,true,false -BAG124,Bob Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,false,false,true -BAT123,James Jones,Bob Avenue,France,Germany,ABC123,Foo,Fred Food,used in car,false,false,true +reference,regime_reg_ref,name,address,notifying_government,country,item_list_codes,item_description,end_use,reason_for_refusal,spire_entity_id +DN2000/0000,AB-CD-EF-000,Organisation Name,"1000 Street Name, City Name",Country Name,Country Name,0A00100,Medium Size Widget,Used in industry,Risk of outcome,123 +DN2000/0010,AB-CD-EF-300,Organisation Name 3,"2001 Street Name, City Name 3",Country Name 3,Country Name 3,0A00201,Unspecified Size Widget,Used in other industry,Risk of outcome 3,125 +DN2010/0001,AB-XY-EF-900,The Widget Company,"2 Example Road, Example City",Example Country,Country Name X,"catch all",Extra Large Size Widget,Used in unknown industry,Risk of outcome 4,126 +DN3000/0000,AB-CD-EF-100,Organisation Name XYZ,"2000 Street Name, City Name 2",Country Name 2,Country Name 2,0A00200,Large Size Widget,Used in other industry,Risk of outcome 2,124 diff --git a/api/external_data/tests/test_views.py b/api/external_data/tests/test_views.py index 661e9c563b..23f80f4517 100644 --- a/api/external_data/tests/test_views.py +++ b/api/external_data/tests/test_views.py @@ -11,6 +11,16 @@ from api.external_data import documents, models, serializers from test_helpers.clients import DataTestClient +denial_data_fields = [ + "reference", + "regime_reg_ref", + "notifying_government", + "item_list_codes", + "item_description", + "end_use", + "reason_for_refusal", +] + class DenialViewSetTests(DataTestClient): def test_create_success(self): @@ -26,57 +36,65 @@ def test_create_success(self): list(models.DenialEntity.objects.values(*serializers.DenialFromCSVFileSerializer.required_headers, "data")), [ { - "address": "123 fake street", - "consignee_name": "Fred Food", - "data": {"end_user_flag": "true", "consignee_flag": "true", "other_role": "false"}, - "country": "Germany", - "item_description": "Foo", - "item_list_codes": "ABC123", - "name": "Jim Example", - "notifying_government": "France", - "end_use": "used in car", - "reference": "FOO123", + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "name": "Organisation Name", + "address": "1000 Street Name, City Name", + "notifying_government": "Country Name", + "country": "Country Name", + "item_list_codes": "0A00100", + "item_description": "Medium Size Widget", + "end_use": "Used in industry", + "reason_for_refusal": "Risk of outcome", + "spire_entity_id": 123, + "data": {}, }, { - "address": "123 fake street", - "consignee_name": "Fred Food", - "data": {"end_user_flag": "false", "consignee_flag": "true", "other_role": "false"}, - "country": "Germany", - "item_description": "Foo", - "item_list_codes": "ABC123", - "name": "Jak Example", - "notifying_government": "France", - "end_use": "used in car", - "reference": "BAR123", + "reference": "DN2000/0010", + "regime_reg_ref": "AB-CD-EF-300", + "name": "Organisation Name 3", + "address": "2001 Street Name, City Name 3", + "notifying_government": "Country Name 3", + "country": "Country Name 3", + "item_list_codes": "0A00201", + "item_description": "Unspecified Size Widget", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 3", + "spire_entity_id": 125, + "data": {}, }, { - "address": "123 fake street", - "consignee_name": "Fred Food", - "data": {"end_user_flag": "false", "consignee_flag": "false", "other_role": "true"}, - "country": "Germany", - "item_description": "Foo", - "item_list_codes": "ABC123", - "name": "Bob Example", - "notifying_government": "France", - "end_use": "used in car", - "reference": "BAG124", + "reference": "DN2010/0001", + "regime_reg_ref": "AB-XY-EF-900", + "name": "The Widget Company", + "address": "2 Example Road, Example City", + "notifying_government": "Example Country", + "country": "Country Name X", + "item_list_codes": "catch all", + "item_description": "Extra Large Size Widget", + "end_use": "Used in unknown industry", + "reason_for_refusal": "Risk of outcome 4", + "spire_entity_id": 126, + "data": {}, }, { - "address": "Bob Avenue", - "consignee_name": "Fred Food", - "data": {"end_user_flag": "false", "consignee_flag": "false", "other_role": "true"}, - "country": "Germany", - "item_description": "Foo", - "item_list_codes": "ABC123", - "name": "James Jones", - "notifying_government": "France", - "end_use": "used in car", - "reference": "BAT123", + "reference": "DN3000/0000", + "regime_reg_ref": "AB-CD-EF-100", + "name": "Organisation Name XYZ", + "address": "2000 Street Name, City Name 2", + "notifying_government": "Country Name 2", + "country": "Country Name 2", + "item_list_codes": "0A00200", + "item_description": "Large Size Widget", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 2", + "spire_entity_id": 124, + "data": {}, }, ], ) - def test_create_validation_error(self): + def test_create_error_missing_required_headers(self): url = reverse("external_data:denial-list") file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_invalid.csv") with open(file_path, "rb") as f: @@ -84,18 +102,101 @@ def test_create_validation_error(self): response = self.client.post(url, {"csv_file": content}, **self.gov_headers) self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {"errors": {"csv_file": ["Missing required headers in CSV file"]}}) + + def test_update_success(self): + url = reverse("external_data:denial-list") + content = """ + reference,regime_reg_ref,name,address,notifying_government,country,item_list_codes,item_description,end_use,reason_for_refusal,spire_entity_id + DN2000/0000,AB-CD-EF-000,Organisation Name,"1000 Street Name, City Name",Country Name,Country Name,0A00100,Medium Size Widget,Used in industry,Risk of outcome,123 + """ + response = self.client.post(url, {"csv_file": content}, **self.gov_headers) + self.assertEqual(response.status_code, 201) + self.assertEqual(models.Denial.objects.count(), 1) + self.assertEqual(models.DenialEntity.objects.count(), 1) self.assertEqual( - response.json(), - { - "errors": { - "csv_file": [ - "[Row 2] reference: This field may not be null.", - "[Row 3] reference: This field may not be null.", - "[Row 4] reference: This field may not be null.", - ] - } - }, + list(models.Denial.objects.values(*denial_data_fields)), + [ + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "notifying_government": "Country Name", + "item_list_codes": "0A00100", + "item_description": "Medium Size Widget", + "end_use": "Used in industry", + "reason_for_refusal": "Risk of outcome", + }, + ], + ) + self.assertEqual( + list(models.DenialEntity.objects.values(*serializers.DenialFromCSVFileSerializer.required_headers, "data")), + [ + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "name": "Organisation Name", + "address": "1000 Street Name, City Name", + "notifying_government": "Country Name", + "country": "Country Name", + "item_list_codes": "0A00100", + "item_description": "Medium Size Widget", + "end_use": "Used in industry", + "reason_for_refusal": "Risk of outcome", + "spire_entity_id": 123, + "data": {}, + }, + ], ) + updated_content = """ + reference,regime_reg_ref,name,address,notifying_government,country,item_list_codes,item_description,end_use,reason_for_refusal,spire_entity_id + DN2000/0000,AB-CD-EF-000,Organisation Name,"1000 Street Name, City Name",Country Name 2,Country Name 2,0A00200,Medium Size Widget 2,Used in industry 2,Risk of outcome 2,124 + """ + response = self.client.post(url, {"csv_file": updated_content}, **self.gov_headers) + self.assertEqual(response.status_code, 201) + self.assertEqual(models.Denial.objects.count(), 1) + self.assertEqual(models.DenialEntity.objects.count(), 1) + self.assertEqual( + list(models.Denial.objects.values(*denial_data_fields)), + [ + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "notifying_government": "Country Name 2", + "item_list_codes": "0A00200", + "item_description": "Medium Size Widget 2", + "end_use": "Used in industry 2", + "reason_for_refusal": "Risk of outcome 2", + }, + ], + ) + self.assertEqual( + list(models.DenialEntity.objects.values(*serializers.DenialFromCSVFileSerializer.required_headers, "data")), + [ + { + "reference": "DN2000/0000", + "regime_reg_ref": "AB-CD-EF-000", + "name": "Organisation Name", + "address": "1000 Street Name, City Name", + "notifying_government": "Country Name 2", + "country": "Country Name 2", + "item_list_codes": "0A00200", + "item_description": "Medium Size Widget 2", + "end_use": "Used in industry 2", + "reason_for_refusal": "Risk of outcome 2", + "spire_entity_id": 124, + "data": {}, + }, + ], + ) + + def test_create_error_serializer_errors(self): + url = reverse("external_data:denial-list") + content = """reference,regime_reg_ref,name,address,notifying_government,country,item_list_codes,item_description,end_use,reason_for_refusal,spire_entity_id + ,AB-CD-EF-000,Organisation Name,"1000 Street Name, City Name",Country Name,Country Name,0A00100,Medium Size Widget,Used in industry,Risk of outcome,123 + """ + response = self.client.post(url, {"csv_file": content}, **self.gov_headers) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json(), {"errors": {"csv_file": ["[Row 1] reference: This field may not be blank."]}}) @pytest.mark.skip( reason="Unique constraint on reference is removed temporarily, enable this test once we reinstate that constraint" @@ -125,7 +226,7 @@ def test_create_validation_error_duplicate(self): ) -class DenialSearchView(DataTestClient): +class DenialSearchViewTests(DataTestClient): @pytest.mark.elasticsearch @parameterized.expand( [ @@ -133,7 +234,7 @@ class DenialSearchView(DataTestClient): ({"page": 1},), ] ) - def test_populate_denials(self, page_query): + def test_populate_denial_entity_objects(self, page_query): call_command("search_index", models=["external_data.denialentity"], action="rebuild", force=True) url = reverse("external_data:denial-list") file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") @@ -143,26 +244,26 @@ def test_populate_denials(self, page_query): self.assertEqual(response.status_code, 201) self.assertEqual(models.DenialEntity.objects.count(), 4) - # and one of them is revoked - denial = models.DenialEntity.objects.get(name="Jak Example") - denial.is_revoked = True - denial.save() + # Set one of them as revoked + denial_entity = models.DenialEntity.objects.get(name="Organisation Name") + denial_entity.is_revoked = True + denial_entity.save() - # then only 2 denials will be returned when searching + # Then only 2 denial entity objects will be returned when searching url = reverse("external_data:denial_search-list") - response = self.client.get(url, {**page_query, "search": "name:Example"}, **self.gov_headers) + response = self.client.get(url, {**page_query, "search": "name:Organisation Name XYZ"}, **self.gov_headers) self.assertEqual(response.status_code, 200) response_json = response.json() expected_result = { - "address": "123 fake street", - "country": "Germany", - "item_description": "Foo", - "item_list_codes": "ABC123", - "name": "Jim Example", - "notifying_government": "France", - "end_use": "used in car", - "reference": "FOO123", + "address": "2000 Street Name, City Name 2", + "country": "Country Name 2", + "item_description": "Large Size Widget", + "item_list_codes": "0A00200", + "name": "Organisation Name XYZ", + "notifying_government": "Country Name 2", + "end_use": "Used in other industry", + "reference": "DN3000/0000", } for key, value in expected_result.items(): @@ -174,14 +275,14 @@ def test_populate_denials(self, page_query): @pytest.mark.elasticsearch @parameterized.expand( [ - ({"search": "name:Bob"}, 1), - ({"search": "name:Example"}, 3), - ({"search": "name:Jones"}, 1), - ({"search": "address:123 fake street"}, 3), - ({"search": "address:Bob Avenue"}, 1), + ({"search": "name:Organisation Name"}, 3), + ({"search": "name:The Widget Company"}, 1), + ({"search": "name:XYZ"}, 1), + ({"search": "address:Street Name"}, 3), + ({"search": "address:Example"}, 1), ] ) - def test_denial_search(self, query, quantity): + def test_denial_entity_search(self, query, quantity): call_command("search_index", models=["external_data.denialentity"], action="rebuild", force=True) url = reverse("external_data:denial-list") file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") @@ -198,8 +299,6 @@ def test_denial_search(self, query, quantity): response_json = response.json() self.assertEqual(len(response_json["results"]), quantity) - -class DenialSearchViewTests(DataTestClient): @pytest.mark.elasticsearch def test_search(self): Index("sanctions-alias-test").create(ignore=[400]) diff --git a/lite_routing b/lite_routing index 1ccd775cff..92bfcb2c8f 160000 --- a/lite_routing +++ b/lite_routing @@ -1 +1 @@ -Subproject commit 1ccd775cff757c1e2587576a78d529ab3f3d40d0 +Subproject commit 92bfcb2c8f3acbfd72ed9f668d2daad374f9bbef