diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index d4607f9316..6d2397e9a2 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -44,7 +44,7 @@ def login_dot_gov_logout(request): } query = urlencode(params) op_logout_url = OIDC_OP_LOGOUT_ENDPOINT - redirect_url = "{url}?{query}".format(url=op_logout_url, query=query) + redirect_url = f"{op_logout_url}?{query}" return redirect_url @@ -54,13 +54,13 @@ def generate_username(uuid): def handle_valid_login(user): - logger.debug("Successful login: {}".format(user)) + logger.debug(f"Successful login: {user}") response = HttpResponse() return response def handle_invalid_login(username): - logger.debug("Unauthorized login attempt: {}".format(username)) + logger.debug(f"Unauthorized login attempt: {username}") return HttpResponse('Unauthorized', status=401) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py b/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py new file mode 100644 index 0000000000..266317e055 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/migrations/0004_remove_duplicate_memberships.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.7 on 2024-02-16 20:43 + +from django.db import migrations + + +def delete_memberships_with_overlapping_emails(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") # noqa + Committee = apps.get_model("committee_accounts", "CommitteeAccount") # noqa + + for committee in Committee.objects.all(): + committee_memberships = Membership.objects.filter(committee_account=committee) + + unique_pending_emails = set() + emails_to_prune = set() + for membership in committee_memberships: + pending_email = str(membership.pending_email).lower() + if pending_email not in unique_pending_emails: + unique_pending_emails.add(pending_email) + else: + emails_to_prune.add(pending_email) + + for email in list(emails_to_prune): + # ordering by user places any memberships with a user first + overlapping_memberships = list( + committee_memberships.filter(pending_email__iexact=email).order_by('user') + ) + for membership_to_delete in overlapping_memberships[1:]: + membership_to_delete.delete() + + +class Migration(migrations.Migration): + + dependencies = [( + 'committee_accounts', + '0003_membership_pending_email_alter_membership_id_and_more' + )] + + operations = [ + migrations.RunPython( + delete_memberships_with_overlapping_emails, + migrations.RunPython.noop, + ), + ] diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 7e17550976..19131c6d84 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -53,7 +53,7 @@ def test_remove_member(self): membership_uuid = UUID("136a21f2-66fe-4d56-89e9-0d1d4612741c") view = CommitteeMembershipViewSet() request = self.factory.get( - "/api/v1/committee-members/{membership_uuid}/remove-member" + f"/api/v1/committee-members/{membership_uuid}/remove-member" ) request.user = self.user request.session = { diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 9492501a9e..01b4bc3b1a 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,6 +1,6 @@ from uuid import UUID from fecfiler.user.models import User -from rest_framework import filters, viewsets, mixins +from rest_framework import filters, viewsets, mixins, pagination from django.contrib.sessions.exceptions import SuspiciousSession from fecfiler.transactions.models import ( Transaction, @@ -27,6 +27,11 @@ logger = structlog.get_logger(__name__) +class CommitteeMemberListPagination(pagination.PageNumberPagination): + page_size = 10 + page_size_query_param = "page_size" + + class CommitteeViewSet(viewsets.GenericViewSet, mixins.ListModelMixin): serializer_class = CommitteeAccountSerializer @@ -86,9 +91,21 @@ def get_committee_id(self): raise SuspiciousSession("session has invalid committee_id") return committee_id + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + if "page" in request.query_params: + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + class CommitteeMembershipViewSet(CommitteeOwnedViewMixin, viewsets.ModelViewSet): serializer_class = CommitteeMembershipSerializer + pagination_class = CommitteeMemberListPagination filter_backends = [filters.OrderingFilter] ordering_fields = ["name", "email", "role", "is_active", "created"] ordering = ["-created"] @@ -110,28 +127,12 @@ def get_queryset(self): Value(""), output_field=TextField(), ), - email=Coalesce( - "user__email", "pending_email", output_field=TextField() - ), + email=Coalesce("user__email", "pending_email", output_field=TextField()), is_active=~Q(user=None), ) ) - def list(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - - if "page" in request.query_params: - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - - @action( - detail=False, methods=["post"], url_path="add-member", url_name="add_member" - ) + @action(detail=False, methods=["post"], url_path="add-member", url_name="add_member") def add_member(self, request): committee_uuid = self.request.session["committee_uuid"] committee = CommitteeAccount.objects.filter(id=committee_uuid).first() @@ -161,7 +162,7 @@ def add_member(self, request): # Check for pre-existing membership matching_memberships = self.get_queryset().filter( - Q(pending_email=email) | Q(user__email=email) + Q(pending_email__iexact=email) | Q(user__email__iexact=email) ) if matching_memberships.count() > 0: return Response( @@ -219,12 +220,18 @@ def register_committee(committee_id, user): f1_email = f1_line.split(FS_STR)[11] failure_reason = None - if f1_email != email: + + if ";" in f1_email: + f1_email = f1_email.split(";") + elif "," in f1_email: + f1_email = f1_email.split(",") + else: + f1_email = [f1_email] + + if email not in f1_email: failure_reason = f"Email {email} does not match committee email" - existing_account = CommitteeAccount.objects.filter( - committee_id=committee_id - ).first() + existing_account = CommitteeAccount.objects.filter(committee_id=committee_id).first() if existing_account: failure_reason = f"Committee {committee_id} already registered" diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index 244113ba60..bf0738f6cf 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -84,7 +84,8 @@ def to_representation(self, instance, depth=0): ), ) - representation["has_transaction_or_report"] = query.exists() + if self.context.get("request") and "contacts" in self.context["request"].path: + representation["has_transaction_or_report"] = query.exists() return representation diff --git a/django-backend/fecfiler/contacts/tests/__init__.py b/django-backend/fecfiler/contacts/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/contacts/test_models.py b/django-backend/fecfiler/contacts/tests/test_models.py similarity index 90% rename from django-backend/fecfiler/contacts/test_models.py rename to django-backend/fecfiler/contacts/tests/test_models.py index a8773b1597..e1f719f80e 100644 --- a/django-backend/fecfiler/contacts/test_models.py +++ b/django-backend/fecfiler/contacts/tests/test_models.py @@ -1,5 +1,5 @@ from django.test import TestCase -from .models import Contact +from ..models import Contact class ContactTestCase(TestCase): @@ -34,6 +34,4 @@ def test_save_and_delete(self): self.assertEquals(soft_deleted_contact.first_name, "First") self.assertIsNotNone(soft_deleted_contact.deleted) soft_deleted_contact.hard_delete() - self.assertRaises( - Contact.DoesNotExist, Contact.all_objects.get, last_name="Last" - ) + self.assertRaises(Contact.DoesNotExist, Contact.all_objects.get, last_name="Last") diff --git a/django-backend/fecfiler/contacts/test_serializers.py b/django-backend/fecfiler/contacts/tests/test_serializers.py similarity index 96% rename from django-backend/fecfiler/contacts/test_serializers.py rename to django-backend/fecfiler/contacts/tests/test_serializers.py index e36bdf7975..e60e88c272 100644 --- a/django-backend/fecfiler/contacts/test_serializers.py +++ b/django-backend/fecfiler/contacts/tests/test_serializers.py @@ -1,6 +1,6 @@ from django.test import TestCase -from .models import Contact -from .serializers import ContactSerializer +from ..models import Contact +from ..serializers import ContactSerializer from rest_framework.request import HttpRequest, Request from fecfiler.user.models import User diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/tests/test_views.py similarity index 86% rename from django-backend/fecfiler/contacts/test_views.py rename to django-backend/fecfiler/contacts/tests/test_views.py index cc6cd99509..51c21a9324 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/tests/test_views.py @@ -1,12 +1,12 @@ from unittest import mock - from django.test import RequestFactory, TestCase from rest_framework.test import force_authenticate import uuid from fecfiler.user.models import User -from .models import Contact -from .views import ContactViewSet, DeletedContactsViewSet +from ..models import Contact +from ..views import ContactViewSet, DeletedContactsViewSet +from .utils import create_test_individual_contact mock_results = { "results": [ @@ -69,9 +69,7 @@ def test_candidate_no_candidate_id(self, mock_get): @mock.patch("requests.get", side_effect=mocked_requests_get_candidates) def test_candidate(self, mock_get): - request = self.factory.get( - "/api/v1/contacts/candidate?" "candidate_id=P60012143" - ) + request = self.factory.get("/api/v1/contacts/candidate?candidate_id=P60012143") request.user = self.user response = ContactViewSet.as_view({"get": "candidate"})(request) @@ -219,7 +217,7 @@ def test_individual_lookup_no_q(self): def test_individual_lookup_happy_path(self): request = self.factory.get( - "/api/v1/contacts/individual_lookup?" "q=Lastname&max_fecfile_results=5" + "/api/v1/contacts/individual_lookup?q=Lastname&max_fecfile_results=5" ) request.user = self.user request.session = { @@ -229,7 +227,7 @@ def test_individual_lookup_happy_path(self): response = ContactViewSet.as_view({"get": "individual_lookup"})(request) - expected_json_fragment = '"last_name": "Lastname", "first_name": ' '"Firstname"' + expected_json_fragment = '"last_name": "Lastname", "first_name": "Firstname"' self.assertEqual(response.status_code, 200) self.assertIn(expected_json_fragment, str(response.content, encoding="utf8")) @@ -248,7 +246,7 @@ def test_organization_lookup_no_q(self): def test_organization_lookup_happy_path(self): request = self.factory.get( - "/api/v1/contacts/organization_lookup?" "q=test&max_fecfile_results=5" + "/api/v1/contacts/organization_lookup?q=test&max_fecfile_results=5" ) request.user = self.user request.session = { @@ -282,9 +280,7 @@ def test_get_contact_id_finds_contact(self): response = ContactViewSet.as_view({"get": "get_contact_id"})(request) self.assertEqual(response.status_code, 200) - self.assertEqual( - response.data, uuid.UUID("a03a141a-d2df-402c-93c6-e705ec6007f3") - ) + self.assertEqual(response.data, uuid.UUID("a03a141a-d2df-402c-93c6-e705ec6007f3")) def test_get_contact_id_no_match(self): request = self.factory.get( @@ -363,3 +359,42 @@ def test_update(self): response = ContactViewSet.as_view({"put": "update"})(request, pk=contact.id) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["first_name"], "Other") + + def test_list_paginated(self): + view = ContactViewSet() + view.format_kwarg = "format" + request = self.factory.get("/api/v1/contacts") + request.user = self.user + request.session = { + "committee_uuid": uuid.UUID("11111111-2222-3333-4444-555555555555"), + "committee_id": "C01234567", + } + + for i in range(10): + create_test_individual_contact( + f"last{i}", f"first{i}", "11111111-2222-3333-4444-555555555555" + ) + request.method = "GET" + request.query_params = {"page": 1} + view.request = request + response = view.list(request) + self.assertEqual(len(response.data["results"]), 10) + + def test_list_no_pagination(self): + view = ContactViewSet() + view.format_kwarg = "format" + request = self.factory.get("/api/v1/contacts") + request.user = self.user + request.session = { + "committee_uuid": uuid.UUID("11111111-2222-3333-4444-555555555555"), + "committee_id": "C01234567", + } + request.method = "GET" + request.query_params = {} + view.request = request + response = view.list(request) + try: + response.data["results"] # A non-paginated response will throw an error here + self.assertTrue(response is None) + except TypeError: + self.assertTrue(response is not None) diff --git a/django-backend/fecfiler/contacts/tests/utils.py b/django-backend/fecfiler/contacts/tests/utils.py new file mode 100644 index 0000000000..669775584f --- /dev/null +++ b/django-backend/fecfiler/contacts/tests/utils.py @@ -0,0 +1,11 @@ +from ..models import Contact + + +def create_test_individual_contact(last_name, first_name, committee_account_id): + contact = Contact.objects.create( + type=Contact.ContactType.INDIVIDUAL, + last_name=last_name, + first_name=first_name, + committee_account_id=committee_account_id, + ) + return contact diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 4b33977aab..f6a0e3b37d 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -6,7 +6,7 @@ from django.db.models import CharField, Q, Value from django.db.models.functions import Concat, Lower, Coalesce from django.http import HttpResponseBadRequest, JsonResponse -from rest_framework import viewsets +from rest_framework import viewsets, pagination from fecfiler.committee_accounts.views import ( CommitteeOwnedViewMixin, ) @@ -40,6 +40,11 @@ def validate_and_sanitize_candidate(candidate_id): return candidate_id +class ContactListPagination(pagination.PageNumberPagination): + page_size = 10 + page_size_query_param = "page_size" + + class ContactViewSet(CommitteeOwnedViewMixin, viewsets.ModelViewSet): """ This viewset automatically provides `list`, `create`, `retrieve`, @@ -47,6 +52,7 @@ class ContactViewSet(CommitteeOwnedViewMixin, viewsets.ModelViewSet): """ serializer_class = ContactSerializer + pagination_class = ContactListPagination """Note that this ViewSet inherits from CommitteeOwnedViewMixin The queryset will be further limmited by the user's committee @@ -182,9 +188,7 @@ def committee_lookup(self, request): fecfile_committees = list( self.get_queryset() - .filter( - Q(type="COM") & (Q(committee_id__icontains=q) | Q(name__icontains=q)) - ) + .filter(Q(type="COM") & (Q(committee_id__icontains=q) | Q(name__icontains=q))) .exclude(id__in=exclude_ids) .values() .order_by("-committee_id") @@ -313,6 +317,7 @@ class DeletedContactsViewSet( GenericViewSet, ): serializer_class = ContactSerializer + pagination_class = ContactListPagination queryset = ( Contact.all_objects.filter(deleted__isnull=False) diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index 95cb5384ac..73033cd9bd 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -51,9 +51,7 @@ class Form1MSerializer(ModelSerializer): contact_candidate_V_id = UUIDField(allow_null=True, required=False) # noqa: N815 contact_affiliated = ContactSerializer(allow_null=True, required=False) - contact_candidate_I = ContactSerializer( # noqa: N815 - allow_null=True, required=False - ) + contact_candidate_I = ContactSerializer(allow_null=True, required=False) # noqa: N815 contact_candidate_II = ContactSerializer( # noqa: N815 allow_null=True, required=False ) @@ -63,9 +61,7 @@ class Form1MSerializer(ModelSerializer): contact_candidate_IV = ContactSerializer( # noqa: N815 allow_null=True, required=False ) - contact_candidate_V = ContactSerializer( # noqa: N815 - allow_null=True, required=False - ) + contact_candidate_V = ContactSerializer(allow_null=True, required=False) # noqa: N815 class Meta: fields = [ @@ -100,9 +96,8 @@ class ReportSerializer(CommitteeOwnedSerializer, FecSchemaValidatorSerializerMix report_status = CharField( read_only=True, ) - report_code_label = CharField( - read_only=True, - ) + report_code_label = CharField(read_only=True) + version_label = CharField(read_only=True) is_first = BooleanField(read_only=True) form_3x = Form3XSerializer(required=False) @@ -110,13 +105,12 @@ class ReportSerializer(CommitteeOwnedSerializer, FecSchemaValidatorSerializerMix form_99 = Form99Serializer(required=False) form_1m = Form1MSerializer(required=False) - def to_representation(self, instance, depth=0): + def to_representation(self, instance: Report, depth=0): representation = super().to_representation(instance) form_3x = representation.pop("form_3x") or [] form_24 = representation.pop("form_24") or [] form_99 = representation.pop("form_99") or [] form_1m = representation.pop("form_1m") or [] - if form_3x: representation["report_type"] = "F3X" for property in form_3x: @@ -206,7 +200,13 @@ def get_fields(): "report", "reporttransaction", ] - ] + ["report_status", "fields_to_validate", "report_code_label", "is_first"] + ] + [ + "report_status", + "fields_to_validate", + "report_code_label", + "version_label", + "is_first", + ] fields = get_fields() read_only_fields = ["id", "created", "updated", "is_first"] diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 12e818be20..547a45956c 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -1,4 +1,4 @@ -from rest_framework import filters, status +from rest_framework import filters, status, pagination from rest_framework.response import Response from rest_framework.decorators import action from rest_framework.viewsets import GenericViewSet, ModelViewSet @@ -8,10 +8,10 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import ReportSerializer -from django.db.models import Case, Value, When +from django.db.models import Case, Value, When, CharField, F +from django.db.models.functions import Concat, Trim import structlog - logger = structlog.get_logger(__name__) report_code_label_mapping = Case( @@ -43,6 +43,23 @@ ) +version_labels = { + "F3XN": "Original", + "F3XA": "Amendment", + "F3XT": "Termination", + "F24N": "Original", + "F24A": "Amendment", + "F1MN": "Original", + "F1MA": "Amendment", + "F99": "Original", +} + + +class ReportListPagination(pagination.PageNumberPagination): + page_size = 10 + page_size_query_param = "page_size" + + class ReportViewSet(CommitteeOwnedViewMixin, ModelViewSet): """ This viewset automatically provides `list`, `create`, `retrieve`, @@ -53,11 +70,38 @@ class ReportViewSet(CommitteeOwnedViewMixin, ModelViewSet): in CommitteeOwnedViewMixin's implementation of get_queryset() """ - queryset = Report.objects.annotate( - report_code_label=report_code_label_mapping - ).all() + whens = [When(form_type=k, then=Value(v)) for k, v in version_labels.items()] + + queryset = ( + Report.objects.annotate(report_code_label=report_code_label_mapping) + # alias fields used by the version_label annotation only. not part of payload + .alias( + form_type_label=Case( + *whens, + default=Value(""), + output_field=CharField(), + ), + report_version_label=Case( + When(report_version__isnull=True, then=Value("")), + default=F("report_version"), + output_field=CharField(), + ), + ) + .annotate( + version_label=Trim( + Concat( + F("form_type_label"), + Value(" "), + F("report_version_label"), + output_field=CharField(), + ) + ) + ) + .all() + ) serializer_class = ReportSerializer + pagination_class = ReportListPagination filter_backends = [filters.OrderingFilter] ordering_fields = [ "report_code_label", @@ -65,6 +109,7 @@ class ReportViewSet(CommitteeOwnedViewMixin, ModelViewSet): "form_type", "upload_submission__created", "report_status", + "version_label", ] ordering = ["form_type"] @@ -146,17 +191,6 @@ def partial_update(self, request, pk=None): response = {"message": "Update function is not offered in this path."} return Response(response, status=status.HTTP_405_METHOD_NOT_ALLOWED) - def list(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - if "page" in request.query_params: - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - class ReportViewMixin(GenericViewSet): def get_queryset(self): diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 2d485a05e6..f55b1dbe8a 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -376,7 +376,6 @@ def get_queryset(self): "_calendar_ytd_per_election_office", ), line_label=self.LINE_LABEL_CLAUSE(), - line_label_order_key=self.LINE_LABEL_ORDER_CLAUSE(), ) .alias(order_key=self.ORDER_KEY_CLAUSE()) .order_by("order_key") @@ -429,12 +428,6 @@ def ORDER_KEY_CLAUSE(self): # noqa: N802 D = ["SD9", "SD10"] E = ["SE"] - def LINE_LABEL_ORDER_CLAUSE(self): # noqa: N802 - order = self.C + self.D + self.A_11 + self.A + self.B + self.E - return Case( - *[When(form_type=line, then=Value(i)) for i, line in enumerate(order)] - ) - def LINE_LABEL_CLAUSE(self): # noqa: N802 label_map = { **line_labels_a, diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 7584b30bbc..b366c64028 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -260,10 +260,14 @@ def to_representation(self, instance): representation["reports"] = [] representation["report_ids"] = [] for report in instance.reports.all(): - representation["reports"].append( - ReportSerializer().to_representation(report) - ) representation["report_ids"].append(report.id) + representation["reports"].append({ + "id": report.id, + "coverage_from_date": report.coverage_from_date, + "coverage_through_date": report.coverage_through_date, + "report_code": report.report_code, + "report_type": report.report_type + }) return representation diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index d93ee95581..fbcacbbbac 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -175,33 +175,6 @@ def test_inherited_election_aggregate(self): transaction = response.data self.assertEqual(transaction.get("calendar_ytd_per_election_office"), "58.00") - def test_multisave_transactions(self): - txn1 = deepcopy(self.payloads["IN_KIND"]) - txn1["contact_1"]["last_name"] = "one" - txn2 = deepcopy(self.payloads["IN_KIND"]) - txn2["contact_1"]["last_name"] = "two" - txn3 = deepcopy(self.payloads["IN_KIND"]) - txn3["contact_1"]["last_name"] = "three" - - payload = [txn1, txn2, txn3] - - view_set = TransactionViewSet() - view_set.format_kwarg = {} - request = self.factory.put( - "/api/v1/transactions/multisave/", - json.dumps(payload), - content_type=self.json_content_type, - ) - request.user = self.user - request.data = deepcopy(payload) - view_set.request = request - - view_set = TransactionViewSet() - response = view_set.save_transactions(self.request(payload)) - transactions = response.data - self.assertEqual(len(transactions), 3) - self.assertEqual("one", transactions[0]["contributor_last_name"]) - def test_reatt_redes_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) txn1["contributor_last_name"] = "one" @@ -291,7 +264,7 @@ def test_save_debt(self): report_coverage_from_date = Report.objects.get( id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" ).coverage_from_date - debt_id = response.data["id"] + debt_id = response.data self.assertEqual(response.status_code, 200) debt = Transaction.objects.get(id=debt_id) self.assertEqual( diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 58dcbd9ec0..738e6b1514 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -42,7 +42,7 @@ class TransactionViewSet(CommitteeOwnedViewMixin, ModelViewSet): pagination_class = TransactionListPagination filter_backends = [filters.OrderingFilter] ordering_fields = [ - "line_label_order_key", + "line_label", "created", "transaction_type_identifier", "memo_code", @@ -81,9 +81,7 @@ def get_queryset(self): schedule_filters = self.request.query_params.get("schedules") if schedule_filters is not None: - schedules_to_include = ( - schedule_filters.split(",") if schedule_filters else [] - ) + schedules_to_include = schedule_filters.split(",") if schedule_filters else [] queryset = queryset.filter( schedule__in=[ Schedule[schedule].value for schedule in schedules_to_include @@ -108,12 +106,12 @@ def create(self, request, *args, **kwargs): saved_transaction = self.save_transaction(request.data, request) print(f"transaction ID: {saved_transaction.id}") # transaction_view = self.get_queryset().get(id=saved_transaction.id) - return Response(TransactionSerializer().to_representation(saved_transaction)) + return Response(saved_transaction.id) def update(self, request, *args, **kwargs): with db_transaction.atomic(): saved_transaction = self.save_transaction(request.data, request) - return Response(TransactionSerializer().to_representation(saved_transaction)) + return Response(saved_transaction.id) def partial_update(self, request, pk=None): response = {"message": "Update function is not offered in this path."} @@ -279,9 +277,7 @@ def save_transaction(self, transaction_data, request): child_transaction.parent_transaction_id = transaction_instance.id child_transaction.save() else: - child_transaction_data["parent_transaction_id"] = ( - transaction_instance.id - ) + child_transaction_data["parent_transaction_id"] = transaction_instance.id child_transaction_data.pop("parent_transaction", None) if child_transaction_data.get("use_parent_contact", None): child_transaction_data["contact_1_id"] = ( @@ -293,25 +289,6 @@ def save_transaction(self, transaction_data, request): return self.queryset.get(id=transaction_instance.id) - @action(detail=False, methods=["put"], url_path=r"multisave") - def save_transactions(self, request): - with db_transaction.atomic(): - saved_data = [self.save_transaction(data, request) for data in request.data] - return Response( - [TransactionSerializer().to_representation(data) for data in saved_data] - ) - - def list(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - @action(detail=False, methods=["put"], url_path=r"multisave/reattribution") def save_reatt_redes_transactions(self, request): with db_transaction.atomic(): @@ -324,14 +301,15 @@ def save_reatt_redes_transactions(self, request): request.data[1]["reatt_redes"] = reatt_redes request.data[1]["reatt_redes_id"] = reatt_redes.id child = request.data[1].get("children", [])[0] - child["reatt_redes"] = reatt_redes + child[" "] = reatt_redes child["reatt_redes_id"] = reatt_redes.id request.data[1]["children"] = [child] to = self.save_transaction(request.data[1], request) saved_data = [reatt_redes, to] - return Response( - [TransactionSerializer().to_representation(data) for data in saved_data] - ) + ids = [] + for data in saved_data: + ids.append(data.id) + return Response(ids) @action(detail=False, methods=["post"], url_path=r"add-to-report") def add_transaction_to_report(self, request): diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py index a7ae2ca563..5dcc0c23d7 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py @@ -63,7 +63,7 @@ def submit(self, dot_fec_bytes, json_payload, fec_report_id=None): "report_id": fec_report_id or str(uuid()), } ) - logger.debug("FEC upload response: {response}") + logger.debug(f"FEC upload response: {response}") return response def poll_status(self, submission_id): diff --git a/django-backend/fecfiler/web_services/dot_fec/web_print_submitter.py b/django-backend/fecfiler/web_services/dot_fec/web_print_submitter.py index 5ed78d3796..ec42554fe0 100644 --- a/django-backend/fecfiler/web_services/dot_fec/web_print_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/web_print_submitter.py @@ -33,7 +33,7 @@ def submit(self, email, dot_fec_bytes): "batch_id": 123, } ) - logger.debug("FEC upload response: {response}") + logger.debug(f"FEC upload response: {response}") return response def poll_status(self, batch_id, submission_id): diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 0b06278f0f..bf8010e6cd 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -26,6 +26,7 @@ def create_dot_fec( upload_submission_id=None, webprint_submission_id=None, force_write_to_disk=False, + file_name=None, ): if upload_submission_id: submission = UploadSubmission.objects.get(id=upload_submission_id) @@ -35,9 +36,10 @@ def create_dot_fec( submission.save_state(FECSubmissionState.CREATING_FILE) try: file_content = compose_dot_fec(report_id, upload_submission_id) - file_name = f"{report_id}_{math.floor(datetime.now().timestamp())}.fec" + if file_name is None: + file_name = f"{report_id}_{math.floor(datetime.now().timestamp())}.fec" - if not file_content or not file_name: + if not file_content: raise Exception("No file created") store_file(file_content, file_name, force_write_to_disk) dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) diff --git a/django-backend/fecfiler/web_services/test_views.py b/django-backend/fecfiler/web_services/test_views.py index 7974b21a53..2207ee7682 100644 --- a/django-backend/fecfiler/web_services/test_views.py +++ b/django-backend/fecfiler/web_services/test_views.py @@ -8,10 +8,16 @@ from fecfiler.reports.tests.utils import create_form3x from fecfiler.web_services.summary.tasks import CalculationState +from unittest.mock import patch +from uuid import UUID + @override_settings(CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPOGATES=True) class WebServicesViewSetTest(TestCase): - fixtures = ["C01234567_user_and_committee"] + fixtures = [ + "C01234567_user_and_committee", + "test_f3x_reports", + ] def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") @@ -19,6 +25,8 @@ def setUp(self): create_committee_view(self.committee.id) self.committee.members.add(self.user) self.factory = RequestFactory() + self.view = WebServicesViewSet() + self.task_id = "testTaskId" def test_create_dot_fec(self): report = create_form3x( @@ -30,6 +38,7 @@ def test_create_dot_fec(self): ) force_authenticate(request, self.user) request.session = {"committee_uuid": self.committee.id} + response = WebServicesViewSet.as_view({"post": "create_dot_fec"})(request) self.assertEqual(response.status_code, 200) report.refresh_from_db() @@ -97,3 +106,41 @@ def test_submit_to_fec(self): report.refresh_from_db() # assert that summary was caclulated self.assertEqual(report.form_3x.L8_cash_on_hand_at_close_period, 1) + + @patch("fecfiler.web_services.views.AsyncResult") + def test_check_dot_fec_ready(self, mock_async_result): + request = self.factory.get(f"api/v1/web-services/dot-fec/check/{self.task_id}") + mock_instance = mock_async_result.return_value + mock_instance.ready.return_value = True + mock_instance.get.return_value = "testId" + self.view.request = request + response = self.view.check_dot_fec(request, self.task_id) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, {"done": True, "id": "testId"}) + + @patch("fecfiler.web_services.views.AsyncResult") + def test_check_dot_fec_not_ready(self, mock_async_result): + request = self.factory.get(f"api/v1/web-services/dot-fec/check/{self.task_id}") + mock_instance = mock_async_result.return_value + mock_instance.ready.return_value = False + mock_instance.get.return_value = "testId" + self.view.request = request + response = self.view.check_dot_fec(request, self.task_id) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data, {"done": False}) + + def test_get_dot_fec_not_exists(self): + id = "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" + + request = self.factory.get(f"api/v1/web-services/dot-fec/{id}") + request.session = { + "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), + "committee_id": "C01234567", + } + force_authenticate(request, user=self.user) + response = self.view.get_dot_fec(request, id) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, f"No .FEC was found for id: {id}") diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index 38bbb86032..f68856a42c 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -1,3 +1,5 @@ +import math +from datetime import datetime from wsgiref.util import FileWrapper from rest_framework import viewsets, status from rest_framework.response import Response @@ -14,6 +16,8 @@ from .web_service_storage import get_file from .models import DotFEC, UploadSubmission, WebPrintSubmission from fecfiler.reports.models import Report +from celery.result import AsyncResult + import structlog logger = structlog.get_logger(__name__) @@ -40,30 +44,44 @@ def create_dot_fec(self, request): report_id = serializer.validated_data["report_id"] logger.debug(f"Starting Celery Task create_dot_fec for report :{report_id}") + file_name = f"{report_id}_{math.floor(datetime.now().timestamp())}.fec" task = self.get_calculation_task(request, report_id) if task: - task = (task | create_dot_fec.s()).apply_async() + task = (task | create_dot_fec.s(None, None, False, file_name)).apply_async() else: - task = create_dot_fec.apply_async((report_id, None, None), retry=False) + task = create_dot_fec.apply_async( + (report_id, None, None, False, file_name), retry=False + ) logger.debug(f"Status from create_dot_fec report {report_id}: {task.status}") - return Response({"status": ".FEC task created"}) + return Response({"file_name": file_name, "task_id": task.task_id}) + + @action( + detail=False, + methods=["get"], + url_path="dot-fec/check/(?P[a-z0-9-]+)", + ) + def check_dot_fec(self, request, task_id): + res = AsyncResult(task_id) + if res.ready(): + return Response({"done": True, "id": res.get()}) + return Response({"done": False}) @action( detail=False, methods=["get"], - url_path="dot-fec/(?P[a-z0-9-]+)", + url_path="dot-fec/(?P[a-z0-9-]+)", renderer_classes=(DotFECRenderer,), ) - def get_dot_fec(self, request, report_id): + def get_dot_fec(self, request, id): """Download the most recent .FEC created for a report Currently only useful for testing purposes """ committee_uuid = request.session["committee_uuid"] dot_fec_record = DotFEC.objects.filter( - report__id=report_id, report__committee_account_id=committee_uuid - ).order_by("-file_name") + id=id, report__committee_account_id=committee_uuid + ) if not dot_fec_record.exists(): - not_found_msg = f"No .FEC was found for report id: {report_id}" + not_found_msg = f"No .FEC was found for id: {id}" logger.error(not_found_msg) return Response(not_found_msg, status=status.HTTP_400_BAD_REQUEST) file_name = dot_fec_record.first().file_name